<commit_before>//
// Aspia Project
// Copyright (C) 2016-2022 Dmitry Chapyshev <dmitry@aspia.ru>
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU General Public License for more details.
//
// You should have received a copy of the GNU General Public License
// along with this program. If not, see <https://www.gnu.org/licenses/>.
//

#include "common/ui/text_chat_widget.h"

#include "base/logging.h"
#include "common/ui/text_chat_incoming_message.h"
#include "common/ui/text_chat_outgoing_message.h"
#include "common/ui/text_chat_status_message.h"
#include "ui_text_chat_widget.h"

#include <QDateTime>
#include <QFile>
#include <QFileDialog>
#include <QHostInfo>
#include <QKeyEvent>
#include <QMenu>
#include <QMessageBox>
#include <QScrollBar>
#include <QTimer>

namespace common {

namespace {

const int kMaxMessageLength = 2048;

} // namespace

TextChatWidget::TextChatWidget(QWidget* parent)
    : QWidget(parent),
      ui(std::make_unique<Ui::TextChatWidget>()),
      host_name_(QHostInfo::localHostName().toStdString()),
      status_clear_timer_(new QTimer(this))
{
    ui->setupUi(this);
    ui->edit_message->installEventFilter(this);
    ui->list_messages->horizontalScrollBar()->installEventFilter(this);
    ui->list_messages->verticalScrollBar()->installEventFilter(this);

    connect(status_clear_timer_, &QTimer::timeout, ui->label_status, &QLabel::clear);
    connect(ui->button_send, &QToolButton::clicked, this, &TextChatWidget::onSendMessage);
    connect(ui->button_tools, &QToolButton::clicked, this, [=]()
    {
        QMenu menu;
        QAction* save_chat_action = menu.addAction(tr("Save chat..."));
        QAction* clear_history_action = menu.addAction(tr("Clear chat"));

        menu.show();

        QPoint pos = ui->button_tools->mapToGlobal(ui->button_tools->rect().topRight());
        pos.setX(pos.x() - menu.rect().width());
        pos.setY(pos.y() - menu.rect().height());

        QAction* action = menu.exec(pos);
        if (action == clear_history_action)
        {
            onClearHistory();
        }
        else if (action == save_chat_action)
        {
            onSaveChat();
        }
    });

    ui->edit_message->setFocus();
}

TextChatWidget::~TextChatWidget() = default;

void TextChatWidget::readMessage(const proto::TextChatMessage& message)
{
    QListWidget* list_messages = ui->list_messages;
    TextChatIncomingMessage* message_widget = new TextChatIncomingMessage(list_messages);

    message_widget->setTimestamp(message.timestamp());
    message_widget->setSource(QString::fromStdString(message.source()));
    message_widget->setMessageText(QString::fromStdString(message.text()));

    QListWidgetItem* item = new QListWidgetItem(list_messages);

    list_messages->addItem(item);
    list_messages->setItemWidget(item, message_widget);

    onUpdateSize();
}

void TextChatWidget::readStatus(const proto::TextChatStatus& status)
{
    status_clear_timer_->stop();

    QString user_name = QString::fromStdString(status.source());

    switch (status.status())
    {
        case proto::TextChatStatus::STATUS_TYPING:
            ui->label_status->setText(tr("%1 is typing...").arg(user_name));
            break;

        case proto::TextChatStatus::STATUS_STARTED:
        {
            QString time = QLocale::system().toString(QTime::currentTime(), QLocale::ShortFormat);
            addStatusMessage(tr("User %1 has joined the chat (%2)").arg(user_name, time));
        }
        break;

        case proto::TextChatStatus::STATUS_STOPPED:
        {
            QString time = QLocale::system().toString(QTime::currentTime(), QLocale::ShortFormat);
            addStatusMessage(tr("User %1 has left the chat (%2)").arg(user_name, time));
        }
        break;

        default:
            LOG(LS_WARNING) << "Unhandled status code: " << static_cast<int>(status.status());
            return;
    }

    status_clear_timer_->start(std::chrono::seconds(1));
}

bool TextChatWidget::eventFilter(QObject* object, QEvent* event)
{
    if (object == ui->edit_message && event->type() == QEvent::KeyPress)
    {
        QKeyEvent* key_event = static_cast<QKeyEvent*>(event);
        if (key_event->key() == Qt::Key_Return)
        {
            onSendMessage();
            return true;
        }

        onSendStatus(proto::TextChatStatus::STATUS_TYPING);
    }
    else if (object == ui->list_messages->horizontalScrollBar() ||
             object == ui->list_messages->verticalScrollBar())
    {
        if (event->type() == QEvent::Show || event->type() == QEvent::Hide)
        {
            onUpdateSize();
        }
    }

    return QWidget::eventFilter(object, event);
}

void TextChatWidget::resizeEvent(QResizeEvent* /* event */)
{
    onUpdateSize();
}

void TextChatWidget::closeEvent(QCloseEvent* event)
{
    onSendStatus(proto::TextChatStatus::STATUS_STOPPED);
    emit textChatClosed();

    QWidget::closeEvent(event);
}

void TextChatWidget::addOutgoingMessage(time_t timestamp, const QString& message)
{
    QListWidget* list_messages = ui->list_messages;
    TextChatOutgoingMessage* message_widget = new TextChatOutgoingMessage(list_messages);

    message_widget->setTimestamp(timestamp);
    message_widget->setMessageText(message);

    QListWidgetItem* item = new QListWidgetItem(list_messages);

    list_messages->addItem(item);
    list_messages->setItemWidget(item, message_widget);

    onUpdateSize();
}

void TextChatWidget::addStatusMessage(const QString& message)
{
    QListWidget* list_messages = ui->list_messages;
    TextChatStatusMessage* message_widget = new TextChatStatusMessage(list_messages);

    message_widget->setMessageText(message);

    QListWidgetItem* item = new QListWidgetItem(list_messages);

    list_messages->addItem(item);
    list_messages->setItemWidget(item, message_widget);

    onUpdateSize();
}

void TextChatWidget::onSendMessage()
{
    QLineEdit* edit_message = ui->edit_message;
    QString message = edit_message->text();
    if (message.isEmpty())
        return;

    if (message.length() > kMaxMessageLength)
    {
        QMessageBox::warning(this,
                             tr("Warning"),
                             tr("The message is too long. The maximum message length is %n "
                                "characters.", "", kMaxMessageLength),
                             QMessageBox::Ok);
        return;
    }

    int64_t timestamp = QDateTime::currentSecsSinceEpoch();

    addOutgoingMessage(timestamp, message);
    edit_message->clear();
    edit_message->setFocus();

    proto::TextChatMessage text_chat_message;
    text_chat_message.set_timestamp(timestamp);
    text_chat_message.set_source(host_name_);
    text_chat_message.set_text(message.toStdString());

    emit sendMessage(text_chat_message);
}

void TextChatWidget::onSendStatus(proto::TextChatStatus::Status status)
{
    proto::TextChatStatus text_chat_status;
    text_chat_status.set_timestamp(QDateTime::currentSecsSinceEpoch());
    text_chat_status.set_source(host_name_);
    text_chat_status.set_status(status);

    emit sendStatus(text_chat_status);
}

void TextChatWidget::onClearHistory()
{
    QListWidget* list_messages = ui->list_messages;
    for (int i = list_messages->count() - 1; i >= 0; --i)
        delete list_messages->item(i);
}

void TextChatWidget::onSaveChat()
{
    QString selected_filter;
    QString file_path = QFileDialog::getSaveFileName(this,
                                                     tr("Save File"),
                                                     QString(),
                                                     tr("TXT files (*.txt)"),
                                                     &selected_filter);
    if (file_path.isEmpty() || selected_filter.isEmpty())
        return;

    QFile file(file_path);
    if (!file.open(QIODevice::WriteOnly | QIODevice::Text))
    {
        QMessageBox::warning(this,
                             tr("Warning"),
                             tr("Could not open file for writing."),
                             QMessageBox::Ok);
        return;
    }

    QListWidget* list_messages = ui->list_messages;
    QTextStream stream(&file);

    for (int i = 0; i < list_messages->count(); ++i)
    {
        QListWidgetItem* item = list_messages->item(i);
        TextChatMessage* message_widget =
            static_cast<TextChatMessage*>(list_messages->itemWidget(item));

        if (message_widget->direction() == TextChatMessage::Direction::INCOMING)
        {
            TextChatIncomingMessage* incoming_message_widget =
                static_cast<TextChatIncomingMessage*>(message_widget);

            stream << "[" << incoming_message_widget->messageTime() << "] "
                   << incoming_message_widget->source() << Qt::endl;
            stream << incoming_message_widget->messageText() << Qt::endl;
            stream << Qt::endl;
        }
        else
        {
            DCHECK_EQ(message_widget->direction(), TextChatMessage::Direction::OUTGOING);

            TextChatOutgoingMessage* outgoing_message_widget =
                static_cast<TextChatOutgoingMessage*>(message_widget);

            stream << "[" << outgoing_message_widget->messageTime() << "] " << Qt::endl;
            stream << outgoing_message_widget->messageText() << Qt::endl;
            stream << Qt::endl;
        }
    }
}

void TextChatWidget::onUpdateSize()
{
    QListWidget* list_messages = ui->list_messages;
    int count = list_messages->count();

    for (int i = 0; i < count; ++i)
    {
        QListWidgetItem* item = list_messages->item(i);

        TextChatMessage* message_widget =
            static_cast<TextChatMessage*>(list_messages->itemWidget(item));
        int viewport_width = list_messages->viewport()->width();

        message_widget->setFixedWidth(viewport_width);
        message_widget->setFixedHeight(message_widget->heightForWidth(viewport_width));

        item->setSizeHint(message_widget->size());
    }

    list_messages->scrollToBottom();
}

} // namespace common
<commit_msg>Saving status messages.<commit_after>//
// Aspia Project
// Copyright (C) 2016-2022 Dmitry Chapyshev <dmitry@aspia.ru>
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU General Public License for more details.
//
// You should have received a copy of the GNU General Public License
// along with this program. If not, see <https://www.gnu.org/licenses/>.
//

#include "common/ui/text_chat_widget.h"

#include "base/logging.h"
#include "common/ui/text_chat_incoming_message.h"
#include "common/ui/text_chat_outgoing_message.h"
#include "common/ui/text_chat_status_message.h"
#include "ui_text_chat_widget.h"

#include <QDateTime>
#include <QFile>
#include <QFileDialog>
#include <QHostInfo>
#include <QKeyEvent>
#include <QMenu>
#include <QMessageBox>
#include <QScrollBar>
#include <QTimer>

namespace common {

namespace {

const int kMaxMessageLength = 2048;

} // namespace

TextChatWidget::TextChatWidget(QWidget* parent)
    : QWidget(parent),
      ui(std::make_unique<Ui::TextChatWidget>()),
      host_name_(QHostInfo::localHostName().toStdString()),
      status_clear_timer_(new QTimer(this))
{
    ui->setupUi(this);
    ui->edit_message->installEventFilter(this);
    ui->list_messages->horizontalScrollBar()->installEventFilter(this);
    ui->list_messages->verticalScrollBar()->installEventFilter(this);

    connect(status_clear_timer_, &QTimer::timeout, ui->label_status, &QLabel::clear);
    connect(ui->button_send, &QToolButton::clicked, this, &TextChatWidget::onSendMessage);
    connect(ui->button_tools, &QToolButton::clicked, this, [=]()
    {
        QMenu menu;
        QAction* save_chat_action = menu.addAction(tr("Save chat..."));
        QAction* clear_history_action = menu.addAction(tr("Clear chat"));

        menu.show();

        QPoint pos = ui->button_tools->mapToGlobal(ui->button_tools->rect().topRight());
        pos.setX(pos.x() - menu.rect().width());
        pos.setY(pos.y() - menu.rect().height());

        QAction* action = menu.exec(pos);
        if (action == clear_history_action)
        {
            onClearHistory();
        }
        else if (action == save_chat_action)
        {
            onSaveChat();
        }
    });

    ui->edit_message->setFocus();
}

TextChatWidget::~TextChatWidget() = default;

void TextChatWidget::readMessage(const proto::TextChatMessage& message)
{
    QListWidget* list_messages = ui->list_messages;
    TextChatIncomingMessage* message_widget = new TextChatIncomingMessage(list_messages);

    message_widget->setTimestamp(message.timestamp());
    message_widget->setSource(QString::fromStdString(message.source()));
    message_widget->setMessageText(QString::fromStdString(message.text()));

    QListWidgetItem* item = new QListWidgetItem(list_messages);

    list_messages->addItem(item);
    list_messages->setItemWidget(item, message_widget);

    onUpdateSize();
}

void TextChatWidget::readStatus(const proto::TextChatStatus& status)
{
    status_clear_timer_->stop();

    QString user_name = QString::fromStdString(status.source());

    switch (status.status())
    {
        case proto::TextChatStatus::STATUS_TYPING:
            ui->label_status->setText(tr("%1 is typing...").arg(user_name));
            break;

        case proto::TextChatStatus::STATUS_STARTED:
        {
            QString time = QLocale::system().toString(QTime::currentTime(), QLocale::ShortFormat);
            addStatusMessage(tr("User %1 has joined the chat (%2)").arg(user_name, time));
        }
        break;

        case proto::TextChatStatus::STATUS_STOPPED:
        {
            QString time = QLocale::system().toString(QTime::currentTime(), QLocale::ShortFormat);
            addStatusMessage(tr("User %1 has left the chat (%2)").arg(user_name, time));
        }
        break;

        default:
            LOG(LS_WARNING) << "Unhandled status code: " << static_cast<int>(status.status());
            return;
    }

    status_clear_timer_->start(std::chrono::seconds(1));
}

bool TextChatWidget::eventFilter(QObject* object, QEvent* event)
{
    if (object == ui->edit_message && event->type() == QEvent::KeyPress)
    {
        QKeyEvent* key_event = static_cast<QKeyEvent*>(event);
        if (key_event->key() == Qt::Key_Return)
        {
            onSendMessage();
            return true;
        }

        onSendStatus(proto::TextChatStatus::STATUS_TYPING);
    }
    else if (object == ui->list_messages->horizontalScrollBar() ||
             object == ui->list_messages->verticalScrollBar())
    {
        if (event->type() == QEvent::Show || event->type() == QEvent::Hide)
        {
            onUpdateSize();
        }
    }

    return QWidget::eventFilter(object, event);
}

void TextChatWidget::resizeEvent(QResizeEvent* /* event */)
{
    onUpdateSize();
}

void TextChatWidget::closeEvent(QCloseEvent* event)
{
    onSendStatus(proto::TextChatStatus::STATUS_STOPPED);
    emit textChatClosed();

    QWidget::closeEvent(event);
}

void TextChatWidget::addOutgoingMessage(time_t timestamp, const QString& message)
{
    QListWidget* list_messages = ui->list_messages;
    TextChatOutgoingMessage* message_widget = new TextChatOutgoingMessage(list_messages);

    message_widget->setTimestamp(timestamp);
    message_widget->setMessageText(message);

    QListWidgetItem* item = new QListWidgetItem(list_messages);

    list_messages->addItem(item);
    list_messages->setItemWidget(item, message_widget);

    onUpdateSize();
}

void TextChatWidget::addStatusMessage(const QString& message)
{
    QListWidget* list_messages = ui->list_messages;
    TextChatStatusMessage* message_widget = new TextChatStatusMessage(list_messages);

    message_widget->setMessageText(message);

    QListWidgetItem* item = new QListWidgetItem(list_messages);

    list_messages->addItem(item);
    list_messages->setItemWidget(item, message_widget);

    onUpdateSize();
}

void TextChatWidget::onSendMessage()
{
    QLineEdit* edit_message = ui->edit_message;
    QString message = edit_message->text();
    if (message.isEmpty())
        return;

    if (message.length() > kMaxMessageLength)
    {
        QMessageBox::warning(this,
                             tr("Warning"),
                             tr("The message is too long. The maximum message length is %n "
                                "characters.", "", kMaxMessageLength),
                             QMessageBox::Ok);
        return;
    }

    int64_t timestamp = QDateTime::currentSecsSinceEpoch();

    addOutgoingMessage(timestamp, message);
    edit_message->clear();
    edit_message->setFocus();

    proto::TextChatMessage text_chat_message;
    text_chat_message.set_timestamp(timestamp);
    text_chat_message.set_source(host_name_);
    text_chat_message.set_text(message.toStdString());

    emit sendMessage(text_chat_message);
}

void TextChatWidget::onSendStatus(proto::TextChatStatus::Status status)
{
    proto::TextChatStatus text_chat_status;
    text_chat_status.set_timestamp(QDateTime::currentSecsSinceEpoch());
    text_chat_status.set_source(host_name_);
    text_chat_status.set_status(status);

    emit sendStatus(text_chat_status);
}

void TextChatWidget::onClearHistory()
{
    QListWidget* list_messages = ui->list_messages;
    for (int i = list_messages->count() - 1; i >= 0; --i)
        delete list_messages->item(i);
}

void TextChatWidget::onSaveChat()
{
    QString selected_filter;
    QString file_path = QFileDialog::getSaveFileName(this,
                                                     tr("Save File"),
                                                     QString(),
                                                     tr("TXT files (*.txt)"),
                                                     &selected_filter);
    if (file_path.isEmpty() || selected_filter.isEmpty())
        return;

    QFile file(file_path);
    if (!file.open(QIODevice::WriteOnly | QIODevice::Text))
    {
        QMessageBox::warning(this,
                             tr("Warning"),
                             tr("Could not open file for writing."),
                             QMessageBox::Ok);
        return;
    }

    QListWidget* list_messages = ui->list_messages;
    QTextStream stream(&file);

    for (int i = 0; i < list_messages->count(); ++i)
    {
        QListWidgetItem* item = list_messages->item(i);
        TextChatMessage* message_widget =
            static_cast<TextChatMessage*>(list_messages->itemWidget(item));

        if (message_widget->direction() == TextChatMessage::Direction::INCOMING)
        {
            TextChatIncomingMessage* incoming_message_widget =
                static_cast<TextChatIncomingMessage*>(message_widget);

            stream << "[" << incoming_message_widget->messageTime() << "] "
                   << incoming_message_widget->source() << Qt::endl;
            stream << incoming_message_widget->messageText() << Qt::endl;
            stream << Qt::endl;
        }
        else if (message_widget->direction() == TextChatMessage::Direction::OUTGOING)
        {
            TextChatOutgoingMessage* outgoing_message_widget =
                static_cast<TextChatOutgoingMessage*>(message_widget);

            stream << "[" << outgoing_message_widget->messageTime() << "] " << Qt::endl;
            stream << outgoing_message_widget->messageText() << Qt::endl;
            stream << Qt::endl;
        }
        else
        {
            DCHECK_EQ(message_widget->direction(), TextChatMessage::Direction::STATUS);

            TextChatStatusMessage* status_message_widget =
                static_cast<TextChatStatusMessage*>(message_widget);

            stream << status_message_widget->messageText() << Qt::endl;
            stream << Qt::endl;
        }
    }
}

void TextChatWidget::onUpdateSize()
{
    QListWidget* list_messages = ui->list_messages;
    int count = list_messages->count();

    for (int i = 0; i < count; ++i)
    {
        QListWidgetItem* item = list_messages->item(i);

        TextChatMessage* message_widget =
            static_cast<TextChatMessage*>(list_messages->itemWidget(item));
        int viewport_width = list_messages->viewport()->width();

        message_widget->setFixedWidth(viewport_width);
        message_widget->setFixedHeight(message_widget->heightForWidth(viewport_width));

        item->setSizeHint(message_widget->size());
    }

    list_messages->scrollToBottom();
}

} // namespace common
<|endoftext|>


<commit_before>/*
 * Copyright (C) 2011 Clearspring Technologies, Inc.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 * http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

/*
 * Copyright (C) 2018 ScyllaDB
 */

/*
 * This file is part of Scylla.
 *
 * Scylla is free software: you can redistribute it and/or modify
 * it under the terms of the GNU Affero General Public License as published by
 * the Free Software Foundation, either version 3 of the License, or
 * (at your option) any later version.
 *
 * Scylla is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 * GNU General Public License for more details.
 *
 * You should have received a copy of the GNU General Public License
 * along with Scylla.  If not, see <http://www.gnu.org/licenses/>.
 */

#pragma once

/*
 Based on the following implementation ([2]) for the Space-Saving algorithm from [1].

 [1] Metwally, A., Agrawal, D., & El Abbadi, A. (2005, January).
     Efficient computation of frequent and top-k elements in data streams.
     In International Conference on Database Theory (pp. 398-412). Springer, Berlin, Heidelberg.
     http://www.cse.ust.hk/~raywong/comp5331/References/EfficientComputationOfFrequentAndTop-kElementsInDataStreams.pdf

 [2] https://github.com/addthis/stream-lib/blob/master/src/main/java/com/clearspring/analytics/stream/StreamSummary.java

 The algorithm keeps a map between keys seen and their counts, keeping a bound on the number of tracked keys.
 Replacement policy evicts the key with the lowest count while inheriting its count, and recording an estimation
 of the error which results from that.
 This error estimation can be later used to prove if the distribution we arrived at corresponds to the real top-K,
 which we can display alongside the results.
 Accuracy depends on the number of tracked keys.

*/

#include <cstdio>
#include <list>
#include <unordered_map>
#include <memory>
#include <tuple>
#include <assert.h>

#include <seastar/core/shared_ptr.hh>
#include <seastar/core/sstring.hh>
#include "utils/chunked_vector.hh"

namespace utils {

using namespace seastar;

template <class T, class Hash = std::hash<T>, class KeyEqual = std::equal_to<T>>
class space_saving_top_k {
private:
    struct bucket;
    using buckets_iterator = typename std::list<bucket>::iterator;

    struct counter {
        buckets_iterator bucket_it;
        T item;
        unsigned count = 0;
        unsigned error = 0;

        counter(T item) : item(item) {}
    };

    using counter_ptr = lw_shared_ptr<counter>;

    using counters = std::list<counter_ptr>;
    using counters_iterator = typename counters::iterator;

    using counters_map = std::unordered_map<T, counters_iterator, Hash, KeyEqual>;
    using counters_map_iterator = typename counters_map::iterator;

    struct bucket {
        std::list<counter_ptr> counters;
        unsigned count;

        bucket(counter_ptr ctr) {
            count = ctr->count;
            counters.push_back(ctr);
        }

        bucket(T item, unsigned count) {
            counters.push_back(make_lw_shared<counter>(item));
            this->count = count;
        }
    };

    using buckets = std::list<bucket>;

    size_t _capacity;
    counters_map _counters_map;
    buckets _buckets; // buckets list in ascending order
    bool _valid = true;

public:
    /// capacity: maximum number of elements to be tracked
    space_saving_top_k(size_t capacity = 256) : _capacity(capacity) {}

    size_t capacity() const { return _capacity; }

    size_t size() const {
        if (!_valid) {
            throw std::runtime_error("space_saving_top_k state is invalid");
        }
        return _counters_map.size();
    }

    bool valid() const { return _valid; }

    // returns true if item is a new one
    bool append(T item, unsigned inc = 1) { return std::get<0>(append_return_all(std::move(item), inc)); }

    // returns optionally dropped item (due to capacity overflow)
    std::optional<T> append_return_dropped(T item, unsigned inc = 1) { return std::get<1>(append_return_all(std::move(item), inc)); }

    // returns whether an element is new and an optionally dropped item (due to capacity overflow)
    std::tuple<bool, std::optional<T>> append_return_all(T item, unsigned inc = 1) {
        if (!_valid) {
            return {false, std::optional<T>()};
        }
        try {
            counters_map_iterator cmap_it = _counters_map.find(item);
            bool is_new_item = cmap_it == _counters_map.end();
            std::optional<T> dropped_item;
            counters_iterator counter_it;
            if (is_new_item) {
                if (size() < _capacity) {
                    _buckets.emplace_front(bucket(item, 0)); // count set to 0, increment_counter() required to complete bucket construction
                    buckets_iterator new_bucket_it = _buckets.begin();
                    counter_it = new_bucket_it->counters.begin();
                    (*counter_it)->bucket_it = new_bucket_it;
                } else {
                    buckets_iterator min_bucket = _buckets.begin();
                    assert(min_bucket != _buckets.end());
                    counter_it = min_bucket->counters.begin();
                    assert(counter_it != min_bucket->counters.end());
                    counter_ptr ctr = *counter_it;
                    _counters_map.erase(ctr->item);
                    dropped_item = std::exchange(ctr->item, std::move(item));
                    ctr->error = min_bucket->count;
                }
                _counters_map[item] = std::move(counter_it);
            } else {
                counter_it = cmap_it->second;
            }

            increment_counter(counter_it, inc);

            return {is_new_item, std::move(dropped_item)};
        } catch (...) {
            _valid = false;
            std::rethrow_exception(std::current_exception());
        }
    }

private:
    void increment_counter(counters_iterator counter_it, unsigned inc) {
        counter_ptr ctr = *counter_it;

        buckets_iterator old_bucket_it = ctr->bucket_it;
        auto& old_buck = *old_bucket_it;
        old_buck.counters.erase(counter_it);

        ctr->count += inc;

        buckets_iterator bi_prev = old_bucket_it;
        buckets_iterator bi_next = std::next(old_bucket_it);
        while (bi_next != _buckets.end()) {
            bucket& buck = *bi_next;
            if (ctr->count == buck.count) {
                buck.counters.push_back(ctr);
                counter_it = std::prev(buck.counters.end());
                break;
            } else if (ctr->count > buck.count) {
                bi_prev = bi_next;
                bi_next = std::next(bi_prev);
            } else {
                bi_next = _buckets.end(); // create new bucket
            }
        }

        if (bi_next == _buckets.end()) {
            bucket buck{ctr};
            counter_it = buck.counters.begin();
            bi_next = _buckets.insert(std::next(bi_prev), std::move(buck));
        }
        ctr->bucket_it = bi_next;
        _counters_map[ctr->item] = std::move(counter_it);

        if (old_buck.counters.empty()) {
            _buckets.erase(old_bucket_it);
        }
    }

    //-----------------------------------------------------------------------------------------
    // Results
public:
    struct result {
        T item;
        unsigned count;
        unsigned error;
    };

    using results = chunked_vector<result>;

    results top(unsigned k) const
    {
        if (!_valid) {
            throw std::runtime_error("space_saving_top_k state is invalid");
        }

        results list;
        // _buckets are in ascending order
        for (auto b_it = _buckets.rbegin(); b_it != _buckets.rend(); ++b_it) {
            auto& b = *b_it;
            for (auto& c: b.counters) {
                if (list.size() == k) {
                    return list;
                }
                list.emplace_back(result{c->item, c->count, c->error});
            }
        }
        return list;
    }

    //-----------------------------------------------------------------------------------------
    // Diagnostics
public:
    template <class TT>
    friend std::ostream& operator<<(std::ostream& out, const typename space_saving_top_k<TT>::counter& c);
    template <class TT>
    friend std::ostream& operator<<(std::ostream& out, const typename space_saving_top_k<TT>::counters_map& counters_map);
    template <class TT>
    friend std::ostream& operator<<(std::ostream& out, const typename space_saving_top_k<TT>::buckets& buckets);
    template <class TT>
    friend std::ostream& operator<<(std::ostream& out, const space_saving_top_k<TT>& top_k);
};

//---------------------------------------------------------------------------------------------

template <class T>
std::ostream& operator<<(std::ostream& out, const typename space_saving_top_k<T>::counter& c) {
    out << c.item << " " << c.count << "/" << c.error << " " << &*c.bucket_it;
    return out;
}

template <class T>
std::ostream& operator<<(std::ostream& out, const typename space_saving_top_k<T>::counters_map& counters_map) {
    out << "{\n";
    for (auto const& [item, counter_i]: counters_map) {
        out << item << " => " << **counter_i << "\n";
    }
    out << "}\n";
    return out;
}

template <class T>
std::ostream& operator<<(std::ostream& out, const typename space_saving_top_k<T>::buckets& buckets) {
    for (auto& b: buckets) {
        out << &b << " " << b.count << " [";
        for (auto& c: b.counters) {
            out << *c << " ";
        }
        out << "]\n";
    }
    return out;
}

template <class T>
std::ostream& operator<<(std::ostream& out, const space_saving_top_k<T>& top_k) {
    out << top_k._buckets;
    out << top_k._counters_map;
    out << "---\n";
    return out;
}

} // namespace utils
<commit_msg>top_k: support for appending top_k results<commit_after>/*
 * Copyright (C) 2011 Clearspring Technologies, Inc.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 * http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

/*
 * Copyright (C) 2018 ScyllaDB
 */

/*
 * This file is part of Scylla.
 *
 * Scylla is free software: you can redistribute it and/or modify
 * it under the terms of the GNU Affero General Public License as published by
 * the Free Software Foundation, either version 3 of the License, or
 * (at your option) any later version.
 *
 * Scylla is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 * GNU General Public License for more details.
 *
 * You should have received a copy of the GNU General Public License
 * along with Scylla.  If not, see <http://www.gnu.org/licenses/>.
 */

#pragma once

/*
 Based on the following implementation ([2]) for the Space-Saving algorithm from [1].

 [1] Metwally, A., Agrawal, D., & El Abbadi, A. (2005, January).
     Efficient computation of frequent and top-k elements in data streams.
     In International Conference on Database Theory (pp. 398-412). Springer, Berlin, Heidelberg.
     http://www.cse.ust.hk/~raywong/comp5331/References/EfficientComputationOfFrequentAndTop-kElementsInDataStreams.pdf

 [2] https://github.com/addthis/stream-lib/blob/master/src/main/java/com/clearspring/analytics/stream/StreamSummary.java

 The algorithm keeps a map between keys seen and their counts, keeping a bound on the number of tracked keys.
 Replacement policy evicts the key with the lowest count while inheriting its count, and recording an estimation
 of the error which results from that.
 This error estimation can be later used to prove if the distribution we arrived at corresponds to the real top-K,
 which we can display alongside the results.
 Accuracy depends on the number of tracked keys.

*/

#include <cstdio>
#include <list>
#include <unordered_map>
#include <memory>
#include <tuple>
#include <assert.h>

#include <seastar/core/shared_ptr.hh>
#include <seastar/core/sstring.hh>
#include "utils/chunked_vector.hh"

namespace utils {

using namespace seastar;

template <class T, class Hash = std::hash<T>, class KeyEqual = std::equal_to<T>>
class space_saving_top_k {
private:
    struct bucket;
    using buckets_iterator = typename std::list<bucket>::iterator;

    struct counter {
        buckets_iterator bucket_it;
        T item;
        unsigned count = 0;
        unsigned error = 0;

        counter(T item, unsigned count = 0, unsigned error = 0) : item(item), count(count), error(error) {}
    };

    using counter_ptr = lw_shared_ptr<counter>;

    using counters = std::list<counter_ptr>;
    using counters_iterator = typename counters::iterator;

    using counters_map = std::unordered_map<T, counters_iterator, Hash, KeyEqual>;
    using counters_map_iterator = typename counters_map::iterator;

    struct bucket {
        std::list<counter_ptr> counters;
        unsigned count;

        bucket(counter_ptr ctr) {
            count = ctr->count;
            counters.push_back(ctr);
        }

        bucket(T item, unsigned count, unsigned error) {
            counters.push_back(make_lw_shared<counter>(item, count, error));
            this->count = count;
        }
    };

    using buckets = std::list<bucket>;

    size_t _capacity;
    counters_map _counters_map;
    buckets _buckets; // buckets list in ascending order
    bool _valid = true;

public:
    /// capacity: maximum number of elements to be tracked
    space_saving_top_k(size_t capacity = 256) : _capacity(capacity) {}

    size_t capacity() const { return _capacity; }

    size_t size() const {
        if (!_valid) {
            throw std::runtime_error("space_saving_top_k state is invalid");
        }
        return _counters_map.size();
    }

    bool valid() const { return _valid; }

    // returns true if item is a new one
    bool append(T item, unsigned inc = 1, unsigned err = 0) {
        return std::get<0>(append_return_all(std::move(item), inc, err));
    }

    // returns optionally dropped item (due to capacity overflow)
    std::optional<T> append_return_dropped(T item, unsigned inc = 1, unsigned err = 0) {
        return std::get<1>(append_return_all(std::move(item), inc, err));
    }

    // returns whether an element is new and an optionally dropped item (due to capacity overflow)
    std::tuple<bool, std::optional<T>> append_return_all(T item, unsigned inc = 1, unsigned err = 0) {
        if (!_valid) {
            return {false, std::optional<T>()};
        }
        try {
            counters_map_iterator cmap_it = _counters_map.find(item);
            bool is_new_item = cmap_it == _counters_map.end();
            std::optional<T> dropped_item;
            counters_iterator counter_it;
            if (is_new_item) {
                if (size() < _capacity) {
                    _buckets.emplace_front(bucket(std::move(item), 0, err)); // inc added later via increment_counter
                    buckets_iterator new_bucket_it = _buckets.begin();
                    counter_it = new_bucket_it->counters.begin();
                    (*counter_it)->bucket_it = new_bucket_it;
                } else {
                    buckets_iterator min_bucket = _buckets.begin();
                    assert(min_bucket != _buckets.end());
                    counter_it = min_bucket->counters.begin();
                    assert(counter_it != min_bucket->counters.end());
                    counter_ptr ctr = *counter_it;
                    _counters_map.erase(ctr->item);
                    dropped_item = std::exchange(ctr->item, std::move(item));
                    ctr->error = min_bucket->count + err;
                }
                _counters_map[item] = std::move(counter_it);
            } else {
                counter_it = cmap_it->second;
            }

            increment_counter(counter_it, inc);

            return {is_new_item, std::move(dropped_item)};
        } catch (...) {
            _valid = false;
            std::rethrow_exception(std::current_exception());
        }
    }

private:
    void increment_counter(counters_iterator counter_it, unsigned inc) {
        counter_ptr ctr = *counter_it;

        buckets_iterator old_bucket_it = ctr->bucket_it;
        auto& old_buck = *old_bucket_it;
        old_buck.counters.erase(counter_it);

        ctr->count += inc;

        buckets_iterator bi_prev = old_bucket_it;
        buckets_iterator bi_next = std::next(old_bucket_it);
        while (bi_next != _buckets.end()) {
            bucket& buck = *bi_next;
            if (ctr->count == buck.count) {
                buck.counters.push_back(ctr);
                counter_it = std::prev(buck.counters.end());
                break;
            } else if (ctr->count > buck.count) {
                bi_prev = bi_next;
                bi_next = std::next(bi_prev);
            } else {
                bi_next = _buckets.end(); // create new bucket
            }
        }

        if (bi_next == _buckets.end()) {
            bucket buck{ctr};
            counter_it = buck.counters.begin();
            bi_next = _buckets.insert(std::next(bi_prev), std::move(buck));
        }
        ctr->bucket_it = bi_next;
        _counters_map[ctr->item] = std::move(counter_it);

        if (old_buck.counters.empty()) {
            _buckets.erase(old_bucket_it);
        }
    }

    //-----------------------------------------------------------------------------------------
    // Results
public:
    struct result {
        T item;
        unsigned count;
        unsigned error;
    };

    using results = chunked_vector<result>;

    results top(unsigned k) const
    {
        if (!_valid) {
            throw std::runtime_error("space_saving_top_k state is invalid");
        }

        results list;
        // _buckets are in ascending order
        for (auto b_it = _buckets.rbegin(); b_it != _buckets.rend(); ++b_it) {
            auto& b = *b_it;
            for (auto& c: b.counters) {
                if (list.size() == k) {
                    return list;
                }
                list.emplace_back(result{c->item, c->count, c->error});
            }
        }
        return list;
    }

    void append(const results& res) {
        for (auto& r: res) {
            append(r.item, r.count, r.error);
        }
    }

    //-----------------------------------------------------------------------------------------
    // Diagnostics
public:
    template <class TT>
    friend std::ostream& operator<<(std::ostream& out, const typename space_saving_top_k<TT>::counter& c);
    template <class TT>
    friend std::ostream& operator<<(std::ostream& out, const typename space_saving_top_k<TT>::counters_map& counters_map);
    template <class TT>
    friend std::ostream& operator<<(std::ostream& out, const typename space_saving_top_k<TT>::buckets& buckets);
    template <class TT>
    friend std::ostream& operator<<(std::ostream& out, const space_saving_top_k<TT>& top_k);
};

//---------------------------------------------------------------------------------------------

template <class T>
std::ostream& operator<<(std::ostream& out, const typename space_saving_top_k<T>::counter& c) {
    out << c.item << " " << c.count << "/" << c.error << " " << &*c.bucket_it;
    return out;
}

template <class T>
std::ostream& operator<<(std::ostream& out, const typename space_saving_top_k<T>::counters_map& counters_map) {
    out << "{\n";
    for (auto const& [item, counter_i]: counters_map) {
        out << item << " => " << **counter_i << "\n";
    }
    out << "}\n";
    return out;
}

template <class T>
std::ostream& operator<<(std::ostream& out, const typename space_saving_top_k<T>::buckets& buckets) {
    for (auto& b: buckets) {
        out << &b << " " << b.count << " [";
        for (auto& c: b.counters) {
            out << *c << " ";
        }
        out << "]\n";
    }
    return out;
}

template <class T>
std::ostream& operator<<(std::ostream& out, const space_saving_top_k<T>& top_k) {
    out << top_k._buckets;
    out << top_k._counters_map;
    out << "---\n";
    return out;
}

} // namespace utils
<|endoftext|>


<commit_before>#include "3rd-party/catch.hpp"

#include <memory>

#include "cache.h"
#include "configcontainer.h"
#include "feedhandler.h"
#include "rss.h"
#include "test-helpers.h"

using namespace newsboat;

namespace {

std::vector<std::shared_ptr<rss_feed>> get_five_empty_feeds()
{
	std::vector<std::shared_ptr<rss_feed>> feeds;
	TestHelpers::TempFile dbfile;
	std::unique_ptr<configcontainer> cfg(new configcontainer());
	std::unique_ptr<cache> rsscache(new cache(dbfile.getPath(), cfg.get()));
	for (int i = 0; i < 5; ++i) {
		const auto feed = std::make_shared<rss_feed>(rsscache.get());
		feeds.push_back(feed);
	}
	return feeds;
}

} // anonymous namespace

TEST_CASE("Correctly returns desired feed", "[feedhandler]")
{
	FeedHandler feedhandler;
	const auto feeds = get_five_empty_feeds();
	int i = 0;
	for (const auto& feed : feeds) {
		feed->set_title(std::to_string(i));
		feed->set_rssurl("url/" + std::to_string(i));
		i++;
	}
	feedhandler.set_feeds(feeds);

	SECTION("By pos")
	{
		auto feed = feedhandler.get_feed(0);
		REQUIRE(feed->title_raw() == "0");

		feed = feedhandler.get_feed(4);
		REQUIRE(feed->title_raw() == "4");
	}

	SECTION("By rssurl")
	{
		auto feed = feedhandler.get_feed_by_url("url/1");
		REQUIRE(feed->title_raw() == "1");

		feed = feedhandler.get_feed_by_url("url/4");
		REQUIRE(feed->title_raw() == "4");
	}
}

TEST_CASE("Throws on get_feed() with pos out of range", "[feedhandler]")
{
	FeedHandler feedhandler;
	feedhandler.set_feeds(get_five_empty_feeds());

	REQUIRE_NOTHROW(feedhandler.get_feed(4));
	CHECK_THROWS_AS(feedhandler.get_feed(5), std::out_of_range);
	CHECK_THROWS_AS(feedhandler.get_feed(-1), std::out_of_range);
}

TEST_CASE("Returns correct number using get_feed_count_by_tag()", "[feedhandler]")
{
	FeedHandler feedhandler;
	feedhandler.set_feeds(get_five_empty_feeds());
	feedhandler.get_feed(0)->set_tags({"Chicken", "Horse"});
	feedhandler.get_feed(1)->set_tags({"Horse", "Duck"});
	feedhandler.get_feed(2)->set_tags({"Duck", "Frog"});
	feedhandler.get_feed(3)->set_tags({"Duck", "Hawk"});

	REQUIRE(feedhandler.get_feed_count_per_tag("Ice Cream") == 0);
	REQUIRE(feedhandler.get_feed_count_per_tag("Chicken") == 1);
	REQUIRE(feedhandler.get_feed_count_per_tag("Horse") == 2);
	REQUIRE(feedhandler.get_feed_count_per_tag("Duck") == 3);
}

TEST_CASE("Correctly returns pos of next unread item", "[feedhandler]")
{
	FeedHandler feedhandler;
	const auto feeds = get_five_empty_feeds();
	int i = 0;
	for (const auto& feed : feeds) {
		const auto item = std::make_shared<rss_item>(nullptr);
		if ((i % 2) == 0)
			item->set_unread_nowrite(true);
		else
			item->set_unread_nowrite(false);
		feed->add_item(item);
		i++;
	}
	feedhandler.set_feeds(feeds);

	REQUIRE(feedhandler.get_pos_of_next_unread(0) == 2);
	REQUIRE(feedhandler.get_pos_of_next_unread(2) == 4);
}

TEST_CASE("Correctly sorts feeds", "[feedhandler]")
{
	SECTION("by none asc")
	{
	}

	SECTION("by none desc")
	{
	}

	SECTION("by firsttag asc")
	{
	}

	SECTION("by firsttag desc")
	{
	}

	SECTION("by title asc")
	{
	}

	SECTION("by title desc")
	{
	}

	SECTION("by articlecount asc")
	{
	}

	SECTION("by articlecount desc")
	{
	}

	SECTION("by unreadarticlecount asc")
	{
	}

	SECTION("by unreadarticlecount desc")
	{
	}

	SECTION("by lastupdated asc")
	{
	}

	SECTION("by lastupdated desc")
	{
	}
}
<commit_msg>Split get_feed() test<commit_after>#include "3rd-party/catch.hpp"

#include <memory>

#include "cache.h"
#include "configcontainer.h"
#include "feedhandler.h"
#include "rss.h"
#include "test-helpers.h"

using namespace newsboat;

namespace {

std::vector<std::shared_ptr<rss_feed>> get_five_empty_feeds()
{
	std::vector<std::shared_ptr<rss_feed>> feeds;
	TestHelpers::TempFile dbfile;
	std::unique_ptr<configcontainer> cfg(new configcontainer());
	std::unique_ptr<cache> rsscache(new cache(dbfile.getPath(), cfg.get()));
	for (int i = 0; i < 5; ++i) {
		const auto feed = std::make_shared<rss_feed>(rsscache.get());
		feeds.push_back(feed);
	}
	return feeds;
}

} // anonymous namespace

TEST_CASE("get_feed() returns feed by its position number", "[feedhandler]")
{
	FeedHandler feedhandler;
	const auto feeds = get_five_empty_feeds();
	int i = 0;
	for (const auto& feed : feeds) {
		feed->set_title(std::to_string(i));
		feed->set_rssurl("url/" + std::to_string(i));
		i++;
	}
	feedhandler.set_feeds(feeds);

	auto feed = feedhandler.get_feed(0);
	REQUIRE(feed->title_raw() == "0");

	feed = feedhandler.get_feed(4);
	REQUIRE(feed->title_raw() == "4");
}

TEST_CASE("get_feed_by_url() returns feed by its URL", "[feedhandler]")
{
	FeedHandler feedhandler;
	const auto feeds = get_five_empty_feeds();
	int i = 0;
	for (const auto& feed : feeds) {
		feed->set_title(std::to_string(i));
		feed->set_rssurl("url/" + std::to_string(i));
		i++;
	}
	feedhandler.set_feeds(feeds);

	auto feed = feedhandler.get_feed_by_url("url/1");
	REQUIRE(feed->title_raw() == "1");

	feed = feedhandler.get_feed_by_url("url/4");
	REQUIRE(feed->title_raw() == "4");
}

TEST_CASE("Throws on get_feed() with pos out of range", "[feedhandler]")
{
	FeedHandler feedhandler;
	feedhandler.set_feeds(get_five_empty_feeds());

	REQUIRE_NOTHROW(feedhandler.get_feed(4));
	CHECK_THROWS_AS(feedhandler.get_feed(5), std::out_of_range);
	CHECK_THROWS_AS(feedhandler.get_feed(-1), std::out_of_range);
}

TEST_CASE("Returns correct number using get_feed_count_by_tag()", "[feedhandler]")
{
	FeedHandler feedhandler;
	feedhandler.set_feeds(get_five_empty_feeds());
	feedhandler.get_feed(0)->set_tags({"Chicken", "Horse"});
	feedhandler.get_feed(1)->set_tags({"Horse", "Duck"});
	feedhandler.get_feed(2)->set_tags({"Duck", "Frog"});
	feedhandler.get_feed(3)->set_tags({"Duck", "Hawk"});

	REQUIRE(feedhandler.get_feed_count_per_tag("Ice Cream") == 0);
	REQUIRE(feedhandler.get_feed_count_per_tag("Chicken") == 1);
	REQUIRE(feedhandler.get_feed_count_per_tag("Horse") == 2);
	REQUIRE(feedhandler.get_feed_count_per_tag("Duck") == 3);
}

TEST_CASE("Correctly returns pos of next unread item", "[feedhandler]")
{
	FeedHandler feedhandler;
	const auto feeds = get_five_empty_feeds();
	int i = 0;
	for (const auto& feed : feeds) {
		const auto item = std::make_shared<rss_item>(nullptr);
		if ((i % 2) == 0)
			item->set_unread_nowrite(true);
		else
			item->set_unread_nowrite(false);
		feed->add_item(item);
		i++;
	}
	feedhandler.set_feeds(feeds);

	REQUIRE(feedhandler.get_pos_of_next_unread(0) == 2);
	REQUIRE(feedhandler.get_pos_of_next_unread(2) == 4);
}

TEST_CASE("Correctly sorts feeds", "[feedhandler]")
{
	SECTION("by none asc")
	{
	}

	SECTION("by none desc")
	{
	}

	SECTION("by firsttag asc")
	{
	}

	SECTION("by firsttag desc")
	{
	}

	SECTION("by title asc")
	{
	}

	SECTION("by title desc")
	{
	}

	SECTION("by articlecount asc")
	{
	}

	SECTION("by articlecount desc")
	{
	}

	SECTION("by unreadarticlecount asc")
	{
	}

	SECTION("by unreadarticlecount desc")
	{
	}

	SECTION("by lastupdated asc")
	{
	}

	SECTION("by lastupdated desc")
	{
	}
}
<|endoftext|>


<commit_before>/*******************************************************************************
 *
 * MIT License
 *
 * Copyright (c) 2020 Advanced Micro Devices, Inc.
 *
 * Permission is hereby granted, free of charge, to any person obtaining a copy
 * of this software and associated documentation files (the "Software"), to deal
 * in the Software without restriction, including without limitation the rights
 * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
 * copies of the Software, and to permit persons to whom the Software is
 * furnished to do so, subject to the following conditions:
 *
 * The above copyright notice and this permission notice shall be included in all
 * copies or substantial portions of the Software.
 *
 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
 * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
 * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
 * SOFTWARE.
 *
 *******************************************************************************/

/// SWDEV-257056?focusedCommentId=6654244&page=com.atlassian.jira.plugin.system.issuetabpanels%3Acomment-tabpanel#comment-6654244
/// \todo Create dedicated ticket and rename macro.
#define WORKAROUND_SWDEV_257056_PCH_MISSING_MACROS 1

#include <miopen/config.h>
#include <miopen/handle.hpp>
#include <miopen/execution_context.hpp>

#if WORKAROUND_SWDEV_257056_PCH_MISSING_MACROS
#include <miopen/hip_build_utils.hpp>
#endif

#include "get_handle.hpp"
#include <vector>
#include <thread>
#include "test.hpp"

/// In comgr-driven OCL and HIP builds, -Werror is not working.
/// Also -Wunused-parameter is not enabled by default and can't be enabled.
/// Let's skip the test for now.
/// \todo Create ticket for comgr.
#define WORKAROUND_COMGR_WARNING_ISSUES MIOPEN_USE_COMGR

enum kernel_type_t
{
    miopenHIPKernelType,
    miopenOpenCLKernelType
};

std::string Write2s(kernel_type_t kern_type)
{
    if(kern_type == miopenHIPKernelType)
        return "#ifndef MIOPEN_DONT_USE_HIP_RUNTIME_HEADERS\n"
               "#include <hip/hip_runtime.h>\n"
#if WORKAROUND_SWDEV_257056_PCH_MISSING_MACROS
               "#else\n"
               "#ifdef hipThreadIdx_x\n"
               "#undef hipThreadIdx_x\n"
               "#endif\n"
               "#define hipThreadIdx_x threadIdx.x\n"
               "\n"
               "#ifdef hipBlockDim_x\n"
               "#undef hipBlockDim_x\n"
               "#endif\n"
               "#define hipBlockDim_x blockDim.x\n"
               "\n"
               "#ifdef hipBlockIdx_x\n"
               "#undef hipBlockIdx_x\n"
               "#endif\n"
               "#define hipBlockIdx_x blockIdx.x\n"
#endif
               "#endif\n"
               "extern \"C\" {\n"
               "__global__ void write(int* data) {\n"
               "    int num = hipThreadIdx_x + hipBlockDim_x * hipBlockIdx_x;\n"
               "    data[num] *= 2;\n"
               "}\n"
               "}\n";
    else if(kern_type == miopenOpenCLKernelType)
        return "__kernel void write(__global int* data) { data[get_global_id(0)] *= 2; }\n";
    else
        MIOPEN_THROW("Unsupported kernel type");
}

void run2s(miopen::Handle& h, std::size_t n, kernel_type_t kern_type)
{
    std::vector<int> data_in(n, 1);
    auto data_dev = h.Write(data_in);
    if(kern_type == miopenOpenCLKernelType)
        h.AddKernel("GEMM", "", Write2s(miopenOpenCLKernelType), "write", {n, 1, 1}, {n, 1, 1}, "")(
            data_dev.get());
    else if(kern_type == miopenHIPKernelType)
        h.AddKernel("NoAlgo",
                    "",
                    "test_hip.cpp",
                    "write",
                    {n, 1, 1},
                    {n, 1, 1},
                    "",
                    0,
                    false,
                    Write2s(miopenHIPKernelType))(data_dev.get());
    else
        MIOPEN_THROW("Unsupported kernel type");
    std::fill(data_in.begin(), data_in.end(), 2);

    auto data_out = h.Read<int>(data_dev, n);
    CHECK(data_out == data_in);
}

void test_multithreads(kernel_type_t kern_type, const bool with_stream = false)
{
    auto&& h1 = get_handle();
    auto&& h2 = get_handle_with_stream(h1);
    std::thread([&] { run2s(with_stream ? h2 : h1, 16, kern_type); }).join();
    std::thread([&] { run2s(with_stream ? h2 : h1, 32, kern_type); }).join();
    std::thread([&] {
        std::thread([&] { run2s(with_stream ? h2 : h1, 64, kern_type); }).join();
    }).join();
    run2s(with_stream ? h2 : h1, 4, kern_type);
}

std::string WriteError(kernel_type_t kern_type)
{
    if(kern_type == miopenOpenCLKernelType)
        return "__kernel void write(__global int* data) { data[i] = 0; }\n";
    else if(kern_type == miopenHIPKernelType)
        return "#ifndef MIOPEN_DONT_USE_HIP_RUNTIME_HEADERS\n"
               "#include <hip/hip_runtime.h>\n"
               "#endif\n"
               "extern \"C\" {\n"
               "__global__ void write(int* data) {\n"
               "    data[num] *= 2;\n"
               "}\n"
               "}\n";
    else
        MIOPEN_THROW("Unsupported kernel type");
}

void test_errors(kernel_type_t kern_type)
{
    auto&& h = get_handle();
    if(kern_type == miopenOpenCLKernelType)
    {
        EXPECT(throws([&] {
            h.AddKernel("GEMM", "", WriteError(kern_type), "write", {1, 1, 1}, {1, 1, 1}, "");
        }));
        try
        {
            h.AddKernel("GEMM", "", WriteError(kern_type), "write", {1, 1, 1}, {1, 1, 1}, "");
        }
        catch(miopen::Exception& e)
        {
            EXPECT(!std::string(e.what()).empty());
        }
    }
    else if(kern_type == miopenHIPKernelType)
    {
        EXPECT(throws([&] {
            h.AddKernel("NoAlgo",
                        "",
                        "error_hip.cpp",
                        "write",
                        {1, 1, 1},
                        {1, 1, 1},
                        "",
                        0,
                        false,
                        WriteError(miopenHIPKernelType));
        }));
        try
        {
            h.AddKernel("NoAlgo",
                        "",
                        "error_hip.cpp",
                        "write",
                        {1, 1, 1},
                        {1, 1, 1},
                        "",
                        0,
                        false,
                        WriteError(miopenHIPKernelType));
        }
        catch(miopen::Exception& e)
        {
            EXPECT(!std::string(e.what()).empty());
        }
    }
}

std::string WriteNop(kernel_type_t kern_type)
{
    if(kern_type == miopenOpenCLKernelType)
        return "__kernel void write(__global int* data) {}\n";
    else if(kern_type == miopenHIPKernelType)
        return "#ifndef MIOPEN_DONT_USE_HIP_RUNTIME_HEADERS\n"
               "#include <hip/hip_runtime.h>\n"
               "#endif\n"
               "extern \"C\" {\n"
               "__global__ void write(int* data) {\n"
               "}\n"
               "}\n";
    else
        MIOPEN_THROW("Unsupported kernel type");
}

void test_warnings(kernel_type_t kern_type)
{
    auto&& h = get_handle();
#if MIOPEN_BUILD_DEV && !WORKAROUND_COMGR_WARNING_ISSUES
    if(kern_type == miopenOpenCLKernelType)
        EXPECT(throws([&] {
            h.AddKernel("GEMM", "", WriteNop(kern_type), "write", {1, 1, 1}, {1, 1, 1}, "");
        }));
    else if(kern_type == miopenHIPKernelType)
        EXPECT(throws([&] {
            h.AddKernel("NoAlgo",
                        "",
                        "nop_hip.cpp",
                        "write",
                        {1, 1, 1},
                        {1, 1, 1},
                        "",
                        0,
                        false,
                        WriteNop(miopenHIPKernelType));
        }));
#else
    (void)kern_type;
    (void)h; // To silence warnings.
#endif
}

void test_arch_name()
{
    auto&& h        = get_handle();
    auto known_arch = {"gfx908", "gfx906", "gfx900", "gfx803", "gfx1030"};
    auto this_arch  = h.GetDeviceName();
    EXPECT(std::any_of(
        known_arch.begin(), known_arch.end(), [&](std::string arch) { return arch == this_arch; }));
}

int main()
{
    auto&& h = get_handle();
    if(h.GetDeviceName() != "gfx803" && miopen::IsHipKernelsEnabled())
    {
        test_multithreads(miopenHIPKernelType);
        test_errors(miopenHIPKernelType);
// Warnings currently dont work in opencl
#if !MIOPEN_BACKEND_OPENCL
        test_warnings(miopenHIPKernelType);
#endif
    }
    test_multithreads(miopenOpenCLKernelType);
    test_multithreads(miopenOpenCLKernelType, true);
    test_errors(miopenOpenCLKernelType);
    test_arch_name();
// Warnings currently dont work in opencl
#if !MIOPEN_BACKEND_OPENCL
    test_warnings(miopenOpenCLKernelType);
#endif
}
<commit_msg>[TESTS] Add gfx90a support in handle_test (#1009)<commit_after>/*******************************************************************************
 *
 * MIT License
 *
 * Copyright (c) 2020 Advanced Micro Devices, Inc.
 *
 * Permission is hereby granted, free of charge, to any person obtaining a copy
 * of this software and associated documentation files (the "Software"), to deal
 * in the Software without restriction, including without limitation the rights
 * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
 * copies of the Software, and to permit persons to whom the Software is
 * furnished to do so, subject to the following conditions:
 *
 * The above copyright notice and this permission notice shall be included in all
 * copies or substantial portions of the Software.
 *
 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
 * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
 * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
 * SOFTWARE.
 *
 *******************************************************************************/

/// SWDEV-257056?focusedCommentId=6654244&page=com.atlassian.jira.plugin.system.issuetabpanels%3Acomment-tabpanel#comment-6654244
/// \todo Create dedicated ticket and rename macro.
#define WORKAROUND_SWDEV_257056_PCH_MISSING_MACROS 1

#include <miopen/config.h>
#include <miopen/handle.hpp>
#include <miopen/execution_context.hpp>

#if WORKAROUND_SWDEV_257056_PCH_MISSING_MACROS
#include <miopen/hip_build_utils.hpp>
#endif

#include "get_handle.hpp"
#include <vector>
#include <thread>
#include "test.hpp"

/// In comgr-driven OCL and HIP builds, -Werror is not working.
/// Also -Wunused-parameter is not enabled by default and can't be enabled.
/// Let's skip the test for now.
/// \todo Create ticket for comgr.
#define WORKAROUND_COMGR_WARNING_ISSUES MIOPEN_USE_COMGR

enum kernel_type_t
{
    miopenHIPKernelType,
    miopenOpenCLKernelType
};

std::string Write2s(kernel_type_t kern_type)
{
    if(kern_type == miopenHIPKernelType)
        return "#ifndef MIOPEN_DONT_USE_HIP_RUNTIME_HEADERS\n"
               "#include <hip/hip_runtime.h>\n"
#if WORKAROUND_SWDEV_257056_PCH_MISSING_MACROS
               "#else\n"
               "#ifdef hipThreadIdx_x\n"
               "#undef hipThreadIdx_x\n"
               "#endif\n"
               "#define hipThreadIdx_x threadIdx.x\n"
               "\n"
               "#ifdef hipBlockDim_x\n"
               "#undef hipBlockDim_x\n"
               "#endif\n"
               "#define hipBlockDim_x blockDim.x\n"
               "\n"
               "#ifdef hipBlockIdx_x\n"
               "#undef hipBlockIdx_x\n"
               "#endif\n"
               "#define hipBlockIdx_x blockIdx.x\n"
#endif
               "#endif\n"
               "extern \"C\" {\n"
               "__global__ void write(int* data) {\n"
               "    int num = hipThreadIdx_x + hipBlockDim_x * hipBlockIdx_x;\n"
               "    data[num] *= 2;\n"
               "}\n"
               "}\n";
    else if(kern_type == miopenOpenCLKernelType)
        return "__kernel void write(__global int* data) { data[get_global_id(0)] *= 2; }\n";
    else
        MIOPEN_THROW("Unsupported kernel type");
}

void run2s(miopen::Handle& h, std::size_t n, kernel_type_t kern_type)
{
    std::vector<int> data_in(n, 1);
    auto data_dev = h.Write(data_in);
    if(kern_type == miopenOpenCLKernelType)
        h.AddKernel("GEMM", "", Write2s(miopenOpenCLKernelType), "write", {n, 1, 1}, {n, 1, 1}, "")(
            data_dev.get());
    else if(kern_type == miopenHIPKernelType)
        h.AddKernel("NoAlgo",
                    "",
                    "test_hip.cpp",
                    "write",
                    {n, 1, 1},
                    {n, 1, 1},
                    "",
                    0,
                    false,
                    Write2s(miopenHIPKernelType))(data_dev.get());
    else
        MIOPEN_THROW("Unsupported kernel type");
    std::fill(data_in.begin(), data_in.end(), 2);

    auto data_out = h.Read<int>(data_dev, n);
    CHECK(data_out == data_in);
}

void test_multithreads(kernel_type_t kern_type, const bool with_stream = false)
{
    auto&& h1 = get_handle();
    auto&& h2 = get_handle_with_stream(h1);
    std::thread([&] { run2s(with_stream ? h2 : h1, 16, kern_type); }).join();
    std::thread([&] { run2s(with_stream ? h2 : h1, 32, kern_type); }).join();
    std::thread([&] {
        std::thread([&] { run2s(with_stream ? h2 : h1, 64, kern_type); }).join();
    }).join();
    run2s(with_stream ? h2 : h1, 4, kern_type);
}

std::string WriteError(kernel_type_t kern_type)
{
    if(kern_type == miopenOpenCLKernelType)
        return "__kernel void write(__global int* data) { data[i] = 0; }\n";
    else if(kern_type == miopenHIPKernelType)
        return "#ifndef MIOPEN_DONT_USE_HIP_RUNTIME_HEADERS\n"
               "#include <hip/hip_runtime.h>\n"
               "#endif\n"
               "extern \"C\" {\n"
               "__global__ void write(int* data) {\n"
               "    data[num] *= 2;\n"
               "}\n"
               "}\n";
    else
        MIOPEN_THROW("Unsupported kernel type");
}

void test_errors(kernel_type_t kern_type)
{
    auto&& h = get_handle();
    if(kern_type == miopenOpenCLKernelType)
    {
        EXPECT(throws([&] {
            h.AddKernel("GEMM", "", WriteError(kern_type), "write", {1, 1, 1}, {1, 1, 1}, "");
        }));
        try
        {
            h.AddKernel("GEMM", "", WriteError(kern_type), "write", {1, 1, 1}, {1, 1, 1}, "");
        }
        catch(miopen::Exception& e)
        {
            EXPECT(!std::string(e.what()).empty());
        }
    }
    else if(kern_type == miopenHIPKernelType)
    {
        EXPECT(throws([&] {
            h.AddKernel("NoAlgo",
                        "",
                        "error_hip.cpp",
                        "write",
                        {1, 1, 1},
                        {1, 1, 1},
                        "",
                        0,
                        false,
                        WriteError(miopenHIPKernelType));
        }));
        try
        {
            h.AddKernel("NoAlgo",
                        "",
                        "error_hip.cpp",
                        "write",
                        {1, 1, 1},
                        {1, 1, 1},
                        "",
                        0,
                        false,
                        WriteError(miopenHIPKernelType));
        }
        catch(miopen::Exception& e)
        {
            EXPECT(!std::string(e.what()).empty());
        }
    }
}

std::string WriteNop(kernel_type_t kern_type)
{
    if(kern_type == miopenOpenCLKernelType)
        return "__kernel void write(__global int* data) {}\n";
    else if(kern_type == miopenHIPKernelType)
        return "#ifndef MIOPEN_DONT_USE_HIP_RUNTIME_HEADERS\n"
               "#include <hip/hip_runtime.h>\n"
               "#endif\n"
               "extern \"C\" {\n"
               "__global__ void write(int* data) {\n"
               "}\n"
               "}\n";
    else
        MIOPEN_THROW("Unsupported kernel type");
}

void test_warnings(kernel_type_t kern_type)
{
    auto&& h = get_handle();
#if MIOPEN_BUILD_DEV && !WORKAROUND_COMGR_WARNING_ISSUES
    if(kern_type == miopenOpenCLKernelType)
        EXPECT(throws([&] {
            h.AddKernel("GEMM", "", WriteNop(kern_type), "write", {1, 1, 1}, {1, 1, 1}, "");
        }));
    else if(kern_type == miopenHIPKernelType)
        EXPECT(throws([&] {
            h.AddKernel("NoAlgo",
                        "",
                        "nop_hip.cpp",
                        "write",
                        {1, 1, 1},
                        {1, 1, 1},
                        "",
                        0,
                        false,
                        WriteNop(miopenHIPKernelType));
        }));
#else
    (void)kern_type;
    (void)h; // To silence warnings.
#endif
}

void test_arch_name()
{
    auto&& h        = get_handle();
    auto known_arch = {"gfx908", "gfx90a", "gfx906", "gfx900", "gfx803", "gfx1030"};
    auto this_arch  = h.GetDeviceName();
    EXPECT(std::any_of(
        known_arch.begin(), known_arch.end(), [&](std::string arch) { return arch == this_arch; }));
}

int main()
{
    auto&& h = get_handle();
    if(h.GetDeviceName() != "gfx803" && miopen::IsHipKernelsEnabled())
    {
        test_multithreads(miopenHIPKernelType);
        test_errors(miopenHIPKernelType);
// Warnings currently dont work in opencl
#if !MIOPEN_BACKEND_OPENCL
        test_warnings(miopenHIPKernelType);
#endif
    }
    test_multithreads(miopenOpenCLKernelType);
    test_multithreads(miopenOpenCLKernelType, true);
    test_errors(miopenOpenCLKernelType);
    test_arch_name();
// Warnings currently dont work in opencl
#if !MIOPEN_BACKEND_OPENCL
    test_warnings(miopenOpenCLKernelType);
#endif
}
<|endoftext|>


<commit_before>//MIT License
//
//Copyright(c) 2016 Matthias Moeller
//
//Permission is hereby granted, free of charge, to any person obtaining a copy
//of this software and associated documentation files(the "Software"), to deal
//in the Software without restriction, including without limitation the rights
//to use, copy, modify, merge, publish, distribute, sublicense, and / or sell
//copies of the Software, and to permit persons to whom the Software is
//furnished to do so, subject to the following conditions :
//
//The above copyright notice and this permission notice shall be included in all
//copies or substantial portions of the Software.
//
//THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
//IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
//FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.IN NO EVENT SHALL THE
//AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
//LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
//OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
//SOFTWARE.

#ifndef __TYTI_STEAM_VDF_PARSER_H__
#define __TYTI_STEAM_VDF_PARSER_H__

#include <unordered_map>
#include <utility>
#include <fstream>
#include <memory>

#include <system_error>
#include <exception>

//for wstring support
#include <locale>
#include <codecvt>
#include <string>

// internal
#include <stack>

namespace tyti
{
    namespace vdf
    {
        namespace detail
        {
            ///////////////////////////////////////////////////////////////////////////
            //  Helper functions selecting the right encoding (char/wchar_T)
            ///////////////////////////////////////////////////////////////////////////

            template<typename T>
            struct literal_macro_help
            {
                static constexpr const char* result(const char* c, const wchar_t* wc) noexcept
                {
                    return c;
                }
                static constexpr const char result(char c, wchar_t wc) noexcept
                {
                    return c;
                }
            };

            template<>
            struct literal_macro_help<wchar_t>
            {
                static constexpr const wchar_t* result(const char* c, const wchar_t* wc) noexcept
                {
                    return wc;
                }
                static constexpr const wchar_t result(char c, wchar_t wc) noexcept
                {
                    return wc;
                }
            };
#define TYTI_L(type, text) vdf::detail::literal_macro_help<type>::result(text, L##text)

            inline std::string string_converter(std::string& w)
            {
                return w;
            }

            inline std::string string_converter(std::wstring& w)
            {
                std::wstring_convert<std::codecvt_utf8<wchar_t>> conv1; // maybe wrong econding
                return conv1.to_bytes(w);
            }

            ///////////////////////////////////////////////////////////////////////////
            //  Writer helper functions
            ///////////////////////////////////////////////////////////////////////////

            struct tabs
            {
                size_t t;
                tabs(size_t i) :t(i) {}
            };

            template<typename oStreamT>
            oStreamT& operator<<(oStreamT& s, tabs t)
            {
                for (; t.t > 0; --t.t)
                    s << "\t";
                return s;
            }
        } // end namespace detail

        ///////////////////////////////////////////////////////////////////////////
        //  Interface
        ///////////////////////////////////////////////////////////////////////////

        /// basic object node. Every object has a name and can contains attributes saved as key_value pairs or childrens
        template<typename CharT>
        struct basic_object
        {
            typedef CharT char_type;
            std::basic_string<char_type> name;
            std::unordered_map<std::basic_string<char_type>, std::basic_string<char_type> > attribs;
            std::unordered_map<std::basic_string<char_type>, std::shared_ptr< basic_object<char_type> > > childs;
        };

        typedef basic_object<char> object;
        typedef basic_object<wchar_t> wobject;

        /** \brief writes given object tree in vdf format to given stream.
        Uses tabs instead of whitespaces.
        */
        template<typename oStreamT, typename charT = typename oStreamT::char_type>
        void write(oStreamT& s, const basic_object<charT>& r, size_t t = 0)
        {
            using namespace detail;
            s << tabs(t) << TYTI_L(charT, '"') << r.name << TYTI_L(charT, "\"\n") << tabs(t) << TYTI_L(charT, "{\n");
            for (auto& i : r.attribs)
                s << tabs(t + 1) << TYTI_L(charT, '"') << i.first << TYTI_L(charT, "\"\t\t\"") << i.second << TYTI_L(charT, "\"\n");
            for (auto& i : r.childs)
                write(s, i, t + 1);
            s << tabs(t) << TYTI_L(charT, "}\n");
        }

        /** \brief Read VDF formatted sequences defined by the range [first, last).
        If the file is mailformatted, parser will try to read it until it can.
        @param first begin iterator
        @param end end iterator
        @param ec output bool. 0 if ok, otherwise, holds an system error code

        Possible error codes:
            std::errc::protocol_error: file is mailformatted
            std::errc::not_enough_memory: not enough space
            std::errc::invalid_argument: iterators throws e.g. out of range
        */

        template<typename IterT, typename charT = typename IterT::value_type>
        basic_object<charT> read(IterT first, IterT last, std::error_code& ec) noexcept
        {
            ec.clear();

            basic_object<charT> root;
            basic_object<charT>* cur = &root;
            std::stack< basic_object<charT>* > lvls;
            //read header
            // first, quoted name
            auto b = std::find(first, last, TYTI_L(charT, '\"'));
            auto bend = std::find(b + 1, last, TYTI_L(charT, '\"'));
            root.name = std::basic_string<charT>(b + 1, bend);
            // second, get {}
            b = std::find(bend, last, TYTI_L(charT, '{'));
            try
            {
                if (b == last)
                {
                    ec = std::make_error_code(std::errc::protocol_error);
                    return root;
                }
                else
                    lvls.push(&root);
                while (!lvls.empty() && b != last)
                {
                    const std::basic_string<charT> startsym = TYTI_L(charT, "\"}");

                    //find first starting attrib/child, or ending
                    b = std::find_first_of(b, last, std::cbegin(startsym), std::cend(startsym));
                    if (*b == '\"')
                    {
                        bend = std::find(b + 1, last, TYTI_L(charT, '\"'));
                        if (bend == last)
                        {
                            ec = std::make_error_code(std::errc::protocol_error);// could not find end of name
                            return root;
                        }

                        std::basic_string<charT> curName(b + 1, bend);
                        b = bend + 1;

                        const std::basic_string<charT> ecspsym = TYTI_L(charT, "\"{");
                        b = std::find_first_of(b, last, std::cbegin(ecspsym), std::cend(ecspsym));
                        if (b == last)
                        {
                            ec = std::make_error_code(std::errc::protocol_error);// could not find 2nd part of pair
                            return root;
                        }

                        if (*b == '\"')
                        {
                            bend = std::find(b + 1, last, TYTI_L(charT, '\"'));
                            if (bend == last)
                            {
                                ec = std::make_error_code(std::errc::protocol_error);//could not find end of name
                                return root;
                            }

                            auto value = std::basic_string<charT>(b + 1, bend);
                            b = bend + 1;

                            if (curName != TYTI_L(charT, "#include") && curName != TYTI_L(charT, "#base"))
                                cur->attribs[curName] = value;
                            else
                            {
                                std::basic_ifstream<charT> i(detail::string_converter(value));
                                auto n = std::make_shared<basic_object<charT>>(read(i, ec));
                                if (ec) return root;
                                cur->childs[n->name] = n;
                            }
                        }
                        else if (*b == '{')
                        {
                            lvls.push(cur);
                            auto n = std::make_shared<basic_object<charT>>();
                            cur->childs[curName] = n;
                            cur = n.get();
                            cur->name = curName;
                            ++b;
                        }
                    }
                    else if (*b == '}')
                    {
                        cur = lvls.top();
                        lvls.pop();
                        ++b;
                    }
                }

            }
            catch (std::bad_alloc& )
            {
                ec = std::make_error_code(std::errc::not_enough_memory);
            }
            catch (...)
            {
                ec = std::make_error_code(std::errc::invalid_argument);
            }
            return root;
        }

        /** \brief Read VDF formatted sequences defined by the range [first, last).
        If the file is mailformatted, parser will try to read it until it can.
        @param first begin iterator
        @param end end iterator
        @param ok output bool. true, if parser successed, false, if parser failed
        */
        template<typename IterT, typename charT = typename IterT::value_type>
        basic_object<charT> read(IterT first, IterT last, bool* ok) noexcept
        {
            std::error_code ec;
            auto r = read(first, last, ec);
            if (ok) *ok = !ec;
            return r;
        }

        /** \brief Read VDF formatted sequences defined by the range [first, last).
        If the file is mailformatted, parser will try to read it until it can.
        @param first begin iterator
        @param end end iterator

        throws a "std::system_error" if a parsing error occured
        */
        template<typename IterT, typename charT = typename IterT::value_type>
        basic_object<charT> read(IterT first, IterT last)
        {
            std::error_code ec;
            const auto r = read(first, last, ec);
            if (!ec)
                throw std::system_error(ec);
            return r;
        }

        /** \brief Loads a stream (e.g. filestream) into the memory and parses the vdf formatted data.
            throws "std::bad_alloc" if file buffer could not be allocated
        */
        template<typename iStreamT, typename charT = iStreamT::char_type >
        basic_object<charT> read(iStreamT& inStream, std::error_code& ec)
        {
            // cache the file
            std::basic_string<charT> str;
            inStream.seekg(0, std::ios::end);
            str.resize(static_cast<size_t>(inStream.tellg()));
            if (str.empty())
                return basic_object<charT>();

            inStream.seekg(0, std::ios::beg);
            inStream.read(&str[0], str.size());
            inStream.close();

            // parse it
            return read(str.begin(), str.end(), ec);
        }

        /** \brief Loads a stream (e.g. filestream) into the memory and parses the vdf formatted data.
            throws "std::bad_alloc" if file buffer could not be allocated
            ok == false, if a parsing error occured
        */
        template<typename iStreamT, typename charT = typename iStreamT::char_type >
        basic_object<charT> read(iStreamT& inStream, bool* ok)
        {
            std::error_code ec;
            const auto r = read(inStream, ec);
            if (ok) *ok = !ec;
            return r;
        }

        /** \brief Loads a stream (e.g. filestream) into the memory and parses the vdf formatted data.
            throws "std::bad_alloc" if file buffer could not be allocated
            throws "std::system_error" if a parsing error occured
        */
        template<typename iStreamT, typename charT = typename iStreamT::char_type >
        basic_object<charT> read(iStreamT& inStream)
        {
            std::error_code ec;
            const auto r = read(inStream, ec);
            if (!ec) throw std::system_error(ec);
            return r;
        }

    } // end namespace vdf
} // end namespace tyti
#ifndef TYTI_NO_L_UNDEF
#undef TYTI_L
#endif

#endif //__TYTI_STEAM_VDF_PARSER_H__
<commit_msg>missing typename<commit_after>//MIT License
//
//Copyright(c) 2016 Matthias Moeller
//
//Permission is hereby granted, free of charge, to any person obtaining a copy
//of this software and associated documentation files(the "Software"), to deal
//in the Software without restriction, including without limitation the rights
//to use, copy, modify, merge, publish, distribute, sublicense, and / or sell
//copies of the Software, and to permit persons to whom the Software is
//furnished to do so, subject to the following conditions :
//
//The above copyright notice and this permission notice shall be included in all
//copies or substantial portions of the Software.
//
//THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
//IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
//FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.IN NO EVENT SHALL THE
//AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
//LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
//OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
//SOFTWARE.

#ifndef __TYTI_STEAM_VDF_PARSER_H__
#define __TYTI_STEAM_VDF_PARSER_H__

#include <unordered_map>
#include <utility>
#include <fstream>
#include <memory>

#include <system_error>
#include <exception>

//for wstring support
#include <locale>
#include <codecvt>
#include <string>

// internal
#include <stack>

namespace tyti
{
    namespace vdf
    {
        namespace detail
        {
            ///////////////////////////////////////////////////////////////////////////
            //  Helper functions selecting the right encoding (char/wchar_T)
            ///////////////////////////////////////////////////////////////////////////

            template<typename T>
            struct literal_macro_help
            {
                static constexpr const char* result(const char* c, const wchar_t* wc) noexcept
                {
                    return c;
                }
                static constexpr const char result(char c, wchar_t wc) noexcept
                {
                    return c;
                }
            };

            template<>
            struct literal_macro_help<wchar_t>
            {
                static constexpr const wchar_t* result(const char* c, const wchar_t* wc) noexcept
                {
                    return wc;
                }
                static constexpr const wchar_t result(char c, wchar_t wc) noexcept
                {
                    return wc;
                }
            };
#define TYTI_L(type, text) vdf::detail::literal_macro_help<type>::result(text, L##text)

            inline std::string string_converter(std::string& w)
            {
                return w;
            }

            inline std::string string_converter(std::wstring& w)
            {
                std::wstring_convert<std::codecvt_utf8<wchar_t>> conv1; // maybe wrong econding
                return conv1.to_bytes(w);
            }

            ///////////////////////////////////////////////////////////////////////////
            //  Writer helper functions
            ///////////////////////////////////////////////////////////////////////////

            struct tabs
            {
                size_t t;
                tabs(size_t i) :t(i) {}
            };

            template<typename oStreamT>
            oStreamT& operator<<(oStreamT& s, tabs t)
            {
                for (; t.t > 0; --t.t)
                    s << "\t";
                return s;
            }
        } // end namespace detail

        ///////////////////////////////////////////////////////////////////////////
        //  Interface
        ///////////////////////////////////////////////////////////////////////////

        /// basic object node. Every object has a name and can contains attributes saved as key_value pairs or childrens
        template<typename CharT>
        struct basic_object
        {
            typedef CharT char_type;
            std::basic_string<char_type> name;
            std::unordered_map<std::basic_string<char_type>, std::basic_string<char_type> > attribs;
            std::unordered_map<std::basic_string<char_type>, std::shared_ptr< basic_object<char_type> > > childs;
        };

        typedef basic_object<char> object;
        typedef basic_object<wchar_t> wobject;

        /** \brief writes given object tree in vdf format to given stream.
        Uses tabs instead of whitespaces.
        */
        template<typename oStreamT, typename charT = typename oStreamT::char_type>
        void write(oStreamT& s, const basic_object<charT>& r, size_t t = 0)
        {
            using namespace detail;
            s << tabs(t) << TYTI_L(charT, '"') << r.name << TYTI_L(charT, "\"\n") << tabs(t) << TYTI_L(charT, "{\n");
            for (auto& i : r.attribs)
                s << tabs(t + 1) << TYTI_L(charT, '"') << i.first << TYTI_L(charT, "\"\t\t\"") << i.second << TYTI_L(charT, "\"\n");
            for (auto& i : r.childs)
                write(s, i, t + 1);
            s << tabs(t) << TYTI_L(charT, "}\n");
        }

        /** \brief Read VDF formatted sequences defined by the range [first, last).
        If the file is mailformatted, parser will try to read it until it can.
        @param first begin iterator
        @param end end iterator
        @param ec output bool. 0 if ok, otherwise, holds an system error code

        Possible error codes:
            std::errc::protocol_error: file is mailformatted
            std::errc::not_enough_memory: not enough space
            std::errc::invalid_argument: iterators throws e.g. out of range
        */

        template<typename IterT, typename charT = typename IterT::value_type>
        basic_object<charT> read(IterT first, IterT last, std::error_code& ec) noexcept
        {
            ec.clear();

            basic_object<charT> root;
            basic_object<charT>* cur = &root;
            std::stack< basic_object<charT>* > lvls;
            //read header
            // first, quoted name
            auto b = std::find(first, last, TYTI_L(charT, '\"'));
            auto bend = std::find(b + 1, last, TYTI_L(charT, '\"'));
            root.name = std::basic_string<charT>(b + 1, bend);
            // second, get {}
            b = std::find(bend, last, TYTI_L(charT, '{'));
            try
            {
                if (b == last)
                {
                    ec = std::make_error_code(std::errc::protocol_error);
                    return root;
                }
                else
                    lvls.push(&root);
                while (!lvls.empty() && b != last)
                {
                    const std::basic_string<charT> startsym = TYTI_L(charT, "\"}");

                    //find first starting attrib/child, or ending
                    b = std::find_first_of(b, last, std::cbegin(startsym), std::cend(startsym));
                    if (*b == '\"')
                    {
                        bend = std::find(b + 1, last, TYTI_L(charT, '\"'));
                        if (bend == last)
                        {
                            ec = std::make_error_code(std::errc::protocol_error);// could not find end of name
                            return root;
                        }

                        std::basic_string<charT> curName(b + 1, bend);
                        b = bend + 1;

                        const std::basic_string<charT> ecspsym = TYTI_L(charT, "\"{");
                        b = std::find_first_of(b, last, std::cbegin(ecspsym), std::cend(ecspsym));
                        if (b == last)
                        {
                            ec = std::make_error_code(std::errc::protocol_error);// could not find 2nd part of pair
                            return root;
                        }

                        if (*b == '\"')
                        {
                            bend = std::find(b + 1, last, TYTI_L(charT, '\"'));
                            if (bend == last)
                            {
                                ec = std::make_error_code(std::errc::protocol_error);//could not find end of name
                                return root;
                            }

                            auto value = std::basic_string<charT>(b + 1, bend);
                            b = bend + 1;

                            if (curName != TYTI_L(charT, "#include") && curName != TYTI_L(charT, "#base"))
                                cur->attribs[curName] = value;
                            else
                            {
                                std::basic_ifstream<charT> i(detail::string_converter(value));
                                auto n = std::make_shared<basic_object<charT>>(read(i, ec));
                                if (ec) return root;
                                cur->childs[n->name] = n;
                            }
                        }
                        else if (*b == '{')
                        {
                            lvls.push(cur);
                            auto n = std::make_shared<basic_object<charT>>();
                            cur->childs[curName] = n;
                            cur = n.get();
                            cur->name = curName;
                            ++b;
                        }
                    }
                    else if (*b == '}')
                    {
                        cur = lvls.top();
                        lvls.pop();
                        ++b;
                    }
                }

            }
            catch (std::bad_alloc& )
            {
                ec = std::make_error_code(std::errc::not_enough_memory);
            }
            catch (...)
            {
                ec = std::make_error_code(std::errc::invalid_argument);
            }
            return root;
        }

        /** \brief Read VDF formatted sequences defined by the range [first, last).
        If the file is mailformatted, parser will try to read it until it can.
        @param first begin iterator
        @param end end iterator
        @param ok output bool. true, if parser successed, false, if parser failed
        */
        template<typename IterT, typename charT = typename IterT::value_type>
        basic_object<charT> read(IterT first, IterT last, bool* ok) noexcept
        {
            std::error_code ec;
            auto r = read(first, last, ec);
            if (ok) *ok = !ec;
            return r;
        }

        /** \brief Read VDF formatted sequences defined by the range [first, last).
        If the file is mailformatted, parser will try to read it until it can.
        @param first begin iterator
        @param end end iterator

        throws a "std::system_error" if a parsing error occured
        */
        template<typename IterT, typename charT = typename IterT::value_type>
        basic_object<charT> read(IterT first, IterT last)
        {
            std::error_code ec;
            const auto r = read(first, last, ec);
            if (!ec)
                throw std::system_error(ec);
            return r;
        }

        /** \brief Loads a stream (e.g. filestream) into the memory and parses the vdf formatted data.
            throws "std::bad_alloc" if file buffer could not be allocated
        */
        template<typename iStreamT, typename charT = typename iStreamT::char_type >
        basic_object<charT> read(iStreamT& inStream, std::error_code& ec)
        {
            // cache the file
            std::basic_string<charT> str;
            inStream.seekg(0, std::ios::end);
            str.resize(static_cast<size_t>(inStream.tellg()));
            if (str.empty())
                return basic_object<charT>();

            inStream.seekg(0, std::ios::beg);
            inStream.read(&str[0], str.size());
            inStream.close();

            // parse it
            return read(str.begin(), str.end(), ec);
        }

        /** \brief Loads a stream (e.g. filestream) into the memory and parses the vdf formatted data.
            throws "std::bad_alloc" if file buffer could not be allocated
            ok == false, if a parsing error occured
        */
        template<typename iStreamT, typename charT = typename iStreamT::char_type >
        basic_object<charT> read(iStreamT& inStream, bool* ok)
        {
            std::error_code ec;
            const auto r = read(inStream, ec);
            if (ok) *ok = !ec;
            return r;
        }

        /** \brief Loads a stream (e.g. filestream) into the memory and parses the vdf formatted data.
            throws "std::bad_alloc" if file buffer could not be allocated
            throws "std::system_error" if a parsing error occured
        */
        template<typename iStreamT, typename charT = typename iStreamT::char_type >
        basic_object<charT> read(iStreamT& inStream)
        {
            std::error_code ec;
            const auto r = read(inStream, ec);
            if (!ec) throw std::system_error(ec);
            return r;
        }

    } // end namespace vdf
} // end namespace tyti
#ifndef TYTI_NO_L_UNDEF
#undef TYTI_L
#endif

#endif //__TYTI_STEAM_VDF_PARSER_H__
<|endoftext|>


<commit_before>/*********************************************************************
 *
 * Condor ClassAd library
 * Copyright (C) 1990-2003, Condor Team, Computer Sciences Department,
 * University of Wisconsin-Madison, WI and Rajesh Raman.
 *
 * This source code is covered by the Condor Public License, which can
 * be found in the accompanying LICENSE file, or online at
 * www.condorproject.org.
 *
 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
 * AND THE UNIVERSITY OF WISCONSIN-MADISON "AS IS" AND ANY EXPRESS OR
 * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
 * WARRANTIES OF MERCHANTABILITY, OF SATISFACTORY QUALITY, AND FITNESS
 * FOR A PARTICULAR PURPOSE OR USE ARE DISCLAIMED. THE COPYRIGHT
 * HOLDERS AND CONTRIBUTORS AND THE UNIVERSITY OF WISCONSIN-MADISON
 * MAKE NO MAKE NO REPRESENTATION THAT THE SOFTWARE, MODIFICATIONS,
 * ENHANCEMENTS OR DERIVATIVE WORKS THEREOF, WILL NOT INFRINGE ANY
 * PATENT, COPYRIGHT, TRADEMARK, TRADE SECRET OR OTHER PROPRIETARY
 * RIGHT.
 *
 *********************************************************************/

#if defined(CLASSAD_DISTRIBUTION)
#include "classad_distribution.h"
#else
#include "condor_classad.h"
#endif

using namespace std;
#ifdef WANT_NAMESPACES
using namespace classad;
#endif

int main(int argc, char **argv)
{
    string classad_version;

    ClassAdLibraryVersion(classad_version);

    cout << "ClassAd Library v" << classad_version << "\n";

    /* ----- Namespace support ----- */
    cout << "  Classad namespace:               ";
#if defined WANT_NAMESPACES
    cout << "yes\n";
#else
    cout << "no\n";
#endif

    /* ----- Shared library function support ----- */
    cout << "  Shared library function support: ";
#if defined ENABLE_SHARED_LIBRARY_FUNCTIONS
    cout << "yes\n;"
#else
    cout << "no\n";
#endif

    /* ----- Regular expressions ----- */
    cout << "  Regular expression function:     ";
#if defined USE_POSIX_REGEX
    cout << "POSIX\n";
#elif defined USE_PCRE
    cout << "PCRE\n";
#else
    cout << "Disabled\n";
#endif

    /* ----- GNU version, if applicable ----- */
#if defined __GNUC__ && defined __GNUC_MINOR__
    cout << "  GCC version:                     ";
    cout << __GNUC__ << "." << __GNUC_MINOR__;
#if defined __GNUC_PATCHLEVEL__
    cout << "." << __GNUC_PATCHLEVEL__;
#endif
    cout << endl;
#endif

    return 0;
}
<commit_msg>Moved a misplaced semicolon.<commit_after>/*********************************************************************
 *
 * Condor ClassAd library
 * Copyright (C) 1990-2003, Condor Team, Computer Sciences Department,
 * University of Wisconsin-Madison, WI and Rajesh Raman.
 *
 * This source code is covered by the Condor Public License, which can
 * be found in the accompanying LICENSE file, or online at
 * www.condorproject.org.
 *
 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
 * AND THE UNIVERSITY OF WISCONSIN-MADISON "AS IS" AND ANY EXPRESS OR
 * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
 * WARRANTIES OF MERCHANTABILITY, OF SATISFACTORY QUALITY, AND FITNESS
 * FOR A PARTICULAR PURPOSE OR USE ARE DISCLAIMED. THE COPYRIGHT
 * HOLDERS AND CONTRIBUTORS AND THE UNIVERSITY OF WISCONSIN-MADISON
 * MAKE NO MAKE NO REPRESENTATION THAT THE SOFTWARE, MODIFICATIONS,
 * ENHANCEMENTS OR DERIVATIVE WORKS THEREOF, WILL NOT INFRINGE ANY
 * PATENT, COPYRIGHT, TRADEMARK, TRADE SECRET OR OTHER PROPRIETARY
 * RIGHT.
 *
 *********************************************************************/

#if defined(CLASSAD_DISTRIBUTION)
#include "classad_distribution.h"
#else
#include "condor_classad.h"
#endif

using namespace std;
#ifdef WANT_NAMESPACES
using namespace classad;
#endif

int main(int argc, char **argv)
{
    string classad_version;

    ClassAdLibraryVersion(classad_version);

    cout << "ClassAd Library v" << classad_version << "\n";

    /* ----- Namespace support ----- */
    cout << "  Classad namespace:               ";
#if defined WANT_NAMESPACES
    cout << "yes\n";
#else
    cout << "no\n";
#endif

    /* ----- Shared library function support ----- */
    cout << "  Shared library function support: ";
#if defined ENABLE_SHARED_LIBRARY_FUNCTIONS
    cout << "yes\n";
#else
    cout << "no\n";
#endif

    /* ----- Regular expressions ----- */
    cout << "  Regular expression function:     ";
#if defined USE_POSIX_REGEX
    cout << "POSIX\n";
#elif defined USE_PCRE
    cout << "PCRE\n";
#else
    cout << "Disabled\n";
#endif

    /* ----- GNU version, if applicable ----- */
#if defined __GNUC__ && defined __GNUC_MINOR__
    cout << "  GCC version:                     ";
    cout << __GNUC__ << "." << __GNUC_MINOR__;
#if defined __GNUC_PATCHLEVEL__
    cout << "." << __GNUC_PATCHLEVEL__;
#endif
    cout << endl;
#endif

    return 0;
}
<|endoftext|>