<commit_before>/*=====================================================================

QGroundControl Open Source Ground Control Station

(c) 2009 - 2015 QGROUNDCONTROL PROJECT <http://www.qgroundcontrol.org>

This file is part of the QGROUNDCONTROL project

    QGROUNDCONTROL is free software: you can redistribute it and/or modify
    it under the terms of the GNU General Public License as published by
    the Free Software Foundation, either version 3 of the License, or
    (at your option) any later version.

    QGROUNDCONTROL is distributed in the hope that it will be useful,
    but WITHOUT ANY WARRANTY; without even the implied warranty of
    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
    GNU General Public License for more details.

    You should have received a copy of the GNU General Public License
    along with QGROUNDCONTROL. If not, see <http://www.gnu.org/licenses/>.

======================================================================*/

/**
 * @file
 *   @brief Definition of UDP connection (server) for unmanned vehicles
 *   @author Lorenz Meier <mavteam@student.ethz.ch>
 *
 */

#include <QTimer>
#include <QList>
#include <QDebug>
#include <QMutexLocker>
#include <QNetworkProxy>
#include <iostream>

#include "UDPLink.h"
#include "QGC.h"
#include <QHostInfo>

static const char* kZeroconfRegistration = "_qgroundcontrol._udp";

static bool is_ip(const QString& address)
{
    int a,b,c,d;
    if (sscanf(address.toStdString().c_str(), "%d.%d.%d.%d", &a, &b, &c, &d) != 4
            && strcmp("::1", address.toStdString().c_str())) {
        return false;
    } else {
        return true;
    }
}

static QString get_ip_address(const QString& address)
{
    if(is_ip(address))
        return address;
    // Need to look it up
    QHostInfo info = QHostInfo::fromName(address);
    if (info.error() == QHostInfo::NoError)
    {
        QList<QHostAddress> hostAddresses = info.addresses();
        QHostAddress address;
        for (int i = 0; i < hostAddresses.size(); i++)
        {
            // Exclude all IPv6 addresses
            if (!hostAddresses.at(i).toString().contains(":"))
            {
                return hostAddresses.at(i).toString();
            }
        }
    }
    return QString("");
}

UDPLink::UDPLink(UDPConfiguration* config)
    : _socket(NULL)
    , _connectState(false)
    #if defined(QGC_ZEROCONF_ENABLED)
    , _dnssServiceRef(NULL)
    #endif
    , _running(false)
{
    Q_ASSERT(config != NULL);
    _config = config;
    _config->setLink(this);

    // We're doing it wrong - because the Qt folks got the API wrong:
    // http://blog.qt.digia.com/blog/2010/06/17/youre-doing-it-wrong/
    moveToThread(this);

    //qDebug() << "UDP Created " << _config->name();
}

UDPLink::~UDPLink()
{
    // Disconnect link from configuration
    _config->setLink(NULL);
    _disconnect();
    // Tell the thread to exit
    _running = false;
    // Wait for it to exit
    wait();
    while(_outQueue.count() > 0) {
        delete _outQueue.dequeue();
    }
    this->deleteLater();
}

/**
 * @brief Runs the thread
 *
 **/
void UDPLink::run()
{
    if(_hardwareConnect()) {
        while(true) {
            readBytes();
            if(_writeBytes() && _running)
                continue;
            if(!_running)
                break;
            this->msleep(50);
        }
    }
    if (_socket) {
        _deregisterZeroconf();
        _socket->close();
    }
}

void UDPLink::_restartConnection()
{
    if(this->isConnected())
    {
        _disconnect();
        _connect();
    }
}

QString UDPLink::getName() const
{
    return _config->name();
}

void UDPLink::addHost(const QString& host)
{
    _config->addHost(host);
}

void UDPLink::removeHost(const QString& host)
{
    _config->removeHost(host);
}

#define UDPLINK_DEBUG 0

void UDPLink::writeBytes(const char* data, qint64 size)
{
    if (!_socket) {
        return;
    }
    QByteArray* qdata = new QByteArray(data, size);
    QMutexLocker lock(&_mutex);
    _outQueue.enqueue(qdata);
}

bool UDPLink::_writeBytes()
{
    QMutexLocker lock(&_mutex);
    if(_outQueue.count() > 0) {
        QByteArray* qdata = _outQueue.dequeue();
        lock.unlock();
        QStringList goneHosts;
        // Send to all connected systems
        QString host;
        int port;
        if(_config->firstHost(host, port)) {
            do {
                QHostAddress currentHost(host);
                if(_socket->writeDatagram(qdata->data(), qdata->size(), currentHost, (quint16)port) < 0) {
                    // This host is gone. Add to list to be removed
                    goneHosts.append(host);
                }
                // Log the amount and time written out for future data rate calculations.
                QMutexLocker dataRateLocker(&dataRateMutex);
                logDataRateToBuffer(outDataWriteAmounts, outDataWriteTimes, &outDataIndex, qdata->size(), QDateTime::currentMSecsSinceEpoch());
            } while (_config->nextHost(host, port));
            //-- Remove hosts that are no longer there
            foreach (QString ghost, goneHosts) {
                _config->removeHost(host);
            }
        }
        delete qdata;
        lock.relock();
    }
    return (_outQueue.count() > 0);
}

/**
 * @brief Read a number of bytes from the interface.
 **/
void UDPLink::readBytes()
{
    while (_socket->hasPendingDatagrams())
    {
        QByteArray datagram;
        datagram.resize(_socket->pendingDatagramSize());

        QHostAddress sender;
        quint16 senderPort;
        _socket->readDatagram(datagram.data(), datagram.size(), &sender, &senderPort);

        // FIXME TODO Check if this method is better than retrieving the data by individual processes
        emit bytesReceived(this, datagram);

        // Log this data reception for this timestep
        QMutexLocker dataRateLocker(&dataRateMutex);
        logDataRateToBuffer(inDataWriteAmounts, inDataWriteTimes, &inDataIndex, datagram.length(), QDateTime::currentMSecsSinceEpoch());

//        // Echo data for debugging purposes
//        std::cerr << __FILE__ << __LINE__ << "Received datagram:" << std::endl;
//        int i;
//        for (i=0; i<s; i++)
//        {
//            unsigned int v=data[i];
//            fprintf(stderr,"%02x ", v);
//        }
//        std::cerr << std::endl;

        // TODO This doesn't validade the sender. Anything sending UDP packets to this port gets
        // added to the list and will start receiving datagrams from here. Even a port scanner
        // would trigger this.
        // Add host to broadcast list if not yet present, or update its port
        _config->addHost(sender.toString(), (int)senderPort);
        if(!_running)
            break;
    }
}

/**
 * @brief Disconnect the connection.
 *
 * @return True if connection has been disconnected, false if connection couldn't be disconnected.
 **/
bool UDPLink::_disconnect(void)
{
    _running = false;
    wait();
    if (_socket) {
        // Make sure delete happen on correct thread
        _socket->deleteLater();
        _socket = NULL;
        emit disconnected();
    }
    // TODO When would this ever return false?
    _connectState = false;
    return !_connectState;
}

/**
 * @brief Connect the connection.
 *
 * @return True if connection has been established, false if connection couldn't be established.
 **/
bool UDPLink::_connect(void)
{
    if(this->isRunning())
    {
        _running = false;
        wait();
    }
    // TODO When would this ever return false?
    bool connected = true;
    // I see no reason to run this in "HighPriority"
    _running = true;
    start(NormalPriority);
    return connected;
}

bool UDPLink::_hardwareConnect()
{
    if (_socket) {
        delete _socket;
        _socket = NULL;
    }
    QHostAddress host = QHostAddress::AnyIPv4;
    _socket = new QUdpSocket();
    _socket->setProxy(QNetworkProxy::NoProxy);
    _connectState = _socket->bind(host, _config->localPort(), QAbstractSocket::ReuseAddressHint);
    if (_connectState) {
        _registerZeroconf(_config->localPort(), kZeroconfRegistration);
        emit connected();
    } else {
        emit communicationError("UDP Link Error", "Error binding UDP port");
    }
    return _connectState;
}

/**
 * @brief Check if connection is active.
 *
 * @return True if link is connected, false otherwise.
 **/
bool UDPLink::isConnected() const
{
    return _connectState;
}

qint64 UDPLink::getConnectionSpeed() const
{
    return 54000000; // 54 Mbit
}

qint64 UDPLink::getCurrentInDataRate() const
{
    return 0;
}

qint64 UDPLink::getCurrentOutDataRate() const
{
    return 0;
}

void UDPLink::_registerZeroconf(uint16_t port, const std::string &regType)
{
#if defined(QGC_ZEROCONF_ENABLED)
    DNSServiceErrorType result = DNSServiceRegister(&_dnssServiceRef, 0, 0, 0,
        regType.c_str(),
        NULL,
        NULL,
        htons(port),
        0,
        NULL,
        NULL,
        NULL);
    if (result != kDNSServiceErr_NoError)
    {
        emit communicationError("UDP Link Error", "Error registering Zeroconf");
        _dnssServiceRef = NULL;
    }
#else
    Q_UNUSED(port);
    Q_UNUSED(regType);
#endif
}

void UDPLink::_deregisterZeroconf()
{
#if defined(QGC_ZEROCONF_ENABLED)
    if (_dnssServiceRef)
     {
         DNSServiceRefDeallocate(_dnssServiceRef);
         _dnssServiceRef = NULL;
     }
#endif
}

//--------------------------------------------------------------------------
//-- UDPConfiguration

UDPConfiguration::UDPConfiguration(const QString& name) : LinkConfiguration(name)
{
    _localPort = QGC_UDP_LOCAL_PORT;
}

UDPConfiguration::UDPConfiguration(UDPConfiguration* source) : LinkConfiguration(source)
{
    _localPort = source->localPort();
    _hosts.clear();
    QString host;
    int port;
    if(source->firstHost(host, port)) {
        do {
            addHost(host, port);
        } while(source->nextHost(host, port));
    }
}

void UDPConfiguration::copyFrom(LinkConfiguration *source)
{
    LinkConfiguration::copyFrom(source);
    UDPConfiguration* usource = dynamic_cast<UDPConfiguration*>(source);
    Q_ASSERT(usource != NULL);
    _localPort = usource->localPort();
    _hosts.clear();
    QString host;
    int port;
    if(usource->firstHost(host, port)) {
        do {
            addHost(host, port);
        } while(usource->nextHost(host, port));
    }
}

/**
 * @param host Hostname in standard formatt, e.g. localhost:14551 or 192.168.1.1:14551
 */
void UDPConfiguration::addHost(const QString& host)
{
    // Handle x.x.x.x:p
    if (host.contains(":"))
    {
        addHost(host.split(":").first(), host.split(":").last().toInt());
    }
    // If no port, use default
    else
    {
        addHost(host, (int)_localPort);
    }
}

void UDPConfiguration::addHost(const QString& host, int port)
{
    QMutexLocker locker(&_confMutex);
    if(_hosts.contains(host)) {
        if(_hosts[host] != port) {
            _hosts[host] = port;
        }
    } else {
        QString ipAdd = get_ip_address(host);
        if(ipAdd.isEmpty()) {
            qWarning() << "UDP:" << "Could not resolve host:" << host << "port:" << port;
        } else {
            _hosts[ipAdd] = port;
            //qDebug() << "UDP:" << "Adding Host:" << ipAdd << ":" << port;
        }
    }
}

void UDPConfiguration::removeHost(const QString& host)
{
    QMutexLocker locker(&_confMutex);
    QString tHost = host;
    if (tHost.contains(":")) {
        tHost = tHost.split(":").first();
    }
    tHost = tHost.trimmed();
    QMap<QString, int>::iterator i = _hosts.find(tHost);
    if(i != _hosts.end()) {
        //qDebug() << "UDP:" << "Removed host:" << host;
        _hosts.erase(i);
    } else {
        qWarning() << "UDP:" << "Could not remove unknown host:" << host;
    }
}

bool UDPConfiguration::firstHost(QString& host, int& port)
{
    _confMutex.lock();
    _it = _hosts.begin();
    if(_it == _hosts.end()) {
        _confMutex.unlock();
        return false;
    }
    _confMutex.unlock();
    return nextHost(host, port);
}

bool UDPConfiguration::nextHost(QString& host, int& port)
{
    QMutexLocker locker(&_confMutex);
    if(_it != _hosts.end()) {
        host = _it.key();
        port = _it.value();
        _it++;
        return true;
    }
    return false;
}

void UDPConfiguration::setLocalPort(quint16 port)
{
    _localPort = port;
}

void UDPConfiguration::saveSettings(QSettings& settings, const QString& root)
{
    _confMutex.lock();
    settings.beginGroup(root);
    settings.setValue("port", (int)_localPort);
    settings.setValue("hostCount", _hosts.count());
    int index = 0;
    QMap<QString, int>::const_iterator it = _hosts.begin();
    while(it != _hosts.end()) {
        QString hkey = QString("host%1").arg(index);
        settings.setValue(hkey, it.key());
        QString pkey = QString("port%1").arg(index);
        settings.setValue(pkey, it.value());
        it++;
        index++;
    }
    settings.endGroup();
    _confMutex.unlock();
}

void UDPConfiguration::loadSettings(QSettings& settings, const QString& root)
{
    _confMutex.lock();
    _hosts.clear();
    _confMutex.unlock();
    settings.beginGroup(root);
    _localPort = (quint16)settings.value("port", QGC_UDP_LOCAL_PORT).toUInt();
    int hostCount = settings.value("hostCount", 0).toInt();
    for(int i = 0; i < hostCount; i++) {
        QString hkey = QString("host%1").arg(i);
        QString pkey = QString("port%1").arg(i);
        if(settings.contains(hkey) && settings.contains(pkey)) {
            addHost(settings.value(hkey).toString(), settings.value(pkey).toInt());
        }
    }
    settings.endGroup();
}

void UDPConfiguration::updateSettings()
{
    if(_link) {
        UDPLink* ulink = dynamic_cast<UDPLink*>(_link);
        if(ulink) {
            ulink->_restartConnection();
        }
    }
}
<commit_msg>Oops, removing a host instead of a ghost :)<commit_after>/*=====================================================================

QGroundControl Open Source Ground Control Station

(c) 2009 - 2015 QGROUNDCONTROL PROJECT <http://www.qgroundcontrol.org>

This file is part of the QGROUNDCONTROL project

    QGROUNDCONTROL is free software: you can redistribute it and/or modify
    it under the terms of the GNU General Public License as published by
    the Free Software Foundation, either version 3 of the License, or
    (at your option) any later version.

    QGROUNDCONTROL is distributed in the hope that it will be useful,
    but WITHOUT ANY WARRANTY; without even the implied warranty of
    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
    GNU General Public License for more details.

    You should have received a copy of the GNU General Public License
    along with QGROUNDCONTROL. If not, see <http://www.gnu.org/licenses/>.

======================================================================*/

/**
 * @file
 *   @brief Definition of UDP connection (server) for unmanned vehicles
 *   @author Lorenz Meier <mavteam@student.ethz.ch>
 *
 */

#include <QTimer>
#include <QList>
#include <QDebug>
#include <QMutexLocker>
#include <QNetworkProxy>
#include <iostream>

#include "UDPLink.h"
#include "QGC.h"
#include <QHostInfo>

static const char* kZeroconfRegistration = "_qgroundcontrol._udp";

static bool is_ip(const QString& address)
{
    int a,b,c,d;
    if (sscanf(address.toStdString().c_str(), "%d.%d.%d.%d", &a, &b, &c, &d) != 4
            && strcmp("::1", address.toStdString().c_str())) {
        return false;
    } else {
        return true;
    }
}

static QString get_ip_address(const QString& address)
{
    if(is_ip(address))
        return address;
    // Need to look it up
    QHostInfo info = QHostInfo::fromName(address);
    if (info.error() == QHostInfo::NoError)
    {
        QList<QHostAddress> hostAddresses = info.addresses();
        QHostAddress address;
        for (int i = 0; i < hostAddresses.size(); i++)
        {
            // Exclude all IPv6 addresses
            if (!hostAddresses.at(i).toString().contains(":"))
            {
                return hostAddresses.at(i).toString();
            }
        }
    }
    return QString("");
}

UDPLink::UDPLink(UDPConfiguration* config)
    : _socket(NULL)
    , _connectState(false)
    #if defined(QGC_ZEROCONF_ENABLED)
    , _dnssServiceRef(NULL)
    #endif
    , _running(false)
{
    Q_ASSERT(config != NULL);
    _config = config;
    _config->setLink(this);

    // We're doing it wrong - because the Qt folks got the API wrong:
    // http://blog.qt.digia.com/blog/2010/06/17/youre-doing-it-wrong/
    moveToThread(this);

    //qDebug() << "UDP Created " << _config->name();
}

UDPLink::~UDPLink()
{
    // Disconnect link from configuration
    _config->setLink(NULL);
    _disconnect();
    // Tell the thread to exit
    _running = false;
    // Wait for it to exit
    wait();
    while(_outQueue.count() > 0) {
        delete _outQueue.dequeue();
    }
    this->deleteLater();
}

/**
 * @brief Runs the thread
 *
 **/
void UDPLink::run()
{
    if(_hardwareConnect()) {
        while(true) {
            readBytes();
            if(_writeBytes() && _running)
                continue;
            if(!_running)
                break;
            this->msleep(50);
        }
    }
    if (_socket) {
        _deregisterZeroconf();
        _socket->close();
    }
}

void UDPLink::_restartConnection()
{
    if(this->isConnected())
    {
        _disconnect();
        _connect();
    }
}

QString UDPLink::getName() const
{
    return _config->name();
}

void UDPLink::addHost(const QString& host)
{
    _config->addHost(host);
}

void UDPLink::removeHost(const QString& host)
{
    _config->removeHost(host);
}

#define UDPLINK_DEBUG 0

void UDPLink::writeBytes(const char* data, qint64 size)
{
    if (!_socket) {
        return;
    }
    QByteArray* qdata = new QByteArray(data, size);
    QMutexLocker lock(&_mutex);
    _outQueue.enqueue(qdata);
}

bool UDPLink::_writeBytes()
{
    QMutexLocker lock(&_mutex);
    if(_outQueue.count() > 0) {
        QByteArray* qdata = _outQueue.dequeue();
        lock.unlock();
        QStringList goneHosts;
        // Send to all connected systems
        QString host;
        int port;
        if(_config->firstHost(host, port)) {
            do {
                QHostAddress currentHost(host);
                if(_socket->writeDatagram(qdata->data(), qdata->size(), currentHost, (quint16)port) < 0) {
                    // This host is gone. Add to list to be removed
                    goneHosts.append(host);
                }
                // Log the amount and time written out for future data rate calculations.
                QMutexLocker dataRateLocker(&dataRateMutex);
                logDataRateToBuffer(outDataWriteAmounts, outDataWriteTimes, &outDataIndex, qdata->size(), QDateTime::currentMSecsSinceEpoch());
            } while (_config->nextHost(host, port));
            //-- Remove hosts that are no longer there
            foreach (QString ghost, goneHosts) {
                _config->removeHost(ghost);
            }
        }
        delete qdata;
        lock.relock();
    }
    return (_outQueue.count() > 0);
}

/**
 * @brief Read a number of bytes from the interface.
 **/
void UDPLink::readBytes()
{
    while (_socket->hasPendingDatagrams())
    {
        QByteArray datagram;
        datagram.resize(_socket->pendingDatagramSize());

        QHostAddress sender;
        quint16 senderPort;
        _socket->readDatagram(datagram.data(), datagram.size(), &sender, &senderPort);

        // FIXME TODO Check if this method is better than retrieving the data by individual processes
        emit bytesReceived(this, datagram);

        // Log this data reception for this timestep
        QMutexLocker dataRateLocker(&dataRateMutex);
        logDataRateToBuffer(inDataWriteAmounts, inDataWriteTimes, &inDataIndex, datagram.length(), QDateTime::currentMSecsSinceEpoch());

//        // Echo data for debugging purposes
//        std::cerr << __FILE__ << __LINE__ << "Received datagram:" << std::endl;
//        int i;
//        for (i=0; i<s; i++)
//        {
//            unsigned int v=data[i];
//            fprintf(stderr,"%02x ", v);
//        }
//        std::cerr << std::endl;

        // TODO This doesn't validade the sender. Anything sending UDP packets to this port gets
        // added to the list and will start receiving datagrams from here. Even a port scanner
        // would trigger this.
        // Add host to broadcast list if not yet present, or update its port
        _config->addHost(sender.toString(), (int)senderPort);
        if(!_running)
            break;
    }
}

/**
 * @brief Disconnect the connection.
 *
 * @return True if connection has been disconnected, false if connection couldn't be disconnected.
 **/
bool UDPLink::_disconnect(void)
{
    _running = false;
    wait();
    if (_socket) {
        // Make sure delete happen on correct thread
        _socket->deleteLater();
        _socket = NULL;
        emit disconnected();
    }
    // TODO When would this ever return false?
    _connectState = false;
    return !_connectState;
}

/**
 * @brief Connect the connection.
 *
 * @return True if connection has been established, false if connection couldn't be established.
 **/
bool UDPLink::_connect(void)
{
    if(this->isRunning())
    {
        _running = false;
        wait();
    }
    // TODO When would this ever return false?
    bool connected = true;
    // I see no reason to run this in "HighPriority"
    _running = true;
    start(NormalPriority);
    return connected;
}

bool UDPLink::_hardwareConnect()
{
    if (_socket) {
        delete _socket;
        _socket = NULL;
    }
    QHostAddress host = QHostAddress::AnyIPv4;
    _socket = new QUdpSocket();
    _socket->setProxy(QNetworkProxy::NoProxy);
    _connectState = _socket->bind(host, _config->localPort(), QAbstractSocket::ReuseAddressHint);
    if (_connectState) {
        _registerZeroconf(_config->localPort(), kZeroconfRegistration);
        emit connected();
    } else {
        emit communicationError("UDP Link Error", "Error binding UDP port");
    }
    return _connectState;
}

/**
 * @brief Check if connection is active.
 *
 * @return True if link is connected, false otherwise.
 **/
bool UDPLink::isConnected() const
{
    return _connectState;
}

qint64 UDPLink::getConnectionSpeed() const
{
    return 54000000; // 54 Mbit
}

qint64 UDPLink::getCurrentInDataRate() const
{
    return 0;
}

qint64 UDPLink::getCurrentOutDataRate() const
{
    return 0;
}

void UDPLink::_registerZeroconf(uint16_t port, const std::string &regType)
{
#if defined(QGC_ZEROCONF_ENABLED)
    DNSServiceErrorType result = DNSServiceRegister(&_dnssServiceRef, 0, 0, 0,
        regType.c_str(),
        NULL,
        NULL,
        htons(port),
        0,
        NULL,
        NULL,
        NULL);
    if (result != kDNSServiceErr_NoError)
    {
        emit communicationError("UDP Link Error", "Error registering Zeroconf");
        _dnssServiceRef = NULL;
    }
#else
    Q_UNUSED(port);
    Q_UNUSED(regType);
#endif
}

void UDPLink::_deregisterZeroconf()
{
#if defined(QGC_ZEROCONF_ENABLED)
    if (_dnssServiceRef)
     {
         DNSServiceRefDeallocate(_dnssServiceRef);
         _dnssServiceRef = NULL;
     }
#endif
}

//--------------------------------------------------------------------------
//-- UDPConfiguration

UDPConfiguration::UDPConfiguration(const QString& name) : LinkConfiguration(name)
{
    _localPort = QGC_UDP_LOCAL_PORT;
}

UDPConfiguration::UDPConfiguration(UDPConfiguration* source) : LinkConfiguration(source)
{
    _localPort = source->localPort();
    _hosts.clear();
    QString host;
    int port;
    if(source->firstHost(host, port)) {
        do {
            addHost(host, port);
        } while(source->nextHost(host, port));
    }
}

void UDPConfiguration::copyFrom(LinkConfiguration *source)
{
    LinkConfiguration::copyFrom(source);
    UDPConfiguration* usource = dynamic_cast<UDPConfiguration*>(source);
    Q_ASSERT(usource != NULL);
    _localPort = usource->localPort();
    _hosts.clear();
    QString host;
    int port;
    if(usource->firstHost(host, port)) {
        do {
            addHost(host, port);
        } while(usource->nextHost(host, port));
    }
}

/**
 * @param host Hostname in standard formatt, e.g. localhost:14551 or 192.168.1.1:14551
 */
void UDPConfiguration::addHost(const QString& host)
{
    // Handle x.x.x.x:p
    if (host.contains(":"))
    {
        addHost(host.split(":").first(), host.split(":").last().toInt());
    }
    // If no port, use default
    else
    {
        addHost(host, (int)_localPort);
    }
}

void UDPConfiguration::addHost(const QString& host, int port)
{
    QMutexLocker locker(&_confMutex);
    if(_hosts.contains(host)) {
        if(_hosts[host] != port) {
            _hosts[host] = port;
        }
    } else {
        QString ipAdd = get_ip_address(host);
        if(ipAdd.isEmpty()) {
            qWarning() << "UDP:" << "Could not resolve host:" << host << "port:" << port;
        } else {
            _hosts[ipAdd] = port;
            //qDebug() << "UDP:" << "Adding Host:" << ipAdd << ":" << port;
        }
    }
}

void UDPConfiguration::removeHost(const QString& host)
{
    QMutexLocker locker(&_confMutex);
    QString tHost = host;
    if (tHost.contains(":")) {
        tHost = tHost.split(":").first();
    }
    tHost = tHost.trimmed();
    QMap<QString, int>::iterator i = _hosts.find(tHost);
    if(i != _hosts.end()) {
        //qDebug() << "UDP:" << "Removed host:" << host;
        _hosts.erase(i);
    } else {
        qWarning() << "UDP:" << "Could not remove unknown host:" << host;
    }
}

bool UDPConfiguration::firstHost(QString& host, int& port)
{
    _confMutex.lock();
    _it = _hosts.begin();
    if(_it == _hosts.end()) {
        _confMutex.unlock();
        return false;
    }
    _confMutex.unlock();
    return nextHost(host, port);
}

bool UDPConfiguration::nextHost(QString& host, int& port)
{
    QMutexLocker locker(&_confMutex);
    if(_it != _hosts.end()) {
        host = _it.key();
        port = _it.value();
        _it++;
        return true;
    }
    return false;
}

void UDPConfiguration::setLocalPort(quint16 port)
{
    _localPort = port;
}

void UDPConfiguration::saveSettings(QSettings& settings, const QString& root)
{
    _confMutex.lock();
    settings.beginGroup(root);
    settings.setValue("port", (int)_localPort);
    settings.setValue("hostCount", _hosts.count());
    int index = 0;
    QMap<QString, int>::const_iterator it = _hosts.begin();
    while(it != _hosts.end()) {
        QString hkey = QString("host%1").arg(index);
        settings.setValue(hkey, it.key());
        QString pkey = QString("port%1").arg(index);
        settings.setValue(pkey, it.value());
        it++;
        index++;
    }
    settings.endGroup();
    _confMutex.unlock();
}

void UDPConfiguration::loadSettings(QSettings& settings, const QString& root)
{
    _confMutex.lock();
    _hosts.clear();
    _confMutex.unlock();
    settings.beginGroup(root);
    _localPort = (quint16)settings.value("port", QGC_UDP_LOCAL_PORT).toUInt();
    int hostCount = settings.value("hostCount", 0).toInt();
    for(int i = 0; i < hostCount; i++) {
        QString hkey = QString("host%1").arg(i);
        QString pkey = QString("port%1").arg(i);
        if(settings.contains(hkey) && settings.contains(pkey)) {
            addHost(settings.value(hkey).toString(), settings.value(pkey).toInt());
        }
    }
    settings.endGroup();
}

void UDPConfiguration::updateSettings()
{
    if(_link) {
        UDPLink* ulink = dynamic_cast<UDPLink*>(_link);
        if(ulink) {
            ulink->_restartConnection();
        }
    }
}
<|endoftext|>


<commit_before>#include "search.hpp"
#include "idastar.hpp"
#include "astar.hpp"
#include "astar-dump.hpp"
#include "wastar.hpp"
#include "greedy.hpp"
#include "bugsy.hpp"
#include "bugsy-slim.hpp"
#include "arastar.hpp"
#include "lsslrtastar2.hpp"
#include "fhatlrtastar.hpp"
#include "dtastar-dump.hpp"
#include "dtastar.hpp"
#include "monstar-dump.hpp"
#include "rtastar.hpp"

/*
#include "multilrtastar.hpp"
#include "lsslrtastar.hpp"
#include "flrtastar.hpp"
#include "flrtastar2.hpp"
#include "cautiouslrtastar.hpp"
#include "frankenlrtastar.hpp"
#include "wlrtastar.hpp"
#include "dflrtastar.hpp"
#include "mrastar.hpp"
#include "greedylrtastar.hpp"
#include "uclrtastar.hpp"
*/

#include <cstddef>
#include <cstdio>

// Functions for conveniently defining a new main
// function for a domain.

void dfheader(FILE*);
void dffooter(FILE*);
void dfpair(FILE *, const char *, const char *, ...);
void dfprocstatus(FILE*);
void fatal(const char*, ...);

template<class D> SearchAlgorithm<D> *getsearch(int argc, const char *argv[]);

template<class D> Result<D> search(D &d, int argc, const char *argv[]) {
	return searchGet(getsearch, d, argc, argv);
}

template<class D> Result<D> searchGet(SearchAlgorithm<D>*(*get)(int, const char *[]), D &d, int argc, const char *argv[]) {
	SearchAlgorithm<D> *srch = get(argc, argv);
	if (!srch && argc > 1)
		fatal("Unknow search algorithm: %s", argv[1]);
	if (!srch)
		fatal("Must specify a search algorithm");

	typename D::State s0 = d.initialstate();
	dfpair(stdout, "initial heuristic", "%f", (double) d.h(s0));
	dfpair(stdout, "initial distance", "%f", (double) d.d(s0));
	dfpair(stdout, "algorithm", argv[1]);

	try {
		srch->search(d, s0);
	} catch (std::bad_alloc&) {
		dfpair(stdout, "out of memory", "%s", "true");
		srch->res.path.clear();
		srch->res.ops.clear();
		srch->finish();
	}
	if (srch->res.path.size() > 0) {
		dfpair(stdout, "final sol cost", "%f",
			(double) d.pathcost(srch->res.path, srch->res.ops));
	} else {
		dfpair(stdout, "final sol cost", "%f", -1.0);
	}
	srch->output(stdout);

	Result<D> res = srch->res;
	delete srch;

	return res;
}

template<class D> SearchAlgorithm<D> *getsearch(int argc, const char *argv[]) {
	if (argc < 2)
		fatal("No algorithm specified");

	if (strcmp(argv[1], "idastar") == 0)
		return new Idastar<D>(argc, argv);
	else if (strcmp(argv[1], "astar") == 0)
		return new Astar<D>(argc, argv);
	else if (strcmp(argv[1], "astar-dump") == 0)
		return new Astar_dump<D>(argc, argv);
	else if (strcmp(argv[1], "wastar") == 0)
		return new Wastar<D>(argc, argv);
	else if (strcmp(argv[1], "greedy") == 0)
		return new Greedy<D>(argc, argv);
	else if (strcmp(argv[1], "speedy") == 0)
		return new Greedy<D, true>(argc, argv);
	else if (strcmp(argv[1], "bugsy") == 0)
		return new Bugsy<D>(argc, argv);
	else if (strcmp(argv[1], "bugsy-slim") == 0)
		return new Bugsy_slim<D>(argc, argv);
	else if (strcmp(argv[1], "arastar") == 0)
		return new Arastar<D>(argc, argv);
	else if (strcmp(argv[1], "arastarmon") == 0)
		return new ArastarMon<D>(argc, argv);
	else if (strcmp(argv[1], "dtastar-dump") == 0)
		return new Dtastar_dump<D>(argc, argv);
	else if (strcmp(argv[1], "lsslrtastar2") == 0)
		return new Lsslrtastar2<D>(argc, argv);
	else if (strcmp(argv[1], "fhatlrtastar") == 0)
		return new Fhatlrtastar<D>(argc, argv);
	else if (strcmp(argv[1], "dtastar") == 0)
		return new Dtastar<D>(argc, argv);
	else if (strcmp(argv[1], "monstar-dump") == 0)
		return new Monstar_dump<D>(argc, argv);
	else if (strcmp(argv[1], "rtastar") == 0)
		return new Rtastar<D>(argc, argv);

/*
	else if (strcmp(argv[1], "mrastar") == 0)
		return new Mrastar<D>(argc, argv);
	else if (strcmp(argv[1], "dflrtastar") == 0)
		return new Dflrtastar<D>(argc, argv);
	else if (strcmp(argv[1], "greedylrtastar") == 0)
		return new Greedylrtastar<D>(argc, argv);
	else if (strcmp(argv[1], "uclrtastar") == 0)
		return new Uclrtastar<D>(argc, argv);
	else if (strcmp(argv[1], "multilrtastar") == 0)
		return new Multilrtastar<D>(argc, argv);
	else if (strcmp(argv[1], "cautiouslrtastar") == 0)
		return new Cautiouslrtastar<D>(argc, argv);
	else if (strcmp(argv[1], "frankenlrtastar") == 0)
		return new Frankenlrtastar<D>(argc, argv);
	else if (strcmp(argv[1], "wlrtastar") == 0)
		return new Wlrtastar<D>(argc, argv);
	else if (strcmp(argv[1], "lsslrtastar") == 0)
		return new Lsslrtastar<D>(argc, argv);
	else if (strcmp(argv[1], "flrtastar") == 0)
		return new Flrtastar<D>(argc, argv);
	else if (strcmp(argv[1], "flrtastar2") == 0)
		return new Flrtastar2<D>(argc, argv);
*/

	fatal("Unknown algorithm: %s", argv[1]);
	return NULL;	// Unreachable
}
<commit_msg>search: reenable some algorithms.<commit_after>#include "search.hpp"
#include "idastar.hpp"
#include "astar.hpp"
#include "astar-dump.hpp"
#include "wastar.hpp"
#include "greedy.hpp"
#include "bugsy.hpp"
#include "bugsy-slim.hpp"
#include "arastar.hpp"
#include "lsslrtastar2.hpp"
#include "fhatlrtastar.hpp"
#include "dtastar-dump.hpp"
#include "dtastar.hpp"
#include "multilrtastar.hpp"

/*
#include "monstar-dump.hpp"
#include "rtastar.hpp"
#include "lsslrtastar.hpp"
#include "flrtastar.hpp"
#include "flrtastar2.hpp"
#include "cautiouslrtastar.hpp"
#include "frankenlrtastar.hpp"
#include "wlrtastar.hpp"
#include "dflrtastar.hpp"
#include "mrastar.hpp"
#include "greedylrtastar.hpp"
#include "uclrtastar.hpp"
*/

#include <cstddef>
#include <cstdio>

// Functions for conveniently defining a new main
// function for a domain.

void dfheader(FILE*);
void dffooter(FILE*);
void dfpair(FILE *, const char *, const char *, ...);
void dfprocstatus(FILE*);
void fatal(const char*, ...);

template<class D> SearchAlgorithm<D> *getsearch(int argc, const char *argv[]);

template<class D> Result<D> search(D &d, int argc, const char *argv[]) {
	return searchGet(getsearch, d, argc, argv);
}

template<class D> Result<D> searchGet(SearchAlgorithm<D>*(*get)(int, const char *[]), D &d, int argc, const char *argv[]) {
	SearchAlgorithm<D> *srch = get(argc, argv);
	if (!srch && argc > 1)
		fatal("Unknow search algorithm: %s", argv[1]);
	if (!srch)
		fatal("Must specify a search algorithm");

	typename D::State s0 = d.initialstate();
	dfpair(stdout, "initial heuristic", "%f", (double) d.h(s0));
	dfpair(stdout, "initial distance", "%f", (double) d.d(s0));
	dfpair(stdout, "algorithm", argv[1]);

	try {
		srch->search(d, s0);
	} catch (std::bad_alloc&) {
		dfpair(stdout, "out of memory", "%s", "true");
		srch->res.path.clear();
		srch->res.ops.clear();
		srch->finish();
	}
	if (srch->res.path.size() > 0) {
		dfpair(stdout, "final sol cost", "%f",
			(double) d.pathcost(srch->res.path, srch->res.ops));
	} else {
		dfpair(stdout, "final sol cost", "%f", -1.0);
	}
	srch->output(stdout);

	Result<D> res = srch->res;
	delete srch;

	return res;
}

template<class D> SearchAlgorithm<D> *getsearch(int argc, const char *argv[]) {
	if (argc < 2)
		fatal("No algorithm specified");

	if (strcmp(argv[1], "idastar") == 0)
		return new Idastar<D>(argc, argv);
	else if (strcmp(argv[1], "astar") == 0)
		return new Astar<D>(argc, argv);
	else if (strcmp(argv[1], "astar-dump") == 0)
		return new Astar_dump<D>(argc, argv);
	else if (strcmp(argv[1], "wastar") == 0)
		return new Wastar<D>(argc, argv);
	else if (strcmp(argv[1], "greedy") == 0)
		return new Greedy<D>(argc, argv);
	else if (strcmp(argv[1], "speedy") == 0)
		return new Greedy<D, true>(argc, argv);
	else if (strcmp(argv[1], "bugsy") == 0)
		return new Bugsy<D>(argc, argv);
	else if (strcmp(argv[1], "bugsy-slim") == 0)
		return new Bugsy_slim<D>(argc, argv);
	else if (strcmp(argv[1], "arastar") == 0)
		return new Arastar<D>(argc, argv);
	else if (strcmp(argv[1], "arastarmon") == 0)
		return new ArastarMon<D>(argc, argv);
	else if (strcmp(argv[1], "dtastar-dump") == 0)
		return new Dtastar_dump<D>(argc, argv);
	else if (strcmp(argv[1], "lsslrtastar2") == 0)
		return new Lsslrtastar2<D>(argc, argv);
	else if (strcmp(argv[1], "fhatlrtastar") == 0)
		return new Fhatlrtastar<D>(argc, argv);
	else if (strcmp(argv[1], "dtastar") == 0)
		return new Dtastar<D>(argc, argv);
	else if (strcmp(argv[1], "multilrtastar") == 0)
		return new Multilrtastar<D>(argc, argv);

/*
	else if (strcmp(argv[1], "monstar-dump") == 0)
		return new Monstar_dump<D>(argc, argv);
	else if (strcmp(argv[1], "rtastar") == 0)
		return new Rtastar<D>(argc, argv);
	else if (strcmp(argv[1], "mrastar") == 0)
		return new Mrastar<D>(argc, argv);
	else if (strcmp(argv[1], "dflrtastar") == 0)
		return new Dflrtastar<D>(argc, argv);
	else if (strcmp(argv[1], "greedylrtastar") == 0)
		return new Greedylrtastar<D>(argc, argv);
	else if (strcmp(argv[1], "uclrtastar") == 0)
		return new Uclrtastar<D>(argc, argv);
	else if (strcmp(argv[1], "cautiouslrtastar") == 0)
		return new Cautiouslrtastar<D>(argc, argv);
	else if (strcmp(argv[1], "frankenlrtastar") == 0)
		return new Frankenlrtastar<D>(argc, argv);
	else if (strcmp(argv[1], "wlrtastar") == 0)
		return new Wlrtastar<D>(argc, argv);
	else if (strcmp(argv[1], "lsslrtastar") == 0)
		return new Lsslrtastar<D>(argc, argv);
	else if (strcmp(argv[1], "flrtastar") == 0)
		return new Flrtastar<D>(argc, argv);
	else if (strcmp(argv[1], "flrtastar2") == 0)
		return new Flrtastar2<D>(argc, argv);
*/

	fatal("Unknown algorithm: %s", argv[1]);
	return NULL;	// Unreachable
}
<|endoftext|>


<commit_before>/**************************************************************************
 *
 * Copyright 2017 VMware, Inc.
 * All Rights Reserved.
 *
 * Permission is hereby granted, free of charge, to any person obtaining a copy
 * of this software and associated documentation files (the "Software"), to deal
 * in the Software without restriction, including without limitation the rights
 * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
 * copies of the Software, and to permit persons to whom the Software is
 * furnished to do so, subject to the following conditions:
 *
 * The above copyright notice and this permission notice shall be included in
 * all copies or substantial portions of the Software.
 *
 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
 * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
 * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
 * THE SOFTWARE.
 *
 **************************************************************************/

#pragma once

#ifdef _MSC_VER
#define ReturnAddress() _ReturnAddress()
#else
#define ReturnAddress() __builtin_return_address(0)
#endif

#include <string>

#ifndef _WIN32

#include <dlfcn.h>

inline std::string
getModuleFromAddress(const void *address)
{
    Dl_info info;
    if (dladdr(address, &info)) {
        return info.dli_fname;
    }
    return "";
}

#endif

<commit_msg>os: Ensure intrin.h is included for _ReturnAddress.<commit_after>/**************************************************************************
 *
 * Copyright 2017 VMware, Inc.
 * All Rights Reserved.
 *
 * Permission is hereby granted, free of charge, to any person obtaining a copy
 * of this software and associated documentation files (the "Software"), to deal
 * in the Software without restriction, including without limitation the rights
 * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
 * copies of the Software, and to permit persons to whom the Software is
 * furnished to do so, subject to the following conditions:
 *
 * The above copyright notice and this permission notice shall be included in
 * all copies or substantial portions of the Software.
 *
 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
 * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
 * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
 * THE SOFTWARE.
 *
 **************************************************************************/

#pragma once

#ifdef _MSC_VER
#include <intrin.h>
#define ReturnAddress() _ReturnAddress()
#else
#define ReturnAddress() __builtin_return_address(0)
#endif

#include <string>

#ifndef _WIN32

#include <dlfcn.h>

inline std::string
getModuleFromAddress(const void *address)
{
    Dl_info info;
    if (dladdr(address, &info)) {
        return info.dli_fname;
    }
    return "";
}

#endif

<|endoftext|>


<commit_before>#include "search.hpp"
#include "astar.hpp"
#include "idastar.hpp"
#include "../utils/utils.hpp"
#include <cstddef>
#include <cstdio>

template<class D> Search<D> *getsearch(int argc, char *argv[]);

template<class D> void search(D &d, int argc, char *argv[]) {
	Search<D> *srch = getsearch<D>(argc, argv);
	typename D::State s0 = d.initialstate();
	dfpair(stdout, "initial heuristic", "%d", d.h(s0));
	srch->search(d, s0);
	srch->output(stdout);
	dfprocstatus(stdout);

	delete srch;
}

template<class D> Search<D> *getsearch(int argc, char *argv[]) {
	if (strcmp(argv[1], "idastar") == 0)
		return new Idastar<D>(argc, argv);
	else if (strcmp(argv[1], "astar") == 0)
		return new Astar<D>(argc, argv);

	fatal("Unknown algorithm: %s\n", argv[1]);
	return NULL;	// Unreachable
}<commit_msg>More comments.<commit_after>#include "search.hpp"
#include "astar.hpp"
#include "idastar.hpp"
#include "../utils/utils.hpp"
#include <cstddef>
#include <cstdio>

// Functions for conveniently defining a new main
// function for a domain.

template<class D> Search<D> *getsearch(int argc, char *argv[]);

template<class D> void search(D &d, int argc, char *argv[]) {
	Search<D> *srch = getsearch<D>(argc, argv);
	typename D::State s0 = d.initialstate();
	dfpair(stdout, "initial heuristic", "%d", d.h(s0));
	srch->search(d, s0);
	srch->output(stdout);
	dfprocstatus(stdout);

	delete srch;
}

template<class D> Search<D> *getsearch(int argc, char *argv[]) {
	if (strcmp(argv[1], "idastar") == 0)
		return new Idastar<D>(argc, argv);
	else if (strcmp(argv[1], "astar") == 0)
		return new Astar<D>(argc, argv);

	fatal("Unknown algorithm: %s\n", argv[1]);
	return NULL;	// Unreachable
}<|endoftext|>


<commit_before>
#include "include/types.h"
#include "config.h"
#include "debug.h"
#include "Mutex.h"
#include "Clock.h"

#include "ceph_ver.h"

#include <errno.h>
#include <fstream>
#include <iostream>
using namespace std;

#define _STR(x) #x
#define STRINGIFY(x) _STR(x)

// debug output
Mutex _dout_lock("_dout_lock", false, false /* no lockdep */);
ostream *_dout = &std::cout;
ostream *_derr = &std::cerr;
char _dout_dir[PATH_MAX] = {0};
char _dout_symlink_dir[PATH_MAX] = {0};
char _dout_file[PATH_MAX] = {0};
char _dout_path[PATH_MAX] = {0};
char _dout_rank_symlink_path[PATH_MAX] = {0};
char _dout_name_symlink_path[PATH_MAX] = {0};
char *_dout_symlink_target = 0;   // _dout_path or _dout_file
bool _dout_is_open = false;
bool _dout_need_open = true;
std::ofstream _dout_out;

static void normalize_relative(const char *from, char *to, int tolen)
{
  if (from[0] == '/')
    strncpy(to, from, tolen);
  else {
    char *c = getcwd(to, tolen);
    assert(c);
    strncat(to, "/", tolen);
    strncat(to, from, tolen);
  }
}

static void build_log_paths()
{
  if (g_conf.log_file && g_conf.log_file[0]) {
    normalize_relative(g_conf.log_file, _dout_path, sizeof(_dout_path));
  } else {
    if (g_conf.log_per_instance) {
      char hostname[80];
      gethostname(hostname, 79);
      snprintf(_dout_file, sizeof(_dout_file), "%s.%d", hostname, getpid());
    } else {
      snprintf(_dout_file, sizeof(_dout_file), "%s.%s.log", g_conf.type, g_conf.id);
    }
    snprintf(_dout_path, sizeof(_dout_path), "%s/%s", _dout_dir, _dout_file);
  }
}

static bool log_to_file()
{
  return (g_conf.log_dir || g_conf.log_file) && !g_conf.log_to_stdout;
}

static int create_symlink(const char *from)
{
  ::unlink(from);
  int r = ::symlink(_dout_symlink_target, from);
  if (r) {
    char buf[80];
    *_dout << "---- " << getpid() << " failed to symlink " << _dout_symlink_target
	   << " from " << from
	   << ": " << strerror_r(errno, buf, sizeof(buf)) << std::endl;
  }
  return r;
}

static void rotate_file(const char *fn, int max)
{
  char a[200], b[200];
  // rotate out old
  int n = 0;
  while (1) {
    struct stat st;
    snprintf(a, sizeof(a), "%s.%lld", fn, (long long)n);
    if (::lstat(a, &st) != 0)
      break;
    n++;
  }
  while (n >= 0) {
    if (n)
      snprintf(a, sizeof(a), "%s.%lld", fn, (long long)n-1);
    else
      snprintf(a, sizeof(a), "%s", fn);
    if (n >= max) {
      ::unlink(a);
      *_dout << "---- " << getpid() << " removed " << a << " ----" << std::endl;
    } else {
      snprintf(b, sizeof(b), "%s.%lld", fn, (long long)n);
      ::rename(a, b);
      *_dout << "---- " << getpid() << " renamed " << a << " -> " << b << " ----" << std::endl;
    }
    n--;
  }
}

static int create_name_symlink()
{
  int r = 0;
  if (log_to_file() && g_conf.log_per_instance && !(g_conf.log_file && g_conf.log_file[0])) {
    snprintf(_dout_name_symlink_path, sizeof(_dout_name_symlink_path),
	     "%s/%s.%s", _dout_symlink_dir, g_conf.type, g_conf.id);

    rotate_file(_dout_name_symlink_path, g_conf.log_sym_history);
    r = create_symlink(_dout_name_symlink_path);
  }
  return r;
}

void _dout_open_log()
{
  bool need_symlink = false;

  // logging enabled?
  if (!log_to_file()) {
    _dout_need_open = false;
    return;
  }

  // calculate log dir, filename, etc.
  // do this _once_.
  if (!_dout_path[0]) {

    // normalize paths
    normalize_relative(g_conf.log_dir, _dout_dir, sizeof(_dout_dir));
    if (!g_conf.log_sym_dir)
      g_conf.log_sym_dir = strdup(g_conf.log_dir);
    normalize_relative(g_conf.log_sym_dir, _dout_symlink_dir, sizeof(_dout_symlink_dir));

    // make symlink targets absolute or relative?
    if ((g_conf.log_file && g_conf.log_file[0]) ||
	strcmp(_dout_symlink_dir, _dout_dir) == 0)
      _dout_symlink_target = _dout_file;
    else
      _dout_symlink_target = _dout_path;

    build_log_paths();

    need_symlink = true;
  }

  _dout_out.close();
  // only truncate if log_per_instance is set.
  _dout_out.open(_dout_path, g_conf.log_per_instance ? (ios::trunc | ios::out) : (ios::out | ios::app));
  if (!_dout_out.is_open()) {
    std::cerr << "error opening output file " << _dout_path << std::endl;
    _dout = &std::cout;
  } else {
    _dout_need_open = false;
    _dout_is_open = true;
    _dout = &_dout_out;
    *_dout << g_clock.now() << " --- " << getpid()
	   << (g_conf.log_per_instance ? " created new log " : " appending to log ")
	   << _dout_path << " ---" << std::endl;
  }
  *_dout << "ceph version " << VERSION << " (" << STRINGIFY(CEPH_GIT_VER) << ")" << std::endl;

  if (need_symlink)
    create_name_symlink();
}

int dout_rename_output_file()  // after calling daemon()
{
  Mutex::Locker l(_dout_lock);
  if (log_to_file() && g_conf.log_per_instance) {
    char oldpath[PATH_MAX];
    char hostname[80];
    gethostname(hostname, 79);

    strcpy(oldpath, _dout_path);

    build_log_paths();

    *_dout << "---- " << getpid() << " renamed log " << oldpath << " -> " << _dout_path << " ----" << std::endl;
    ::rename(oldpath, _dout_path);

    // $type.$id symlink
    if (g_conf.log_per_instance && _dout_name_symlink_path[0])
      create_symlink(_dout_name_symlink_path);
    if (_dout_rank_symlink_path[0])
      create_symlink(_dout_rank_symlink_path);
  }
  return 0;
}

int dout_create_rank_symlink(int64_t n)
{
  Mutex::Locker l(_dout_lock);
  int r = 0;
  if (log_to_file() && !(g_conf.log_file && g_conf.log_file[0])) {
    if (_dout_need_open)
      _dout_open_log();

    snprintf(_dout_rank_symlink_path, sizeof(_dout_rank_symlink_path),
	     "%s/%s%lld", _dout_symlink_dir, g_conf.type, (long long)n);
    r = create_symlink(_dout_rank_symlink_path);
  }
  return r;
}

<commit_msg>debug: always append to log<commit_after>
#include "include/types.h"
#include "config.h"
#include "debug.h"
#include "Mutex.h"
#include "Clock.h"

#include "ceph_ver.h"

#include <errno.h>
#include <fstream>
#include <iostream>
using namespace std;

#define _STR(x) #x
#define STRINGIFY(x) _STR(x)

// debug output
Mutex _dout_lock("_dout_lock", false, false /* no lockdep */);
ostream *_dout = &std::cout;
ostream *_derr = &std::cerr;
char _dout_dir[PATH_MAX] = {0};
char _dout_symlink_dir[PATH_MAX] = {0};
char _dout_file[PATH_MAX] = {0};
char _dout_path[PATH_MAX] = {0};
char _dout_rank_symlink_path[PATH_MAX] = {0};
char _dout_name_symlink_path[PATH_MAX] = {0};
char *_dout_symlink_target = 0;   // _dout_path or _dout_file
bool _dout_is_open = false;
bool _dout_need_open = true;
std::ofstream _dout_out;

static void normalize_relative(const char *from, char *to, int tolen)
{
  if (from[0] == '/')
    strncpy(to, from, tolen);
  else {
    char *c = getcwd(to, tolen);
    assert(c);
    strncat(to, "/", tolen);
    strncat(to, from, tolen);
  }
}

static void build_log_paths()
{
  if (g_conf.log_file && g_conf.log_file[0]) {
    normalize_relative(g_conf.log_file, _dout_path, sizeof(_dout_path));
  } else {
    if (g_conf.log_per_instance) {
      char hostname[80];
      gethostname(hostname, 79);
      snprintf(_dout_file, sizeof(_dout_file), "%s.%d", hostname, getpid());
    } else {
      snprintf(_dout_file, sizeof(_dout_file), "%s.%s.log", g_conf.type, g_conf.id);
    }
    snprintf(_dout_path, sizeof(_dout_path), "%s/%s", _dout_dir, _dout_file);
  }
}

static bool log_to_file()
{
  return (g_conf.log_dir || g_conf.log_file) && !g_conf.log_to_stdout;
}

static int create_symlink(const char *from)
{
  ::unlink(from);
  int r = ::symlink(_dout_symlink_target, from);
  if (r) {
    char buf[80];
    *_dout << "---- " << getpid() << " failed to symlink " << _dout_symlink_target
	   << " from " << from
	   << ": " << strerror_r(errno, buf, sizeof(buf)) << std::endl;
  }
  return r;
}

static void rotate_file(const char *fn, int max)
{
  char a[200], b[200];
  // rotate out old
  int n = 0;
  while (1) {
    struct stat st;
    snprintf(a, sizeof(a), "%s.%lld", fn, (long long)n);
    if (::lstat(a, &st) != 0)
      break;
    n++;
  }
  while (n >= 0) {
    if (n)
      snprintf(a, sizeof(a), "%s.%lld", fn, (long long)n-1);
    else
      snprintf(a, sizeof(a), "%s", fn);
    if (n >= max) {
      ::unlink(a);
      *_dout << "---- " << getpid() << " removed " << a << " ----" << std::endl;
    } else {
      snprintf(b, sizeof(b), "%s.%lld", fn, (long long)n);
      ::rename(a, b);
      *_dout << "---- " << getpid() << " renamed " << a << " -> " << b << " ----" << std::endl;
    }
    n--;
  }
}

static int create_name_symlink()
{
  int r = 0;
  if (log_to_file() && g_conf.log_per_instance && !(g_conf.log_file && g_conf.log_file[0])) {
    snprintf(_dout_name_symlink_path, sizeof(_dout_name_symlink_path),
	     "%s/%s.%s", _dout_symlink_dir, g_conf.type, g_conf.id);

    rotate_file(_dout_name_symlink_path, g_conf.log_sym_history);
    r = create_symlink(_dout_name_symlink_path);
  }
  return r;
}

void _dout_open_log()
{
  bool need_symlink = false;

  // logging enabled?
  if (!log_to_file()) {
    _dout_need_open = false;
    return;
  }

  // calculate log dir, filename, etc.
  // do this _once_.
  if (!_dout_path[0]) {

    // normalize paths
    normalize_relative(g_conf.log_dir, _dout_dir, sizeof(_dout_dir));
    if (!g_conf.log_sym_dir)
      g_conf.log_sym_dir = strdup(g_conf.log_dir);
    normalize_relative(g_conf.log_sym_dir, _dout_symlink_dir, sizeof(_dout_symlink_dir));

    // make symlink targets absolute or relative?
    if ((g_conf.log_file && g_conf.log_file[0]) ||
	strcmp(_dout_symlink_dir, _dout_dir) == 0)
      _dout_symlink_target = _dout_file;
    else
      _dout_symlink_target = _dout_path;

    build_log_paths();

    need_symlink = true;
  }

  _dout_out.close();
  _dout_out.open(_dout_path, ios::out | ios::app);
  if (!_dout_out.is_open()) {
    std::cerr << "error opening output file " << _dout_path << std::endl;
    _dout = &std::cout;
  } else {
    _dout_need_open = false;
    _dout_is_open = true;
    _dout = &_dout_out;
    *_dout << g_clock.now() << " --- " << getpid() << " opened log "
	   << _dout_path << " ---" << std::endl;
  }
  *_dout << "ceph version " << VERSION << " (" << STRINGIFY(CEPH_GIT_VER) << ")" << std::endl;

  if (need_symlink)
    create_name_symlink();
}

int dout_rename_output_file()  // after calling daemon()
{
  Mutex::Locker l(_dout_lock);
  if (log_to_file() && g_conf.log_per_instance) {
    char oldpath[PATH_MAX];
    char hostname[80];
    gethostname(hostname, 79);

    strcpy(oldpath, _dout_path);

    build_log_paths();

    *_dout << "---- " << getpid() << " renamed log " << oldpath << " -> " << _dout_path << " ----" << std::endl;
    ::rename(oldpath, _dout_path);

    // $type.$id symlink
    if (g_conf.log_per_instance && _dout_name_symlink_path[0])
      create_symlink(_dout_name_symlink_path);
    if (_dout_rank_symlink_path[0])
      create_symlink(_dout_rank_symlink_path);
  }
  return 0;
}

int dout_create_rank_symlink(int64_t n)
{
  Mutex::Locker l(_dout_lock);
  int r = 0;
  if (log_to_file() && !(g_conf.log_file && g_conf.log_file[0])) {
    if (_dout_need_open)
      _dout_open_log();

    snprintf(_dout_rank_symlink_path, sizeof(_dout_rank_symlink_path),
	     "%s/%s%lld", _dout_symlink_dir, g_conf.type, (long long)n);
    r = create_symlink(_dout_rank_symlink_path);
  }
  return r;
}

<|endoftext|>


<commit_before>#ifndef CONFIG_ARGS_HPP_
#define CONFIG_ARGS_HPP_

#define KILOBYTE 1024L
#define MEGABYTE (KILOBYTE*1024L)
#define GIGABYTE (MEGABYTE*1024L)
#define TERABYTE (GIGABYTE*1024L)

/*!
 * Version strings
 */

#define SOFTWARE_NAME_STRING "RethinkDB"
#define VERSION_STRING "0.4"

/**
 * Basic configuration parameters.
 * TODO: Many of these should be runtime switches.
 */
// Max concurrent IO requests per event queue
#define MAX_CONCURRENT_IO_REQUESTS                64

// Don't send more IO requests to the system until the per-thread
// queue of IO requests is higher than this depth
#define TARGET_IO_QUEUE_DEPTH                     64

// Defines the maximum size of the batch of IO events to process on
// each loop iteration. A larger number will increase throughput but
// decrease concurrency
#define MAX_IO_EVENT_PROCESSING_BATCH_SIZE        50

// The io batch factor ensures a minimum number of i/o operations
// which are picked from any specific i/o account consecutively.
// A higher value might be advantageous for throughput if seek times
// matter on the underlying i/o system. A low value improves latency.
// The advantage only holds as long as each account has a tendentially
// sequential set of i/o operations though. If access patterns are random
// for all accounts, a low io batch factor does just as well (as bad) when it comes
// to the number of random seeks, but might still provide a lower latency
// than a high batch factor. Our serializer writes usually generate a sequential
// access pattern and therefore take considerable advantage from a high io batch
// factor.
#define DEFAULT_IO_BATCH_FACTOR                   8

// Currently, each cache uses two IO accounts:
// one account for writes, and one account for reads.
// By adjusting the priorities of these accounts, reads
// can be prioritized over writes or the other way around.
//
// This is a one-per-serializer/file priority.
// The per-cache priorities are dynamically derived by dividing these priorities
// by the number of slices on a specific file.
#define CACHE_READS_IO_PRIORITY                   512
#define CACHE_WRITES_IO_PRIORITY                  64

// Garbage Colletion uses its own two IO accounts.
// There is one low-priority account that is meant to guarantee
// (performance-wise) unintrusive garbage collection.
// If the garbage ratio keeps growing,
// GC starts using the high priority account instead, which
// might have a negative influence on database performance
// under i/o heavy workloads but guarantees that the database
// doesn't grow indefinitely.
//
// This is a one-per-serializer/file priority.
#define GC_IO_PRIORITY_NICE                       8
#define GC_IO_PRIORITY_HIGH                       (2 * CACHE_WRITES_IO_PRIORITY)

// Size of the buffer used to perform IO operations (in bytes).
#define IO_BUFFER_SIZE                            (4 * KILOBYTE)

// Size of the device block size (in bytes)
#define DEVICE_BLOCK_SIZE                         (4 * KILOBYTE)

// Size of each btree node (in bytes) on disk
#define DEFAULT_BTREE_BLOCK_SIZE                  (4 * KILOBYTE)

// Maximum number of data blocks
#define MAX_DATA_EXTENTS                          (TERABYTE / (16 * KILOBYTE))

// Size of each extent (in bytes)
#define DEFAULT_EXTENT_SIZE                       (512 * KILOBYTE)

// Max number of blocks which can be read ahead in one i/o transaction (if enabled)
#define MAX_READ_AHEAD_BLOCKS 32

// Ratio of free ram to use for the cache by default
// TODO: DEFAULT_MAX_CACHE_RATIO is unused. Should it be deleted?
#define DEFAULT_MAX_CACHE_RATIO                   0.5

// Maximum number of threads we support
// TODO: make this dynamic where possible
#define MAX_THREADS                               128

// Ticks (in milliseconds) the internal timed tasks are performed at
#define TIMER_TICKS_IN_MS                         5

// How many milliseconds to allow changes to sit in memory before flushing to disk
#define DEFAULT_FLUSH_TIMER_MS                    1000

// flush_waiting_threshold is the maximal number of transactions which can wait
// for a sync before a flush gets triggered on any single slice. As transactions only wait for
// sync with wait_for_flush enabled, this option plays a role only then.
#define DEFAULT_FLUSH_WAITING_THRESHOLD           8

// If wait_for_flush is true, concurrent flushing can be used to reduce the latency
// of each single flush. max_concurrent_flushes controls how many flushes can be active
// on a specific slice at any given time.
#define DEFAULT_MAX_CONCURRENT_FLUSHES            1

// If the size of the data affected by the current set of patches in a block is larger than
// block size / MAX_PATCHES_SIZE_RATIO, we flush the block instead of waiting for
// more patches to come. Flushing the block means that we rewrite the actual data
// block (including all patches). In this case, we don't have to store any of the
// patches for that block, because to block will be re-written anyway. This results
// in improved CPU efficiency, because we save the overhead of storing and managing
// patches.
// In summary: larger max patches size ratio -> less usage of patches, more i/o
//     smaller max patches size ratio -> more usage of patches, less i/o, more CPU usage
//
// Note: An average write transaction under canonical workload leads to patches of about 75
// bytes of affected data.
// The actual value is constantly adjusted between MAX_PATCHES_SIZE_RATIO_MIN and
// MAX_PATCHES_SIZE_RATIO_MAX depending on whether the system is i/o bound or not.
// An exception from this is operating in durability mode, where any write
// operation is ultimately i/o bound anyway. In this case, the patches size ratio
// does not get dynamically adjusted but stays constant at MAX_PATCHES_SIZE_RATIO_DURABILITY.
#define MAX_PATCHES_SIZE_RATIO_MIN                100
#define MAX_PATCHES_SIZE_RATIO_MAX                2
#define MAX_PATCHES_SIZE_RATIO_DURABILITY         5
#define RAISE_PATCHES_RATIO_AT_FRACTION_OF_UNSAVED_DATA_LIMIT 0.6

// If more than this many bytes of dirty data accumulate in the cache, then write
// transactions will be throttled.
// A value of 0 means that it will automatically be set to MAX_UNSAVED_DATA_LIMIT_FRACTION
// times the max cache size
#define DEFAULT_UNSAVED_DATA_LIMIT                4096 * MEGABYTE

// The unsaved data limit cannot exceed this fraction of the max cache size
#define MAX_UNSAVED_DATA_LIMIT_FRACTION           0.9

// We start flushing dirty pages as soon as we hit this fraction of the unsaved data limit
#define FLUSH_AT_FRACTION_OF_UNSAVED_DATA_LIMIT   0.2

// How many times the page replacement algorithm tries to find an eligible page before giving up.
// Note that (MAX_UNSAVED_DATA_LIMIT_FRACTION ** PAGE_REPL_NUM_TRIES) is the probability that the
// page replacement algorithm will succeed on a given try, and if that probability is less than 1/2
// then the page replacement algorithm will on average be unable to evict pages from the cache.
#define PAGE_REPL_NUM_TRIES                       10

// How large can the key be, in bytes?  This value needs to fit in a byte.
#define MAX_KEY_SIZE                              250

// Any values of this size or less will be directly stored in btree leaf nodes.
// Values greater than this size will be stored in overflow blocks. This value
// needs to fit in a byte.
#define MAX_IN_NODE_VALUE_SIZE                    250

// memcached specifies the maximum value size to be 1MB, but customers asked this to be much higher
#define MAX_VALUE_SIZE                            10 * MEGABYTE

// Values larger than this will be streamed in a get operation
#define MAX_BUFFERED_GET_SIZE                     MAX_VALUE_SIZE // streaming is too slow for now, so we disable it completely

// If a single connection sends this many 'noreply' commands, the next command will
// have to wait until the first one finishes
#define MAX_CONCURRENT_QUERIES_PER_CONNECTION     500

// The number of concurrent queries when loading memcached operations from a file.
#define MAX_CONCURRENT_QUEURIES_ON_IMPORT         1000

// How many timestamps we store in a leaf node.  We store the
// NUM_LEAF_NODE_EARLIER_TIMES+1 most-recent timestamps.
#define NUM_LEAF_NODE_EARLIER_TIMES               4

// We assume there will never be more than this many blocks. The value
// is computed by dividing 1 TB by the smallest reasonable block size.
// This value currently fits in 32 bits, and so block_id_t is a uint32_t.
#define MAX_BLOCK_ID                              (TERABYTE / KILOBYTE)

// We assume that there will never be more than this many blocks held in memory by the cache at
// any one time. The value is computed by dividing 50 GB by the smallest reasonable block size.
#define MAX_BLOCKS_IN_MEMORY                      (50 * GIGABYTE / KILOBYTE)

// Special block IDs.  These don't really belong here because they're
// more magic constants than tunable parameters.

// The btree superblock, which has a reference to the root node block
// id.
#define SUPERBLOCK_ID                             0
// HEY: This is kind of fragile because some patch disk storage code
// expects this value to be 1 (since the free list returns 1 the first
// time a block id is generated, or something).
#define MC_CONFIGBLOCK_ID                         (SUPERBLOCK_ID + 1)

// The ratio at which we should start GCing.  (HEY: What's the extra
// 0.000001 in MAX_GC_HIGH_RATIO for?  Is it because we told the user
// that 0.99 was too high?)
#define DEFAULT_GC_HIGH_RATIO                     0.25
// TODO: MAX_GC_HIGH_RATIO is unused.  Use it!
// TODO: Probably this value is way too high.
//  - Keeping this around because if it becomes configurable again, we
//    should have these limits.  Before then at least rassert it.
#define MAX_GC_HIGH_RATIO                         0.990001

// The ratio at which we don't want to keep GC'ing.
#define DEFAULT_GC_LOW_RATIO                      0.15
// TODO: MIN_GC_LOW_RATIO is unused.  Use it!
//  - Keeping this around because if it becomes configurable again, we
//    should have these limits.  Before then at least rassert it.
#define MIN_GC_LOW_RATIO                          0.099999

// What's the maximum number of "young" extents we can have?
#define GC_YOUNG_EXTENT_MAX_SIZE                  50
// What's the definition of a "young" extent in microseconds?
#define GC_YOUNG_EXTENT_TIMELIMIT_MICROS          50000

// If the size of the LBA on a given disk exceeds LBA_MIN_SIZE_FOR_GC, then the fraction of the
// entries that are live and not garbage should be at least LBA_MIN_UNGARBAGE_FRACTION.
// TODO: Maybe change this back to 20 megabytes?
#define LBA_MIN_SIZE_FOR_GC                       (MEGABYTE * 1)
// TODO: This used to be 0.15 but we figured why not do the opposite of our well-tested parameter?
#define LBA_MIN_UNGARBAGE_FRACTION                0.85

// How many LBA structures to have for each file
// TODO: LBA_SHARD_FACTOR used to be 16.
#define LBA_SHARD_FACTOR                          4

// How many bytes of buffering space we can use per disk when reading the LBA. If it's set
// too high, then RethinkDB will eat a lot of memory at startup. This is bad because tcmalloc
// doesn't return memory to the OS. If it's set too low, startup will take a longer time.
#define LBA_READ_BUFFER_SIZE                      GIGABYTE

// How many different places in each file we should be writing to at once, not counting the
// metablock or LBA
#define MAX_ACTIVE_DATA_EXTENTS                   64
#define DEFAULT_ACTIVE_DATA_EXTENTS               1

// The size of zones the serializer will divide a block device into
#define DEFAULT_FILE_ZONE_SIZE                    GIGABYTE

#define COROUTINE_STACK_SIZE                      131072

#define MAX_COROS_PER_THREAD                      10000

// Size of a cache line (used in cache_line_padded_t).
#define CACHE_LINE_SIZE                           64

#endif  // CONFIG_ARGS_HPP_

<commit_msg>Made extent size 32KB.<commit_after>#ifndef CONFIG_ARGS_HPP_
#define CONFIG_ARGS_HPP_

#define KILOBYTE 1024L
#define MEGABYTE (KILOBYTE*1024L)
#define GIGABYTE (MEGABYTE*1024L)
#define TERABYTE (GIGABYTE*1024L)

/*!
 * Version strings
 */

#define SOFTWARE_NAME_STRING "RethinkDB"
#define VERSION_STRING "0.4"

/**
 * Basic configuration parameters.
 * TODO: Many of these should be runtime switches.
 */
// Max concurrent IO requests per event queue
#define MAX_CONCURRENT_IO_REQUESTS                64

// Don't send more IO requests to the system until the per-thread
// queue of IO requests is higher than this depth
#define TARGET_IO_QUEUE_DEPTH                     64

// Defines the maximum size of the batch of IO events to process on
// each loop iteration. A larger number will increase throughput but
// decrease concurrency
#define MAX_IO_EVENT_PROCESSING_BATCH_SIZE        50

// The io batch factor ensures a minimum number of i/o operations
// which are picked from any specific i/o account consecutively.
// A higher value might be advantageous for throughput if seek times
// matter on the underlying i/o system. A low value improves latency.
// The advantage only holds as long as each account has a tendentially
// sequential set of i/o operations though. If access patterns are random
// for all accounts, a low io batch factor does just as well (as bad) when it comes
// to the number of random seeks, but might still provide a lower latency
// than a high batch factor. Our serializer writes usually generate a sequential
// access pattern and therefore take considerable advantage from a high io batch
// factor.
#define DEFAULT_IO_BATCH_FACTOR                   8

// Currently, each cache uses two IO accounts:
// one account for writes, and one account for reads.
// By adjusting the priorities of these accounts, reads
// can be prioritized over writes or the other way around.
//
// This is a one-per-serializer/file priority.
// The per-cache priorities are dynamically derived by dividing these priorities
// by the number of slices on a specific file.
#define CACHE_READS_IO_PRIORITY                   512
#define CACHE_WRITES_IO_PRIORITY                  64

// Garbage Colletion uses its own two IO accounts.
// There is one low-priority account that is meant to guarantee
// (performance-wise) unintrusive garbage collection.
// If the garbage ratio keeps growing,
// GC starts using the high priority account instead, which
// might have a negative influence on database performance
// under i/o heavy workloads but guarantees that the database
// doesn't grow indefinitely.
//
// This is a one-per-serializer/file priority.
#define GC_IO_PRIORITY_NICE                       8
#define GC_IO_PRIORITY_HIGH                       (2 * CACHE_WRITES_IO_PRIORITY)

// Size of the buffer used to perform IO operations (in bytes).
#define IO_BUFFER_SIZE                            (4 * KILOBYTE)

// Size of the device block size (in bytes)
#define DEVICE_BLOCK_SIZE                         (4 * KILOBYTE)

// Size of each btree node (in bytes) on disk
#define DEFAULT_BTREE_BLOCK_SIZE                  (4 * KILOBYTE)

// Maximum number of data blocks
#define MAX_DATA_EXTENTS                          (TERABYTE / (16 * KILOBYTE))

// Size of each extent (in bytes)
#define DEFAULT_EXTENT_SIZE                       (32 * KILOBYTE)

// Max number of blocks which can be read ahead in one i/o transaction (if enabled)
#define MAX_READ_AHEAD_BLOCKS 32

// Ratio of free ram to use for the cache by default
// TODO: DEFAULT_MAX_CACHE_RATIO is unused. Should it be deleted?
#define DEFAULT_MAX_CACHE_RATIO                   0.5

// Maximum number of threads we support
// TODO: make this dynamic where possible
#define MAX_THREADS                               128

// Ticks (in milliseconds) the internal timed tasks are performed at
#define TIMER_TICKS_IN_MS                         5

// How many milliseconds to allow changes to sit in memory before flushing to disk
#define DEFAULT_FLUSH_TIMER_MS                    1000

// flush_waiting_threshold is the maximal number of transactions which can wait
// for a sync before a flush gets triggered on any single slice. As transactions only wait for
// sync with wait_for_flush enabled, this option plays a role only then.
#define DEFAULT_FLUSH_WAITING_THRESHOLD           8

// If wait_for_flush is true, concurrent flushing can be used to reduce the latency
// of each single flush. max_concurrent_flushes controls how many flushes can be active
// on a specific slice at any given time.
#define DEFAULT_MAX_CONCURRENT_FLUSHES            1

// If the size of the data affected by the current set of patches in a block is larger than
// block size / MAX_PATCHES_SIZE_RATIO, we flush the block instead of waiting for
// more patches to come. Flushing the block means that we rewrite the actual data
// block (including all patches). In this case, we don't have to store any of the
// patches for that block, because to block will be re-written anyway. This results
// in improved CPU efficiency, because we save the overhead of storing and managing
// patches.
// In summary: larger max patches size ratio -> less usage of patches, more i/o
//     smaller max patches size ratio -> more usage of patches, less i/o, more CPU usage
//
// Note: An average write transaction under canonical workload leads to patches of about 75
// bytes of affected data.
// The actual value is constantly adjusted between MAX_PATCHES_SIZE_RATIO_MIN and
// MAX_PATCHES_SIZE_RATIO_MAX depending on whether the system is i/o bound or not.
// An exception from this is operating in durability mode, where any write
// operation is ultimately i/o bound anyway. In this case, the patches size ratio
// does not get dynamically adjusted but stays constant at MAX_PATCHES_SIZE_RATIO_DURABILITY.
#define MAX_PATCHES_SIZE_RATIO_MIN                100
#define MAX_PATCHES_SIZE_RATIO_MAX                2
#define MAX_PATCHES_SIZE_RATIO_DURABILITY         5
#define RAISE_PATCHES_RATIO_AT_FRACTION_OF_UNSAVED_DATA_LIMIT 0.6

// If more than this many bytes of dirty data accumulate in the cache, then write
// transactions will be throttled.
// A value of 0 means that it will automatically be set to MAX_UNSAVED_DATA_LIMIT_FRACTION
// times the max cache size
#define DEFAULT_UNSAVED_DATA_LIMIT                4096 * MEGABYTE

// The unsaved data limit cannot exceed this fraction of the max cache size
#define MAX_UNSAVED_DATA_LIMIT_FRACTION           0.9

// We start flushing dirty pages as soon as we hit this fraction of the unsaved data limit
#define FLUSH_AT_FRACTION_OF_UNSAVED_DATA_LIMIT   0.2

// How many times the page replacement algorithm tries to find an eligible page before giving up.
// Note that (MAX_UNSAVED_DATA_LIMIT_FRACTION ** PAGE_REPL_NUM_TRIES) is the probability that the
// page replacement algorithm will succeed on a given try, and if that probability is less than 1/2
// then the page replacement algorithm will on average be unable to evict pages from the cache.
#define PAGE_REPL_NUM_TRIES                       10

// How large can the key be, in bytes?  This value needs to fit in a byte.
#define MAX_KEY_SIZE                              250

// Any values of this size or less will be directly stored in btree leaf nodes.
// Values greater than this size will be stored in overflow blocks. This value
// needs to fit in a byte.
#define MAX_IN_NODE_VALUE_SIZE                    250

// memcached specifies the maximum value size to be 1MB, but customers asked this to be much higher
#define MAX_VALUE_SIZE                            10 * MEGABYTE

// Values larger than this will be streamed in a get operation
#define MAX_BUFFERED_GET_SIZE                     MAX_VALUE_SIZE // streaming is too slow for now, so we disable it completely

// If a single connection sends this many 'noreply' commands, the next command will
// have to wait until the first one finishes
#define MAX_CONCURRENT_QUERIES_PER_CONNECTION     500

// The number of concurrent queries when loading memcached operations from a file.
#define MAX_CONCURRENT_QUEURIES_ON_IMPORT         1000

// How many timestamps we store in a leaf node.  We store the
// NUM_LEAF_NODE_EARLIER_TIMES+1 most-recent timestamps.
#define NUM_LEAF_NODE_EARLIER_TIMES               4

// We assume there will never be more than this many blocks. The value
// is computed by dividing 1 TB by the smallest reasonable block size.
// This value currently fits in 32 bits, and so block_id_t is a uint32_t.
#define MAX_BLOCK_ID                              (TERABYTE / KILOBYTE)

// We assume that there will never be more than this many blocks held in memory by the cache at
// any one time. The value is computed by dividing 50 GB by the smallest reasonable block size.
#define MAX_BLOCKS_IN_MEMORY                      (50 * GIGABYTE / KILOBYTE)

// Special block IDs.  These don't really belong here because they're
// more magic constants than tunable parameters.

// The btree superblock, which has a reference to the root node block
// id.
#define SUPERBLOCK_ID                             0
// HEY: This is kind of fragile because some patch disk storage code
// expects this value to be 1 (since the free list returns 1 the first
// time a block id is generated, or something).
#define MC_CONFIGBLOCK_ID                         (SUPERBLOCK_ID + 1)

// The ratio at which we should start GCing.  (HEY: What's the extra
// 0.000001 in MAX_GC_HIGH_RATIO for?  Is it because we told the user
// that 0.99 was too high?)
#define DEFAULT_GC_HIGH_RATIO                     0.25
// TODO: MAX_GC_HIGH_RATIO is unused.  Use it!
// TODO: Probably this value is way too high.
//  - Keeping this around because if it becomes configurable again, we
//    should have these limits.  Before then at least rassert it.
#define MAX_GC_HIGH_RATIO                         0.990001

// The ratio at which we don't want to keep GC'ing.
#define DEFAULT_GC_LOW_RATIO                      0.15
// TODO: MIN_GC_LOW_RATIO is unused.  Use it!
//  - Keeping this around because if it becomes configurable again, we
//    should have these limits.  Before then at least rassert it.
#define MIN_GC_LOW_RATIO                          0.099999

// What's the maximum number of "young" extents we can have?
#define GC_YOUNG_EXTENT_MAX_SIZE                  50
// What's the definition of a "young" extent in microseconds?
#define GC_YOUNG_EXTENT_TIMELIMIT_MICROS          50000

// If the size of the LBA on a given disk exceeds LBA_MIN_SIZE_FOR_GC, then the fraction of the
// entries that are live and not garbage should be at least LBA_MIN_UNGARBAGE_FRACTION.
// TODO: Maybe change this back to 20 megabytes?
#define LBA_MIN_SIZE_FOR_GC                       (MEGABYTE * 1)
// TODO: This used to be 0.15 but we figured why not do the opposite of our well-tested parameter?
#define LBA_MIN_UNGARBAGE_FRACTION                0.85

// How many LBA structures to have for each file
// TODO: LBA_SHARD_FACTOR used to be 16.
#define LBA_SHARD_FACTOR                          4

// How many bytes of buffering space we can use per disk when reading the LBA. If it's set
// too high, then RethinkDB will eat a lot of memory at startup. This is bad because tcmalloc
// doesn't return memory to the OS. If it's set too low, startup will take a longer time.
#define LBA_READ_BUFFER_SIZE                      GIGABYTE

// How many different places in each file we should be writing to at once, not counting the
// metablock or LBA
#define MAX_ACTIVE_DATA_EXTENTS                   64
#define DEFAULT_ACTIVE_DATA_EXTENTS               1

// The size of zones the serializer will divide a block device into
#define DEFAULT_FILE_ZONE_SIZE                    GIGABYTE

#define COROUTINE_STACK_SIZE                      131072

#define MAX_COROS_PER_THREAD                      10000

// Size of a cache line (used in cache_line_padded_t).
#define CACHE_LINE_SIZE                           64

#endif  // CONFIG_ARGS_HPP_

<|endoftext|>


<commit_before>/**
 * @file cache.cpp
 *
 */

#include "cache.h"
#include "info.h"
#include "logger.h"
#include "plugin_factory.h"
#include <time.h>

using namespace std;
using namespace himan::plugin;

typedef lock_guard<mutex> Lock;

cache::cache()
{
	itsLogger = logger("cache");
}
string cache::UniqueName(const info& info)
{
	stringstream ss;

	// clang-format off

	ss << info.Producer().Id() << "_"
	   << info.Time().OriginDateTime().String("%Y-%m-%d %H:%M:%S") << "_"
	   << info.Time().ValidDateTime().String("%Y-%m-%d %H:%M:%S") << "_"
	   << info.Param().Name() << "_"
	   << static_cast<string>(info.Level()) << "_"
	   << info.ForecastType().Type() << "_"
	   << info.ForecastType().Value();

	// clang-format on

	return ss.str();
}

string cache::UniqueNameFromOptions(search_options& options)
{
	ASSERT(options.configuration->DatabaseType() == kNoDatabase || options.prod.Id() != kHPMissingInt);
	stringstream ss;

	// clang-format off

	ss << options.prod.Id() << "_"
	   << options.time.OriginDateTime().String("%Y-%m-%d %H:%M:%S") << "_"
	   << options.time.ValidDateTime().String("%Y-%m-%d %H:%M:%S") << "_"
	   << options.param.Name() << "_"
	   << static_cast<string>(options.level) << "_"
	   << options.ftype.Type() << "_"
	   << options.ftype.Value();

	// clang-format on

	return ss.str();
}

void cache::Insert(info_t anInfo, bool pin)
{
	SplitToPool(anInfo, pin);
}
void cache::SplitToPool(info_t anInfo, bool pin)
{
	auto localInfo = make_shared<info>(*anInfo);

	// Cached data is never replaced by another data that has
	// the same uniqueName

	const string uniqueName = UniqueName(*localInfo);

	if (cache_pool::Instance()->Exists(uniqueName))
	{
		itsLogger.Trace("Data with key " + uniqueName + " already exists at cache");

		// Update timestamp of this cache item
		cache_pool::Instance()->UpdateTime(uniqueName);
		return;
	}

#ifdef HAVE_CUDA
	if (localInfo->Grid()->IsPackedData())
	{
		itsLogger.Trace("Removing packed data from cached info");
		localInfo->Grid()->PackedData().Clear();
	}
#endif

	ASSERT(!localInfo->Grid()->IsPackedData());

	// localInfo might contain multiple grids. When adding data to cache, we need
	// to make sure that single info contains only single grid.

	if (localInfo->DimensionSize() > 1)
	{
		auto newInfo =
		    make_shared<info>(localInfo->ForecastType(), localInfo->Time(), localInfo->Level(), localInfo->Param());
		newInfo->Grid(localInfo->SharedGrid());
		localInfo = newInfo;
	}

	ASSERT(localInfo->DimensionSize() == 1);
	cache_pool::Instance()->Insert(uniqueName, localInfo, pin);
}

vector<shared_ptr<himan::info>> cache::GetInfo(search_options& options)
{
	string uniqueName = UniqueNameFromOptions(options);

	vector<shared_ptr<himan::info>> info;

	auto foundInfo = cache_pool::Instance()->GetInfo(uniqueName);
	if (foundInfo)
	{
		info.push_back(foundInfo);
		itsLogger.Trace("Found matching data for " + uniqueName);
	}

	return info;
}

void cache::Clean()
{
	cache_pool::Instance()->Clean();
}
size_t cache::Size() const
{
	return cache_pool::Instance()->Size();
}
cache_pool* cache_pool::itsInstance = NULL;

cache_pool::cache_pool() : itsCacheLimit(-1)
{
	itsLogger = logger("cache_pool");
}

cache_pool* cache_pool::Instance()
{
	if (!itsInstance)
	{
		itsInstance = new cache_pool();
	}

	return itsInstance;
}

void cache_pool::CacheLimit(int theCacheLimit)
{
	itsCacheLimit = theCacheLimit;
}
bool cache_pool::Exists(const string& uniqueName)
{
	Lock lock(itsAccessMutex);
	return itsCache.count(uniqueName) > 0;
}
void cache_pool::Insert(const string& uniqueName, shared_ptr<himan::info> anInfo, bool pin)
{
	cache_item item;
	item.info = anInfo;
	item.access_time = time(nullptr);
	item.pinned = pin;

	{
		Lock lock(itsAccessMutex);

		itsCache.insert(pair<string, cache_item>(uniqueName, item));
	}

	itsLogger.Trace("Data added to cache with name: " + uniqueName + ", pinned: " + to_string(pin));

	if (itsCacheLimit > -1 && itsCache.size() > static_cast<size_t>(itsCacheLimit))
	{
		Clean();
	}
}

void cache_pool::UpdateTime(const std::string& uniqueName)
{
	Lock lock(itsAccessMutex);
	itsCache[uniqueName].access_time = time(nullptr);
}
void cache_pool::Clean()
{
	ASSERT(itsCacheLimit > 0);
	if (itsCache.size() <= static_cast<size_t>(itsCacheLimit))
	{
		return;
	}

	string oldestName;
	time_t oldestTime = INT_MAX;

	{
		Lock lock(itsAccessMutex);

		for (const auto& kv : itsCache)
		{
			if (kv.second.access_time < oldestTime && !kv.second.pinned)
			{
				oldestName = kv.first;
				oldestTime = kv.second.access_time;
			}
		}

		ASSERT(!oldestName.empty());

		itsCache.erase(oldestName);
	}

	itsLogger.Trace("Data cleared from cache: " + oldestName + " with time: " + to_string(oldestTime));
	itsLogger.Trace("Cache size: " + to_string(itsCache.size()));
}

shared_ptr<himan::info> cache_pool::GetInfo(const string& uniqueName)
{
	Lock lock(itsAccessMutex);

	auto it = itsCache.find(uniqueName);
	if (it != itsCache.end())
	{
		return make_shared<info>(*it->second.info);
	}
	return nullptr;
}

size_t cache_pool::Size() const
{
	return itsCache.size();
}
<commit_msg>Log if data not found from cache at trace level<commit_after>/**
 * @file cache.cpp
 *
 */

#include "cache.h"
#include "info.h"
#include "logger.h"
#include "plugin_factory.h"
#include <time.h>

using namespace std;
using namespace himan::plugin;

typedef lock_guard<mutex> Lock;

cache::cache()
{
	itsLogger = logger("cache");
}
string cache::UniqueName(const info& info)
{
	stringstream ss;

	// clang-format off

	ss << info.Producer().Id() << "_"
	   << info.Time().OriginDateTime().String("%Y-%m-%d %H:%M:%S") << "_"
	   << info.Time().ValidDateTime().String("%Y-%m-%d %H:%M:%S") << "_"
	   << info.Param().Name() << "_"
	   << static_cast<string>(info.Level()) << "_"
	   << info.ForecastType().Type() << "_"
	   << info.ForecastType().Value();

	// clang-format on

	return ss.str();
}

string cache::UniqueNameFromOptions(search_options& options)
{
	ASSERT(options.configuration->DatabaseType() == kNoDatabase || options.prod.Id() != kHPMissingInt);
	stringstream ss;

	// clang-format off

	ss << options.prod.Id() << "_"
	   << options.time.OriginDateTime().String("%Y-%m-%d %H:%M:%S") << "_"
	   << options.time.ValidDateTime().String("%Y-%m-%d %H:%M:%S") << "_"
	   << options.param.Name() << "_"
	   << static_cast<string>(options.level) << "_"
	   << options.ftype.Type() << "_"
	   << options.ftype.Value();

	// clang-format on

	return ss.str();
}

void cache::Insert(info_t anInfo, bool pin)
{
	SplitToPool(anInfo, pin);
}
void cache::SplitToPool(info_t anInfo, bool pin)
{
	auto localInfo = make_shared<info>(*anInfo);

	// Cached data is never replaced by another data that has
	// the same uniqueName

	const string uniqueName = UniqueName(*localInfo);

	if (cache_pool::Instance()->Exists(uniqueName))
	{
		itsLogger.Trace("Data with key " + uniqueName + " already exists at cache");

		// Update timestamp of this cache item
		cache_pool::Instance()->UpdateTime(uniqueName);
		return;
	}

#ifdef HAVE_CUDA
	if (localInfo->Grid()->IsPackedData())
	{
		itsLogger.Trace("Removing packed data from cached info");
		localInfo->Grid()->PackedData().Clear();
	}
#endif

	ASSERT(!localInfo->Grid()->IsPackedData());

	// localInfo might contain multiple grids. When adding data to cache, we need
	// to make sure that single info contains only single grid.

	if (localInfo->DimensionSize() > 1)
	{
		auto newInfo =
		    make_shared<info>(localInfo->ForecastType(), localInfo->Time(), localInfo->Level(), localInfo->Param());
		newInfo->Grid(localInfo->SharedGrid());
		localInfo = newInfo;
	}

	ASSERT(localInfo->DimensionSize() == 1);
	cache_pool::Instance()->Insert(uniqueName, localInfo, pin);
}

vector<shared_ptr<himan::info>> cache::GetInfo(search_options& options)
{
	string uniqueName = UniqueNameFromOptions(options);

	vector<shared_ptr<himan::info>> info;

	auto foundInfo = cache_pool::Instance()->GetInfo(uniqueName);
	if (foundInfo)
	{
		info.push_back(foundInfo);
	}

	itsLogger.Trace("Data " + string(foundInfo ? "found" : "not found") + " for " + uniqueName);

	return info;
}

void cache::Clean()
{
	cache_pool::Instance()->Clean();
}
size_t cache::Size() const
{
	return cache_pool::Instance()->Size();
}
cache_pool* cache_pool::itsInstance = NULL;

cache_pool::cache_pool() : itsCacheLimit(-1)
{
	itsLogger = logger("cache_pool");
}

cache_pool* cache_pool::Instance()
{
	if (!itsInstance)
	{
		itsInstance = new cache_pool();
	}

	return itsInstance;
}

void cache_pool::CacheLimit(int theCacheLimit)
{
	itsCacheLimit = theCacheLimit;
}
bool cache_pool::Exists(const string& uniqueName)
{
	Lock lock(itsAccessMutex);
	return itsCache.count(uniqueName) > 0;
}
void cache_pool::Insert(const string& uniqueName, shared_ptr<himan::info> anInfo, bool pin)
{
	cache_item item;
	item.info = anInfo;
	item.access_time = time(nullptr);
	item.pinned = pin;

	{
		Lock lock(itsAccessMutex);

		itsCache.insert(pair<string, cache_item>(uniqueName, item));
	}

	itsLogger.Trace("Data added to cache with name: " + uniqueName + ", pinned: " + to_string(pin));

	if (itsCacheLimit > -1 && itsCache.size() > static_cast<size_t>(itsCacheLimit))
	{
		Clean();
	}
}

void cache_pool::UpdateTime(const std::string& uniqueName)
{
	Lock lock(itsAccessMutex);
	itsCache[uniqueName].access_time = time(nullptr);
}
void cache_pool::Clean()
{
	ASSERT(itsCacheLimit > 0);
	if (itsCache.size() <= static_cast<size_t>(itsCacheLimit))
	{
		return;
	}

	string oldestName;
	time_t oldestTime = INT_MAX;

	{
		Lock lock(itsAccessMutex);

		for (const auto& kv : itsCache)
		{
			if (kv.second.access_time < oldestTime && !kv.second.pinned)
			{
				oldestName = kv.first;
				oldestTime = kv.second.access_time;
			}
		}

		ASSERT(!oldestName.empty());

		itsCache.erase(oldestName);
	}

	itsLogger.Trace("Data cleared from cache: " + oldestName + " with time: " + to_string(oldestTime));
	itsLogger.Trace("Cache size: " + to_string(itsCache.size()));
}

shared_ptr<himan::info> cache_pool::GetInfo(const string& uniqueName)
{
	Lock lock(itsAccessMutex);

	auto it = itsCache.find(uniqueName);
	if (it != itsCache.end())
	{
		return make_shared<info>(*it->second.info);
	}
	return nullptr;
}

size_t cache_pool::Size() const
{
	return itsCache.size();
}
<|endoftext|>