<commit_before>/****************************************************************************
 *
 *   (c) 2009-2016 QGROUNDCONTROL PROJECT <http://www.qgroundcontrol.org>
 *
 * QGroundControl is licensed according to the terms in the file
 * COPYING.md in the root of the source code directory.
 *
 ****************************************************************************/

#include "QGCFileDownload.h"

#include <QFileInfo>
#include <QStandardPaths>
#include <QNetworkProxy>

QGCFileDownload::QGCFileDownload(QObject* parent)
    : QNetworkAccessManager(parent)
{

}

bool QGCFileDownload::download(const QString& remoteFile, bool redirect)
{
    if (!redirect) {
        _originalRemoteFile = remoteFile;
    }

    if (remoteFile.isEmpty()) {
        qWarning() << "downloadFile empty";
        return false;
    }

    // Split out filename from path
    QString remoteFileName = QFileInfo(remoteFile).fileName();
    if (remoteFileName.isEmpty()) {
        qWarning() << "Unabled to parse filename from downloadFile" << remoteFile;
        return false;
    }

    // Strip out parameters from remote filename
    int parameterIndex = remoteFileName.indexOf("?");
    if (parameterIndex != -1) {
        remoteFileName  = remoteFileName.left(parameterIndex);
    }

    // Determine location to download file to
    QString localFile = QStandardPaths::writableLocation(QStandardPaths::TempLocation);
    if (localFile.isEmpty()) {
        localFile = QStandardPaths::writableLocation(QStandardPaths::DownloadLocation);
        if (localFile.isEmpty()) {
            qDebug() << "Unabled to find writable download location. Tried downloads and temp directory.";
            return false;
        }
    }
    localFile += "/"  + remoteFileName;

    QUrl remoteUrl;
    if (remoteFile.startsWith("http:") || remoteFile.startsWith("https:")) {
        remoteUrl.setUrl(remoteFile);
    } else {
        remoteUrl = QUrl::fromLocalFile(remoteFile);
    }
    if (!remoteUrl.isValid()) {
        qWarning() << "Remote URL is invalid" << remoteFile;
        return false;
    }

    QNetworkRequest networkRequest(remoteUrl);

    QNetworkProxy tProxy;
    tProxy.setType(QNetworkProxy::DefaultProxy);
    setProxy(tProxy);

    // Store local file location in user attribute so we can retrieve when the download finishes
    networkRequest.setAttribute(QNetworkRequest::User, localFile);

    QNetworkReply* networkReply = get(networkRequest);
    if (!networkReply) {
        qWarning() << "QNetworkAccessManager::get failed";
        return false;
    }

    connect(networkReply, &QNetworkReply::downloadProgress, this, &QGCFileDownload::downloadProgress);
    connect(networkReply, &QNetworkReply::finished, this, &QGCFileDownload::_downloadFinished);
    connect(networkReply, static_cast<void (QNetworkReply::*)(QNetworkReply::NetworkError)>(&QNetworkReply::error),
            this, &QGCFileDownload::_downloadError);

    return true;
}

void QGCFileDownload::_downloadFinished(void)
{
    QNetworkReply* reply = qobject_cast<QNetworkReply*>(QObject::sender());

    // When an error occurs or the user cancels the download, we still end up here. So bail out in
    // those cases.
    if (reply->error() != QNetworkReply::NoError) {
        return;
    }

    // Check for redirection
    QVariant redirectionTarget = reply->attribute(QNetworkRequest::RedirectionTargetAttribute);
    if (!redirectionTarget.isNull()) {
        QUrl redirectUrl = reply->url().resolved(redirectionTarget.toUrl());
        download(redirectUrl.toString(), true /* redirect */);
        reply->deleteLater();
        return;
    }

    // Download file location is in user attribute
    QString downloadFilename = reply->request().attribute(QNetworkRequest::User).toString();

    if (!downloadFilename.isEmpty()) {
        // Store downloaded file in download location
        QFile file(downloadFilename);
        if (!file.open(QIODevice::WriteOnly)) {
            emit error(tr("Could not save downloaded file to %1. Error: %2").arg(downloadFilename).arg(file.errorString()));
            return;
        }

        file.write(reply->readAll());
        file.close();

        emit downloadFinished(_originalRemoteFile, downloadFilename);
    } else {
        QString errorMsg = "Internal error";
        qWarning() << errorMsg;
        emit error(errorMsg);
    }
}

/// @brief Called when an error occurs during download
void QGCFileDownload::_downloadError(QNetworkReply::NetworkError code)
{
    QString errorMsg;

    if (code == QNetworkReply::OperationCanceledError) {
        errorMsg = tr("Download cancelled");

    } else if (code == QNetworkReply::ContentNotFoundError) {
        errorMsg = tr("Error: File Not Found");

    } else {
        errorMsg = tr("Error during download. Error: %1").arg(code);
    }

    emit error(errorMsg);
}
<commit_msg>Fix leaking QNetworkReply<commit_after>/****************************************************************************
 *
 *   (c) 2009-2016 QGROUNDCONTROL PROJECT <http://www.qgroundcontrol.org>
 *
 * QGroundControl is licensed according to the terms in the file
 * COPYING.md in the root of the source code directory.
 *
 ****************************************************************************/

#include "QGCFileDownload.h"

#include <QFileInfo>
#include <QStandardPaths>
#include <QNetworkProxy>

QGCFileDownload::QGCFileDownload(QObject* parent)
    : QNetworkAccessManager(parent)
{

}

bool QGCFileDownload::download(const QString& remoteFile, bool redirect)
{
    if (!redirect) {
        _originalRemoteFile = remoteFile;
    }

    if (remoteFile.isEmpty()) {
        qWarning() << "downloadFile empty";
        return false;
    }

    // Split out filename from path
    QString remoteFileName = QFileInfo(remoteFile).fileName();
    if (remoteFileName.isEmpty()) {
        qWarning() << "Unabled to parse filename from downloadFile" << remoteFile;
        return false;
    }

    // Strip out parameters from remote filename
    int parameterIndex = remoteFileName.indexOf("?");
    if (parameterIndex != -1) {
        remoteFileName  = remoteFileName.left(parameterIndex);
    }

    // Determine location to download file to
    QString localFile = QStandardPaths::writableLocation(QStandardPaths::TempLocation);
    if (localFile.isEmpty()) {
        localFile = QStandardPaths::writableLocation(QStandardPaths::DownloadLocation);
        if (localFile.isEmpty()) {
            qDebug() << "Unabled to find writable download location. Tried downloads and temp directory.";
            return false;
        }
    }
    localFile += "/"  + remoteFileName;

    QUrl remoteUrl;
    if (remoteFile.startsWith("http:") || remoteFile.startsWith("https:")) {
        remoteUrl.setUrl(remoteFile);
    } else {
        remoteUrl = QUrl::fromLocalFile(remoteFile);
    }
    if (!remoteUrl.isValid()) {
        qWarning() << "Remote URL is invalid" << remoteFile;
        return false;
    }

    QNetworkRequest networkRequest(remoteUrl);

    QNetworkProxy tProxy;
    tProxy.setType(QNetworkProxy::DefaultProxy);
    setProxy(tProxy);

    // Store local file location in user attribute so we can retrieve when the download finishes
    networkRequest.setAttribute(QNetworkRequest::User, localFile);

    QNetworkReply* networkReply = get(networkRequest);
    if (!networkReply) {
        qWarning() << "QNetworkAccessManager::get failed";
        return false;
    }

    connect(networkReply, &QNetworkReply::downloadProgress, this, &QGCFileDownload::downloadProgress);
    connect(networkReply, &QNetworkReply::finished, this, &QGCFileDownload::_downloadFinished);
    connect(networkReply, static_cast<void (QNetworkReply::*)(QNetworkReply::NetworkError)>(&QNetworkReply::error),
            this, &QGCFileDownload::_downloadError);

    return true;
}

void QGCFileDownload::_downloadFinished(void)
{
    QNetworkReply* reply = qobject_cast<QNetworkReply*>(QObject::sender());

    // When an error occurs or the user cancels the download, we still end up here. So bail out in
    // those cases.
    if (reply->error() != QNetworkReply::NoError) {
        reply->deleteLater();
        return;
    }

    // Check for redirection
    QVariant redirectionTarget = reply->attribute(QNetworkRequest::RedirectionTargetAttribute);
    if (!redirectionTarget.isNull()) {
        QUrl redirectUrl = reply->url().resolved(redirectionTarget.toUrl());
        download(redirectUrl.toString(), true /* redirect */);
        reply->deleteLater();
        return;
    }

    // Download file location is in user attribute
    QString downloadFilename = reply->request().attribute(QNetworkRequest::User).toString();

    if (!downloadFilename.isEmpty()) {
        // Store downloaded file in download location
        QFile file(downloadFilename);
        if (!file.open(QIODevice::WriteOnly)) {
            emit error(tr("Could not save downloaded file to %1. Error: %2").arg(downloadFilename).arg(file.errorString()));
            return;
        }

        file.write(reply->readAll());
        file.close();

        emit downloadFinished(_originalRemoteFile, downloadFilename);
    } else {
        QString errorMsg = "Internal error";
        qWarning() << errorMsg;
        emit error(errorMsg);
    }

    reply->deleteLater();
}

/// @brief Called when an error occurs during download
void QGCFileDownload::_downloadError(QNetworkReply::NetworkError code)
{
    QString errorMsg;

    if (code == QNetworkReply::OperationCanceledError) {
        errorMsg = tr("Download cancelled");

    } else if (code == QNetworkReply::ContentNotFoundError) {
        errorMsg = tr("Error: File Not Found");

    } else {
        errorMsg = tr("Error during download. Error: %1").arg(code);
    }

    emit error(errorMsg);
}
<|endoftext|>


<commit_before>#include <common/FSUtils.h>

#include <concurrent_unordered_map.h>
#include <future>

#include <common/CapricaConfig.h>
#include <common/CaselessStringComparer.h>

namespace caprica { namespace FSUtils {

static Concurrency::concurrent_unordered_map<std::string, std::future<std::string>, CaselessStringHasher, CaselessStringEqual> futureFileReadMap{ };
static Concurrency::concurrent_unordered_map<std::string, std::string, CaselessStringHasher, CaselessStringEqual> readFilesMap{ };
std::string Cache::readFile(const std::string& filename) {
  std::ifstream inFile{ filename, std::ifstream::binary };
  std::stringstream strStream;
  strStream << inFile.rdbuf();
  auto str = strStream.str();
  readFilesMap[filename] = str;
  return str;
}

void Cache::waitForAll() {
  for (auto& f : futureFileReadMap) {
    f.second.get();
  }
}

void Cache::push_need(const std::string& filename) {
  if (CapricaConfig::asyncFileRead) {
    auto abs = canonical(filename).string();
    if (!futureFileReadMap.count(abs)) {
      futureFileReadMap[abs] = std::async([abs]() {
        return readFile(abs);
      });
    }
  }
}

std::string Cache::cachedReadFull(const std::string& filename) {
  auto abs = canonical(filename).string();
  push_need(abs);
  auto f = readFilesMap.find(abs);
  if (f != readFilesMap.end())
    return f->second;

  if (!CapricaConfig::asyncFileRead)
    return readFile(abs);

  return futureFileReadMap[abs].get();
}

static Concurrency::concurrent_unordered_map<std::string, std::future<void>, CaselessStringHasher, CaselessStringEqual> futureFileWriteMap{ };
static void writeFile(const std::string& filename, const std::string& value) {
  if (!CapricaConfig::performanceTestMode) {
    std::ofstream destFile{ filename, std::ifstream::binary };
    destFile << value;
  }
}

void async_write(const std::string& filename, const std::string& value) {
  if (!CapricaConfig::asyncFileWrite) {
    writeFile(filename, value);
  } else {
    futureFileWriteMap[filename] = std::async([](const std::string& filename, const std::string& value) {
      writeFile(filename, value);
    }, filename, value);
  }
}

// Borrowed and modified from http://stackoverflow.com/a/1750710/776797
boost::filesystem::path canonical(const boost::filesystem::path& path) {
  auto absPath = boost::filesystem::absolute(path);
  if (CapricaConfig::resolveSymlinks) {
    return boost::filesystem::canonical(absPath).make_preferred();
  } else {
    boost::filesystem::path result;
    for (auto it = absPath.begin(); it != absPath.end(); ++it) {
      if (*it == "..") {
        // /a/b/../.. is not /a/b/.. under most circumstances
        // We can end up with ..s in our result because of symbolic links
        if (result.filename() == "..")
          result /= *it;
        // Otherwise it should be safe to resolve the parent
        else
          result = result.parent_path();
      } else if (*it == ".") {
        // Ignore
      } else {
        // Just cat other path entries
        result /= *it;
      }
    }
    return result.make_preferred();
  }
}

// Borrowed and slightly modified from http://stackoverflow.com/a/5773008/776797
boost::filesystem::path naive_uncomplete(const boost::filesystem::path p, const boost::filesystem::path base) {
  using boost::filesystem::path;

  if (p == base)
    return "./";
  /*!! this breaks stuff if path is a filename rather than a directory,
  which it most likely is... but then base shouldn't be a filename so... */

  boost::filesystem::path from_path, from_base, output;

  boost::filesystem::path::iterator path_it = p.begin(), path_end = p.end();
  boost::filesystem::path::iterator base_it = base.begin(), base_end = base.end();

  // check for emptiness
  if ((path_it == path_end) || (base_it == base_end))
    throw std::runtime_error("path or base was empty; couldn't generate relative path");

#ifdef WIN32
  // drive letters are different; don't generate a relative path
  if (*path_it != *base_it)
    return p;

  // now advance past drive letters; relative paths should only go up
  // to the root of the drive and not past it
  ++path_it, ++base_it;
#endif

  // Cache system-dependent dot, double-dot and slash strings
  const std::string _dot = ".";
  const std::string _dots = "..";
  const std::string _sep = "\\";

  // iterate over path and base
  while (true) {

    // compare all elements so far of path and base to find greatest common root;
    // when elements of path and base differ, or run out:
    if ((path_it == path_end) || (base_it == base_end) || (*path_it != *base_it)) {

      // write to output, ../ times the number of remaining elements in base;
      // this is how far we've had to come down the tree from base to get to the common root
      for (; base_it != base_end; ++base_it) {
        if (*base_it == _dot)
          continue;
        else if (*base_it == _sep)
          continue;

        output /= "../";
      }

      // write to output, the remaining elements in path;
      // this is the path relative from the common root
      boost::filesystem::path::iterator path_it_start = path_it;
      for (; path_it != path_end; ++path_it) {

        if (path_it != path_it_start)
          output /= "/";

        if (*path_it == _dot)
          continue;
        if (*path_it == _sep)
          continue;

        output /= *path_it;
      }

      break;
    }

    // add directory level to both paths and continue iteration
    from_path /= path(*path_it);
    from_base /= path(*base_it);

    ++path_it, ++base_it;
  }

  return output;
}

}}<commit_msg>Finally fix a bug that was caused by the value in a concurrent_unordered_map not being available when the key was already findable.<commit_after>#include <common/FSUtils.h>

#include <concurrent_unordered_map.h>
#include <future>

#include <common/CapricaConfig.h>
#include <common/CaselessStringComparer.h>

namespace caprica { namespace FSUtils {

static Concurrency::concurrent_unordered_map<std::string, std::future<std::string>, CaselessStringHasher, CaselessStringEqual> futureFileReadMap{ };
static Concurrency::concurrent_unordered_map<std::string, std::string, CaselessStringHasher, CaselessStringEqual> readFilesMap{ };
std::string Cache::readFile(const std::string& filename) {
  std::ifstream inFile{ filename, std::ifstream::binary };
  std::stringstream strStream;
  strStream << inFile.rdbuf();
  auto str = strStream.str();
  readFilesMap.insert({ filename, str });
  return str;
}

void Cache::waitForAll() {
  for (auto& f : futureFileReadMap) {
    f.second.get();
  }
}

void Cache::push_need(const std::string& filename) {
  if (CapricaConfig::asyncFileRead) {
    auto abs = canonical(filename).string();
    if (!futureFileReadMap.count(abs)) {
      futureFileReadMap[abs] = std::async([abs]() {
        return readFile(abs);
      });
    }
  }
}

std::string Cache::cachedReadFull(const std::string& filename) {
  auto abs = canonical(filename).string();
  push_need(abs);
  auto f = readFilesMap.find(abs);
  if (f != readFilesMap.end())
    return f->second;

  if (!CapricaConfig::asyncFileRead)
    return readFile(abs);

  return futureFileReadMap[abs].get();
}

static Concurrency::concurrent_unordered_map<std::string, std::future<void>, CaselessStringHasher, CaselessStringEqual> futureFileWriteMap{ };
static void writeFile(const std::string& filename, const std::string& value) {
  if (!CapricaConfig::performanceTestMode) {
    std::ofstream destFile{ filename, std::ifstream::binary };
    destFile << value;
  }
}

void async_write(const std::string& filename, const std::string& value) {
  if (!CapricaConfig::asyncFileWrite) {
    writeFile(filename, value);
  } else {
    futureFileWriteMap[filename] = std::async([](const std::string& filename, const std::string& value) {
      writeFile(filename, value);
    }, filename, value);
  }
}

// Borrowed and modified from http://stackoverflow.com/a/1750710/776797
boost::filesystem::path canonical(const boost::filesystem::path& path) {
  auto absPath = boost::filesystem::absolute(path);
  if (CapricaConfig::resolveSymlinks) {
    return boost::filesystem::canonical(absPath).make_preferred();
  } else {
    boost::filesystem::path result;
    for (auto it = absPath.begin(); it != absPath.end(); ++it) {
      if (*it == "..") {
        // /a/b/../.. is not /a/b/.. under most circumstances
        // We can end up with ..s in our result because of symbolic links
        if (result.filename() == "..")
          result /= *it;
        // Otherwise it should be safe to resolve the parent
        else
          result = result.parent_path();
      } else if (*it == ".") {
        // Ignore
      } else {
        // Just cat other path entries
        result /= *it;
      }
    }
    return result.make_preferred();
  }
}

// Borrowed and slightly modified from http://stackoverflow.com/a/5773008/776797
boost::filesystem::path naive_uncomplete(const boost::filesystem::path p, const boost::filesystem::path base) {
  using boost::filesystem::path;

  if (p == base)
    return "./";
  /*!! this breaks stuff if path is a filename rather than a directory,
  which it most likely is... but then base shouldn't be a filename so... */

  boost::filesystem::path from_path, from_base, output;

  boost::filesystem::path::iterator path_it = p.begin(), path_end = p.end();
  boost::filesystem::path::iterator base_it = base.begin(), base_end = base.end();

  // check for emptiness
  if ((path_it == path_end) || (base_it == base_end))
    throw std::runtime_error("path or base was empty; couldn't generate relative path");

#ifdef WIN32
  // drive letters are different; don't generate a relative path
  if (*path_it != *base_it)
    return p;

  // now advance past drive letters; relative paths should only go up
  // to the root of the drive and not past it
  ++path_it, ++base_it;
#endif

  // Cache system-dependent dot, double-dot and slash strings
  const std::string _dot = ".";
  const std::string _dots = "..";
  const std::string _sep = "\\";

  // iterate over path and base
  while (true) {

    // compare all elements so far of path and base to find greatest common root;
    // when elements of path and base differ, or run out:
    if ((path_it == path_end) || (base_it == base_end) || (*path_it != *base_it)) {

      // write to output, ../ times the number of remaining elements in base;
      // this is how far we've had to come down the tree from base to get to the common root
      for (; base_it != base_end; ++base_it) {
        if (*base_it == _dot)
          continue;
        else if (*base_it == _sep)
          continue;

        output /= "../";
      }

      // write to output, the remaining elements in path;
      // this is the path relative from the common root
      boost::filesystem::path::iterator path_it_start = path_it;
      for (; path_it != path_end; ++path_it) {

        if (path_it != path_it_start)
          output /= "/";

        if (*path_it == _dot)
          continue;
        if (*path_it == _sep)
          continue;

        output /= *path_it;
      }

      break;
    }

    // add directory level to both paths and continue iteration
    from_path /= path(*path_it);
    from_base /= path(*base_it);

    ++path_it, ++base_it;
  }

  return output;
}

}}<|endoftext|>


<commit_before>#include "ProtocolSimple.h"
#include "ThorsNisse/NisseService.h"

using namespace ThorsAnvil::Nisse::ProtocolSimple;

std::string const ReadMessageHandler::failSizeMessage       = "Failed: Reading Size Object";
std::string const ReadMessageHandler::failIncompleteMessage = "Failed: Size OK. But message incomplete";
std::string const WriteMessageHandler::messageSuffix        = "-> 200 OK Replied";

ReadMessageHandler::ReadMessageHandler(NisseService& parent, ThorsAnvil::Socket::DataSocket&& so)
    : NisseHandler(parent, so.getSocketId(), EV_READ | EV_PERSIST)
    , socket(std::move(so))
    , readSizeObject(0)
    , readBuffer(0)
{}

short ReadMessageHandler::eventActivate(LibSocketId /*sockId*/, short /*eventType*/)
{
    bool more;
    std::size_t read;
    if (readSizeObject != sizeof(readSizeObject))
    {
        std::tie(more, read) = socket.getMessageData(reinterpret_cast<char*>(&bufferSize), sizeof(bufferSize), readSizeObject);
        readSizeObject += read;
        if (readSizeObject != sizeof(readSizeObject))
        {
            if (!more)
            {
                // Did not get a size object
                // And the stream was closed.
                std::string fail = failSizeMessage;
                moveHandler<WriteMessageHandler>(std::move(socket), std::move(fail), false);
            }
            // We have not received all of the size object
            // Return for now. When more data arrives we will
            // try and read more when this function is re-called.
            return 0;
        }
        buffer.resize(bufferSize);
    }
    std::tie(more, read) = socket.getMessageData(&buffer[0], bufferSize, readBuffer);
    readBuffer += read;
    if (readBuffer != bufferSize)
    {
        if (!more)
        {
            // We did not get the full message.
            // There is no more data on the buffer.
            // We are going to give up and drop the
            std::string fail = failIncompleteMessage;
            moveHandler<WriteMessageHandler>(std::move(socket), std::move(fail), false);
        }
        // We have not received all of the message
        // Return for now. When more data arrives we will
        // try and read more when this function is re-called.
        return 0;
    }
    moveHandler<WriteMessageHandler>(std::move(socket), std::move(buffer), true);
    return 0;
}

WriteMessageHandler::WriteMessageHandler(NisseService& parent, ThorsAnvil::Socket::DataSocket&& so, std::string const& m, bool ok)
    : NisseHandler(parent, so.getSocketId(), EV_WRITE | EV_PERSIST)
    , socket(std::move(so))
    , writeSizeObject(0)
    , writeBuffer(0)
    , message(m)
{
    if (ok)
    {
        message += messageSuffix;
    }
}

short WriteMessageHandler::eventActivate(LibSocketId /*sockId*/, short /*eventType*/)
{
    bool        more;
    std::size_t written;
    if (writeSizeObject != sizeof(writeSizeObject))
    {
        std::size_t bufferSize = message.size();
        std::tie(more, written) = socket.putMessageData(reinterpret_cast<char*>(&bufferSize), sizeof(bufferSize), writeSizeObject);
        writeSizeObject += written;
        if (writeSizeObject != sizeof(writeSizeObject))
        {
            if (!more)
            {
                dropHandler();
            }
            return 0;
        }
    }
    std::tie(more, written) = socket.putMessageData(message.c_str(), message.size(), writeBuffer);
    writeBuffer += written;
    if (writeBuffer != message.size())
    {
        if (!more)
        {
            dropHandler();
        }
        return 0;
    }
    dropHandler();
    return 0;
}

#ifdef COVERAGE_TEST
/*
 * This code is only compiled into the unit tests for code coverage purposes
 * It is not part of the live code.
 */
#include "ThorsNisse/NisseService.tpp"
template void ThorsAnvil::Nisse::NisseService::listenOn<ReadMessageHandler>(int);
template void ThorsAnvil::Nisse::NisseService::listenOn<WriteMessageHandler, std::string>(int, std::string&);
#endif
<commit_msg>Update ProtocolSimple Reader to not use EV_PERSIST<commit_after>#include "ProtocolSimple.h"
#include "ThorsNisse/NisseService.h"

using namespace ThorsAnvil::Nisse::ProtocolSimple;

std::string const ReadMessageHandler::failSizeMessage       = "Failed: Reading Size Object";
std::string const ReadMessageHandler::failIncompleteMessage = "Failed: Size OK. But message incomplete";
std::string const WriteMessageHandler::messageSuffix        = "-> 200 OK Replied";

ReadMessageHandler::ReadMessageHandler(NisseService& parent, ThorsAnvil::Socket::DataSocket&& so)
    : NisseHandler(parent, so.getSocketId(), EV_READ)
    , socket(std::move(so))
    , readSizeObject(0)
    , readBuffer(0)
{}

short ReadMessageHandler::eventActivate(LibSocketId /*sockId*/, short /*eventType*/)
{
    bool more;
    std::size_t read;
    if (readSizeObject != sizeof(readSizeObject))
    {
        std::tie(more, read) = socket.getMessageData(reinterpret_cast<char*>(&bufferSize), sizeof(bufferSize), readSizeObject);
        readSizeObject += read;
        if (readSizeObject != sizeof(readSizeObject))
        {
            if (!more)
            {
                // Did not get a size object
                // And the stream was closed.
                std::string fail = failSizeMessage;
                moveHandler<WriteMessageHandler>(std::move(socket), std::move(fail), false);
                return 0;
            }
            // We have not received all of the size object
            // Return for now. When more data arrives we will
            // try and read more when this function is re-called.
            return EV_READ;
        }
        buffer.resize(bufferSize);
    }
    std::tie(more, read) = socket.getMessageData(&buffer[0], bufferSize, readBuffer);
    readBuffer += read;
    if (readBuffer != bufferSize)
    {
        if (!more)
        {
            // We did not get the full message.
            // There is no more data on the buffer.
            // We are going to give up and drop the
            std::string fail = failIncompleteMessage;
            moveHandler<WriteMessageHandler>(std::move(socket), std::move(fail), false);
            return 0;
        }
        // We have not received all of the message
        // Return for now. When more data arrives we will
        // try and read more when this function is re-called.
        return EV_READ;
    }
    moveHandler<WriteMessageHandler>(std::move(socket), std::move(buffer), true);
    return 0;
}

WriteMessageHandler::WriteMessageHandler(NisseService& parent, ThorsAnvil::Socket::DataSocket&& so, std::string const& m, bool ok)
    : NisseHandler(parent, so.getSocketId(), EV_WRITE | EV_PERSIST)
    , socket(std::move(so))
    , writeSizeObject(0)
    , writeBuffer(0)
    , message(m)
{
    if (ok)
    {
        message += messageSuffix;
    }
}

short WriteMessageHandler::eventActivate(LibSocketId /*sockId*/, short /*eventType*/)
{
    bool        more;
    std::size_t written;
    if (writeSizeObject != sizeof(writeSizeObject))
    {
        std::size_t bufferSize = message.size();
        std::tie(more, written) = socket.putMessageData(reinterpret_cast<char*>(&bufferSize), sizeof(bufferSize), writeSizeObject);
        writeSizeObject += written;
        if (writeSizeObject != sizeof(writeSizeObject))
        {
            if (!more)
            {
                dropHandler();
            }
            return 0;
        }
    }
    std::tie(more, written) = socket.putMessageData(message.c_str(), message.size(), writeBuffer);
    writeBuffer += written;
    if (writeBuffer != message.size())
    {
        if (!more)
        {
            dropHandler();
        }
        return 0;
    }
    dropHandler();
    return 0;
}

#ifdef COVERAGE_TEST
/*
 * This code is only compiled into the unit tests for code coverage purposes
 * It is not part of the live code.
 */
#include "ThorsNisse/NisseService.tpp"
template void ThorsAnvil::Nisse::NisseService::listenOn<ReadMessageHandler>(int);
template void ThorsAnvil::Nisse::NisseService::listenOn<WriteMessageHandler, std::string>(int, std::string&);
#endif
<|endoftext|>


<commit_before>#include "replication/delete_queue.hpp"

#include "btree/node.hpp"
#include "buffer_cache/buf_lock.hpp"
#include "buffer_cache/co_functions.hpp"
#include "containers/scoped_malloc.hpp"
#include "logger.hpp"
#include "store.hpp"

namespace replication {

namespace delete_queue {

// The offset of the primal offset.
const int PRIMAL_OFFSET_OFFSET = sizeof(block_magic_t);
const int TIMESTAMPS_AND_OFFSETS_OFFSET = PRIMAL_OFFSET_OFFSET + sizeof(off64_t);
const int TIMESTAMPS_AND_OFFSETS_SIZE = sizeof(large_buf_ref) + 3 * sizeof(block_id_t);

off64_t *primal_offset(void *root_buffer) {
    return reinterpret_cast<off64_t *>(reinterpret_cast<char *>(root_buffer) + PRIMAL_OFFSET_OFFSET);
}

large_buf_ref *timestamps_and_offsets_largebuf(void *root_buffer) {
    char *p = reinterpret_cast<char *>(root_buffer);
    return reinterpret_cast<large_buf_ref *>(p + TIMESTAMPS_AND_OFFSETS_OFFSET);
}

large_buf_ref *keys_largebuf(void *root_buffer) {
    char *p = reinterpret_cast<char *>(root_buffer);
    return reinterpret_cast<large_buf_ref *>(p + (TIMESTAMPS_AND_OFFSETS_OFFSET + TIMESTAMPS_AND_OFFSETS_SIZE));
}

int keys_largebuf_ref_size(block_size_t block_size) {
    return block_size.value() - (TIMESTAMPS_AND_OFFSETS_OFFSET + TIMESTAMPS_AND_OFFSETS_SIZE);
}

}  // namespace delete_queue

void add_key_to_delete_queue(int64_t delete_queue_limit, boost::shared_ptr<transactor_t>& txor, block_id_t queue_root_id, repli_timestamp timestamp, const store_key_t *key) {
    thread_saver_t saver;

    // Beware: Right now, some aspects of correctness depend on the
    // fact that we hold the queue_root lock for the entire operation.
    buf_lock_t queue_root(saver, *txor, queue_root_id, rwi_write);

    // TODO this could be a non-major write?
    void *queue_root_buf = queue_root->get_data_major_write();

    off64_t *primal_offset = delete_queue::primal_offset(queue_root_buf);
    large_buf_ref *t_o_ref = delete_queue::timestamps_and_offsets_largebuf(queue_root_buf);
    large_buf_ref *keys_ref = delete_queue::keys_largebuf(queue_root_buf);

    rassert(t_o_ref->size % sizeof(delete_queue::t_and_o) == 0);

    // Figure out what we need to do.
    bool will_want_to_dequeue = (keys_ref->size + 1 + int64_t(key->size) > delete_queue_limit);
    bool will_actually_dequeue = false;

    delete_queue::t_and_o second_tao = { repli_timestamp::invalid, -1 };

    // Possibly update the (timestamp, offset) queue.  (This happens at most once per second.)
    {
        if (t_o_ref->size == 0) {
            // HEY: Why must we allocate large_buf_t's with new?
            boost::scoped_ptr<large_buf_t> t_o_largebuf(new large_buf_t(txor, t_o_ref, lbref_limit_t(delete_queue::TIMESTAMPS_AND_OFFSETS_SIZE), rwi_write));

            // The size is only zero in the unallocated state.  (Large
            // bufs can't actually handle size zero, so we can't let
            // the large buf shrink to that size.)
            delete_queue::t_and_o tao;
            tao.timestamp = timestamp;
            tao.offset = *primal_offset + keys_ref->size;
            t_o_largebuf->allocate(sizeof(tao));
            t_o_largebuf->fill_at(0, &tao, sizeof(tao));
            rassert(keys_ref->size == 0);
            rassert(!will_want_to_dequeue);
        } else {

            delete_queue::t_and_o last_tao;

            {
                boost::scoped_ptr<large_buf_t> t_o_largebuf(new large_buf_t(txor, t_o_ref, lbref_limit_t(delete_queue::TIMESTAMPS_AND_OFFSETS_SIZE), rwi_write));

                co_acquire_large_buf_slice(saver, t_o_largebuf.get(), t_o_ref->size - sizeof(last_tao), sizeof(last_tao));

                t_o_largebuf->read_at(t_o_ref->size - sizeof(last_tao), &last_tao, sizeof(last_tao));

                if (last_tao.timestamp.time > timestamp.time) {
                    logWRN("The delete queue is receiving updates out of order (t1 = %d, t2 = %d), or the system clock has been set back!  Bringing up a replica may be excessively inefficient.\n", last_tao.timestamp.time, timestamp.time);

                    // Timestamps must be monotonically increasing, so sorry.
                    timestamp = last_tao.timestamp;
                }

                if (last_tao.timestamp.time != timestamp.time) {
                    delete_queue::t_and_o tao;
                    tao.timestamp = timestamp;
                    tao.offset = *primal_offset + keys_ref->size;
                    int refsize_adjustment_dontcare;

                    // It's okay to append because we acquired the rhs
                    // of the large buf.
                    t_o_largebuf->append(sizeof(tao), &refsize_adjustment_dontcare);
                    t_o_largebuf->fill_at(t_o_ref->size - sizeof(tao), &tao, sizeof(tao));
                }
            }

            if (will_want_to_dequeue && t_o_ref->size >= int64_t(2 * sizeof(second_tao))) {
                boost::scoped_ptr<large_buf_t> t_o_largebuf(new large_buf_t(txor, t_o_ref, lbref_limit_t(delete_queue::TIMESTAMPS_AND_OFFSETS_SIZE), rwi_write));

                co_acquire_large_buf_slice(saver, t_o_largebuf.get(), 0, 2 * sizeof(second_tao));

                t_o_largebuf->read_at(sizeof(second_tao), &second_tao, sizeof(second_tao));
                will_actually_dequeue = true;

                // It's okay to unprepend because we acquired the lhs of the large buf.
                int refsize_adjustment_dontcare;
                t_o_largebuf->unprepend(sizeof(second_tao), &refsize_adjustment_dontcare);
            }
        }
    }

    // Update the keys list.

    {
        lbref_limit_t reflimit = lbref_limit_t(delete_queue::keys_largebuf_ref_size((*txor)->cache->get_block_size()));

        int64_t amount_to_unprepend = will_actually_dequeue ? second_tao.offset - *primal_offset : 0;
        large_buf_t::co_enqueue(txor, keys_ref, reflimit, amount_to_unprepend, key, 1 + key->size);
        *primal_offset += amount_to_unprepend;
    }

}

bool dump_keys_from_delete_queue(boost::shared_ptr<transactor_t>& txor, block_id_t queue_root_id, repli_timestamp begin_timestamp, deletion_key_stream_receiver_t *recipient) {
    thread_saver_t saver;

    // Beware: Right now, some aspects of correctness depend on the
    // fact that we hold the queue_root lock for the entire operation.
    buf_lock_t queue_root(saver, *txor, queue_root_id, rwi_read);

    void *queue_root_buf = const_cast<void *>(queue_root->get_data_read());

    off64_t *primal_offset = delete_queue::primal_offset(queue_root_buf);
    large_buf_ref *t_o_ref = delete_queue::timestamps_and_offsets_largebuf(queue_root_buf);
    large_buf_ref *keys_ref = delete_queue::keys_largebuf(queue_root_buf);

    if (t_o_ref->size != 0 && keys_ref->size != 0) {

        rassert(t_o_ref->size % sizeof(delete_queue::t_and_o) == 0);

        // TODO: DON'T hold the queue_root lock for the entire operation.  Sheesh.

        int64_t begin_offset = 0;
        int64_t end_offset = keys_ref->size;

        {
            boost::scoped_ptr<large_buf_t> t_o_largebuf(new large_buf_t(txor, t_o_ref, lbref_limit_t(delete_queue::TIMESTAMPS_AND_OFFSETS_SIZE), rwi_read));
            co_acquire_large_buf(saver, t_o_largebuf.get());

            delete_queue::t_and_o tao;
            int64_t i = 0, ie = t_o_ref->size;
            bool begin_found = false;
            while (i < ie) {
                t_o_largebuf->read_at(i, &tao, sizeof(tao));
                if (!begin_found && begin_timestamp.time <= tao.timestamp.time) {
                    begin_offset = tao.offset - *primal_offset;
                    begin_found = true;
                    break;
                }
                i += sizeof(tao);
            }

            if (!begin_found && ie > 0 && begin_timestamp.time > tao.timestamp.time) {
                begin_offset = end_offset;
                begin_found = true;
            }

            if (!recipient->should_send_deletion_keys(begin_found)) {
                return false;
            }

            // So we have a begin_offset and an end_offset.
        }

        rassert(begin_offset <= end_offset);

        if (begin_offset < end_offset) {
            boost::scoped_ptr<large_buf_t> keys_largebuf(new large_buf_t(txor, keys_ref, lbref_limit_t(delete_queue::keys_largebuf_ref_size((*txor)->cache->get_block_size())), rwi_read_outdated_ok));

            // TODO: acquire subinterval.
            co_acquire_large_buf_slice(saver, keys_largebuf.get(), begin_offset, end_offset - begin_offset);

            int64_t n = end_offset - begin_offset;

            // TODO: don't copy needlessly... sheesh.  This is a fake
            // implementation, make something that actually streams later.
            scoped_malloc<char> buf(n);

            keys_largebuf->read_at(begin_offset, buf.get(), n);

            char *p = buf.get();
            char *e = p + n;
            while (p < e) {
                btree_key_t *k = reinterpret_cast<btree_key_t *>(p);
                rassert(k->size + 1 <= e - p);

                recipient->deletion_key(k);
                p += k->size + 1;
            }
        }
    }

    recipient->done_deletion_keys();
    return true;
}

// TODO: maybe this function should be somewhere else.  Well,
// certainly.  Right now we don't have a notion of an "empty"
// largebuf, so we'll know that we have to ->allocate the largebuf
// when we see a size of 0 in the large_buf_ref.
void initialize_large_buf_ref(large_buf_ref *ref, int size_in_bytes) {
    int ids_bytes = size_in_bytes - offsetof(large_buf_ref, block_ids);
    rassert(ids_bytes > 0);

    ref->offset = 0;
    ref->size = 0;
    for (int i = 0, e = ids_bytes / sizeof(block_id_t); i < e; ++i) {
        ref->block_ids[i] = NULL_BLOCK_ID;
    }
}

void initialize_empty_delete_queue(delete_queue_block_t *dqb, block_size_t block_size) {
    dqb->magic = delete_queue_block_t::expected_magic;
    *delete_queue::primal_offset(dqb) = 0;
    large_buf_ref *t_and_o = delete_queue::timestamps_and_offsets_largebuf(dqb);
    initialize_large_buf_ref(t_and_o, delete_queue::TIMESTAMPS_AND_OFFSETS_SIZE);
    large_buf_ref *k = delete_queue::keys_largebuf(dqb);
    initialize_large_buf_ref(k, delete_queue::keys_largebuf_ref_size(block_size));
}

const block_magic_t delete_queue_block_t::expected_magic = { { 'D', 'e', 'l', 'Q' } };

}  // namespace replication
<commit_msg>Removed b.s. assertion about whether we will want to dequeue in delete queue.<commit_after>#include "replication/delete_queue.hpp"

#include "btree/node.hpp"
#include "buffer_cache/buf_lock.hpp"
#include "buffer_cache/co_functions.hpp"
#include "containers/scoped_malloc.hpp"
#include "logger.hpp"
#include "store.hpp"

namespace replication {

namespace delete_queue {

// The offset of the primal offset.
const int PRIMAL_OFFSET_OFFSET = sizeof(block_magic_t);
const int TIMESTAMPS_AND_OFFSETS_OFFSET = PRIMAL_OFFSET_OFFSET + sizeof(off64_t);
const int TIMESTAMPS_AND_OFFSETS_SIZE = sizeof(large_buf_ref) + 3 * sizeof(block_id_t);

off64_t *primal_offset(void *root_buffer) {
    return reinterpret_cast<off64_t *>(reinterpret_cast<char *>(root_buffer) + PRIMAL_OFFSET_OFFSET);
}

large_buf_ref *timestamps_and_offsets_largebuf(void *root_buffer) {
    char *p = reinterpret_cast<char *>(root_buffer);
    return reinterpret_cast<large_buf_ref *>(p + TIMESTAMPS_AND_OFFSETS_OFFSET);
}

large_buf_ref *keys_largebuf(void *root_buffer) {
    char *p = reinterpret_cast<char *>(root_buffer);
    return reinterpret_cast<large_buf_ref *>(p + (TIMESTAMPS_AND_OFFSETS_OFFSET + TIMESTAMPS_AND_OFFSETS_SIZE));
}

int keys_largebuf_ref_size(block_size_t block_size) {
    return block_size.value() - (TIMESTAMPS_AND_OFFSETS_OFFSET + TIMESTAMPS_AND_OFFSETS_SIZE);
}

}  // namespace delete_queue

void add_key_to_delete_queue(int64_t delete_queue_limit, boost::shared_ptr<transactor_t>& txor, block_id_t queue_root_id, repli_timestamp timestamp, const store_key_t *key) {
    thread_saver_t saver;

    // Beware: Right now, some aspects of correctness depend on the
    // fact that we hold the queue_root lock for the entire operation.
    buf_lock_t queue_root(saver, *txor, queue_root_id, rwi_write);

    // TODO this could be a non-major write?
    void *queue_root_buf = queue_root->get_data_major_write();

    off64_t *primal_offset = delete_queue::primal_offset(queue_root_buf);
    large_buf_ref *t_o_ref = delete_queue::timestamps_and_offsets_largebuf(queue_root_buf);
    large_buf_ref *keys_ref = delete_queue::keys_largebuf(queue_root_buf);

    rassert(t_o_ref->size % sizeof(delete_queue::t_and_o) == 0);

    // Figure out what we need to do.
    bool will_want_to_dequeue = (keys_ref->size + 1 + int64_t(key->size) > delete_queue_limit);
    bool will_actually_dequeue = false;

    delete_queue::t_and_o second_tao = { repli_timestamp::invalid, -1 };

    // Possibly update the (timestamp, offset) queue.  (This happens at most once per second.)
    {
        if (t_o_ref->size == 0) {
            // HEY: Why must we allocate large_buf_t's with new?
            boost::scoped_ptr<large_buf_t> t_o_largebuf(new large_buf_t(txor, t_o_ref, lbref_limit_t(delete_queue::TIMESTAMPS_AND_OFFSETS_SIZE), rwi_write));

            // The size is only zero in the unallocated state.  (Large
            // bufs can't actually handle size zero, so we can't let
            // the large buf shrink to that size.)
            delete_queue::t_and_o tao;
            tao.timestamp = timestamp;
            tao.offset = *primal_offset + keys_ref->size;
            t_o_largebuf->allocate(sizeof(tao));
            t_o_largebuf->fill_at(0, &tao, sizeof(tao));
            rassert(keys_ref->size == 0);
        } else {

            delete_queue::t_and_o last_tao;

            {
                boost::scoped_ptr<large_buf_t> t_o_largebuf(new large_buf_t(txor, t_o_ref, lbref_limit_t(delete_queue::TIMESTAMPS_AND_OFFSETS_SIZE), rwi_write));

                co_acquire_large_buf_slice(saver, t_o_largebuf.get(), t_o_ref->size - sizeof(last_tao), sizeof(last_tao));

                t_o_largebuf->read_at(t_o_ref->size - sizeof(last_tao), &last_tao, sizeof(last_tao));

                if (last_tao.timestamp.time > timestamp.time) {
                    logWRN("The delete queue is receiving updates out of order (t1 = %d, t2 = %d), or the system clock has been set back!  Bringing up a replica may be excessively inefficient.\n", last_tao.timestamp.time, timestamp.time);

                    // Timestamps must be monotonically increasing, so sorry.
                    timestamp = last_tao.timestamp;
                }

                if (last_tao.timestamp.time != timestamp.time) {
                    delete_queue::t_and_o tao;
                    tao.timestamp = timestamp;
                    tao.offset = *primal_offset + keys_ref->size;
                    int refsize_adjustment_dontcare;

                    // It's okay to append because we acquired the rhs
                    // of the large buf.
                    t_o_largebuf->append(sizeof(tao), &refsize_adjustment_dontcare);
                    t_o_largebuf->fill_at(t_o_ref->size - sizeof(tao), &tao, sizeof(tao));
                }
            }

            if (will_want_to_dequeue && t_o_ref->size >= int64_t(2 * sizeof(second_tao))) {
                boost::scoped_ptr<large_buf_t> t_o_largebuf(new large_buf_t(txor, t_o_ref, lbref_limit_t(delete_queue::TIMESTAMPS_AND_OFFSETS_SIZE), rwi_write));

                co_acquire_large_buf_slice(saver, t_o_largebuf.get(), 0, 2 * sizeof(second_tao));

                t_o_largebuf->read_at(sizeof(second_tao), &second_tao, sizeof(second_tao));
                will_actually_dequeue = true;

                // It's okay to unprepend because we acquired the lhs of the large buf.
                int refsize_adjustment_dontcare;
                t_o_largebuf->unprepend(sizeof(second_tao), &refsize_adjustment_dontcare);
            }
        }
    }

    // Update the keys list.

    {
        lbref_limit_t reflimit = lbref_limit_t(delete_queue::keys_largebuf_ref_size((*txor)->cache->get_block_size()));

        int64_t amount_to_unprepend = will_actually_dequeue ? second_tao.offset - *primal_offset : 0;
        large_buf_t::co_enqueue(txor, keys_ref, reflimit, amount_to_unprepend, key, 1 + key->size);
        *primal_offset += amount_to_unprepend;
    }

}

bool dump_keys_from_delete_queue(boost::shared_ptr<transactor_t>& txor, block_id_t queue_root_id, repli_timestamp begin_timestamp, deletion_key_stream_receiver_t *recipient) {
    thread_saver_t saver;

    // Beware: Right now, some aspects of correctness depend on the
    // fact that we hold the queue_root lock for the entire operation.
    buf_lock_t queue_root(saver, *txor, queue_root_id, rwi_read);

    void *queue_root_buf = const_cast<void *>(queue_root->get_data_read());

    off64_t *primal_offset = delete_queue::primal_offset(queue_root_buf);
    large_buf_ref *t_o_ref = delete_queue::timestamps_and_offsets_largebuf(queue_root_buf);
    large_buf_ref *keys_ref = delete_queue::keys_largebuf(queue_root_buf);

    if (t_o_ref->size != 0 && keys_ref->size != 0) {

        rassert(t_o_ref->size % sizeof(delete_queue::t_and_o) == 0);

        // TODO: DON'T hold the queue_root lock for the entire operation.  Sheesh.

        int64_t begin_offset = 0;
        int64_t end_offset = keys_ref->size;

        {
            boost::scoped_ptr<large_buf_t> t_o_largebuf(new large_buf_t(txor, t_o_ref, lbref_limit_t(delete_queue::TIMESTAMPS_AND_OFFSETS_SIZE), rwi_read));
            co_acquire_large_buf(saver, t_o_largebuf.get());

            delete_queue::t_and_o tao;
            int64_t i = 0, ie = t_o_ref->size;
            bool begin_found = false;
            while (i < ie) {
                t_o_largebuf->read_at(i, &tao, sizeof(tao));
                if (!begin_found && begin_timestamp.time <= tao.timestamp.time) {
                    begin_offset = tao.offset - *primal_offset;
                    begin_found = true;
                    break;
                }
                i += sizeof(tao);
            }

            if (!begin_found && ie > 0 && begin_timestamp.time > tao.timestamp.time) {
                begin_offset = end_offset;
                begin_found = true;
            }

            if (!recipient->should_send_deletion_keys(begin_found)) {
                return false;
            }

            // So we have a begin_offset and an end_offset.
        }

        rassert(begin_offset <= end_offset);

        if (begin_offset < end_offset) {
            boost::scoped_ptr<large_buf_t> keys_largebuf(new large_buf_t(txor, keys_ref, lbref_limit_t(delete_queue::keys_largebuf_ref_size((*txor)->cache->get_block_size())), rwi_read_outdated_ok));

            // TODO: acquire subinterval.
            co_acquire_large_buf_slice(saver, keys_largebuf.get(), begin_offset, end_offset - begin_offset);

            int64_t n = end_offset - begin_offset;

            // TODO: don't copy needlessly... sheesh.  This is a fake
            // implementation, make something that actually streams later.
            scoped_malloc<char> buf(n);

            keys_largebuf->read_at(begin_offset, buf.get(), n);

            char *p = buf.get();
            char *e = p + n;
            while (p < e) {
                btree_key_t *k = reinterpret_cast<btree_key_t *>(p);
                rassert(k->size + 1 <= e - p);

                recipient->deletion_key(k);
                p += k->size + 1;
            }
        }
    }

    recipient->done_deletion_keys();
    return true;
}

// TODO: maybe this function should be somewhere else.  Well,
// certainly.  Right now we don't have a notion of an "empty"
// largebuf, so we'll know that we have to ->allocate the largebuf
// when we see a size of 0 in the large_buf_ref.
void initialize_large_buf_ref(large_buf_ref *ref, int size_in_bytes) {
    int ids_bytes = size_in_bytes - offsetof(large_buf_ref, block_ids);
    rassert(ids_bytes > 0);

    ref->offset = 0;
    ref->size = 0;
    for (int i = 0, e = ids_bytes / sizeof(block_id_t); i < e; ++i) {
        ref->block_ids[i] = NULL_BLOCK_ID;
    }
}

void initialize_empty_delete_queue(delete_queue_block_t *dqb, block_size_t block_size) {
    dqb->magic = delete_queue_block_t::expected_magic;
    *delete_queue::primal_offset(dqb) = 0;
    large_buf_ref *t_and_o = delete_queue::timestamps_and_offsets_largebuf(dqb);
    initialize_large_buf_ref(t_and_o, delete_queue::TIMESTAMPS_AND_OFFSETS_SIZE);
    large_buf_ref *k = delete_queue::keys_largebuf(dqb);
    initialize_large_buf_ref(k, delete_queue::keys_largebuf_ref_size(block_size));
}

const block_magic_t delete_queue_block_t::expected_magic = { { 'D', 'e', 'l', 'Q' } };

}  // namespace replication
<|endoftext|>


<commit_before>// Copyright 2016 Alessio Sclocco <a.sclocco@vu.nl>
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

#include <vector>
#include <iostream>
#include <exception>
#include <algorithm>
#include <iomanip>

#include <configuration.hpp>

#include <ArgumentList.hpp>
#include <InitializeOpenCL.hpp>
#include <Kernel.hpp>
#include <Reduction.hpp>
#include <utils.hpp>
#include <Timer.hpp>
#include <Stats.hpp>

const inputDataType magicValue = 42;

void initializeDeviceMemory(cl::Context & clContext, cl::CommandQueue * clQueue, std::vector< inputDataType > * input, cl::Buffer * input_d, cl::Buffer * output_d, const unsigned int outputSize);

int main(int argc, char * argv[]) {
  unsigned int nrIterations = 0;
  unsigned int clPlatformID = 0;
  unsigned int clDeviceID = 0;
  unsigned int vectorSize = 0;
  unsigned int maxThreads = 0;
  unsigned int maxItems = 0;
  unsigned int maxVector = 0;
  unsigned int inputSize = 0;
  TuneBench::ReductionConf conf;

  try {
    isa::utils::ArgumentList args(argc, argv);

    clPlatformID = args.getSwitchArgument< unsigned int >("-opencl_platform");
    clDeviceID = args.getSwitchArgument< unsigned int >("-opencl_device");
    nrIterations = args.getSwitchArgument< unsigned int >("-iterations");
    vectorSize = args.getSwitchArgument< unsigned int >("-vector");
    maxThreads = args.getSwitchArgument< unsigned int >("-max_threads");
    maxItems = args.getSwitchArgument< unsigned int >("-max_items");
    maxVector = args.getSwitchArgument< unsigned int >("-max_vector");
    inputSize = args.getSwitchArgument< unsigned int >("-input_size");
  } catch ( isa::utils::EmptyCommandLine & err ) {
    std::cerr << argv[0] << " -opencl_platform ... -opencl_device ... -iterations ... -vector ... -max_threads ... -max_items ... -max_vector ... -input_size ... " << std::endl;
    return 1;
  } catch ( std::exception & err ) {
    std::cerr << err.what() << std::endl;
    return 1;
  }

  cl::Context clContext;
  std::vector< cl::Platform > * clPlatforms = new std::vector< cl::Platform >();
  std::vector< cl::Device > * clDevices = new std::vector< cl::Device >();
  std::vector< std::vector< cl::CommandQueue > > * clQueues = 0;

  // Allocate host memory
  std::vector< inputDataType > input(inputSize);
  cl::Buffer input_d, output_d;

  std::fill(input.begin(), input.end(), magicValue);

  std::cout << std::fixed << std::endl;
  std::cout << "# inputSize outputSize *configuratio* GB/s time stdDeviation COV" << std::endl << std::endl;

  for ( unsigned int threads = vectorSize; threads <= maxThreads; threads += vectorSize ) {
    conf.setNrThreadsD0(threads);
    for ( unsigned int items = 1; items <= maxItems; items++ ) {
      conf.setNrItemsD0(items);
      for ( unsigned int itemsPerBlock = 1; itemsPerBlock < inputSize; itemsPerBlock++ ) {
        conf.setNrItemsPerBlock(itemsPerBlock * (conf.getNrThreadsD0() * conf.getNrItemsD0()));
        for ( unsigned int vector = 1; vector <= maxVector; vector++ ) {
          conf.setVector(vector);
          if ( inputSize % (conf.getNrThreadsD0() * conf.getNrItemsD0() * conf.getVector()) != 0 ) {
            continue;
          } else if ( conf.getNrItemsPerBlock() > (inputSize / conf.getVector()) ) {
            break;
          } else if ( (inputSize / conf.getVector()) % conf.getNrItemsPerBlock() != 0 ) {
            continue;
          } else if ( 1 + (conf.getNrItemsD0() * conf.getVector()) > maxItems ) {
            break;
          }

          // Generate kernel
          unsigned int outputSize = inputSize / conf.getNrItemsPerBlock() /conf.getVector();
          double gbytes = isa::utils::giga((static_cast< uint64_t >(inputSize) * sizeof(inputDataType)) + (static_cast< uint64_t >(outputSize) * sizeof(outputDataType)));
          std::vector< outputDataType > output(outputSize);
          cl::Event clEvent;
          cl::Kernel * kernel;
          isa::utils::Timer timer;
          std::string * code = TuneBench::getReductionOpenCL(conf, inputDataName, outputDataName);

          delete clQueues;
          clQueues = new std::vector< std::vector< cl::CommandQueue > >();
          isa::OpenCL::initializeOpenCL(clPlatformID, 1, clPlatforms, &clContext, clDevices, clQueues);
          try {
            initializeDeviceMemory(clContext, &(clQueues->at(clDeviceID)[0]), &input, &input_d, &output_d, outputSize);
          } catch ( cl::Error & err ) {
            return -1;
          }
          try {
            kernel = isa::OpenCL::compile("reduction", *code, "-cl-mad-enable -Werror", clContext, clDevices->at(clDeviceID));
          } catch ( isa::OpenCL::OpenCLError & err ) {
            std::cerr << err.what() << std::endl;
            delete code;
            break;
          }
          delete code;

          cl::NDRange global(conf.getNrThreadsD0() * (inputSize / conf.getNrItemsPerBlock() / conf.getVector()));
          cl::NDRange local(conf.getNrThreadsD0());

          kernel->setArg(0, input_d);
          kernel->setArg(1, output_d);

          try {
            // Warm-up run
            clQueues->at(clDeviceID)[0].finish();
            clQueues->at(clDeviceID)[0].enqueueNDRangeKernel(*kernel, cl::NullRange, global, local, 0, &clEvent);
            clEvent.wait();
            // Tuning runs
            for ( unsigned int iteration = 0; iteration < nrIterations; iteration++ ) {
              timer.start();
              clQueues->at(clDeviceID)[0].enqueueNDRangeKernel(*kernel, cl::NullRange, global, local, 0, &clEvent);
              clEvent.wait();
              timer.stop();
            }
            clQueues->at(clDeviceID)[0].enqueueReadBuffer(output_d, CL_TRUE, 0, output.size() * sizeof(outputDataType), reinterpret_cast< void * >(output.data()), 0, &clEvent);
            clEvent.wait();
          } catch ( cl::Error & err ) {
            std::cerr << "OpenCL kernel execution error (";
            std::cerr << conf.print();
            std::cerr << "), (";
            std::cerr << isa::utils::toString(conf.getNrThreadsD0() * (inputSize / conf.getNrItemsPerBlock())) << "): ";
            std::cerr << isa::utils::toString(err.err()) << std::endl;
            delete kernel;
            if ( err.err() == -4 || err.err() == -61 ) {
              return -1;
            }
            break;
          }
          delete kernel;

          bool error = false;
          for ( auto item = output.begin(); item != output.end(); ++item ) {
            if ( !isa::utils::same(*item, (magicValue * (conf.getNrItemsPerBlock() * conf.getVector()))) ) {
              std::cerr << "Output error (" << conf.print() << ")." << std::endl;
              error = true;
              break;
            }
          }
          if ( error ) {
            continue;
          }

          std::cout << inputSize << " " << outputSize << " ";
          std::cout << conf.print() << " ";
          std::cout << std::setprecision(3);
          std::cout << gbytes / timer.getAverageTime() << " ";
          std::cout << std::setprecision(6);
          std::cout << timer.getAverageTime() << " " << timer.getStandardDeviation() << " " << timer.getCoefficientOfVariation() << std::endl;
        }
      }
    }
  }
  std::cout << std::endl;

  return 0;
}

void initializeDeviceMemory(cl::Context & clContext, cl::CommandQueue * clQueue, std::vector< inputDataType > * input, cl::Buffer * input_d, cl::Buffer * output_d, const unsigned int outputSize) {
  try {
    *input_d = cl::Buffer(clContext, CL_MEM_READ_ONLY, input->size() * sizeof(inputDataType), 0, 0);
    *output_d = cl::Buffer(clContext, CL_MEM_WRITE_ONLY, outputSize * sizeof(outputDataType), 0, 0);
    clQueue->enqueueWriteBuffer(*input_d, CL_FALSE, 0, input->size() * sizeof(inputDataType), reinterpret_cast< void * >(input->data()));
    clQueue->finish();
  } catch ( cl::Error & err ) {
    std::cerr << "OpenCL error (memory initialization): " << isa::utils::toString(err.err()) << "." << std::endl;
    throw;
  }
}

<commit_msg>Improving the number of registers count.<commit_after>// Copyright 2016 Alessio Sclocco <a.sclocco@vu.nl>
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

#include <vector>
#include <iostream>
#include <exception>
#include <algorithm>
#include <iomanip>

#include <configuration.hpp>

#include <ArgumentList.hpp>
#include <InitializeOpenCL.hpp>
#include <Kernel.hpp>
#include <Reduction.hpp>
#include <utils.hpp>
#include <Timer.hpp>
#include <Stats.hpp>

const inputDataType magicValue = 42;

void initializeDeviceMemory(cl::Context & clContext, cl::CommandQueue * clQueue, std::vector< inputDataType > * input, cl::Buffer * input_d, cl::Buffer * output_d, const unsigned int outputSize);

int main(int argc, char * argv[]) {
  unsigned int nrIterations = 0;
  unsigned int clPlatformID = 0;
  unsigned int clDeviceID = 0;
  unsigned int vectorSize = 0;
  unsigned int maxThreads = 0;
  unsigned int maxItems = 0;
  unsigned int maxVector = 0;
  unsigned int inputSize = 0;
  TuneBench::ReductionConf conf;

  try {
    isa::utils::ArgumentList args(argc, argv);

    clPlatformID = args.getSwitchArgument< unsigned int >("-opencl_platform");
    clDeviceID = args.getSwitchArgument< unsigned int >("-opencl_device");
    nrIterations = args.getSwitchArgument< unsigned int >("-iterations");
    vectorSize = args.getSwitchArgument< unsigned int >("-vector");
    maxThreads = args.getSwitchArgument< unsigned int >("-max_threads");
    maxItems = args.getSwitchArgument< unsigned int >("-max_items");
    maxVector = args.getSwitchArgument< unsigned int >("-max_vector");
    inputSize = args.getSwitchArgument< unsigned int >("-input_size");
  } catch ( isa::utils::EmptyCommandLine & err ) {
    std::cerr << argv[0] << " -opencl_platform ... -opencl_device ... -iterations ... -vector ... -max_threads ... -max_items ... -max_vector ... -input_size ... " << std::endl;
    return 1;
  } catch ( std::exception & err ) {
    std::cerr << err.what() << std::endl;
    return 1;
  }

  cl::Context clContext;
  std::vector< cl::Platform > * clPlatforms = new std::vector< cl::Platform >();
  std::vector< cl::Device > * clDevices = new std::vector< cl::Device >();
  std::vector< std::vector< cl::CommandQueue > > * clQueues = 0;

  // Allocate host memory
  std::vector< inputDataType > input(inputSize);
  cl::Buffer input_d, output_d;

  std::fill(input.begin(), input.end(), magicValue);

  std::cout << std::fixed << std::endl;
  std::cout << "# inputSize outputSize *configuration* GB/s time stdDeviation COV" << std::endl << std::endl;

  for ( unsigned int threads = vectorSize; threads <= maxThreads; threads += vectorSize ) {
    conf.setNrThreadsD0(threads);
    for ( unsigned int items = 1; items <= maxItems; items++ ) {
      conf.setNrItemsD0(items);
      for ( unsigned int itemsPerBlock = 1; itemsPerBlock < inputSize; itemsPerBlock++ ) {
        conf.setNrItemsPerBlock(itemsPerBlock * (conf.getNrThreadsD0() * conf.getNrItemsD0()));
        for ( unsigned int vector = 1; vector <= maxVector; vector++ ) {
          conf.setVector(vector);
          if ( inputSize % (conf.getNrThreadsD0() * conf.getNrItemsD0() * conf.getVector()) != 0 ) {
            continue;
          } else if ( conf.getNrItemsPerBlock() > (inputSize / conf.getVector()) ) {
            break;
          } else if ( (inputSize / conf.getVector()) % conf.getNrItemsPerBlock() != 0 ) {
            continue;
          } else if ( 2 + (conf.getNrItemsD0() * conf.getVector()) > maxItems ) {
            break;
          }

          // Generate kernel
          unsigned int outputSize = inputSize / conf.getNrItemsPerBlock() /conf.getVector();
          double gbytes = isa::utils::giga((static_cast< uint64_t >(inputSize) * sizeof(inputDataType)) + (static_cast< uint64_t >(outputSize) * sizeof(outputDataType)));
          std::vector< outputDataType > output(outputSize);
          cl::Event clEvent;
          cl::Kernel * kernel;
          isa::utils::Timer timer;
          std::string * code = TuneBench::getReductionOpenCL(conf, inputDataName, outputDataName);

          delete clQueues;
          clQueues = new std::vector< std::vector< cl::CommandQueue > >();
          isa::OpenCL::initializeOpenCL(clPlatformID, 1, clPlatforms, &clContext, clDevices, clQueues);
          try {
            initializeDeviceMemory(clContext, &(clQueues->at(clDeviceID)[0]), &input, &input_d, &output_d, outputSize);
          } catch ( cl::Error & err ) {
            return -1;
          }
          try {
            kernel = isa::OpenCL::compile("reduction", *code, "-cl-mad-enable -Werror", clContext, clDevices->at(clDeviceID));
          } catch ( isa::OpenCL::OpenCLError & err ) {
            std::cerr << err.what() << std::endl;
            delete code;
            break;
          }
          delete code;

          cl::NDRange global(conf.getNrThreadsD0() * (inputSize / conf.getNrItemsPerBlock() / conf.getVector()));
          cl::NDRange local(conf.getNrThreadsD0());

          kernel->setArg(0, input_d);
          kernel->setArg(1, output_d);

          try {
            // Warm-up run
            clQueues->at(clDeviceID)[0].finish();
            clQueues->at(clDeviceID)[0].enqueueNDRangeKernel(*kernel, cl::NullRange, global, local, 0, &clEvent);
            clEvent.wait();
            // Tuning runs
            for ( unsigned int iteration = 0; iteration < nrIterations; iteration++ ) {
              timer.start();
              clQueues->at(clDeviceID)[0].enqueueNDRangeKernel(*kernel, cl::NullRange, global, local, 0, &clEvent);
              clEvent.wait();
              timer.stop();
            }
            clQueues->at(clDeviceID)[0].enqueueReadBuffer(output_d, CL_TRUE, 0, output.size() * sizeof(outputDataType), reinterpret_cast< void * >(output.data()), 0, &clEvent);
            clEvent.wait();
          } catch ( cl::Error & err ) {
            std::cerr << "OpenCL kernel execution error (";
            std::cerr << conf.print();
            std::cerr << "), (";
            std::cerr << isa::utils::toString(conf.getNrThreadsD0() * (inputSize / conf.getNrItemsPerBlock())) << "): ";
            std::cerr << isa::utils::toString(err.err()) << std::endl;
            delete kernel;
            if ( err.err() == -4 || err.err() == -61 ) {
              return -1;
            }
            break;
          }
          delete kernel;

          bool error = false;
          for ( auto item = output.begin(); item != output.end(); ++item ) {
            if ( !isa::utils::same(*item, (magicValue * (conf.getNrItemsPerBlock() * conf.getVector()))) ) {
              std::cerr << "Output error (" << conf.print() << ")." << std::endl;
              error = true;
              break;
            }
          }
          if ( error ) {
            continue;
          }

          std::cout << inputSize << " " << outputSize << " ";
          std::cout << conf.print() << " ";
          std::cout << std::setprecision(3);
          std::cout << gbytes / timer.getAverageTime() << " ";
          std::cout << std::setprecision(6);
          std::cout << timer.getAverageTime() << " " << timer.getStandardDeviation() << " " << timer.getCoefficientOfVariation() << std::endl;
        }
      }
    }
  }
  std::cout << std::endl;

  return 0;
}

void initializeDeviceMemory(cl::Context & clContext, cl::CommandQueue * clQueue, std::vector< inputDataType > * input, cl::Buffer * input_d, cl::Buffer * output_d, const unsigned int outputSize) {
  try {
    *input_d = cl::Buffer(clContext, CL_MEM_READ_ONLY, input->size() * sizeof(inputDataType), 0, 0);
    *output_d = cl::Buffer(clContext, CL_MEM_WRITE_ONLY, outputSize * sizeof(outputDataType), 0, 0);
    clQueue->enqueueWriteBuffer(*input_d, CL_FALSE, 0, input->size() * sizeof(inputDataType), reinterpret_cast< void * >(input->data()));
    clQueue->finish();
  } catch ( cl::Error & err ) {
    std::cerr << "OpenCL error (memory initialization): " << isa::utils::toString(err.err()) << "." << std::endl;
    throw;
  }
}

<|endoftext|>


<commit_before>/*
 * ctapdev.cc
 *
 *  Created on: 24.06.2013
 *      Author: andreas
 */

#include "ctapdev.hpp"

using namespace rofcore;

extern int errno;

ctapdev::ctapdev(
		cnetdev_owner *netdev_owner,
		const rofl::cdpid& dpid,
		std::string const& devname,
		uint16_t pvid,
		rofl::cmacaddr const& hwaddr,
		pthread_t tid) :
		cnetdev(netdev_owner, devname, tid),
		fd(-1),
		dpid(dpid),
		devname(devname),
		pvid(pvid),
		hwaddr(hwaddr)
{
	try {
		tap_open(devname, hwaddr);
	} catch (...) {
		port_open_timer_id = register_timer(CTAPDEV_TIMER_OPEN_PORT, 1);
	}
}

ctapdev::~ctapdev()
{
	tap_close();
}

void
ctapdev::tap_open(std::string const& devname, rofl::cmacaddr const& hwaddr)
{
	try {
		struct ifreq ifr;
		int rc;

		if (fd > -1) {
			tap_close();
		}

		if ((fd = open("/dev/net/tun", O_RDWR|O_NONBLOCK)) < 0) {
			throw eTapDevOpenFailed("ctapdev::tap_open() opening /dev/net/tun failed");
		}

		memset(&ifr, 0, sizeof(ifr));

		/* Flags: IFF_TUN   - TUN device (no Ethernet headers)
		 *        IFF_TAP   - TAP device
		 *
		 *        IFF_NO_PI - Do not provide packet information
		 */
		ifr.ifr_flags = IFF_TAP | IFF_NO_PI;
		strncpy(ifr.ifr_name, devname.c_str(), IFNAMSIZ);

		if ((rc = ioctl(fd, TUNSETIFF, (void *)&ifr)) < 0) {
			close(fd);
			throw eTapDevIoctlFailed("ctapdev::tap_open() setting flags IFF_TAP | IFF_NO_PI on /dev/net/tun failed");
		}

		set_hwaddr(hwaddr);

		enable_interface();

		//netdev_owner->netdev_open(this);

		register_filedesc_r(fd);

	} catch (eTapDevOpenFailed& e) {

		rofcore::logging::error << "ctapdev::tap_open() open() failed, dev:" << devname << std::endl;

		throw eNetDevCritical(rofl::eSysCall("open"));

	} catch (eTapDevIoctlFailed& e) {

		rofcore::logging::error << "ctapdev::tap_open() open() failed, dev:" << devname << std::endl;

		throw eNetDevCritical(rofl::eSysCall("ctapdev ioctl"));

	} catch (eNetDevIoctl& e) {

		rofcore::logging::error << "ctapdev::tap_open() open() failed, dev:" << devname << std::endl;

		throw eNetDevCritical(rofl::eSysCall("cnetdev open"));

	}
}

void
ctapdev::tap_close()
{
	try {
		if (fd == -1) {
			return;
		}

		//netdev_owner->netdev_close(this);

		disable_interface();

		deregister_filedesc_r(fd);

	} catch (eNetDevIoctl& e) {
		rofcore::logging::error << "ctapdev::tap_close() failed: dev:" << devname << std::endl;
	}

	close(fd);

	fd = -1;
}

void
ctapdev::enqueue(rofl::cpacket *pkt)
{
	if (fd == -1) {
		cpacketpool::get_instance().release_pkt(pkt);
		return;
	}

	// store pkt in outgoing queue
	pout_queue.push_back(pkt);

	register_filedesc_w(fd);
}

void
ctapdev::enqueue(std::vector<rofl::cpacket*> pkts)
{
	if (fd == -1) {
		for (std::vector<rofl::cpacket*>::iterator
				it = pkts.begin(); it != pkts.end(); ++it) {
			cpacketpool::get_instance().release_pkt((*it));
		}
		return;
	}

	// store pkts in outgoing queue
	for (std::vector<rofl::cpacket*>::iterator
			it = pkts.begin(); it != pkts.end(); ++it) {
		pout_queue.push_back(*it);
	}

	register_filedesc_w(fd);
}

void
ctapdev::handle_revent(int fd)
{
	rofl::cpacket *pkt = (rofl::cpacket*)0;
	try {

		rofl::cmemory mem(1518);

		int rc = read(fd, mem.somem(), mem.memlen());

		// error occured (or non-blocking)
		if (rc < 0) {
			switch (errno) {
			case EAGAIN: 	throw eNetDevAgain("ctapdev::handle_revent() EAGAIN when reading from /dev/net/tun");
			default: 		throw eNetDevCritical("ctapdev::handle_revent() error when reading from /dev/net/tun");
			}
		} else {
			pkt = cpacketpool::get_instance().acquire_pkt();

			pkt->unpack(mem.somem(), rc);

			netdev_owner->enqueue(this, pkt);
		}

	} catch (ePacketPoolExhausted& e) {
		rofcore::logging::error << "ctapdev::handle_revent() packet pool exhausted, no idle slots available" << std::endl;

	} catch (eNetDevAgain& e) {

		rofcore::logging::error << "ctapdev::handle_revent() EAGAIN, retrying later" << std::endl;

		cpacketpool::get_instance().release_pkt(pkt);

	} catch (eNetDevCritical& e) {
		rofcore::logging::error << "ctapdev::handle_revent() error occured" << std::endl;

		cpacketpool::get_instance().release_pkt(pkt);

		delete this; return;
	}
}

void
ctapdev::handle_wevent(int fd)
{
	rofl::cpacket * pkt = (rofl::cpacket*)0;
	try {

		while (not pout_queue.empty()) {

			pkt = pout_queue.front();
			int rc = 0;
			if ((rc = write(fd, pkt->soframe(), pkt->length())) < 0) {
				switch (errno) {
				case EAGAIN: 	throw eNetDevAgain("ctapdev::handle_wevent() EAGAIN");
				default:		throw eNetDevCritical("ctapdev::handle_wevent() error occured");
				}
			}

			cpacketpool::get_instance().release_pkt(pkt);

			pout_queue.pop_front();
		}

		if (pout_queue.empty()) {
			deregister_filedesc_w(fd);
		}

	} catch (eNetDevAgain& e) {

		// keep fd in wfds
		rofcore::logging::error << "ctapdev::handle_wevent() EAGAIN, retrying later" << std::endl;

	} catch (eNetDevCritical& e) {

		rofcore::logging::error << "ctapdev::handle_wevent() critical error occured" << std::endl;
		cpacketpool::get_instance().release_pkt(pkt);

		throw;
		//delete this; return;
	}
}

void
ctapdev::handle_timeout(int opaque, void* data)
{
	switch (opaque) {
	case CTAPDEV_TIMER_OPEN_PORT: {
		try {
			tap_open(devname, hwaddr);
		} catch (...) {
			port_open_timer_id = register_timer(CTAPDEV_TIMER_OPEN_PORT, 1);
		}
	} break;
	}
}

<commit_msg>cnetlink => method route_link_cb() => added catch clause for exception crtlink::eRtLinkNotFound<commit_after>/*
 * ctapdev.cc
 *
 *  Created on: 24.06.2013
 *      Author: andreas
 */

#include "ctapdev.hpp"

using namespace rofcore;

extern int errno;

ctapdev::ctapdev(
		cnetdev_owner *netdev_owner,
		const rofl::cdpid& dpid,
		std::string const& devname,
		uint16_t pvid,
		rofl::cmacaddr const& hwaddr,
		pthread_t tid) :
		cnetdev(netdev_owner, devname, tid),
		fd(-1),
		dpid(dpid),
		devname(devname),
		pvid(pvid),
		hwaddr(hwaddr)
{
	try {
		tap_open(devname, hwaddr);
	} catch (...) {
		port_open_timer_id = register_timer(CTAPDEV_TIMER_OPEN_PORT, 1);
	}
}

ctapdev::~ctapdev()
{
	tap_close();
}

void
ctapdev::tap_open(std::string const& devname, rofl::cmacaddr const& hwaddr)
{
	try {
		struct ifreq ifr;
		int rc;

		if (fd > -1) {
			tap_close();
		}

		if ((fd = open("/dev/net/tun", O_RDWR|O_NONBLOCK)) < 0) {
			throw eTapDevOpenFailed("ctapdev::tap_open() opening /dev/net/tun failed");
		}

		memset(&ifr, 0, sizeof(ifr));

		/* Flags: IFF_TUN   - TUN device (no Ethernet headers)
		 *        IFF_TAP   - TAP device
		 *
		 *        IFF_NO_PI - Do not provide packet information
		 */
		ifr.ifr_flags = IFF_TAP | IFF_NO_PI;
		strncpy(ifr.ifr_name, devname.c_str(), IFNAMSIZ);

		if ((rc = ioctl(fd, TUNSETIFF, (void *)&ifr)) < 0) {
			close(fd);
			throw eTapDevIoctlFailed("ctapdev::tap_open() setting flags IFF_TAP | IFF_NO_PI on /dev/net/tun failed");
		}

		set_hwaddr(hwaddr);

		enable_interface();

		//netdev_owner->netdev_open(this);

		register_filedesc_r(fd);

	} catch (eTapDevOpenFailed& e) {

		rofcore::logging::error << "ctapdev::tap_open() open() failed, dev:" << devname << std::endl;

		throw eNetDevCritical(rofl::eSysCall(std::string("open")));

	} catch (eTapDevIoctlFailed& e) {

		rofcore::logging::error << "ctapdev::tap_open() open() failed, dev:" << devname << std::endl;

		throw eNetDevCritical(rofl::eSysCall(std::string("ctapdev ioctl")));

	} catch (eNetDevIoctl& e) {

		rofcore::logging::error << "ctapdev::tap_open() open() failed, dev:" << devname << std::endl;

		throw eNetDevCritical(rofl::eSysCall(std::string("cnetdev open")));

	}
}

void
ctapdev::tap_close()
{
	try {
		if (fd == -1) {
			return;
		}

		//netdev_owner->netdev_close(this);

		disable_interface();

		deregister_filedesc_r(fd);

	} catch (eNetDevIoctl& e) {
		rofcore::logging::error << "ctapdev::tap_close() failed: dev:" << devname << std::endl;
	}

	close(fd);

	fd = -1;
}

void
ctapdev::enqueue(rofl::cpacket *pkt)
{
	if (fd == -1) {
		cpacketpool::get_instance().release_pkt(pkt);
		return;
	}

	// store pkt in outgoing queue
	pout_queue.push_back(pkt);

	register_filedesc_w(fd);
}

void
ctapdev::enqueue(std::vector<rofl::cpacket*> pkts)
{
	if (fd == -1) {
		for (std::vector<rofl::cpacket*>::iterator
				it = pkts.begin(); it != pkts.end(); ++it) {
			cpacketpool::get_instance().release_pkt((*it));
		}
		return;
	}

	// store pkts in outgoing queue
	for (std::vector<rofl::cpacket*>::iterator
			it = pkts.begin(); it != pkts.end(); ++it) {
		pout_queue.push_back(*it);
	}

	register_filedesc_w(fd);
}

void
ctapdev::handle_revent(int fd)
{
	rofl::cpacket *pkt = (rofl::cpacket*)0;
	try {

		rofl::cmemory mem(1518);

		int rc = read(fd, mem.somem(), mem.memlen());

		// error occured (or non-blocking)
		if (rc < 0) {
			switch (errno) {
			case EAGAIN: 	throw eNetDevAgain("ctapdev::handle_revent() EAGAIN when reading from /dev/net/tun");
			default: 		throw eNetDevCritical("ctapdev::handle_revent() error when reading from /dev/net/tun");
			}
		} else {
			pkt = cpacketpool::get_instance().acquire_pkt();

			pkt->unpack(mem.somem(), rc);

			netdev_owner->enqueue(this, pkt);
		}

	} catch (ePacketPoolExhausted& e) {
		rofcore::logging::error << "ctapdev::handle_revent() packet pool exhausted, no idle slots available" << std::endl;

	} catch (eNetDevAgain& e) {

		rofcore::logging::error << "ctapdev::handle_revent() EAGAIN, retrying later" << std::endl;

		cpacketpool::get_instance().release_pkt(pkt);

	} catch (eNetDevCritical& e) {
		rofcore::logging::error << "ctapdev::handle_revent() error occured" << std::endl;

		cpacketpool::get_instance().release_pkt(pkt);

		delete this; return;
	}
}

void
ctapdev::handle_wevent(int fd)
{
	rofl::cpacket * pkt = (rofl::cpacket*)0;
	try {

		while (not pout_queue.empty()) {

			pkt = pout_queue.front();
			int rc = 0;
			if ((rc = write(fd, pkt->soframe(), pkt->length())) < 0) {
				switch (errno) {
				case EAGAIN: 	throw eNetDevAgain("ctapdev::handle_wevent() EAGAIN");
				default:		throw eNetDevCritical("ctapdev::handle_wevent() error occured");
				}
			}

			cpacketpool::get_instance().release_pkt(pkt);

			pout_queue.pop_front();
		}

		if (pout_queue.empty()) {
			deregister_filedesc_w(fd);
		}

	} catch (eNetDevAgain& e) {

		// keep fd in wfds
		rofcore::logging::error << "ctapdev::handle_wevent() EAGAIN, retrying later" << std::endl;

	} catch (eNetDevCritical& e) {

		rofcore::logging::error << "ctapdev::handle_wevent() critical error occured" << std::endl;
		cpacketpool::get_instance().release_pkt(pkt);

		throw;
		//delete this; return;
	}
}

void
ctapdev::handle_timeout(int opaque, void* data)
{
	switch (opaque) {
	case CTAPDEV_TIMER_OPEN_PORT: {
		try {
			tap_open(devname, hwaddr);
		} catch (...) {
			port_open_timer_id = register_timer(CTAPDEV_TIMER_OPEN_PORT, 1);
		}
	} break;
	}
}

<|endoftext|>


<commit_before>///
/// @file  S2LoadBalancer.cpp
/// @brief The S2LoadBalancer evenly distributes the work load between
///        the threads in the computation of the special leaves.
///
/// Simply parallelizing the computation of the special leaves in the
/// Lagarias-Miller-Odlyzko and Deleglise-Rivat algorithms by
/// subdividing the sieve interval by the number of threads into
/// equally sized subintervals does not scale because the distribution
/// of the special leaves is highly skewed and most special leaves are
/// in the first few segments whereas later on there are very few
/// special leaves.
///
/// Based on the above observations it is clear that we need some kind
/// of load balancing in order to scale our parallel algorithm for
/// computing special leaves. Below are the ideas I used to develop a
/// load balancing algorithm that achieves a high load balance by
/// dynamically increasing or decreasing the interval size based on
/// the relative standard deviation of the thread run-times.
///
/// 1) Start with a tiny segment size of x^(1/3) / (log x * log log x)
///    and one segment per thread. Each thread sieves a distance of
///    segment_size * segments_per_thread. Once all threads have
///    processed their intervals we calculate a new interval size
///    based on the ideas below.
///
/// 2) If the relative standard deviation of the thread run-times is
///    large then we know the special leaves are distributed unevenly,
///    else if the relative standard deviation is low the special
///    leaves are more evenly distributed.
///
/// 3) If the special leaves are distributed unevenly then we can
///    increase the load balance by decreasing the interval size.
///    Contrary if the special leaves are more evenly distributed
///    we can increase the interval size in order to improve the
///    algorithm's efficiency.
///
/// 4) We can't use a static threshold for as to when the relative
///    standard deviation is low or large as this threshold varies for
///    different PC architectures. So instead we compare the current
///    relative standard deviation to the previous one in order to
///    decide whether to increase or decrease the interval size.
///
/// Copyright (C) 2016 Kim Walisch, <kim.walisch@gmail.com>
///
/// This file is distributed under the BSD License. See the COPYING
/// file in the top level directory.
///

#include <S2LoadBalancer.hpp>
#include <primesum-internal.hpp>
#include <aligned_vector.hpp>
#include <imath.hpp>
#include <int128_t.hpp>

#include <stdint.h>
#include <algorithm>
#include <cmath>

using namespace std;
using namespace primesum;

namespace {

double get_avg(aligned_vector<double>& timings)
{
  size_t n = timings.size();
  double sum = 0;

  for (size_t i = 0; i < n; i++)
    sum += timings[i];

  return sum / n;
}

/// Relative standard deviation
double rel_std_dev(aligned_vector<double>& timings)
{
  size_t n = timings.size();
  double avg = get_avg(timings);
  double sum = 0;

  if (avg == 0)
    return 0;

  for (size_t i = 0; i < n; i++)
  {
    double mean = timings[i] - avg;
    sum += mean * mean;
  }

  double std_dev = sqrt(sum / max(1.0, n - 1.0));
  double rsd = 100 * std_dev / avg;

  return rsd;
}

} // namespace

namespace primesum {

S2LoadBalancer::S2LoadBalancer(int128_t x,
                               int64_t y,
                               int64_t z,
                               int64_t threads) :
  x_((double) x),
  y_((double) y),
  z_((double) z),
  rsd_(40),
  count_(0),
  total_seconds_(0),
  sqrtz_(isqrt(z))
{
  init(x, y, threads);
}

void S2LoadBalancer::init(int128_t x,
                          int64_t y,
                          int64_t threads)
{
  // determined by benchmarking
  double log_threads = max(1.0, log((double) threads));
  decrease_dividend_ = max(0.5, log_threads / 3);

  min_seconds_ = 0.01 * log_threads;
  double divisor = log(log(x_)) * log(x_);
  update_min_size(divisor);

  double alpha = get_alpha(x, y);
  smallest_hard_leaf_ = (int64_t) (x / (y * sqrt(alpha) * iroot<6>(x)));
}

double S2LoadBalancer::get_rsd() const
{
  return rsd_;
}

double S2LoadBalancer::get_avg_seconds() const
{
  return total_seconds_ / count_;
}

int64_t S2LoadBalancer::get_min_segment_size() const
{
  return min_size_;
}

/// Increase if relative std dev < pivot.
/// Decrease if relative std dev > pivot.
///
double S2LoadBalancer::get_pivot(double seconds) const
{
  double log_seconds = log(seconds);
  log_seconds = max(min_seconds_, log_seconds);
  double dont_decrease = decrease_dividend_ / (seconds * log_seconds);
  dont_decrease = min(dont_decrease, rsd_);

  return rsd_ + dont_decrease;
}

bool S2LoadBalancer::is_increase(double seconds,
                                 double pivot) const
{
  return (seconds < min_seconds_ ||
          seconds < get_avg_seconds()) &&
         !is_decrease(seconds, pivot);
}

bool S2LoadBalancer::is_decrease(double seconds,
                                 double pivot) const
{
  return seconds > min_seconds_ &&
         rsd_ > pivot;
}

void S2LoadBalancer::update_min_size(double divisor)
{
  int64_t min_size = 1 << 9;
  int64_t size = (int64_t) (sqrtz_ / max(1.0, divisor));
  min_size_ = max(size, min_size);
  min_size_ = next_power_of_2(min_size_);
}

/// Balance the load in the computation of the special leaves
/// by dynamically adjusting the segment_size and segments_per_thread.
/// @param timings  Timings of the threads.
///
void S2LoadBalancer::update(int64_t low,
                            int64_t threads,
                            int64_t* segment_size,
                            int64_t* segments_per_thread,
                            aligned_vector<double>& timings)
{
  count_++;
  double seconds = get_avg(timings);
  total_seconds_ += seconds;
  double pivot = get_pivot(seconds);
  rsd_ = max(0.1, rel_std_dev(timings));

  // 1 segment per thread
  if (*segment_size < sqrtz_)
  {
    if (is_increase(seconds, pivot))
      *segment_size <<= 1;
    else if (is_decrease(seconds, pivot))
      if (*segment_size > min_size_)
        *segment_size >>= 1;
  }
  // many segments per thread
  else if (low > smallest_hard_leaf_)
    update(segments_per_thread, seconds, pivot);

  int64_t thread_distance = *segment_size * *segments_per_thread;
  int64_t high = low + thread_distance * threads;

  // near smallest_hard_leaf_ the hard special leaves
  // are distributed unevenly so use min_size_
  if (low <= smallest_hard_leaf_ &&
      high > smallest_hard_leaf_)
  {
    *segment_size = min_size_;
    thread_distance = *segment_size * *segments_per_thread;
    high = low + thread_distance * threads;
  }

  // slightly increase min_size_
  if (high >= smallest_hard_leaf_)
  {
    update_min_size(log(y_));
    *segment_size = max(min_size_, *segment_size);
  }
}

/// Increase the segments_per_thread if the relative standard
/// deviation of the thread run times is small, or
/// decrease the segments_per_thread if the relative standard
/// deviation of the thread run times is large.
///
void S2LoadBalancer::update(int64_t* segments_per_thread,
                            double seconds,
                            double pivot)
{
  if (is_increase(seconds, pivot) ||
      is_decrease(seconds, pivot))
  {
    if (seconds < min_seconds_)
      *segments_per_thread *= 2;
    else
    {
      double factor = pivot / rsd_;
      factor = in_between(0.5, factor, 2);
      double segments = *segments_per_thread * factor;
      segments = max(1.0, segments);
      *segments_per_thread = (int64_t) segments;
    }
  }
}

} // namespace
<commit_msg>Fix division by 0<commit_after>///
/// @file  S2LoadBalancer.cpp
/// @brief The S2LoadBalancer evenly distributes the work load between
///        the threads in the computation of the special leaves.
///
/// Simply parallelizing the computation of the special leaves in the
/// Lagarias-Miller-Odlyzko and Deleglise-Rivat algorithms by
/// subdividing the sieve interval by the number of threads into
/// equally sized subintervals does not scale because the distribution
/// of the special leaves is highly skewed and most special leaves are
/// in the first few segments whereas later on there are very few
/// special leaves.
///
/// Based on the above observations it is clear that we need some kind
/// of load balancing in order to scale our parallel algorithm for
/// computing special leaves. Below are the ideas I used to develop a
/// load balancing algorithm that achieves a high load balance by
/// dynamically increasing or decreasing the interval size based on
/// the relative standard deviation of the thread run-times.
///
/// 1) Start with a tiny segment size of x^(1/3) / (log x * log log x)
///    and one segment per thread. Each thread sieves a distance of
///    segment_size * segments_per_thread. Once all threads have
///    processed their intervals we calculate a new interval size
///    based on the ideas below.
///
/// 2) If the relative standard deviation of the thread run-times is
///    large then we know the special leaves are distributed unevenly,
///    else if the relative standard deviation is low the special
///    leaves are more evenly distributed.
///
/// 3) If the special leaves are distributed unevenly then we can
///    increase the load balance by decreasing the interval size.
///    Contrary if the special leaves are more evenly distributed
///    we can increase the interval size in order to improve the
///    algorithm's efficiency.
///
/// 4) We can't use a static threshold for as to when the relative
///    standard deviation is low or large as this threshold varies for
///    different PC architectures. So instead we compare the current
///    relative standard deviation to the previous one in order to
///    decide whether to increase or decrease the interval size.
///
/// Copyright (C) 2016 Kim Walisch, <kim.walisch@gmail.com>
///
/// This file is distributed under the BSD License. See the COPYING
/// file in the top level directory.
///

#include <S2LoadBalancer.hpp>
#include <primesum-internal.hpp>
#include <aligned_vector.hpp>
#include <imath.hpp>
#include <int128_t.hpp>

#include <stdint.h>
#include <algorithm>
#include <cmath>

using namespace std;
using namespace primesum;

namespace {

double get_avg(aligned_vector<double>& timings)
{
  size_t n = timings.size();
  double sum = 0;

  for (size_t i = 0; i < n; i++)
    sum += timings[i];

  return sum / n;
}

/// Relative standard deviation
double rel_std_dev(aligned_vector<double>& timings)
{
  size_t n = timings.size();
  double avg = get_avg(timings);
  double sum = 0;

  if (avg == 0)
    return 0;

  for (size_t i = 0; i < n; i++)
  {
    double mean = timings[i] - avg;
    sum += mean * mean;
  }

  double std_dev = sqrt(sum / max(1.0, n - 1.0));
  double rsd = 100 * std_dev / avg;

  return rsd;
}

} // namespace

namespace primesum {

S2LoadBalancer::S2LoadBalancer(int128_t x,
                               int64_t y,
                               int64_t z,
                               int64_t threads) :
  x_((double) x),
  y_((double) y),
  z_((double) z),
  rsd_(40),
  count_(0),
  total_seconds_(0),
  sqrtz_(isqrt(z))
{
  init(x, y, threads);
}

void S2LoadBalancer::init(int128_t x,
                          int64_t y,
                          int64_t threads)
{
  // determined by benchmarking
  double log_threads = max(1.0, log((double) threads));
  decrease_dividend_ = max(0.5, log_threads / 3);

  min_seconds_ = 0.01 * log_threads;
  double divisor = log(log(x_)) * log(x_);
  update_min_size(divisor);

  double alpha = get_alpha(x, y);
  smallest_hard_leaf_ = (int64_t) (x / (y * sqrt(alpha) * iroot<6>(x)));
}

double S2LoadBalancer::get_rsd() const
{
  return rsd_;
}

double S2LoadBalancer::get_avg_seconds() const
{
  return total_seconds_ / count_;
}

int64_t S2LoadBalancer::get_min_segment_size() const
{
  return min_size_;
}

/// Increase if relative std dev < pivot.
/// Decrease if relative std dev > pivot.
///
double S2LoadBalancer::get_pivot(double seconds) const
{
  seconds = max(min_seconds_, seconds);
  double log_seconds = log(seconds);
  log_seconds = max(min_seconds_, log_seconds);
  double dont_decrease = decrease_dividend_ / (seconds * log_seconds);
  dont_decrease = min(dont_decrease, rsd_);

  return rsd_ + dont_decrease;
}

bool S2LoadBalancer::is_increase(double seconds,
                                 double pivot) const
{
  return (seconds < min_seconds_ ||
          seconds < get_avg_seconds()) &&
         !is_decrease(seconds, pivot);
}

bool S2LoadBalancer::is_decrease(double seconds,
                                 double pivot) const
{
  return seconds > min_seconds_ &&
         rsd_ > pivot;
}

void S2LoadBalancer::update_min_size(double divisor)
{
  int64_t min_size = 1 << 9;
  int64_t size = (int64_t) (sqrtz_ / max(1.0, divisor));
  min_size_ = max(size, min_size);
  min_size_ = next_power_of_2(min_size_);
}

/// Balance the load in the computation of the special leaves
/// by dynamically adjusting the segment_size and segments_per_thread.
/// @param timings  Timings of the threads.
///
void S2LoadBalancer::update(int64_t low,
                            int64_t threads,
                            int64_t* segment_size,
                            int64_t* segments_per_thread,
                            aligned_vector<double>& timings)
{
  count_++;
  double seconds = get_avg(timings);
  total_seconds_ += seconds;
  double pivot = get_pivot(seconds);
  rsd_ = max(0.1, rel_std_dev(timings));

  // 1 segment per thread
  if (*segment_size < sqrtz_)
  {
    if (is_increase(seconds, pivot))
      *segment_size <<= 1;
    else if (is_decrease(seconds, pivot))
      if (*segment_size > min_size_)
        *segment_size >>= 1;
  }
  // many segments per thread
  else if (low > smallest_hard_leaf_)
    update(segments_per_thread, seconds, pivot);

  int64_t thread_distance = *segment_size * *segments_per_thread;
  int64_t high = low + thread_distance * threads;

  // near smallest_hard_leaf_ the hard special leaves
  // are distributed unevenly so use min_size_
  if (low <= smallest_hard_leaf_ &&
      high > smallest_hard_leaf_)
  {
    *segment_size = min_size_;
    thread_distance = *segment_size * *segments_per_thread;
    high = low + thread_distance * threads;
  }

  // slightly increase min_size_
  if (high >= smallest_hard_leaf_)
  {
    update_min_size(log(y_));
    *segment_size = max(min_size_, *segment_size);
  }
}

/// Increase the segments_per_thread if the relative standard
/// deviation of the thread run times is small, or
/// decrease the segments_per_thread if the relative standard
/// deviation of the thread run times is large.
///
void S2LoadBalancer::update(int64_t* segments_per_thread,
                            double seconds,
                            double pivot)
{
  if (is_increase(seconds, pivot) ||
      is_decrease(seconds, pivot))
  {
    if (seconds < min_seconds_)
      *segments_per_thread *= 2;
    else
    {
      double factor = pivot / rsd_;
      factor = in_between(0.5, factor, 2);
      double segments = *segments_per_thread * factor;
      segments = max(1.0, segments);
      *segments_per_thread = (int64_t) segments;
    }
  }
}

} // namespace
<|endoftext|>