<commit_before>
// Copyright 2010-2012 University of Washington. All Rights Reserved.
// LICENSE_PLACEHOLDER
// This software was created with Government support under DE
// AC05-76RL01830 awarded by the United States Department of
// Energy. The Government has certain rights in the software.
#include "Addressing.hpp"
#include "Communicator.hpp"
#include "Collective.hpp"
#include "Cache.hpp"
#include "GlobalCompletionEvent.hpp"
#include "ParallelLoop.hpp"

namespace Grappa {

/// Initialize an array of elements of generic type with a given value.
///
/// This version sends a large number of active messages, the same way as the Incoherent
/// releaser, to set each part of a global array. In theory, this version should be able
/// to be called from multiple locations at the same time (to initialize different regions of global memory).
///
/// @param base Base address of the array to be set.
/// @param value Value to set every element of array to (will be copied to all the nodes)
/// @param count Number of elements to set, starting at the base address.
template< typename T, typename S >
void memset(GlobalAddress<T> base, S value, size_t count) {
  call_on_all_cores([base,count,value]{
    T * local_base = base.localize();
    T * local_end = (base+count).localize();
    for (size_t i=0; i<local_end-local_base; i++) {
      local_base[i] = value;
    }
  });
}

namespace impl {
  template< typename T >
  void do_memcpy_locally(GlobalAddress<T> dst, GlobalAddress<T> src, size_t nelem) {
    typedef typename Incoherent<T>::WO Writeback;
    auto src_end = src+nelem;
    // T * local_base = src.localize(), * local_end = (src+nelem).localize();
    int64_t nfirstcore = src.block_max() - src;
    int64_t nlastcore = src_end - src_end.block_min();
    int64_t nmiddle = nelem - nfirstcore - nlastcore;

    const size_t nblock = block_size / sizeof(T);

    CHECK_EQ(nmiddle % nblock, 0);

    auto src_start = src;
    if (src.core() == mycore()) {
      int64_t nfirstcore = src.block_max() - src;
      Writeback w(dst, nfirstcore, src.pointer());
      src_start += nfirstcore;
    }
    if (src_end.core() == mycore()) {
      int64_t nlastcore = src_end - src_end.block_min();
      int64_t index = nelem - nlastcore;
      CHECK((src+index).core() == mycore());
      Writeback w(dst+index, nlastcore, (src+index).pointer());
      src_end -= nlastcore;
    }

    auto * local_base = src_start.localize();
    size_t nlocal_trimmed = src_end.localize() - local_base;
    CHECK_EQ((src_end-src_start) % nblock, 0);
    size_t nlocalblocks = nlocal_trimmed/nblock;
    Writeback * ws = locale_alloc<Writeback>(nlocalblocks);
    for (size_t i=0; i<nlocal_trimmed/nblock; i++) {
      size_t j = make_linear(local_base+(i*nblock))-src;
      new (ws+i) Writeback(dst+j, nblock, local_base+(i*nblock));
      ws[i].start_release();
    }
    for (size_t i=0; i<nlocalblocks; i++) { ws[i].block_until_released(); }
    locale_free(ws);

    // const size_t nlocal = local_end - local_base;
    // const size_t nlocalblocks = nlocal/nblock + ((nlocal%nblock == 0) ? 0 : 1);
    // Writeback ** putters = new Writeback*[nlocalblocks];
    // for (size_t i=0; i < nlocalblocks; i++) {
    //   size_t j = make_linear(local_base+(i*nblock))-src;
    //   size_t n = (i < nlocalblocks-1) ? nblock : (local_end-local_base)-(i*nblock);
    //
    //   // initialize WO cache to read from this block locally and write to corresponding block in dest
    //   putters[i] = new Writeback(dst+j, n, local_base+(i*nblock));
    //   putters[i]->start_release();
    // }
    // for (size_t i=0; i < nlocalblocks; i++) { delete putters[i]; }
    // delete [] putters;
  }
}

/// Memcpy over Grappa global arrays. Arguments `dst` and `src` must point into global arrays
/// (so must be linear addresses) and be non-overlapping, and both must have at least `nelem`
/// elements.
template< typename T >
void memcpy(GlobalAddress<T> dst, GlobalAddress<T> src, size_t nelem) {
  on_all_cores([dst,src,nelem]{
    impl::do_memcpy_locally(dst,src,nelem);
  });
}

/// Asynchronous version of memcpy, spawns only on cores with array elements. Synchronizes
/// with given GlobalCompletionEvent, so memcpy's are known to be complete after GCE->wait().
/// Note: same restrictions on `dst` and `src` as Grappa::memcpy).
template< GlobalCompletionEvent * GCE = &impl::local_gce, typename T = void >
void memcpy_async(GlobalAddress<T> dst, GlobalAddress<T> src, size_t nelem) {
  on_cores_localized_async<GCE>(src, nelem, [dst,src,nelem](T* base, size_t nlocal){
    impl::do_memcpy_locally(dst,src,nelem);
  });
}

/// not implemented yet
template< typename T >
void prefix_sum(GlobalAddress<T> array, size_t nelem) {
  // not implemented
}

namespace util {

  template< int VERBOSITY = 1, typename T = decltype(nullptr)>
  inline void print_array(const char * name, T * base, size_t nelem, int width = 10) {
    std::stringstream ss; ss << "\n" << name << ": [";
    for (size_t i=0; i<nelem; i++) {
      if (i % width == 0) ss << "\n  ";
      ss << " " << base[i];
    }
    ss << "\n]"; VLOG(VERBOSITY) << ss.str();
  }

  template< int VERBOSITY = 1, typename T = decltype(nullptr) >
  inline void print_array(const char * name, GlobalAddress<T> base, size_t nelem, int width = 10) {
    std::stringstream ss; ss << "\n" << name << ": [";
    for (size_t i=0; i<nelem; i++) {
      if (i % width == 0) ss << "\n  ";
      ss << " " << delegate::read(base+i);
    }
    ss << "\n]"; VLOG(VERBOSITY) << ss.str();
  }
}

} // namespace Grappa

/// Legacy: @see { Grappa::memset() }
template< typename T , typename S >
void Grappa_memset(GlobalAddress<T> request_address, S value, size_t count) {
  Grappa::memset(request_address, value, count);
}

/// Legacy: @see { Grappa::memset() }
template< typename T, typename S >
void Grappa_memset_local(GlobalAddress<T> base, S value, size_t count) {
  Grappa::memset(base, value, count);
}

template< typename T >
void Grappa_memcpy(GlobalAddress<T> dst, GlobalAddress<T> src, size_t nelem) {
  Grappa::memcpy(dst,src,nelem);
}
<commit_msg>Array: add helper: util::print_array<commit_after>
// Copyright 2010-2012 University of Washington. All Rights Reserved.
// LICENSE_PLACEHOLDER
// This software was created with Government support under DE
// AC05-76RL01830 awarded by the United States Department of
// Energy. The Government has certain rights in the software.
#include "Addressing.hpp"
#include "Communicator.hpp"
#include "Collective.hpp"
#include "Cache.hpp"
#include "GlobalCompletionEvent.hpp"
#include "ParallelLoop.hpp"

namespace Grappa {

/// Initialize an array of elements of generic type with a given value.
///
/// This version sends a large number of active messages, the same way as the Incoherent
/// releaser, to set each part of a global array. In theory, this version should be able
/// to be called from multiple locations at the same time (to initialize different regions of global memory).
///
/// @param base Base address of the array to be set.
/// @param value Value to set every element of array to (will be copied to all the nodes)
/// @param count Number of elements to set, starting at the base address.
template< typename T, typename S >
void memset(GlobalAddress<T> base, S value, size_t count) {
  call_on_all_cores([base,count,value]{
    T * local_base = base.localize();
    T * local_end = (base+count).localize();
    for (size_t i=0; i<local_end-local_base; i++) {
      local_base[i] = value;
    }
  });
}

namespace impl {
  template< typename T >
  void do_memcpy_locally(GlobalAddress<T> dst, GlobalAddress<T> src, size_t nelem) {
    typedef typename Incoherent<T>::WO Writeback;
    auto src_end = src+nelem;
    // T * local_base = src.localize(), * local_end = (src+nelem).localize();
    int64_t nfirstcore = src.block_max() - src;
    int64_t nlastcore = src_end - src_end.block_min();
    int64_t nmiddle = nelem - nfirstcore - nlastcore;

    const size_t nblock = block_size / sizeof(T);

    CHECK_EQ(nmiddle % nblock, 0);

    auto src_start = src;
    if (src.core() == mycore()) {
      int64_t nfirstcore = src.block_max() - src;
      Writeback w(dst, nfirstcore, src.pointer());
      src_start += nfirstcore;
    }
    if (src_end.core() == mycore()) {
      int64_t nlastcore = src_end - src_end.block_min();
      int64_t index = nelem - nlastcore;
      CHECK((src+index).core() == mycore());
      Writeback w(dst+index, nlastcore, (src+index).pointer());
      src_end -= nlastcore;
    }

    auto * local_base = src_start.localize();
    size_t nlocal_trimmed = src_end.localize() - local_base;
    CHECK_EQ((src_end-src_start) % nblock, 0);
    size_t nlocalblocks = nlocal_trimmed/nblock;
    Writeback * ws = locale_alloc<Writeback>(nlocalblocks);
    for (size_t i=0; i<nlocal_trimmed/nblock; i++) {
      size_t j = make_linear(local_base+(i*nblock))-src;
      new (ws+i) Writeback(dst+j, nblock, local_base+(i*nblock));
      ws[i].start_release();
    }
    for (size_t i=0; i<nlocalblocks; i++) { ws[i].block_until_released(); }
    locale_free(ws);

    // const size_t nlocal = local_end - local_base;
    // const size_t nlocalblocks = nlocal/nblock + ((nlocal%nblock == 0) ? 0 : 1);
    // Writeback ** putters = new Writeback*[nlocalblocks];
    // for (size_t i=0; i < nlocalblocks; i++) {
    //   size_t j = make_linear(local_base+(i*nblock))-src;
    //   size_t n = (i < nlocalblocks-1) ? nblock : (local_end-local_base)-(i*nblock);
    //
    //   // initialize WO cache to read from this block locally and write to corresponding block in dest
    //   putters[i] = new Writeback(dst+j, n, local_base+(i*nblock));
    //   putters[i]->start_release();
    // }
    // for (size_t i=0; i < nlocalblocks; i++) { delete putters[i]; }
    // delete [] putters;
  }
}

/// Memcpy over Grappa global arrays. Arguments `dst` and `src` must point into global arrays
/// (so must be linear addresses) and be non-overlapping, and both must have at least `nelem`
/// elements.
template< typename T >
void memcpy(GlobalAddress<T> dst, GlobalAddress<T> src, size_t nelem) {
  on_all_cores([dst,src,nelem]{
    impl::do_memcpy_locally(dst,src,nelem);
  });
}

/// Asynchronous version of memcpy, spawns only on cores with array elements. Synchronizes
/// with given GlobalCompletionEvent, so memcpy's are known to be complete after GCE->wait().
/// Note: same restrictions on `dst` and `src` as Grappa::memcpy).
template< GlobalCompletionEvent * GCE = &impl::local_gce, typename T = void >
void memcpy_async(GlobalAddress<T> dst, GlobalAddress<T> src, size_t nelem) {
  on_cores_localized_async<GCE>(src, nelem, [dst,src,nelem](T* base, size_t nlocal){
    impl::do_memcpy_locally(dst,src,nelem);
  });
}

/// not implemented yet
template< typename T >
void prefix_sum(GlobalAddress<T> array, size_t nelem) {
  // not implemented
}

namespace util {

  template< int VERBOSITY = 1, typename T = decltype(nullptr)>
  inline void print_array(const char * name, T * base, size_t nelem, int width = 10) {
    std::stringstream ss; ss << "\n" << name << ": [";
    for (size_t i=0; i<nelem; i++) {
      if (i % width == 0) ss << "\n  ";
      ss << " " << base[i];
    }
    ss << "\n]"; VLOG(VERBOSITY) << ss.str();
  }

  template< int VERBOSITY = 1, typename T = decltype(nullptr) >
  inline void print_array(const char * name, GlobalAddress<T> base, size_t nelem, int width = 10) {
    std::stringstream ss; ss << "\n" << name << ": [";
    for (size_t i=0; i<nelem; i++) {
      if (i % width == 0) ss << "\n  ";
      ss << " " << delegate::read(base+i);
    }
    ss << "\n]"; VLOG(VERBOSITY) << ss.str();
  }

}

} // namespace Grappa

/// Legacy: @see { Grappa::memset() }
template< typename T , typename S >
void Grappa_memset(GlobalAddress<T> request_address, S value, size_t count) {
  Grappa::memset(request_address, value, count);
}

/// Legacy: @see { Grappa::memset() }
template< typename T, typename S >
void Grappa_memset_local(GlobalAddress<T> base, S value, size_t count) {
  Grappa::memset(base, value, count);
}

template< typename T >
void Grappa_memcpy(GlobalAddress<T> dst, GlobalAddress<T> src, size_t nelem) {
  Grappa::memcpy(dst,src,nelem);
}
<|endoftext|>


<commit_before>/**
 * @file rodsdownloadthread.h
 * @brief Implementation of class RodsDownloadThread
 *
 * The RodsDownloadThread class extends the Qt thread management class
 * QThread and implements a worker thread for a download (get) operation.
 *
 * Copyright (C) 2014-2015 University of Jyväskylä. All rights reserved.
 * License: The BSD 3-Clause License, see LICENSE file for details.
 *
 * @author Ilari Korhonen
 */

// application class RodsDownloadThread header
#include "rodsdownloadthread.h"

RodsDownloadThread::RodsDownloadThread(Kanki::RodsConnection *theConn, Kanki::RodsObjEntryPtr theObj, const std::string &theDestPath,
                                       bool verifyChecksum, bool allowOverwrite)
    : QThread()
{
    this->conn = new Kanki::RodsConnection(theConn);
    this->objEntry = theObj;
    this->destPath = theDestPath;

    this->verify = verifyChecksum;
    this->overwrite = allowOverwrite;
}

void RodsDownloadThread::run() Q_DECL_OVERRIDE
{
    int status = 0;
    QString statusStr = "Initializing...";

    // signal ui to setup progress display
    progressMarquee(statusStr);

    // open the parallel connection for transfer and authenticate
    if ((status = this->conn->connect()) < 0)
    {
        reportError("Download failed", "Open parallel connection failed", status);
        return;
    }

    else if ((status = this->conn->login()) < 0)
    {
        reportError("Download failed", "Authentication failed", status);
        return;
    }

    // in the case of downloading a collection, do it recursively
    if (this->objEntry->objType == COLL_OBJ_T)
    {
        std::vector<Kanki::RodsObjEntryPtr> collObjs;
        std::string basePath = this->objEntry->collPath;

        // first item to the download object list
        collObjs.push_back(this->objEntry);

        // try to construct the download object list recursively
        if (!(status = makeCollObjList(this->objEntry, &collObjs)))
        {
            // notify ui of progress bar state (object count)
            setupProgressDisplay(statusStr, 0, collObjs.size());

            // iterate thru object list
            for (unsigned int i = 0; i < collObjs.size(); i++)
            {
                Kanki::RodsObjEntryPtr curObj = collObjs.at(i);

                std::string basePath = this->objEntry->getObjectBasePath();
                std::string objPath = curObj->getObjectFullPath();

                objPath.erase(objPath.begin(), objPath.begin() + basePath.size());
                std::string dstPath = this->destPath + objPath;

                // in the case of a data object, we do a get operation
                if (curObj->objType == DATA_OBJ_T)
                {
                    // notify ui of current operation and progress
                    statusStr = "Downloading ";
                    statusStr += curObj->getObjectName().c_str();
                    progressUpdate(statusStr, i+1);

                    // try to do a rods get operation
                    if ((status = this->downloadFile(curObj, dstPath, this->verify, this->overwrite)) < 0)
                        reportError("iRODS get file error", "Get failed", status);
                }

                // for collection objects we create the corresponding directory
                else if (curObj->objType == COLL_OBJ_T)
                {
                    // get directory name for ui
                    std::string dirName = dstPath.substr(dstPath.find_last_of('/') + 1);

                    // notify ui
                    statusStr = "Creating directory ";
                    statusStr += dirName.c_str();
                    progressUpdate(statusStr, i+1);

                    // check if directory exists and if not, make it
                    QDir dstDir(dstPath.c_str());

                    if (!dstDir.exists())
                        dstDir.mkpath(dstPath.c_str());
                }
            }
        }
    }

    // in the case of downloading a single data object, a simple get operation
    else if (this->objEntry->objType == DATA_OBJ_T)
    {
        QString statusStr = "Downloading file: ";

        statusStr += this->objEntry->getObjectName().c_str();
        std::string dstPath = this->destPath + "/" + this->objEntry->getObjectName();
        setupProgressDisplay(statusStr, 1, 1);

        // try to do a rods get operation
        if ((status = this->downloadFile(objEntry, dstPath, this->verify, this->overwrite)) < 0)
            reportError("Download failed", "Kanki data stream error", status);
    }

    this->conn->disconnect();
    delete(this->conn);
}

int RodsDownloadThread::makeCollObjList(Kanki::RodsObjEntryPtr obj, std::vector<Kanki::RodsObjEntryPtr> *objs)
{
    int status = 0;

    // we proceed only for collections
    if (obj->objType == COLL_OBJ_T)
    {
        std::vector<Kanki::RodsObjEntryPtr> curCollObjs;

        // try to read collection
        if ((status = this->conn->readColl(obj->collPath, &curCollObjs)) >= 0)
        {
            // iterate thru current collection
            for (std::vector<Kanki::RodsObjEntryPtr>::iterator i = curCollObjs.begin(); i != curCollObjs.end(); i++)
            {
                // add object to list
                Kanki::RodsObjEntryPtr curObj = *i;
                objs->push_back(curObj);

                // notify ui
                QString statusStr = "Building a list of objects (" + QVariant((int)objs->size()).toString() + ")...";
                progressMarquee(statusStr);

                // recurse on collection objects
                if (curObj->objType == COLL_OBJ_T)
                {
                    status = makeCollObjList(curObj, objs);

                    // on error, back off recursion
                    if (status < 0)
                        return (status);
                }
            }
        }
    }

    return (status);
}

int RodsDownloadThread::downloadFile(Kanki::RodsObjEntryPtr obj, std::string localPath,
                                     bool verifyChecksum, bool allowOverwrite)
{
    Kanki::RodsDataInStream inStream(this->conn, obj);
    long int status = 0, lastRead = 0, lastWrite = 0, totalRead = 0, totalWritten = 0;
    QFile localFile(localPath.c_str());
    long int readSize = 33554432;
    void *buffer = std::malloc(readSize);

    // check if we're allowed to proceed
    if (localFile.exists() && !allowOverwrite)
        return (OVERWRITE_WITHOUT_FORCE_FLAG);

    // try to open local file and the rods data stream
    if (!localFile.open(QIODevice::WriteOnly))
        return (-1);

    // try to initiate get operation and open data stream
    if ((status = inStream.initGetOpr()) < 0)
        return (status);

    if ((status = inStream.openDataObj()) < 0)
        return (status);

    else {
        // update status display only on large enough objects
        if (obj->objSize > readSize)
            setupSubProgressDisplay("Transferring...", 0, 100);

        std::chrono::high_resolution_clock::time_point t0 = std::chrono::high_resolution_clock::now();

        while ((lastRead = inStream.read(buffer, readSize)) > 0)
        {
            std::chrono::high_resolution_clock::time_point t1 = std::chrono::high_resolution_clock::now();
            std::chrono::milliseconds diff = std::chrono::duration_cast<std::chrono::milliseconds>(t1 - t0);

            totalRead += lastRead;

            if ((lastWrite = localFile.write((const char*)buffer, lastRead)) > 0)
            {
                totalWritten += lastWrite;

                double speed = ((double)totalRead / 1048576) / ((double)diff.count() / 1000);
                double percentage = ceil(((double)totalRead / (double)obj->objSize) * 100);

                QString statusStr = "Transferring... " + QVariant((int)percentage).toString() + "%";
                statusStr += " at " + QVariant(ceil(speed)).toString() + " MB/s";

                if (obj->objSize > readSize)
                    subProgressUpdate(statusStr, (int)percentage);
            }

            else {
                status = -1;
                break;
            }
        }
    }

    // close local file and rods data stream
    localFile.close();
    status = inStream.closeDataObj();
    inStream.getOprDone();

    // if verify checksum was required
    if (verifyChecksum && strlen(inStream.checksum()))
    {
        subProgressUpdate("Verifying Checksum...", 100);
        status = verifyChksumLocFile((char*)localPath.c_str(), (char*)inStream.checksum(), NULL);
    }

    std::free(buffer);

    return (status);
}
<commit_msg>do download with the adaptive read request size<commit_after>/**
 * @file rodsdownloadthread.h
 * @brief Implementation of class RodsDownloadThread
 *
 * The RodsDownloadThread class extends the Qt thread management class
 * QThread and implements a worker thread for a download (get) operation.
 *
 * Copyright (C) 2014-2015 University of Jyväskylä. All rights reserved.
 * License: The BSD 3-Clause License, see LICENSE file for details.
 *
 * @author Ilari Korhonen
 */

// application class RodsDownloadThread header
#include "rodsdownloadthread.h"

RodsDownloadThread::RodsDownloadThread(Kanki::RodsConnection *theConn, Kanki::RodsObjEntryPtr theObj, const std::string &theDestPath,
                                       bool verifyChecksum, bool allowOverwrite)
    : QThread()
{
    this->conn = new Kanki::RodsConnection(theConn);
    this->objEntry = theObj;
    this->destPath = theDestPath;

    this->verify = verifyChecksum;
    this->overwrite = allowOverwrite;
}

void RodsDownloadThread::run() Q_DECL_OVERRIDE
{
    int status = 0;
    QString statusStr = "Initializing...";

    // signal ui to setup progress display
    progressMarquee(statusStr);

    // open the parallel connection for transfer and authenticate
    if ((status = this->conn->connect()) < 0)
    {
        reportError("Download failed", "Open parallel connection failed", status);
        return;
    }

    else if ((status = this->conn->login()) < 0)
    {
        reportError("Download failed", "Authentication failed", status);
        return;
    }

    // in the case of downloading a collection, do it recursively
    if (this->objEntry->objType == COLL_OBJ_T)
    {
        std::vector<Kanki::RodsObjEntryPtr> collObjs;
        std::string basePath = this->objEntry->collPath;

        // first item to the download object list
        collObjs.push_back(this->objEntry);

        // try to construct the download object list recursively
        if (!(status = makeCollObjList(this->objEntry, &collObjs)))
        {
            // notify ui of progress bar state (object count)
            setupProgressDisplay(statusStr, 0, collObjs.size());

            // iterate thru object list
            for (unsigned int i = 0; i < collObjs.size(); i++)
            {
                Kanki::RodsObjEntryPtr curObj = collObjs.at(i);

                std::string basePath = this->objEntry->getObjectBasePath();
                std::string objPath = curObj->getObjectFullPath();

                objPath.erase(objPath.begin(), objPath.begin() + basePath.size());
                std::string dstPath = this->destPath + objPath;

                // in the case of a data object, we do a get operation
                if (curObj->objType == DATA_OBJ_T)
                {
                    // notify ui of current operation and progress
                    statusStr = "Downloading ";
                    statusStr += curObj->getObjectName().c_str();
                    progressUpdate(statusStr, i+1);

                    // try to do a rods get operation
                    if ((status = this->downloadFile(curObj, dstPath, this->verify, this->overwrite)) < 0)
                        reportError("iRODS get file error", "Get failed", status);
                }

                // for collection objects we create the corresponding directory
                else if (curObj->objType == COLL_OBJ_T)
                {
                    // get directory name for ui
                    std::string dirName = dstPath.substr(dstPath.find_last_of('/') + 1);

                    // notify ui
                    statusStr = "Creating directory ";
                    statusStr += dirName.c_str();
                    progressUpdate(statusStr, i+1);

                    // check if directory exists and if not, make it
                    QDir dstDir(dstPath.c_str());

                    if (!dstDir.exists())
                        dstDir.mkpath(dstPath.c_str());
                }
            }
        }
    }

    // in the case of downloading a single data object, a simple get operation
    else if (this->objEntry->objType == DATA_OBJ_T)
    {
        QString statusStr = "Downloading file: ";

        statusStr += this->objEntry->getObjectName().c_str();
        std::string dstPath = this->destPath + "/" + this->objEntry->getObjectName();
        setupProgressDisplay(statusStr, 1, 1);

        // try to do a rods get operation
        if ((status = this->downloadFile(objEntry, dstPath, this->verify, this->overwrite)) < 0)
            reportError("Download failed", "Kanki data stream error", status);
    }

    this->conn->disconnect();
    delete(this->conn);
}

int RodsDownloadThread::makeCollObjList(Kanki::RodsObjEntryPtr obj, std::vector<Kanki::RodsObjEntryPtr> *objs)
{
    int status = 0;

    // we proceed only for collections
    if (obj->objType == COLL_OBJ_T)
    {
        std::vector<Kanki::RodsObjEntryPtr> curCollObjs;

        // try to read collection
        if ((status = this->conn->readColl(obj->collPath, &curCollObjs)) >= 0)
        {
            // iterate thru current collection
            for (std::vector<Kanki::RodsObjEntryPtr>::iterator i = curCollObjs.begin(); i != curCollObjs.end(); i++)
            {
                // add object to list
                Kanki::RodsObjEntryPtr curObj = *i;
                objs->push_back(curObj);

                // notify ui
                QString statusStr = "Building a list of objects (" + QVariant((int)objs->size()).toString() + ")...";
                progressMarquee(statusStr);

                // recurse on collection objects
                if (curObj->objType == COLL_OBJ_T)
                {
                    status = makeCollObjList(curObj, objs);

                    // on error, back off recursion
                    if (status < 0)
                        return (status);
                }
            }
        }
    }

    return (status);
}

int RodsDownloadThread::downloadFile(Kanki::RodsObjEntryPtr obj, std::string localPath,
                                     bool verifyChecksum, bool allowOverwrite)
{
    Kanki::RodsDataInStream inStream(this->conn, obj);
    long int status = 0, lastRead = 0, lastWrite = 0, totalRead = 0, totalWritten = 0;
    QFile localFile(localPath.c_str());
    long int readSize = 33554432;
    void *buffer = std::malloc(readSize);

    // check if we're allowed to proceed
    if (localFile.exists() && !allowOverwrite)
        return (OVERWRITE_WITHOUT_FORCE_FLAG);

    // try to open local file and the rods data stream
    if (!localFile.open(QIODevice::WriteOnly))
        return (-1);

    // try to initiate get operation and open data stream
    if ((status = inStream.initGetOpr()) < 0)
        return (status);

    if ((status = inStream.openDataObj()) < 0)
        return (status);

    else {
        // update status display only on large enough objects
        if (obj->objSize > readSize)
            setupSubProgressDisplay("Transferring...", 0, 100);

        std::chrono::high_resolution_clock::time_point t0 = std::chrono::high_resolution_clock::now();

        while ((lastRead = inStream.readAdaptive(buffer, readSize)) > 0)
        {
            std::chrono::high_resolution_clock::time_point t1 = std::chrono::high_resolution_clock::now();
            std::chrono::milliseconds diff = std::chrono::duration_cast<std::chrono::milliseconds>(t1 - t0);

            totalRead += lastRead;

            if ((lastWrite = localFile.write((const char*)buffer, lastRead)) > 0)
            {
                totalWritten += lastWrite;

                double speed = ((double)totalRead / 1048576) / ((double)diff.count() / 1000);
                double percentage = ceil(((double)totalRead / (double)obj->objSize) * 100);

                QString statusStr = "Transferring... " + QVariant((int)percentage).toString() + "%";
                statusStr += " at " + QVariant(ceil(speed)).toString() + " MB/s";

                if (obj->objSize > readSize)
                    subProgressUpdate(statusStr, (int)percentage);
            }

            else {
                status = -1;
                break;
            }
        }
    }

    // close local file and rods data stream
    localFile.close();
    status = inStream.closeDataObj();
    inStream.getOprDone();

    // if verify checksum was required
    if (verifyChecksum && strlen(inStream.checksum()))
    {
        subProgressUpdate("Verifying Checksum...", 100);
        status = verifyChksumLocFile((char*)localPath.c_str(), (char*)inStream.checksum(), NULL);
    }

    std::free(buffer);

    return (status);
}
<|endoftext|>


<commit_before>#include "ROOT/TDataFrame.hxx"
#include "TRandom.h"
#include "TROOT.h"
#include "gtest/gtest.h"
#include <limits>
using namespace ROOT::Experimental;
using namespace ROOT::Experimental::TDF;
using namespace ROOT::Detail::TDF;

/********* FIXTURES *********/
static constexpr ULong64_t gNEvents = 8ull;

// fixture that provides a TDF with no data-source and a single column "x" containing normal-distributed doubles
class TDFCallbacks : public ::testing::Test {
private:
   TDataFrame fLoopManager;
   TInterface<TLoopManager> DefineRandomCol()
   {
      TRandom r;
      return fLoopManager.Define("x", [r]() mutable { return r.Gaus(); });
   }

protected:
   TDFCallbacks() : fLoopManager(gNEvents), tdf(DefineRandomCol()) {}
   TInterface<TLoopManager> tdf;
};

#ifdef R__USE_IMT
static constexpr unsigned int gNSlots = 4u;

// fixture that enables implicit MT and provides a TDF with no data-source and a single column "x" containing
// normal-distributed doubles
class TDFCallbacksMT : public ::testing::Test {
   class TIMTEnabler {
   public:
      TIMTEnabler(unsigned int sl) { ROOT::EnableImplicitMT(sl); }
      ~TIMTEnabler() { ROOT::DisableImplicitMT(); }
   };

private:
   TIMTEnabler fIMTEnabler;
   TDataFrame fLoopManager;
   TInterface<TLoopManager> DefineRandomCol()
   {
      std::vector<TRandom> rs(gNSlots);
      return fLoopManager.DefineSlot("x", [rs](unsigned int slot) mutable { return rs[slot].Gaus(); });
   }

protected:
   TDFCallbacksMT() : fIMTEnabler(gNSlots), fLoopManager(gNEvents), tdf(DefineRandomCol()) {}
   TInterface<TLoopManager> tdf;
};
#endif

/********* TESTS *********/
TEST_F(TDFCallbacks, Histo1DWithFillTOHelper)
{
   // Histo1D<double> + OnPartialResult + FillTOHelper
   auto h = tdf.Histo1D<double>({"", "", 128, -2., 2.}, "x");
   using value_t = typename decltype(h)::Value_t;
   ULong64_t i = 0ull;
   ULong64_t everyN = 1ull;
   h.OnPartialResult(everyN, [&i, &everyN](value_t &h_) {
      i += everyN;
      EXPECT_EQ(h_.GetEntries(), i);
   });
   *h;
   EXPECT_EQ(gNEvents, i);
}

TEST_F(TDFCallbacks, JittedHisto1DWithFillTOHelper)
{
   // Histo1D + Jitting + OnPartialResult + FillTOHelper
   auto h = tdf.Histo1D({"", "", 128, -2., 2.}, "x");
   using value_t = typename decltype(h)::Value_t;
   ULong64_t i = 0ull;
   ULong64_t everyN = 1ull;
   h.OnPartialResult(everyN, [&i, &everyN](value_t &h_) {
      i += everyN;
      EXPECT_EQ(h_.GetEntries(), i);
   });
   *h;
   EXPECT_EQ(gNEvents, i);
}

TEST_F(TDFCallbacks, Histo1DWithFillHelper)
{
   // Histo1D<double> + OnPartialResult + FillHelper
   auto h = tdf.Histo1D<double>("x");
   using value_t = typename decltype(h)::Value_t;
   ULong64_t i = 0ull;
   ULong64_t everyN = 1ull;
   h.OnPartialResult(everyN, [&i, &everyN](value_t &h_) {
      i += everyN;
      EXPECT_EQ(h_.GetEntries(), i);
   });
   *h;
   EXPECT_EQ(gNEvents, i);
}

TEST_F(TDFCallbacks, JittedHisto1DWithFillHelper)
{
   // Histo1D + Jitting + OnPartialResult + FillHelper
   auto h = tdf.Histo1D("x");
   using value_t = typename decltype(h)::Value_t;
   ULong64_t i = 0ull;
   ULong64_t everyN = 1ull;
   h.OnPartialResult(everyN, [&i, &everyN](value_t &h_) {
      i += everyN;
      EXPECT_EQ(h_.GetEntries(), i);
   });
   *h;
   EXPECT_EQ(gNEvents, i);
}

TEST_F(TDFCallbacks, Min)
{
   // Min + OnPartialResult
   auto m = tdf.Min<double>("x");
   double runningMin = std::numeric_limits<double>::max();
   m.OnPartialResult(2, [&runningMin](double x) {
      EXPECT_LE(x, runningMin);
      runningMin = x;
   });
   *m;
   EXPECT_DOUBLE_EQ(runningMin, *m);
}

TEST_F(TDFCallbacks, JittedMin)
{
   // Min + Jitting + OnPartialResult
   auto m = tdf.Min("x");
   double runningMin = std::numeric_limits<double>::max();
   m.OnPartialResult(2, [&runningMin](double x) {
      EXPECT_LE(x, runningMin);
      runningMin = x;
   });
   *m;
   EXPECT_DOUBLE_EQ(runningMin, *m);
}

TEST_F(TDFCallbacks, Max)
{
   // Max + OnPartialResult
   auto m = tdf.Max<double>("x");
   double runningMax = std::numeric_limits<double>::lowest();
   m.OnPartialResult(2, [&runningMax](double x) {
      EXPECT_GE(x, runningMax);
      runningMax = x;
   });
   *m;
   EXPECT_DOUBLE_EQ(runningMax, *m);
}

TEST_F(TDFCallbacks, JittedMax)
{
   // Max + Jitting + OnPartialResult
   auto m = tdf.Max("x");
   double runningMax = std::numeric_limits<double>::lowest();
   m.OnPartialResult(2, [&runningMax](double x) {
      EXPECT_GE(x, runningMax);
      runningMax = x;
   });
   *m;
   EXPECT_DOUBLE_EQ(runningMax, *m);
}

TEST_F(TDFCallbacks, Mean)
{
   // Mean + OnPartialResult
   auto m = tdf.Mean<double>("x");
   // TODO find a better way to check that the running mean makes sense
   bool called = false;
   m.OnPartialResult(gNEvents / 2, [&called](double) { called = true; });
   *m;
   EXPECT_TRUE(called);
}

TEST_F(TDFCallbacks, JittedMean)
{
   // Mean + Jitting + OnPartialResult
   auto m = tdf.Mean("x");
   // TODO find a better way to check that the running mean makes sense
   bool called = false;
   m.OnPartialResult(gNEvents / 2, [&called](double) { called = true; });
   *m;
   EXPECT_TRUE(called);
}

TEST_F(TDFCallbacks, Take)
{
   // Take + OnPartialResult
   auto t = tdf.Take<double>("x");
   unsigned int i = 0u;
   t.OnPartialResult(1, [&](decltype(t)::Value_t &t_) {
      ++i;
      EXPECT_EQ(t_.size(), i);
   });
   *t;
}

TEST_F(TDFCallbacks, Count)
{
   // Count + OnPartialResult
   auto c = tdf.Count();
   ULong64_t i = 0ull;
   c.OnPartialResult(1, [&](decltype(c)::Value_t c_) {
      ++i;
      EXPECT_EQ(c_, i);
   });
   *c;
   EXPECT_EQ(*c, i);
}

TEST_F(TDFCallbacks, Reduce)
{
   // Reduce + OnPartialResult
   double runningMin;
   auto m = tdf.Min<double>("x").OnPartialResult(1, [&runningMin](double m_) { runningMin = m_; });
   auto r = tdf.Reduce([](double x1, double x2) { return std::min(x1, x2); }, {"x"}, 0.);
   r.OnPartialResult(1, [&runningMin](double r_) { EXPECT_DOUBLE_EQ(r_, runningMin); });
   *r;
}

TEST_F(TDFCallbacks, Chaining)
{
   // Chaining of multiple OnPartialResult[Slot] calls
   unsigned int i = 0u;
   auto c = tdf.Count()
               .OnPartialResult(1, [&i](ULong64_t) { ++i; })
               .OnPartialResultSlot(1, [&i](unsigned int, ULong64_t) {++i; });
   *c;
   EXPECT_EQ(i, gNEvents * 2);
}

TEST_F(TDFCallbacks, OrderOfExecution)
{
   // Test that callbacks are executed in the order they are registered
   unsigned int i = 0u;
   auto c = tdf.Count();
   c.OnPartialResult(1, [&i](ULong64_t) {
      EXPECT_EQ(i, 0u);
      i = 42u;
   });
   c.OnPartialResultSlot(1, [&i](unsigned int slot, ULong64_t) {
      if (slot == 0u) {
         EXPECT_EQ(i, 42u);
         i = 0u;
      }
   });
   *c;
   EXPECT_EQ(i, 0u);
}

TEST_F(TDFCallbacks, ExecuteOnce)
{
   // OnPartialResult(kOnce)
   auto c = tdf.Count();
   unsigned int callCount = 0;
   c.OnPartialResult(0, [&callCount](ULong64_t) { callCount++; });
   *c;
   EXPECT_EQ(callCount, 1u);
}

TEST_F(TDFCallbacks, MultipleCallbacks)
{
   // registration of multiple callbacks on the same partial result
   auto h = tdf.Histo1D<double>({"", "", 128, -2., 2.}, "x");
   using value_t = typename decltype(h)::Value_t;
   ULong64_t everyN = 1ull;
   ULong64_t i = 0ull;
   h.OnPartialResult(everyN, [&i, everyN](value_t &h_) {
      i += everyN;
      EXPECT_EQ(h_.GetEntries(), i);
   });

   everyN = 2ull;
   ULong64_t i2 = 0ull;
   h.OnPartialResult(everyN, [&i2, everyN](value_t &h_) {
      i2 += everyN;
      EXPECT_EQ(h_.GetEntries(), i2);
   });

   *h;
}

TEST_F(TDFCallbacks, MultipleEventLoops)
{
   // callbacks must be de-registered after the event-loop is run
   auto h = tdf.Histo1D<double>({"", "", 128, -2., 2.}, "x");
   using value_t = typename decltype(h)::Value_t;
   ULong64_t i = 0ull;
   h.OnPartialResult(1ull, [&i](value_t &) { ++i; });
   *h;

   auto h2 = tdf.Histo1D<double>({"", "", 128, -2., 2.}, "x");
   *h2;

   EXPECT_EQ(i, gNEvents);
}

class FunctorClass {
   unsigned int &i_;

public:
   FunctorClass(unsigned int &i) : i_(i) {}
   void operator()(ULong64_t) { ++i_; }
};

TEST_F(TDFCallbacks, FunctorClass)
{
   unsigned int i = 0;
   *(tdf.Count().OnPartialResult(1, FunctorClass(i)));
   EXPECT_EQ(i, gNEvents);
}

unsigned int freeFunctionCounter = 0;
void FreeFunction(ULong64_t)
{
   freeFunctionCounter++;
}

TEST_F(TDFCallbacks, FreeFunction)
{
   *(tdf.Count().OnPartialResult(1, FreeFunction));
   EXPECT_EQ(freeFunctionCounter, gNEvents);
}

#ifdef R__USE_IMT
/******** Multi-thread tests **********/
TEST_F(TDFCallbacksMT, ExecuteOncePerSlot)
{
   // OnPartialResultSlot(kOnce)
   auto c = tdf.Count();
   std::atomic_uint callCount(0u);
   c.OnPartialResultSlot(c.kOnce, [&callCount](unsigned int, ULong64_t) { callCount++; });
   *c;
   EXPECT_EQ(callCount, gNSlots * 2); // TDF spawns two tasks per slot
}

TEST_F(TDFCallbacksMT, Histo1DWithFillTOHelper)
{
   // Histo1D<double> + OnPartialResultSlot + FillTOHelper
   auto h = tdf.Histo1D<double>({"", "", 128, -2., 2.}, "x");
   using value_t = typename decltype(h)::Value_t;
   std::array<ULong64_t, gNSlots> is;
   is.fill(0ull);
   constexpr ULong64_t everyN = 1ull;
   h.OnPartialResultSlot(everyN, [&is](unsigned int slot, value_t &h_) {
      is[slot] += everyN;
      EXPECT_EQ(h_.GetEntries(), is[slot]);
   });
   *h;
   EXPECT_EQ(gNEvents, std::accumulate(is.begin(), is.end(), 0ull));
}

TEST_F(TDFCallbacksMT, Histo1DWithFillHelper)
{
   // Histo1D<double> + OnPartialResultSlot + FillHelper
   auto h = tdf.Histo1D<double>("x");
   using value_t = typename decltype(h)::Value_t;
   std::array<ULong64_t, gNSlots> is;
   is.fill(0ull);
   constexpr ULong64_t everyN = 1ull;
   h.OnPartialResultSlot(everyN, [&is](unsigned int slot, value_t &h_) {
      is[slot] += everyN;
      EXPECT_EQ(h_.GetEntries(), is[slot]);
   });
   *h;
   EXPECT_EQ(gNEvents, std::accumulate(is.begin(), is.end(), 0ull));
}
#endif
<commit_msg>[TDF] Improve tests for multi-thread one-time execution of callbacks<commit_after>#include "ROOT/TDataFrame.hxx"
#include "TRandom.h"
#include "TROOT.h"
#include "gtest/gtest.h"
#include <limits>
using namespace ROOT::Experimental;
using namespace ROOT::Experimental::TDF;
using namespace ROOT::Detail::TDF;

/********* FIXTURES *********/
static constexpr ULong64_t gNEvents = 8ull;

// fixture that provides a TDF with no data-source and a single column "x" containing normal-distributed doubles
class TDFCallbacks : public ::testing::Test {
private:
   TDataFrame fLoopManager;
   TInterface<TLoopManager> DefineRandomCol()
   {
      TRandom r;
      return fLoopManager.Define("x", [r]() mutable { return r.Gaus(); });
   }

protected:
   TDFCallbacks() : fLoopManager(gNEvents), tdf(DefineRandomCol()) {}
   TInterface<TLoopManager> tdf;
};

#ifdef R__USE_IMT
static constexpr unsigned int gNSlots = 4u;

// fixture that enables implicit MT and provides a TDF with no data-source and a single column "x" containing
// normal-distributed doubles
class TDFCallbacksMT : public ::testing::Test {
   class TIMTEnabler {
   public:
      TIMTEnabler(unsigned int sl) { ROOT::EnableImplicitMT(sl); }
      ~TIMTEnabler() { ROOT::DisableImplicitMT(); }
   };

private:
   TIMTEnabler fIMTEnabler;
   TDataFrame fLoopManager;
   TInterface<TLoopManager> DefineRandomCol()
   {
      std::vector<TRandom> rs(gNSlots);
      return fLoopManager.DefineSlot("x", [rs](unsigned int slot) mutable { return rs[slot].Gaus(); });
   }

protected:
   TDFCallbacksMT() : fIMTEnabler(gNSlots), fLoopManager(gNEvents), tdf(DefineRandomCol()) {}
   TInterface<TLoopManager> tdf;
};
#endif

/********* TESTS *********/
TEST_F(TDFCallbacks, Histo1DWithFillTOHelper)
{
   // Histo1D<double> + OnPartialResult + FillTOHelper
   auto h = tdf.Histo1D<double>({"", "", 128, -2., 2.}, "x");
   using value_t = typename decltype(h)::Value_t;
   ULong64_t i = 0ull;
   ULong64_t everyN = 1ull;
   h.OnPartialResult(everyN, [&i, &everyN](value_t &h_) {
      i += everyN;
      EXPECT_EQ(h_.GetEntries(), i);
   });
   *h;
   EXPECT_EQ(gNEvents, i);
}

TEST_F(TDFCallbacks, JittedHisto1DWithFillTOHelper)
{
   // Histo1D + Jitting + OnPartialResult + FillTOHelper
   auto h = tdf.Histo1D({"", "", 128, -2., 2.}, "x");
   using value_t = typename decltype(h)::Value_t;
   ULong64_t i = 0ull;
   ULong64_t everyN = 1ull;
   h.OnPartialResult(everyN, [&i, &everyN](value_t &h_) {
      i += everyN;
      EXPECT_EQ(h_.GetEntries(), i);
   });
   *h;
   EXPECT_EQ(gNEvents, i);
}

TEST_F(TDFCallbacks, Histo1DWithFillHelper)
{
   // Histo1D<double> + OnPartialResult + FillHelper
   auto h = tdf.Histo1D<double>("x");
   using value_t = typename decltype(h)::Value_t;
   ULong64_t i = 0ull;
   ULong64_t everyN = 1ull;
   h.OnPartialResult(everyN, [&i, &everyN](value_t &h_) {
      i += everyN;
      EXPECT_EQ(h_.GetEntries(), i);
   });
   *h;
   EXPECT_EQ(gNEvents, i);
}

TEST_F(TDFCallbacks, JittedHisto1DWithFillHelper)
{
   // Histo1D + Jitting + OnPartialResult + FillHelper
   auto h = tdf.Histo1D("x");
   using value_t = typename decltype(h)::Value_t;
   ULong64_t i = 0ull;
   ULong64_t everyN = 1ull;
   h.OnPartialResult(everyN, [&i, &everyN](value_t &h_) {
      i += everyN;
      EXPECT_EQ(h_.GetEntries(), i);
   });
   *h;
   EXPECT_EQ(gNEvents, i);
}

TEST_F(TDFCallbacks, Min)
{
   // Min + OnPartialResult
   auto m = tdf.Min<double>("x");
   double runningMin = std::numeric_limits<double>::max();
   m.OnPartialResult(2, [&runningMin](double x) {
      EXPECT_LE(x, runningMin);
      runningMin = x;
   });
   *m;
   EXPECT_DOUBLE_EQ(runningMin, *m);
}

TEST_F(TDFCallbacks, JittedMin)
{
   // Min + Jitting + OnPartialResult
   auto m = tdf.Min("x");
   double runningMin = std::numeric_limits<double>::max();
   m.OnPartialResult(2, [&runningMin](double x) {
      EXPECT_LE(x, runningMin);
      runningMin = x;
   });
   *m;
   EXPECT_DOUBLE_EQ(runningMin, *m);
}

TEST_F(TDFCallbacks, Max)
{
   // Max + OnPartialResult
   auto m = tdf.Max<double>("x");
   double runningMax = std::numeric_limits<double>::lowest();
   m.OnPartialResult(2, [&runningMax](double x) {
      EXPECT_GE(x, runningMax);
      runningMax = x;
   });
   *m;
   EXPECT_DOUBLE_EQ(runningMax, *m);
}

TEST_F(TDFCallbacks, JittedMax)
{
   // Max + Jitting + OnPartialResult
   auto m = tdf.Max("x");
   double runningMax = std::numeric_limits<double>::lowest();
   m.OnPartialResult(2, [&runningMax](double x) {
      EXPECT_GE(x, runningMax);
      runningMax = x;
   });
   *m;
   EXPECT_DOUBLE_EQ(runningMax, *m);
}

TEST_F(TDFCallbacks, Mean)
{
   // Mean + OnPartialResult
   auto m = tdf.Mean<double>("x");
   // TODO find a better way to check that the running mean makes sense
   bool called = false;
   m.OnPartialResult(gNEvents / 2, [&called](double) { called = true; });
   *m;
   EXPECT_TRUE(called);
}

TEST_F(TDFCallbacks, JittedMean)
{
   // Mean + Jitting + OnPartialResult
   auto m = tdf.Mean("x");
   // TODO find a better way to check that the running mean makes sense
   bool called = false;
   m.OnPartialResult(gNEvents / 2, [&called](double) { called = true; });
   *m;
   EXPECT_TRUE(called);
}

TEST_F(TDFCallbacks, Take)
{
   // Take + OnPartialResult
   auto t = tdf.Take<double>("x");
   unsigned int i = 0u;
   t.OnPartialResult(1, [&](decltype(t)::Value_t &t_) {
      ++i;
      EXPECT_EQ(t_.size(), i);
   });
   *t;
}

TEST_F(TDFCallbacks, Count)
{
   // Count + OnPartialResult
   auto c = tdf.Count();
   ULong64_t i = 0ull;
   c.OnPartialResult(1, [&](decltype(c)::Value_t c_) {
      ++i;
      EXPECT_EQ(c_, i);
   });
   *c;
   EXPECT_EQ(*c, i);
}

TEST_F(TDFCallbacks, Reduce)
{
   // Reduce + OnPartialResult
   double runningMin;
   auto m = tdf.Min<double>("x").OnPartialResult(1, [&runningMin](double m_) { runningMin = m_; });
   auto r = tdf.Reduce([](double x1, double x2) { return std::min(x1, x2); }, {"x"}, 0.);
   r.OnPartialResult(1, [&runningMin](double r_) { EXPECT_DOUBLE_EQ(r_, runningMin); });
   *r;
}

TEST_F(TDFCallbacks, Chaining)
{
   // Chaining of multiple OnPartialResult[Slot] calls
   unsigned int i = 0u;
   auto c = tdf.Count()
               .OnPartialResult(1, [&i](ULong64_t) { ++i; })
               .OnPartialResultSlot(1, [&i](unsigned int, ULong64_t) {++i; });
   *c;
   EXPECT_EQ(i, gNEvents * 2);
}

TEST_F(TDFCallbacks, OrderOfExecution)
{
   // Test that callbacks are executed in the order they are registered
   unsigned int i = 0u;
   auto c = tdf.Count();
   c.OnPartialResult(1, [&i](ULong64_t) {
      EXPECT_EQ(i, 0u);
      i = 42u;
   });
   c.OnPartialResultSlot(1, [&i](unsigned int slot, ULong64_t) {
      if (slot == 0u) {
         EXPECT_EQ(i, 42u);
         i = 0u;
      }
   });
   *c;
   EXPECT_EQ(i, 0u);
}

TEST_F(TDFCallbacks, ExecuteOnce)
{
   // OnPartialResult(kOnce)
   auto c = tdf.Count();
   unsigned int callCount = 0;
   c.OnPartialResult(0, [&callCount](ULong64_t) { callCount++; });
   *c;
   EXPECT_EQ(callCount, 1u);
}

TEST_F(TDFCallbacks, MultipleCallbacks)
{
   // registration of multiple callbacks on the same partial result
   auto h = tdf.Histo1D<double>({"", "", 128, -2., 2.}, "x");
   using value_t = typename decltype(h)::Value_t;
   ULong64_t everyN = 1ull;
   ULong64_t i = 0ull;
   h.OnPartialResult(everyN, [&i, everyN](value_t &h_) {
      i += everyN;
      EXPECT_EQ(h_.GetEntries(), i);
   });

   everyN = 2ull;
   ULong64_t i2 = 0ull;
   h.OnPartialResult(everyN, [&i2, everyN](value_t &h_) {
      i2 += everyN;
      EXPECT_EQ(h_.GetEntries(), i2);
   });

   *h;
}

TEST_F(TDFCallbacks, MultipleEventLoops)
{
   // callbacks must be de-registered after the event-loop is run
   auto h = tdf.Histo1D<double>({"", "", 128, -2., 2.}, "x");
   using value_t = typename decltype(h)::Value_t;
   ULong64_t i = 0ull;
   h.OnPartialResult(1ull, [&i](value_t &) { ++i; });
   *h;

   auto h2 = tdf.Histo1D<double>({"", "", 128, -2., 2.}, "x");
   *h2;

   EXPECT_EQ(i, gNEvents);
}

class FunctorClass {
   unsigned int &i_;

public:
   FunctorClass(unsigned int &i) : i_(i) {}
   void operator()(ULong64_t) { ++i_; }
};

TEST_F(TDFCallbacks, FunctorClass)
{
   unsigned int i = 0;
   *(tdf.Count().OnPartialResult(1, FunctorClass(i)));
   EXPECT_EQ(i, gNEvents);
}

unsigned int freeFunctionCounter = 0;
void FreeFunction(ULong64_t)
{
   freeFunctionCounter++;
}

TEST_F(TDFCallbacks, FreeFunction)
{
   *(tdf.Count().OnPartialResult(1, FreeFunction));
   EXPECT_EQ(freeFunctionCounter, gNEvents);
}

#ifdef R__USE_IMT
/******** Multi-thread tests **********/
TEST_F(TDFCallbacksMT, ExecuteOncePerSlot)
{
   // OnPartialResultSlot(kOnce)
   auto c = tdf.Count();
   std::atomic_uint callCount(0u);
   c.OnPartialResultSlot(c.kOnce, [&callCount](unsigned int, ULong64_t) { callCount++; });
   *c;
   // depending on how tasks are dispatched to worker threads and how quickly threads push and pop slot numbers from
   // TSlotStack, the callback might be executed 1 to nSlots times.
   EXPECT_LE(callCount, gNSlots);
   EXPECT_GT(callCount, 0u);
}

TEST_F(TDFCallbacksMT, ExecuteOnce)
{
   // OnPartialResult(kOnce)
   auto c = tdf.Count();
   std::atomic_uint callCount(0u);
   c.OnPartialResult(c.kOnce, [&callCount](ULong64_t) { callCount++; });
   *c;
   EXPECT_EQ(callCount, 1u);
}

TEST_F(TDFCallbacksMT, Histo1DWithFillTOHelper)
{
   // Histo1D<double> + OnPartialResultSlot + FillTOHelper
   auto h = tdf.Histo1D<double>({"", "", 128, -2., 2.}, "x");
   using value_t = typename decltype(h)::Value_t;
   std::array<ULong64_t, gNSlots> is;
   is.fill(0ull);
   constexpr ULong64_t everyN = 1ull;
   h.OnPartialResultSlot(everyN, [&is](unsigned int slot, value_t &h_) {
      is[slot] += everyN;
      EXPECT_EQ(h_.GetEntries(), is[slot]);
   });
   *h;
   EXPECT_EQ(gNEvents, std::accumulate(is.begin(), is.end(), 0ull));
}

TEST_F(TDFCallbacksMT, Histo1DWithFillHelper)
{
   // Histo1D<double> + OnPartialResultSlot + FillHelper
   auto h = tdf.Histo1D<double>("x");
   using value_t = typename decltype(h)::Value_t;
   std::array<ULong64_t, gNSlots> is;
   is.fill(0ull);
   constexpr ULong64_t everyN = 1ull;
   h.OnPartialResultSlot(everyN, [&is](unsigned int slot, value_t &h_) {
      is[slot] += everyN;
      EXPECT_EQ(h_.GetEntries(), is[slot]);
   });
   *h;
   EXPECT_EQ(gNEvents, std::accumulate(is.begin(), is.end(), 0ull));
}

TEST(TDFCallbacksMTMore, LessTasksThanWorkers)
{
   ROOT::EnableImplicitMT(4);
   TDataFrame d(1);
   auto c = d.Count();
   std::atomic_uint counter(0u);
   c.OnPartialResult(c.kOnce, [&counter](ULong64_t) { counter++; });
   *c;
   EXPECT_EQ(counter, 1u);

   ROOT::DisableImplicitMT();
}

#endif
<|endoftext|>


<commit_before>/**
 * @file rodsmetadatawindow.cpp
 * @brief Implementation of class RodsMetadataWindow
 *
 * The RodsMetadataWindow class extends the Qt window class QMainWindow
 * and implements an iRODS object metadata manager window.
 *
 * Copyright (C) 2014-2015 University of Jyväskylä. All rights reserved.
 * License: The BSD 3-Clause License, see LICENSE file for details.
 *
 * @author Ilari Korhonen
 */

// application class RodsMetadataWindow header
#include "rodsmetadatawindow.h"

// generated UI class Ui::RodsMetadataWindow header
#include "ui_rodsmetadatawindow.h"

RodsMetadataWindow::RodsMetadataWindow(Kanki::RodsConnection *theConn, Kanki::RodsObjEntryPtr theObjEntry, QWidget *parent) :
    QMainWindow(parent),
    ui(new Ui::RodsMetadataWindow),
    conn(theConn),
    objEntry(theObjEntry)
{
    ui->setupUi(this);

    // create metadata and schema objects for rods object metadata
    this->metadata = new Kanki::RodsObjMetadata(this->conn, this->objEntry);
    this->schema = new RodsMetadataSchema();

    // set window title to specify object
    std::string title = "iRODS Metadata Manager (Object: " + this->objEntry->getObjectFullPath() + ")";
    this->setWindowTitle(title.c_str());

    // create new metadata model object and link it to the view
    RodsMetadataModel *rodsMetaModel = new RodsMetadataModel(this->objEntry, this->metadata, this->schema);
    ui->treeView->setModel(rodsMetaModel);

    // create new metadata mapper object
    metaMapper = new QDataWidgetMapper(this);
    metaMapper->setOrientation(Qt::Horizontal);
    metaMapper->setModel(rodsMetaModel);

    // set metadata mappings to editor widgets
    metaMapper->addMapping(this->ui->attrNameField, RodsMetadataModel::metaName);
    metaMapper->addMapping(this->ui->attrValueField, RodsMetadataModel::metaValue);
    metaMapper->addMapping(this->ui->attrUnitField, RodsMetadataModel::metaUnit);

    // connect selection changed event signal to handler slot in this class
    connect(this->ui->treeView->selectionModel(), SIGNAL(selectionChanged(const QItemSelection &, const QItemSelection &)),
            this, SLOT(metaTreeSelectionChanged(const QItemSelection &, const QItemSelection &)));

    // refresh view
    this->refresh();
}

RodsMetadataWindow::~RodsMetadataWindow()
{
    delete (this->metadata);
    delete (this->schema);
    delete (this->ui);
}

void RodsMetadataWindow::closeEvent(QCloseEvent *event)
{
    (void)event;

    this->unregister(this->objEntry->getObjectFullPath());
}

void RodsMetadataWindow::metaTreeSelectionChanged(const QItemSelection &newSelection, const QItemSelection &oldSelection)
{
    (void)newSelection;
    (void)oldSelection;

    // get current index from selection model
    const QModelIndex index = this->ui->treeView->selectionModel()->currentIndex();

    // must check index is valid !
    if (index.isValid())
    {
        const RodsMetadataItem *item = static_cast<RodsMetadataItem*>(index.internalPointer());

        // if selection was a namespace
        if (item->childCount() > 0)
        {
            this->ui->metaEditBox->setDisabled(true);
        }

        // if selection was proper metadata item
        else {
            // enable metadata inspector
            this->ui->metaEditBox->setEnabled(true);

            // set root and current index for metadata mapper
            this->metaMapper->setRootIndex(index.parent());
            this->metaMapper->setCurrentModelIndex(index);
        }
    }
}

void RodsMetadataWindow::refresh()
{
    int status = 0;

    QApplication::setOverrideCursor(Qt::WaitCursor);
    QApplication::processEvents();

    // try to refresh object metadata
    if ((status = metadata->refresh()) < 0)
    {
        this->reportError("Error while refreshing metadata - iRODS API error", status);
    }

    QApplication::restoreOverrideCursor();

    // get metadata model object and refresh it
    RodsMetadataModel *model = static_cast<RodsMetadataModel*>(this->ui->treeView->model());
    model->refresh();

    // by default open all namespaces
    this->ui->treeView->expandAll();

    // resize columns after tree view data has been initialized
    for (int i = 0; i < model->columnCount(QModelIndex()); i++)
        this->ui->treeView->resizeColumnToContents(i);
}

void RodsMetadataWindow::addAttribute()
{
    QInputDialog attrName(this);
    QStringList attrs;
    QString selection;
    bool ok = false;
    int status = 0;

    attrs = this->schema->getAttributeList();
    selection = attrName.getItem(this, "Select Schema Attribute", "Attribute:", attrs, 0, true, &ok);

    // if user didn't press cancel and entered a proper attribute string
    if (ok && selection.length())
    {
        if ((status = this->metadata->addAttribute(selection.toStdString(), "MISSING_METADATA")) < 0)
            this->reportError("Error while adding an attribute - iRODS API error", status);

        else
            this->refresh();
    }
}

void RodsMetadataWindow::removeAttribute()
{
    int status = 0;
    QModelIndex curIndex = this->ui->treeView->currentIndex();

    // proceed only if we have a valid index
    if (curIndex.isValid())
    {
        RodsMetadataItem *item = static_cast<RodsMetadataItem*>(curIndex.internalPointer());

        // if the selection was a proper attribute
        if (!item->childCount())
        {
            // remove attribute from metadata
            if ((status = this->metadata->removeAttribute(item->getName().toStdString(),
                                                          item->getValue().toStdString(),
                                                          item->getUnit().toStdString())) < 0)
            {
                this->reportError("Error while removing attribute - iRODS API error", status);
            }

            // if successful, refresh view
            else {
                this->refresh();
            }
        }
    }
}

void RodsMetadataWindow::reportError(QString msg, int status) const
{
    // error reporting via a Qt message box
    QMessageBox errMsg;
    QString errStr = msg + " " + QVariant(status).toString();

    errMsg.setText(errStr);
    errMsg.setIcon(QMessageBox::Critical);
    errMsg.exec();
}

void RodsMetadataWindow::on_actionRefresh_triggered()
{
    this->refresh();
}

void RodsMetadataWindow::on_actionAddAttribute_triggered()
{
    this->addAttribute();
}

void RodsMetadataWindow::on_actionRemove_Attribute_triggered()
{
    this->removeAttribute();
}
<commit_msg>BUG FIX for metadata editor window crashing when last change was not committed before window close, resulted in a bad memory access (boost::shared_ptr got destroyed and mapper object still valid trying to push changes from ui)...<commit_after>/**
 * @file rodsmetadatawindow.cpp
 * @brief Implementation of class RodsMetadataWindow
 *
 * The RodsMetadataWindow class extends the Qt window class QMainWindow
 * and implements an iRODS object metadata manager window.
 *
 * Copyright (C) 2014-2015 University of Jyväskylä. All rights reserved.
 * License: The BSD 3-Clause License, see LICENSE file for details.
 *
 * @author Ilari Korhonen
 */

// application class RodsMetadataWindow header
#include "rodsmetadatawindow.h"

// generated UI class Ui::RodsMetadataWindow header
#include "ui_rodsmetadatawindow.h"

RodsMetadataWindow::RodsMetadataWindow(Kanki::RodsConnection *theConn, Kanki::RodsObjEntryPtr theObjEntry, QWidget *parent) :
    QMainWindow(parent),
    ui(new Ui::RodsMetadataWindow),
    conn(theConn),
    objEntry(theObjEntry)
{
    ui->setupUi(this);

    // create metadata and schema objects for rods object metadata
    this->metadata = new Kanki::RodsObjMetadata(this->conn, this->objEntry);
    this->schema = new RodsMetadataSchema();

    // set window title to specify object
    std::string title = "iRODS Metadata Manager (Object: " + this->objEntry->getObjectFullPath() + ")";
    this->setWindowTitle(title.c_str());

    // create new metadata model object and link it to the view
    RodsMetadataModel *rodsMetaModel = new RodsMetadataModel(this->objEntry, this->metadata, this->schema);
    ui->treeView->setModel(rodsMetaModel);

    // create new metadata mapper object
    metaMapper = new QDataWidgetMapper(this);
    metaMapper->setOrientation(Qt::Horizontal);
    metaMapper->setModel(rodsMetaModel);

    // set metadata mappings to editor widgets
    metaMapper->addMapping(this->ui->attrNameField, RodsMetadataModel::metaName);
    metaMapper->addMapping(this->ui->attrValueField, RodsMetadataModel::metaValue);
    metaMapper->addMapping(this->ui->attrUnitField, RodsMetadataModel::metaUnit);

    // connect selection changed event signal to handler slot in this class
    connect(this->ui->treeView->selectionModel(), SIGNAL(selectionChanged(const QItemSelection &, const QItemSelection &)),
            this, SLOT(metaTreeSelectionChanged(const QItemSelection &, const QItemSelection &)));

    // refresh view
    this->refresh();
}

RodsMetadataWindow::~RodsMetadataWindow()
{
    // destroy mapper first, to prevent access
    delete (this->metaMapper);
    delete (this->ui);

    // cleanup after everything else
    delete (this->metadata);
    delete (this->schema);
}

void RodsMetadataWindow::closeEvent(QCloseEvent *event)
{
    (void)event;

    this->unregister(this->objEntry->getObjectFullPath());
}

void RodsMetadataWindow::metaTreeSelectionChanged(const QItemSelection &newSelection, const QItemSelection &oldSelection)
{
    (void)newSelection;
    (void)oldSelection;

    // get current index from selection model
    const QModelIndex index = this->ui->treeView->selectionModel()->currentIndex();

    // must check index is valid !
    if (index.isValid())
    {
        const RodsMetadataItem *item = static_cast<RodsMetadataItem*>(index.internalPointer());

        // if selection was a namespace
        if (item->childCount() > 0)
        {
            this->ui->metaEditBox->setDisabled(true);
        }

        // if selection was proper metadata item
        else {
            // enable metadata inspector
            this->ui->metaEditBox->setEnabled(true);

            // set root and current index for metadata mapper
            this->metaMapper->setRootIndex(index.parent());
            this->metaMapper->setCurrentModelIndex(index);
        }
    }
}

void RodsMetadataWindow::refresh()
{
    int status = 0;

    QApplication::setOverrideCursor(Qt::WaitCursor);
    QApplication::processEvents();

    // try to refresh object metadata
    if ((status = metadata->refresh()) < 0)
    {
        this->reportError("Error while refreshing metadata - iRODS API error", status);
    }

    QApplication::restoreOverrideCursor();

    // get metadata model object and refresh it
    RodsMetadataModel *model = static_cast<RodsMetadataModel*>(this->ui->treeView->model());
    model->refresh();

    // by default open all namespaces
    this->ui->treeView->expandAll();

    // resize columns after tree view data has been initialized
    for (int i = 0; i < model->columnCount(QModelIndex()); i++)
        this->ui->treeView->resizeColumnToContents(i);
}

void RodsMetadataWindow::addAttribute()
{
    QInputDialog attrName(this);
    QStringList attrs;
    QString selection;
    bool ok = false;
    int status = 0;

    attrs = this->schema->getAttributeList();
    selection = attrName.getItem(this, "Select Schema Attribute", "Attribute:", attrs, 0, true, &ok);

    // if user didn't press cancel and entered a proper attribute string
    if (ok && selection.length())
    {
        if ((status = this->metadata->addAttribute(selection.toStdString(), "MISSING_METADATA")) < 0)
            this->reportError("Error while adding an attribute - iRODS API error", status);

        else
            this->refresh();
    }
}

void RodsMetadataWindow::removeAttribute()
{
    int status = 0;
    QModelIndex curIndex = this->ui->treeView->currentIndex();

    // proceed only if we have a valid index
    if (curIndex.isValid())
    {
        RodsMetadataItem *item = static_cast<RodsMetadataItem*>(curIndex.internalPointer());

        // if the selection was a proper attribute
        if (!item->childCount())
        {
            // remove attribute from metadata
            if ((status = this->metadata->removeAttribute(item->getName().toStdString(),
                                                          item->getValue().toStdString(),
                                                          item->getUnit().toStdString())) < 0)
            {
                this->reportError("Error while removing attribute - iRODS API error", status);
            }

            // if successful, refresh view
            else {
                this->refresh();
            }
        }
    }
}

void RodsMetadataWindow::reportError(QString msg, int status) const
{
    // error reporting via a Qt message box
    QMessageBox errMsg;
    QString errStr = msg + " " + QVariant(status).toString();

    errMsg.setText(errStr);
    errMsg.setIcon(QMessageBox::Critical);
    errMsg.exec();
}

void RodsMetadataWindow::on_actionRefresh_triggered()
{
    this->refresh();
}

void RodsMetadataWindow::on_actionAddAttribute_triggered()
{
    this->addAttribute();
}

void RodsMetadataWindow::on_actionRemove_Attribute_triggered()
{
    this->removeAttribute();
}
<|endoftext|>


<commit_before>/*
 * NovelChapterParser.cpp
 *
 *  Created on: 2014年11月28日
 *      Author: nemo
 */

#include "header/NovelChapterParser.h"
#include "Utility/FileUtils.h"

#include <iostream>
#include <dirent.h>

NovelChapterParser::NovelChapterParser()
{
	_tempDir = "temp";
	_resultDir = "result";

	FileUtils::createDir(_tempDir.c_str());
	FileUtils::createDir(_resultDir.c_str());
}

NovelChapterParser::~NovelChapterParser()
{
	fileClosed();
}

/**
 * @brief	Open in and out file.
 *
 * @param 	novelName	Input file name.
 *
 * @retval	 0	File open success.
 * @retval	-1	File open fail.
 * @retval	-2	File path is empty.
 * @retval	-3	Output file open failed.
 */
int NovelChapterParser::fileOpen(string novelName)
{
	if(!novelName.empty())
	{
		string inFilePath = _tempDir + "/" + novelName;
		string outFilePath = _resultDir + "/" + novelName;

		_fileInStream.open(inFilePath.c_str(), ios::in);

		if (!_fileInStream)
			return -1;
		else
		{
			cout << novelName << endl;
			_fileOutStream.open(outFilePath.c_str(), ios::out);
			if(!_fileOutStream)
				return -3;

			return 0;
		}
	}
	else
	{
		return -2;
	}
}

/**
 * @brief	Close in and out files.
 */
void NovelChapterParser::fileClosed()
{
	_fileInStream.close();
	_fileOutStream.close();
}

/**
 * @brief	Set regular expression.
 *
 * @param regularExpression		Input regular expression.
 */
void NovelChapterParser::setRegularExp(string regularExpression)
{
	_regexList.push_back(regex(regularExpression));
}

/**
 * @brief	Parse the novel chapter.
 *
 * Parse the novel chapter according to the setting regular expression.
 */
void NovelChapterParser::parseChapter()
{
	analysisFile();
	if(_inputNovel.size() == 0)
	{
		cout << "No input file.";
		exit(0);
	}

	for(auto iter = _inputNovel.begin(); iter != _inputNovel.end(); iter++)
	{
		int err = fileOpen(*iter);

		if(err != 0)
		{
			cout << "File open Error(" << err << ")." << endl;
			fileClosed();
			continue;
		}

		parseContents();
		fileClosed();
	}

}

/**
 * @brief	Analysis the temp folder.
 *
 * Analysis the temp folder, and save the file name.
 */
void NovelChapterParser::analysisFile()
{
	struct dirent **namelist;
	int fileCount = scandir(_tempDir.c_str(), &namelist, 0, alphasort);
	if(fileCount < 0)
	{
		perror("scandir");
	}
	else
	{
		for(int i = 0; i < fileCount; i++)
		{
			string fileName = namelist[i]->d_name;
			if(fileName == "." || fileName == "..")
				continue;
			else if(fileName[fileName.length()-1] == '~')
				continue;
			else
				_inputNovel.push_back(namelist[i]->d_name);
		}
	}
	free(namelist);
}

/**
 * @brief	Parse the novel contents.
 */
void NovelChapterParser::parseContents()
{

	unsigned int maxReadSize = 1024;
	char readBuf[maxReadSize];

	while (true)
	{
		if(_fileInStream)
		{
			_fileInStream.getline(readBuf, maxReadSize);
			string readString = readBuf;
			if(readString.empty())
			{
				outputContents(readString);
				continue;
			}

			bool isChapter = false;
			for(auto iter = _regexList.begin(); iter != _regexList.end(); iter++)
			{
				smatch mat;
				regex_search(readString, mat, *iter);

				string matchResult = *(mat.begin());
				if (!matchResult.empty())
				{
					cout << matchResult << endl;

					string chapterString;
					chapterString = "<chapter> " + readString;
					outputContents(chapterString);
					isChapter = true;
				}
			}
			if( !isChapter )
				outputContents(readString);

		}
		else
		{
			cout << "File closed." << endl << endl;
			break;
		}
	}
}

/**
 * @brief	Output contents to the new file.
 */
void NovelChapterParser::outputContents(string contents)
{
	_fileOutStream << contents;
}
<commit_msg>Add endl for the output message.<commit_after>/*
 * NovelChapterParser.cpp
 *
 *  Created on: 2014年11月28日
 *      Author: nemo
 */

#include "header/NovelChapterParser.h"
#include "Utility/FileUtils.h"

#include <iostream>
#include <dirent.h>

NovelChapterParser::NovelChapterParser()
{
	_tempDir = "temp";
	_resultDir = "result";

	FileUtils::createDir(_tempDir.c_str());
	FileUtils::createDir(_resultDir.c_str());
}

NovelChapterParser::~NovelChapterParser()
{
	fileClosed();
}

/**
 * @brief	Open in and out file.
 *
 * @param 	novelName	Input file name.
 *
 * @retval	 0	File open success.
 * @retval	-1	File open fail.
 * @retval	-2	File path is empty.
 * @retval	-3	Output file open failed.
 */
int NovelChapterParser::fileOpen(string novelName)
{
	if(!novelName.empty())
	{
		string inFilePath = _tempDir + "/" + novelName;
		string outFilePath = _resultDir + "/" + novelName;

		_fileInStream.open(inFilePath.c_str(), ios::in);

		if (!_fileInStream)
			return -1;
		else
		{
			cout << novelName << endl;
			_fileOutStream.open(outFilePath.c_str(), ios::out);
			if(!_fileOutStream)
				return -3;

			return 0;
		}
	}
	else
	{
		return -2;
	}
}

/**
 * @brief	Close in and out files.
 */
void NovelChapterParser::fileClosed()
{
	_fileInStream.close();
	_fileOutStream.close();
}

/**
 * @brief	Set regular expression.
 *
 * @param regularExpression		Input regular expression.
 */
void NovelChapterParser::setRegularExp(string regularExpression)
{
	_regexList.push_back(regex(regularExpression));
}

/**
 * @brief	Parse the novel chapter.
 *
 * Parse the novel chapter according to the setting regular expression.
 */
void NovelChapterParser::parseChapter()
{
	analysisFile();
	if(_inputNovel.size() == 0)
	{
		cout << "No input file." << endl;
		exit(0);
	}

	for(auto iter = _inputNovel.begin(); iter != _inputNovel.end(); iter++)
	{
		int err = fileOpen(*iter);

		if(err != 0)
		{
			cout << "File open Error(" << err << ")." << endl;
			fileClosed();
			continue;
		}

		parseContents();
		fileClosed();
	}

}

/**
 * @brief	Analysis the temp folder.
 *
 * Analysis the temp folder, and save the file name.
 */
void NovelChapterParser::analysisFile()
{
	struct dirent **namelist;
	int fileCount = scandir(_tempDir.c_str(), &namelist, 0, alphasort);
	if(fileCount < 0)
	{
		perror("scandir");
	}
	else
	{
		for(int i = 0; i < fileCount; i++)
		{
			string fileName = namelist[i]->d_name;
			if(fileName == "." || fileName == "..")
				continue;
			else if(fileName[fileName.length()-1] == '~')
				continue;
			else
				_inputNovel.push_back(namelist[i]->d_name);
		}
	}
	free(namelist);
}

/**
 * @brief	Parse the novel contents.
 */
void NovelChapterParser::parseContents()
{

	unsigned int maxReadSize = 1024;
	char readBuf[maxReadSize];

	while (true)
	{
		if(_fileInStream)
		{
			_fileInStream.getline(readBuf, maxReadSize);
			string readString = readBuf;
			if(readString.empty())
			{
				outputContents(readString);
				continue;
			}

			bool isChapter = false;
			for(auto iter = _regexList.begin(); iter != _regexList.end(); iter++)
			{
				smatch mat;
				regex_search(readString, mat, *iter);

				string matchResult = *(mat.begin());
				if (!matchResult.empty())
				{
					cout << matchResult << endl;

					string chapterString;
					chapterString = "<chapter> " + readString;
					outputContents(chapterString);
					isChapter = true;
				}
			}
			if( !isChapter )
				outputContents(readString);

		}
		else
		{
			cout << "File closed." << endl << endl;
			break;
		}
	}
}

/**
 * @brief	Output contents to the new file.
 */
void NovelChapterParser::outputContents(string contents)
{
	_fileOutStream << contents;
}
<|endoftext|>


<commit_before>#include <irrlicht/irrlicht.h>
#include <irrlicht/driverChoice.h>

//#pragma comment(lib, "Irrlicht.lib")

using namespace irr;
//using namespace core;
//using namespace scene;
//using namespace video;
//using namespace io;
//using namespace gui;
//
enum {
  // I use this ISceneNode ID to indicate a scene node that is
  // not pickable by getSceneNodeAndCollisionPointFromRay()
  ID_IsNotPickable = 0,

  // I use this flag in ISceneNode IDs to indicate that the
  // scene node can be picked by ray selection.
  IDFlag_IsPickable = 1 << 0,

  // I use this flag in ISceneNode IDs to indicate that the
  // scene node can be highlighted.  In this example, the
  // homonids can be highlighted, but the level mesh can't.
  IDFlag_IsHighlightable = 1 << 1
};

int main() {
  // Uses driverChoiceConsole() from driverChoice.h
  IrrlichtDevice *device = createDevice(driverChoiceConsole(), core::dimension2d<u32>(512, 384), 16, false, false, false, 0);
  device->setWindowCaption(L"Hello World! - Irrlicht Engine Demo");

  // set up video driver, scene manager and gui environment
  video::IVideoDriver* driver = device->getVideoDriver();
  scene::ISceneManager* smgr = device->getSceneManager();
  gui::IGUIEnvironment* guienv = device->getGUIEnvironment();

  // Create the level and the collision selector
  device->getFileSystem()->addFileArchive("models/map-20kdm2.pk3");
  scene::IMeshSceneNode *cube = smgr->addCubeSceneNode(15.0f, 0, -1, core::vector3df(150,10,10), core::vector3df(0,0,0), core::vector3df(10, 10, 1));
  cube->setMaterialFlag(video::EMF_LIGHTING, false);
  scene::IAnimatedMesh* levelMesh = smgr->getMesh("20kdm2.bsp");
  scene::IMeshSceneNode* levelNode = 0;
  scene::ITriangleSelector* selector = 0;
  if (levelMesh) {
    levelNode = smgr->addOctreeSceneNode(levelMesh->getMesh(0), 0, IDFlag_IsPickable);
//      node = smgr->addMeshSceneNode(mesh->getMesh(0));
    if(levelNode){
      levelNode->setPosition(core::vector3df(-1350,-130,-1400));
      selector = smgr->createOctreeTriangleSelector(
                 levelNode->getMesh(), levelNode, 128);
      levelNode->setTriangleSelector(selector);
    }
  }

  //Create camera and pin it to the floor
  scene::ICameraSceneNode *camera = smgr->addCameraSceneNodeFPS(0, 100.0f, .3f, ID_IsNotPickable, 0, 0, true, 3.f);
  camera->setPosition(core::vector3df(50,50,-60));

  //Add collision to the camera
  if (selector) {
    scene::ISceneNodeAnimator* anim = smgr->createCollisionResponseAnimator(
      selector, camera, core::vector3df(30,50,30),
      core::vector3df(0,-10,0), core::vector3df(0,30,0));
    selector->drop(); // As soon as we're done with the selector, drop it.
    camera->addAnimator(anim);
    anim->drop();  // And likewise, drop the animator when we're done referring to it.
  }
  device->getCursorControl()->setVisible(false);
  //camera->setTarget(cube->getAbsolutePosition());
  int lastFPS = -1;
  video::ITexture * texture;
  while(device->run()) {
    device->getVideoDriver()->removeTexture(texture);
    texture = driver->getTexture("/tmp/roomtest/shot.png");
    cube->setMaterialTexture(0, texture);
    if(device->isWindowActive()){
      // TODO: maybe this should be billboard based?
      //int fps = driver->getFPS();
      //core::stringw status;
      //if(fps != lastFPS){
      //  status = L"grimmwa.re VR hackspace mkI [";
      //  status += driver->getName();
      //  status += "] FPS::";
      //  status += fps;
      //  lastFPS = fps;
      //}
      //font->draw(status, core::rect<s32>(10,10,200,22), video::SColor(255,255,255,255));

      driver->beginScene(true, true, video::SColor(255,100,101,140));
      smgr->drawAll();
      guienv->drawAll();
      driver->endScene();
    }
  }
  device->drop();
  return 0;

}
<commit_msg>Resized<commit_after>#include <irrlicht/irrlicht.h>
#include <irrlicht/driverChoice.h>

//#pragma comment(lib, "Irrlicht.lib")

using namespace irr;
//using namespace core;
//using namespace scene;
//using namespace video;
//using namespace io;
//using namespace gui;
//
enum {
  // I use this ISceneNode ID to indicate a scene node that is
  // not pickable by getSceneNodeAndCollisionPointFromRay()
  ID_IsNotPickable = 0,

  // I use this flag in ISceneNode IDs to indicate that the
  // scene node can be picked by ray selection.
  IDFlag_IsPickable = 1 << 0,

  // I use this flag in ISceneNode IDs to indicate that the
  // scene node can be highlighted.  In this example, the
  // homonids can be highlighted, but the level mesh can't.
  IDFlag_IsHighlightable = 1 << 1
};

int main() {
  // Uses driverChoiceConsole() from driverChoice.h
  IrrlichtDevice *device = createDevice(driverChoiceConsole(), core::dimension2d<u32>(512, 384), 16, false, false, false, 0);
  device->setWindowCaption(L"Hello World! - Irrlicht Engine Demo");

  // set up video driver, scene manager and gui environment
  video::IVideoDriver* driver = device->getVideoDriver();
  scene::ISceneManager* smgr = device->getSceneManager();
  gui::IGUIEnvironment* guienv = device->getGUIEnvironment();

  // Create the level and the collision selector
  device->getFileSystem()->addFileArchive("models/map-20kdm2.pk3");
  scene::IMeshSceneNode *cube = smgr->addCubeSceneNode(15.0f, 0, -1, core::vector3df(150,10,10), core::vector3df(0,0,0), core::vector3df(4, 4, 1));
  cube->setMaterialFlag(video::EMF_LIGHTING, false);
  scene::IAnimatedMesh* levelMesh = smgr->getMesh("20kdm2.bsp");
  scene::IMeshSceneNode* levelNode = 0;
  scene::ITriangleSelector* selector = 0;
  if (levelMesh) {
    levelNode = smgr->addOctreeSceneNode(levelMesh->getMesh(0), 0, IDFlag_IsPickable);
//      node = smgr->addMeshSceneNode(mesh->getMesh(0));
    if(levelNode){
      levelNode->setPosition(core::vector3df(-1350,-130,-1400));
      selector = smgr->createOctreeTriangleSelector(
                 levelNode->getMesh(), levelNode, 128);
      levelNode->setTriangleSelector(selector);
    }
  }

  //Create camera and pin it to the floor
  scene::ICameraSceneNode *camera = smgr->addCameraSceneNodeFPS(0, 100.0f, .3f, ID_IsNotPickable, 0, 0, true, 3.f);
  camera->setPosition(core::vector3df(50,50,-60));

  //Add collision to the camera
  if (selector) {
    scene::ISceneNodeAnimator* anim = smgr->createCollisionResponseAnimator(
      selector, camera, core::vector3df(30,50,30),
      core::vector3df(0,-10,0), core::vector3df(0,30,0));
    selector->drop(); // As soon as we're done with the selector, drop it.
    camera->addAnimator(anim);
    anim->drop();  // And likewise, drop the animator when we're done referring to it.
  }
  device->getCursorControl()->setVisible(false);
  //camera->setTarget(cube->getAbsolutePosition());
  int lastFPS = -1;
  video::ITexture * texture;
  while(device->run()) {
    device->getVideoDriver()->removeTexture(texture);
    texture = driver->getTexture("/tmp/roomtest/shot.png");
    cube->setMaterialTexture(0, texture);
    if(device->isWindowActive()){
      // TODO: maybe this should be billboard based?
      //int fps = driver->getFPS();
      //core::stringw status;
      //if(fps != lastFPS){
      //  status = L"grimmwa.re VR hackspace mkI [";
      //  status += driver->getName();
      //  status += "] FPS::";
      //  status += fps;
      //  lastFPS = fps;
      //}
      //font->draw(status, core::rect<s32>(10,10,200,22), video::SColor(255,255,255,255));

      driver->beginScene(true, true, video::SColor(255,100,101,140));
      smgr->drawAll();
      guienv->drawAll();
      driver->endScene();
    }
  }
  device->drop();
  return 0;

}
<|endoftext|>


<commit_before>26b9640f-ad5d-11e7-a968-ac87a332f658<commit_msg>No longer crashes if X<commit_after>272af89e-ad5d-11e7-a599-ac87a332f658<|endoftext|>


<commit_before>3054d542-2748-11e6-a941-e0f84713e7b8<commit_msg>Nope, didn't work, now it does<commit_after>30624e7a-2748-11e6-94e8-e0f84713e7b8<|endoftext|>


<commit_before>//=======================================================================
// Copyright Baptiste Wicht 2011.
// Distributed under the Boost Software License, Version 1.0.
// (See accompanying file LICENSE_1_0.txt or copy at
//  http://www.boost.org/LICENSE_1_0.txt)
//=======================================================================

#include "OptimizationEngine.hpp"

#include <boost/variant/static_visitor.hpp>

#include "Types.hpp"
#include "Options.hpp"

#include "StringPool.hpp"
#include "FunctionTable.hpp"

#include "IsConstantVisitor.hpp"
#include "GetTypeVisitor.hpp"
#include "VisitorUtils.hpp"
#include "ASTVisitor.hpp"
#include "Variable.hpp"

using namespace eddic;

struct GetIntValue : public boost::static_visitor<int> {
    int operator()(ast::ComposedValue& value) const {
        int acc = boost::apply_visitor(*this, value.Content->first);

        for(auto& operation : value.Content->operations){
            int v = boost::apply_visitor(*this, operation.get<1>());
            char op = operation.get<0>();

            switch(op){
                case '+':
                    acc += v;
                    break;
                case '-':
                    acc -= v;
                    break;
                case '*':
                    acc *= v;
                    break;
                case '/':
                    acc /= v;
                    break;
                case '%':
                    acc %= v;
                    break;
            }
        }

        return acc;
    }

    int operator()(ast::Integer& integer) const {
        return integer.value;
    }

    //Other values are not integers
    template<typename T>
    int operator()(T&) const {
        assert(false);

        return -1;
    }
};

struct GetStringValue : public boost::static_visitor<std::string> {
    std::string operator()(ast::ComposedValue& value) const {
        std::string acc = boost::apply_visitor(*this, value.Content->first);

        for(auto& operation : value.Content->operations){
            std::string v = boost::apply_visitor(*this, operation.get<1>());
            acc = acc.substr(0, acc.size() - 1).append(v.substr(1, acc.size() - 1));
        }

        return acc;
    }

    std::string operator()(ast::Litteral& litteral) const {
        return litteral.value;
    }

    //Other values are not strings
    template<typename T>
    std::string operator()(T&) const {
        assert(false);

        return "";
    }
};

struct ValueOptimizer : public boost::static_visitor<ast::Value> {
    private:
        StringPool& pool;

    public:
        ValueOptimizer(StringPool& p) : pool(p) {}

        ast::Value operator()(ast::ComposedValue& value) const {
            if(value.Content->operations.empty()){
                return boost::apply_visitor(*this, value.Content->first);
            }

            //If the value is constant, we can replace it with the results of the computation
            if(IsConstantVisitor()(value)){
                Type type = GetTypeVisitor()(value);

                if(type.base() == BaseType::INT){
                    if (OptimizeIntegers) {
                        ast::Integer integer;
                        integer.value = GetIntValue()(value);
                        return integer;
                    }
                } else if(type.base() == BaseType::STRING){
                    if (OptimizeStrings) {
                        ast::Litteral litteral;
                        litteral.value = GetStringValue()(value);
                        litteral.label = pool.label(litteral.value);
                        return litteral;
                    }
                }
            }

            //Optimize the first value
            value.Content->first = boost::apply_visitor(*this, value.Content->first);

            //We can try to optimize every part of the composed value
            auto start = value.Content->operations.begin();
            auto end = value.Content->operations.end();

            while(start != end){
                start->get<1>() = boost::apply_visitor(*this, start->get<1>());

                ++start;
            }

            assert(value.Content->operations.size() > 0); //Once here, there is no more empty composed value

            //If we get there, that means that no optimization has been (or can be) performed
            return value;
        }

        //No optimizations for other kind of values
        template<typename T>
        ast::Value operator()(T& value) const {
            return value;
        }
};

struct CanBeRemoved : public boost::static_visitor<bool> {
    private:
        FunctionTable& functionTable;

    public:
        CanBeRemoved(FunctionTable& table) : functionTable(table) {}

        bool operator()(ast::FirstLevelBlock block){
            return boost::apply_visitor(*this, block);
        }

        bool optimizeVariable(std::shared_ptr<Context> context, const std::string& variable){
            if(OptimizeUnused){
                if(context->getVariable(variable)->referenceCount() <= 0){
                    //Removing from the AST is not enough, because it is stored in the context now
                    context->removeVariable(variable);

                    return true;
                }
            }

            return false;
        }

        bool operator()(ast::GlobalVariableDeclaration& declaration){
            return optimizeVariable(declaration.Content->context, declaration.Content->variableName);
        }

        bool operator()(ast::Declaration& declaration){
            return optimizeVariable(declaration.Content->context, declaration.Content->variableName);
        }

        bool operator()(ast::FunctionDeclaration& declaration){
            if(OptimizeUnused){
                if(declaration.Content->functionName != "main" && functionTable.referenceCount(declaration.Content->mangledName) <= 0){
                    return true;
                }
            }

            return false;
        }

        bool operator()(ast::Instruction& instruction){
           return boost::apply_visitor(*this, instruction);
        }

        //Nothing to optimize for the other types
        template<typename T>
        bool operator()(T&) {
            return false;
        }
};

struct OptimizationVisitor : public boost::static_visitor<> {
    private:
        FunctionTable& functionTable;
        StringPool& pool;
        ValueOptimizer optimizer;

    public:
        OptimizationVisitor(FunctionTable& t, StringPool& p) : functionTable(t), pool(p), optimizer(ValueOptimizer(pool)) {}

        template<typename T>
        void removeUnused(std::vector<T>& vector){
            auto iter = vector.begin();
            auto end = vector.end();

            CanBeRemoved visitor(functionTable);
            auto newEnd = remove_if(iter, end, visitor);

            vector.erase(newEnd, end);

            visit_each(*this, vector);
        }

        void operator()(ast::Program& program){\
            removeUnused(program.Content->blocks);
        }

        void operator()(ast::FunctionDeclaration& function){
            removeUnused(function.Content->instructions);
        }

        void operator()(ast::If& if_){
            visit(*this, if_.Content->condition);
            removeUnused(if_.Content->instructions);
            visit_each_non_variant(*this, if_.Content->elseIfs);
            visit_optional_non_variant(*this, if_.Content->else_);
        }

        void operator()(ast::ElseIf& elseIf){
            visit(*this, elseIf.condition);
            removeUnused(elseIf.instructions);
        }

        void operator()(ast::Else& else_){
            removeUnused(else_.instructions);
        }

        void operator()(ast::For& for_){
            visit_optional(*this, for_.Content->start);
            visit_optional(*this, for_.Content->condition);
            visit_optional(*this, for_.Content->repeat);
            removeUnused(for_.Content->instructions);
        }

        void operator()(ast::While& while_){
            visit(*this, while_.Content->condition);
            removeUnused(while_.Content->instructions);
        }

        void operator()(ast::Foreach& foreach_){
            removeUnused(foreach_.Content->instructions);
        }

        void operator()(ast::FunctionCall& functionCall){
            auto start = functionCall.Content->values.begin();
            auto end = functionCall.Content->values.end();

            while(start != end){
                *start = boost::apply_visitor(optimizer, *start);

                ++start;
            }
        }

        void operator()(ast::Assignment& assignment){
            assignment.Content->value = boost::apply_visitor(optimizer, assignment.Content->value);
        }

        void operator()(ast::Declaration& declaration){
            declaration.Content->value = boost::apply_visitor(optimizer, *declaration.Content->value);
        }

        void operator()(ast::ArrayValue& value){
            value.Content->indexValue = boost::apply_visitor(optimizer, value.Content->indexValue);
        }

        void operator()(ast::BinaryCondition& binaryCondition){
            binaryCondition.Content->lhs = boost::apply_visitor(optimizer, binaryCondition.Content->lhs);
            binaryCondition.Content->rhs = boost::apply_visitor(optimizer, binaryCondition.Content->rhs);
        }

        template<typename T>
        void operator()(T&){
            //Nothing to optimize for other types
        }
};

void OptimizationEngine::optimize(ast::Program& program, FunctionTable& functionTable, StringPool& pool){
    OptimizationVisitor visitor(functionTable, pool);
    visitor(program);
}
<commit_msg>Fix optimization of composed value inside array value<commit_after>//=======================================================================
// Copyright Baptiste Wicht 2011.
// Distributed under the Boost Software License, Version 1.0.
// (See accompanying file LICENSE_1_0.txt or copy at
//  http://www.boost.org/LICENSE_1_0.txt)
//=======================================================================

#include "OptimizationEngine.hpp"

#include <boost/variant/static_visitor.hpp>

#include "Types.hpp"
#include "Options.hpp"

#include "StringPool.hpp"
#include "FunctionTable.hpp"

#include "IsConstantVisitor.hpp"
#include "GetTypeVisitor.hpp"
#include "VisitorUtils.hpp"
#include "ASTVisitor.hpp"
#include "Variable.hpp"

using namespace eddic;

struct GetIntValue : public boost::static_visitor<int> {
    int operator()(ast::ComposedValue& value) const {
        int acc = boost::apply_visitor(*this, value.Content->first);

        for(auto& operation : value.Content->operations){
            int v = boost::apply_visitor(*this, operation.get<1>());
            char op = operation.get<0>();

            switch(op){
                case '+':
                    acc += v;
                    break;
                case '-':
                    acc -= v;
                    break;
                case '*':
                    acc *= v;
                    break;
                case '/':
                    acc /= v;
                    break;
                case '%':
                    acc %= v;
                    break;
            }
        }

        return acc;
    }

    int operator()(ast::Integer& integer) const {
        return integer.value;
    }

    //Other values are not integers
    template<typename T>
    int operator()(T&) const {
        assert(false);

        return -1;
    }
};

struct GetStringValue : public boost::static_visitor<std::string> {
    std::string operator()(ast::ComposedValue& value) const {
        std::string acc = boost::apply_visitor(*this, value.Content->first);

        for(auto& operation : value.Content->operations){
            std::string v = boost::apply_visitor(*this, operation.get<1>());
            acc = acc.substr(0, acc.size() - 1).append(v.substr(1, acc.size() - 1));
        }

        return acc;
    }

    std::string operator()(ast::Litteral& litteral) const {
        return litteral.value;
    }

    //Other values are not strings
    template<typename T>
    std::string operator()(T&) const {
        assert(false);

        return "";
    }
};

struct ValueOptimizer : public boost::static_visitor<ast::Value> {
    private:
        StringPool& pool;

    public:
        ValueOptimizer(StringPool& p) : pool(p) {}

        ast::Value operator()(ast::ComposedValue& value) const {
            if(value.Content->operations.empty()){
                return boost::apply_visitor(*this, value.Content->first);
            }

            //If the value is constant, we can replace it with the results of the computation
            if(IsConstantVisitor()(value)){
                Type type = GetTypeVisitor()(value);

                if(type.base() == BaseType::INT){
                    if (OptimizeIntegers) {
                        ast::Integer integer;
                        integer.value = GetIntValue()(value);
                        return integer;
                    }
                } else if(type.base() == BaseType::STRING){
                    if (OptimizeStrings) {
                        ast::Litteral litteral;
                        litteral.value = GetStringValue()(value);
                        litteral.label = pool.label(litteral.value);
                        return litteral;
                    }
                }
            }

            //Optimize the first value
            value.Content->first = boost::apply_visitor(*this, value.Content->first);

            //We can try to optimize every part of the composed value
            auto start = value.Content->operations.begin();
            auto end = value.Content->operations.end();

            while(start != end){
                start->get<1>() = boost::apply_visitor(*this, start->get<1>());

                ++start;
            }

            assert(value.Content->operations.size() > 0); //Once here, there is no more empty composed value

            //If we get there, that means that no optimization has been (or can be) performed
            return value;
        }

        ast::Value operator()(ast::ArrayValue& value) const {
            value.Content->indexValue = boost::apply_visitor(*this, value.Content->indexValue);

            return value;
        }

        //No optimizations for other kind of values
        template<typename T>
        ast::Value operator()(T& value) const {
            return value;
        }
};

struct CanBeRemoved : public boost::static_visitor<bool> {
    private:
        FunctionTable& functionTable;

    public:
        CanBeRemoved(FunctionTable& table) : functionTable(table) {}

        bool operator()(ast::FirstLevelBlock block){
            return boost::apply_visitor(*this, block);
        }

        bool optimizeVariable(std::shared_ptr<Context> context, const std::string& variable){
            if(OptimizeUnused){
                if(context->getVariable(variable)->referenceCount() <= 0){
                    //Removing from the AST is not enough, because it is stored in the context now
                    context->removeVariable(variable);

                    return true;
                }
            }

            return false;
        }

        bool operator()(ast::GlobalVariableDeclaration& declaration){
            return optimizeVariable(declaration.Content->context, declaration.Content->variableName);
        }

        bool operator()(ast::Declaration& declaration){
            return optimizeVariable(declaration.Content->context, declaration.Content->variableName);
        }

        bool operator()(ast::FunctionDeclaration& declaration){
            if(OptimizeUnused){
                if(declaration.Content->functionName != "main" && functionTable.referenceCount(declaration.Content->mangledName) <= 0){
                    return true;
                }
            }

            return false;
        }

        bool operator()(ast::Instruction& instruction){
           return boost::apply_visitor(*this, instruction);
        }

        //Nothing to optimize for the other types
        template<typename T>
        bool operator()(T&) {
            return false;
        }
};

struct OptimizationVisitor : public boost::static_visitor<> {
    private:
        FunctionTable& functionTable;
        StringPool& pool;
        ValueOptimizer optimizer;

    public:
        OptimizationVisitor(FunctionTable& t, StringPool& p) : functionTable(t), pool(p), optimizer(ValueOptimizer(pool)) {}

        template<typename T>
        void removeUnused(std::vector<T>& vector){
            auto iter = vector.begin();
            auto end = vector.end();

            CanBeRemoved visitor(functionTable);
            auto newEnd = remove_if(iter, end, visitor);

            vector.erase(newEnd, end);

            visit_each(*this, vector);
        }

        void operator()(ast::Program& program){\
            removeUnused(program.Content->blocks);
        }

        void operator()(ast::FunctionDeclaration& function){
            removeUnused(function.Content->instructions);
        }

        void operator()(ast::If& if_){
            visit(*this, if_.Content->condition);
            removeUnused(if_.Content->instructions);
            visit_each_non_variant(*this, if_.Content->elseIfs);
            visit_optional_non_variant(*this, if_.Content->else_);
        }

        void operator()(ast::ElseIf& elseIf){
            visit(*this, elseIf.condition);
            removeUnused(elseIf.instructions);
        }

        void operator()(ast::Else& else_){
            removeUnused(else_.instructions);
        }

        void operator()(ast::For& for_){
            visit_optional(*this, for_.Content->start);
            visit_optional(*this, for_.Content->condition);
            visit_optional(*this, for_.Content->repeat);
            removeUnused(for_.Content->instructions);
        }

        void operator()(ast::While& while_){
            visit(*this, while_.Content->condition);
            removeUnused(while_.Content->instructions);
        }

        void operator()(ast::Foreach& foreach_){
            removeUnused(foreach_.Content->instructions);
        }

        void operator()(ast::FunctionCall& functionCall){
            auto start = functionCall.Content->values.begin();
            auto end = functionCall.Content->values.end();

            while(start != end){
                *start = boost::apply_visitor(optimizer, *start);

                ++start;
            }
        }

        void operator()(ast::Assignment& assignment){
            assignment.Content->value = boost::apply_visitor(optimizer, assignment.Content->value);
        }

        void operator()(ast::Declaration& declaration){
            declaration.Content->value = boost::apply_visitor(optimizer, *declaration.Content->value);
        }

        void operator()(ast::BinaryCondition& binaryCondition){
            binaryCondition.Content->lhs = boost::apply_visitor(optimizer, binaryCondition.Content->lhs);
            binaryCondition.Content->rhs = boost::apply_visitor(optimizer, binaryCondition.Content->rhs);
        }

        template<typename T>
        void operator()(T&){
            //Nothing to optimize for other types
        }
};

void OptimizationEngine::optimize(ast::Program& program, FunctionTable& functionTable, StringPool& pool){
    OptimizationVisitor visitor(functionTable, pool);
    visitor(program);
}
<|endoftext|>