<commit_before>// This file is part of the HörTech Open Master Hearing Aid (openMHA)
// Copyright © 2008 2009 2010 2011 2013 2014 2015 2016 2018 2019 HörTech gGmbH
//
// openMHA is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
// the Free Software Foundation, version 3 of the License.
//
// openMHA is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
// GNU Affero General Public License, version 3 for more details.
//
// You should have received a copy of the GNU Affero General Public License,
// version 3 along with openMHA.  If not, see <http://www.gnu.org/licenses/>.

#include "mha_toolbox.h"
#include "mha_plugin.hh"
#include "mha_fftfb.hh"
#include <math.h>
#include <fstream>
#include <iostream>
#include "mha_events.h"
#include "mhapluginloader.h"
#include "mha_algo_comm.hh"
#include "dc_afterburn.h"

#include "mha_defs.h"

namespace multibandcompressor {

class plugin_signals_t {
public:
    plugin_signals_t(unsigned int channels,unsigned int bands);
    void update_levels(MHAOvlFilter::fftfb_t*,mha_spec_t* s_in);
    void apply_gains(MHAOvlFilter::fftfb_t*,DynComp::dc_afterburn_t& burn,mha_spec_t* s_out);
private:
    MHASignal::waveform_t plug_level;
    MHASignal::waveform_t gain;
public:
    mha_wave_t* plug_output;
};

plugin_signals_t::plugin_signals_t(unsigned int channels,unsigned int bands)
    : plug_level(1,bands*channels),
      gain(bands,channels),
      plug_output(&plug_level)
{}

void plugin_signals_t::update_levels(MHAOvlFilter::fftfb_t*pFb,mha_spec_t* s_in)
{
    plug_output = &plug_level;
    pFb->get_fbpower(&gain,s_in);
    unsigned int band;
    unsigned int channel;
    for(band=0;band<gain.num_frames;band++)
        for(channel=0;channel<gain.num_channels;channel++){
            gain.value(band,channel) = sqrt(gain.value(band,channel));
            plug_output->buf[band+channel*gain.num_frames] =
                gain.value(band,channel);
        }
}

void plugin_signals_t::apply_gains(MHAOvlFilter::fftfb_t* pFb,DynComp::dc_afterburn_t& burn, mha_spec_t* s_out)
{
    unsigned int band;
    unsigned int channel;
    mha_real_t Lin(0);
    for(band=0;band<gain.num_frames;band++)
        for(channel=0;channel<gain.num_channels;channel++){
            Lin = gain.value(band,channel);
            gain.value(band,channel) =
                plug_output->buf[band+channel*gain.num_frames] /
                std::max(1e-20f,gain.value(band,channel));
            burn.burn(gain.value(band,channel),Lin,band,channel);
        }
    pFb->apply_gains(s_out,s_out,&gain);
}

class fftfb_plug_t : public MHAOvlFilter::fftfb_t {
public:
    fftfb_plug_t(MHAOvlFilter::fftfb_vars_t&,const mhaconfig_t& cfg,algo_comm_t ac,std::string alg);
    void insert();
private:
    /** vector of nominal center frequencies / Hz */
    MHA_AC::waveform_t cfv;
    /** vector of edge frequencies / Hz */
    MHA_AC::waveform_t efv;
    /** vector of band-weigths (sum of squared fft-bin-weigths)/num_frames */
    MHA_AC::waveform_t bwv;
};

fftfb_plug_t::fftfb_plug_t(MHAOvlFilter::fftfb_vars_t& vars,const mhaconfig_t& cfg,algo_comm_t ac,std::string alg)
    : MHAOvlFilter::fftfb_t(vars,cfg.fftlen,cfg.srate),
      cfv(ac,alg+"_cf",nbands(),1,false),
      efv(ac,alg+"_ef",nbands()+1,1,false),
      bwv(ac,alg+"_band_weights",nbands(),1,false)
{
    std::vector<mha_real_t> cfhz = get_cf_hz();
    std::vector<mha_real_t> efhz = get_ef_hz();
    unsigned int kfb, kfr;
    for(kfb=0;kfb<nbands();kfb++) {
        cfv[kfb] = cfhz[kfb];
        efv[kfb] = efhz[kfb];
        bwv[kfb] = 0.0f;
        for (kfr=bin1(kfb); kfr < bin2(kfb); kfr++) {
            bwv[kfb] += w(kfr,kfb) * w(kfr,kfb);
        }
        bwv[kfb] /= (cfg.fftlen/2+1);
    }
    if( nbands() )
        efv[nbands()] = efhz[nbands()];
}

void fftfb_plug_t::insert()
{
    cfv.insert();
    efv.insert();
    bwv.insert();
}

class interface_t : public MHAPlugin::plugin_t<fftfb_plug_t>,
                    public MHAOvlFilter::fftfb_vars_t {
public:
    interface_t(const algo_comm_t&,
                const std::string&,
                const std::string&);
    void prepare(mhaconfig_t&);
    void release();
    mha_spec_t* process(mha_spec_t*);
private:
    int num_channels;
    DynComp::dc_afterburn_t burn;
    MHAEvents::patchbay_t<interface_t> patchbay;
    void update_cfg();
    std::string algo;
    MHAParser::mhapluginloader_t plug;
    plugin_signals_t* plug_sigs;
};

/** \internal

    Default values are set and MHA configuration variables registered into the parser.

    \param ac_     algorithm communication handle
    \param th     chain name
    \param al     algorithm name
*/
interface_t::interface_t(const algo_comm_t& ac_,
                         const std::string& th,
                         const std::string& al)
    : MHAPlugin::plugin_t<fftfb_plug_t>("Multiband compressor framework based on level in overlapping filter bands.",ac_),
      MHAOvlFilter::fftfb_vars_t(static_cast<MHAParser::parser_t&>(*this)),
      num_channels(0),
      algo(al),
      plug(*this,ac),
      plug_sigs(NULL)
{
    set_node_id("multibandcompressor");
    insert_member(burn);
    std::string ch_name(al+"_nch");
    ac.insert_var_int(ac.handle, ch_name.c_str(), &num_channels);
}

void interface_t::prepare(mhaconfig_t& tf)
{
    if( tf.domain != MHA_SPECTRUM )
        throw MHA_ErrorMsg("Only spectral processing is supported.");
    num_channels = input_cfg().channels;
    update_cfg();
    poll_config();
    cfg->insert();
    mhaconfig_t gain_actor_cfg;
    memset(&gain_actor_cfg,0,sizeof(gain_actor_cfg));
    gain_actor_cfg.channels = input_cfg().channels*cfg->nbands();
    gain_actor_cfg.fragsize = 1;
    gain_actor_cfg.srate = input_cfg().srate/input_cfg().fragsize;
    gain_actor_cfg.domain = MHA_WAVEFORM;
    mhaconfig_t gain_actor_cfg_out(gain_actor_cfg);
    plug.prepare(gain_actor_cfg);
    try{
        PluginLoader::mhaconfig_compare(gain_actor_cfg_out,gain_actor_cfg,"Multibandcompressor:gain actor: ");
        plug_sigs = new plugin_signals_t(input_cfg().channels,cfg->nbands());
    }
    catch(...){
        plug.release();
        throw;
    }
}

void interface_t::release()
{
    plug.release();
    if( plug_sigs ){
        delete plug_sigs;
        plug_sigs = NULL;
    }
    burn.unset_fb_pars();
}

void interface_t::update_cfg()
{
    fftfb_plug_t* fb(new fftfb_plug_t(static_cast<MHAOvlFilter::fftfb_vars_t&>(*this),input_cfg(),ac,algo));
    push_config(fb);
    burn.set_fb_pars(fb->get_cf_hz(),input_cfg().channels,input_cfg().srate/input_cfg().fragsize);
}

mha_spec_t* interface_t::process(mha_spec_t* s)
{
    poll_config();
    if( plug_sigs ){
        burn.update_burner();
        plug_sigs->update_levels(cfg,s);
        plug.process(plug_sigs->plug_output,&(plug_sigs->plug_output));
        plug_sigs->apply_gains(cfg,burn,s);
    }
    return s;
}

}

MHAPLUGIN_CALLBACKS(multibandcompressor,multibandcompressor::interface_t,spec,spec)
MHAPLUGIN_DOCUMENTATION\
(multibandcompressor,
 "filterbank compression feature-extraction level-modification level-meter",
 "multibandcompressor provides a complete framework\n"
 "for dynamic range compression in multiple frequency\n"
 "bands.\n"
 "\n"
 "It contains the same filterbank as the\n"
 "fftfilterbank plugin (see there for documentation of\n"
 "the filterbank) and combines the frequency bands\n"
 "again after the compression.\n"
 "\n"
 "For the actual dynamic range compression,\n"
 "multibandcompressor can load any other plugin using\n"
 "the field plugin\\_name.  Common choices for this\n"
 "plugin would be dc\\_simple or dc.\n"
 "\n"
 "Note that the dynamic range compression receives a\n"
 "pseudo time signal where the sampling rate is the\n"
 "rate of the block processing, i.e. in each channel\n"
 "and band, there is exactly 1 signal sample for\n"
 "every block.\n"
 "These samples are a sparse, non-negative\n"
 "representation of the actual signal in the\n"
 "respective frequency band: The magnitude of each\n"
 "sample is chosen by multibandcompressor so that\n"
 "the level computed from this sparse signal is the\n"
 "same as the level computed from the full signal for\n"
 "this frequency band.\n"
 "\n"
 "The dynamic range compression will then apply gain\n"
 "(or attenuation) to the sparse signal in each\n"
 "frequency band.\n"
 "The gain applied to the sparse\n"
 "signal is measured by multibandcompressor\n"
 "and eventually applied to the\n"
 "respective full signal.\n"
 "\n"
 "Before the compressor gain is applied to\n"
 "the full signal, it may be\n"
 "modified by the after-burner built\n"
 "into the multibandcompressor plugin\n"
 "(sub-parser 'burn'): The purpose of\n"
 "the after-burner is to enforce a\n"
 "configurable Maximum Power Output\n"
 "(MPO) for each frequency band, and to\n"
 "compensate for drains and confluxes\n"
 "of sound energy through vents and\n"
 "open fittings.  Note that compensating\n"
 "for drains in this way can easily\n"
 "lead to feedback howling and should\n"
 "be done with caution.  The\n"
 "after-burner can be disabled\n"
 "by setting burn.bypass=yes.\n"
 )

// Local Variables:
// compile-command: "make"
// c-basic-offset: 4
// End:
<commit_msg>Fix AC variable updates in multibandcompressor<commit_after>// This file is part of the HörTech Open Master Hearing Aid (openMHA)
// Copyright © 2008 2009 2010 2011 2013 2014 2015 2016 2018 2019 HörTech gGmbH
// Copyright © 2020 HörTech gGmbH
//
// openMHA is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
// the Free Software Foundation, version 3 of the License.
//
// openMHA is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
// GNU Affero General Public License, version 3 for more details.
//
// You should have received a copy of the GNU Affero General Public License,
// version 3 along with openMHA.  If not, see <http://www.gnu.org/licenses/>.

#include "mha_toolbox.h"
#include "mha_plugin.hh"
#include "mha_fftfb.hh"
#include <math.h>
#include <fstream>
#include <iostream>
#include "mha_events.h"
#include "mhapluginloader.h"
#include "mha_algo_comm.hh"
#include "dc_afterburn.h"

#include "mha_defs.h"

namespace multibandcompressor {

class plugin_signals_t {
public:
    plugin_signals_t(unsigned int channels,unsigned int bands);
    void update_levels(MHAOvlFilter::fftfb_t*,mha_spec_t* s_in);
    void apply_gains(MHAOvlFilter::fftfb_t*,DynComp::dc_afterburn_t& burn,mha_spec_t* s_out);
private:
    MHASignal::waveform_t plug_level;
    MHASignal::waveform_t gain;
public:
    mha_wave_t* plug_output;
};

plugin_signals_t::plugin_signals_t(unsigned int channels,unsigned int bands)
    : plug_level(1,bands*channels),
      gain(bands,channels),
      plug_output(&plug_level)
{}

void plugin_signals_t::update_levels(MHAOvlFilter::fftfb_t*pFb,mha_spec_t* s_in)
{
    plug_output = &plug_level;
    pFb->get_fbpower(&gain,s_in);
    unsigned int band;
    unsigned int channel;
    for(band=0;band<gain.num_frames;band++)
        for(channel=0;channel<gain.num_channels;channel++){
            gain.value(band,channel) = sqrt(gain.value(band,channel));
            plug_output->buf[band+channel*gain.num_frames] =
                gain.value(band,channel);
        }
}

void plugin_signals_t::apply_gains(MHAOvlFilter::fftfb_t* pFb,DynComp::dc_afterburn_t& burn, mha_spec_t* s_out)
{
    unsigned int band;
    unsigned int channel;
    mha_real_t Lin(0);
    for(band=0;band<gain.num_frames;band++)
        for(channel=0;channel<gain.num_channels;channel++){
            Lin = gain.value(band,channel);
            gain.value(band,channel) =
                plug_output->buf[band+channel*gain.num_frames] /
                std::max(1e-20f,gain.value(band,channel));
            burn.burn(gain.value(band,channel),Lin,band,channel);
        }
    pFb->apply_gains(s_out,s_out,&gain);
}

class fftfb_plug_t : public MHAOvlFilter::fftfb_t {
public:
    fftfb_plug_t(MHAOvlFilter::fftfb_vars_t&,const mhaconfig_t& cfg,algo_comm_t ac,std::string alg);
    void insert();
private:
    /** vector of nominal center frequencies / Hz */
    MHA_AC::waveform_t cfv;
    /** vector of edge frequencies / Hz */
    MHA_AC::waveform_t efv;
    /** vector of band-weigths (sum of squared fft-bin-weigths)/num_frames */
    MHA_AC::waveform_t bwv;
};

fftfb_plug_t::fftfb_plug_t(MHAOvlFilter::fftfb_vars_t& vars,const mhaconfig_t& cfg,algo_comm_t ac,std::string alg)
    : MHAOvlFilter::fftfb_t(vars,cfg.fftlen,cfg.srate),
      cfv(ac,alg+"_cf",nbands(),1,false),
      efv(ac,alg+"_ef",nbands()+1,1,false),
      bwv(ac,alg+"_band_weights",nbands(),1,false)
{
    std::vector<mha_real_t> cfhz = get_cf_hz();
    std::vector<mha_real_t> efhz = get_ef_hz();
    unsigned int kfb, kfr;
    for(kfb=0;kfb<nbands();kfb++) {
        cfv[kfb] = cfhz[kfb];
        efv[kfb] = efhz[kfb];
        bwv[kfb] = 0.0f;
        for (kfr=bin1(kfb); kfr < bin2(kfb); kfr++) {
            bwv[kfb] += w(kfr,kfb) * w(kfr,kfb);
        }
        bwv[kfb] /= (cfg.fftlen/2+1);
    }
    if( nbands() )
        efv[nbands()] = efhz[nbands()];
}

void fftfb_plug_t::insert()
{
    cfv.insert();
    efv.insert();
    bwv.insert();
}

class interface_t : public MHAPlugin::plugin_t<fftfb_plug_t>,
                    public MHAOvlFilter::fftfb_vars_t {
public:
    interface_t(const algo_comm_t&,
                const std::string&,
                const std::string&);
    void prepare(mhaconfig_t&);
    void release();
    mha_spec_t* process(mha_spec_t*);
private:
    MHA_AC::int_t num_channels;
    DynComp::dc_afterburn_t burn;
    MHAEvents::patchbay_t<interface_t> patchbay;
    void update_cfg();
    std::string algo;
    MHAParser::mhapluginloader_t plug;
    plugin_signals_t* plug_sigs;
};

/** \internal

    Default values are set and MHA configuration variables registered into the parser.

    \param ac_     algorithm communication handle
    \param th     chain name
    \param al     algorithm name
*/
interface_t::interface_t(const algo_comm_t& ac_,
                         const std::string& th,
                         const std::string& al)
    : MHAPlugin::plugin_t<fftfb_plug_t>("Multiband compressor framework based on level in overlapping filter bands.",ac_),
      MHAOvlFilter::fftfb_vars_t(static_cast<MHAParser::parser_t&>(*this)),
      num_channels(ac_, al + "_nch", 0),
      algo(al),
      plug(*this,ac),
      plug_sigs(NULL)
{
    set_node_id("multibandcompressor");
    insert_member(burn);
}

void interface_t::prepare(mhaconfig_t& tf)
{
    if( tf.domain != MHA_SPECTRUM )
        throw MHA_ErrorMsg("Only spectral processing is supported.");
    num_channels.data = input_cfg().channels;
    num_channels.insert();
    update_cfg();
    poll_config();
    cfg->insert();
    mhaconfig_t gain_actor_cfg;
    memset(&gain_actor_cfg,0,sizeof(gain_actor_cfg));
    gain_actor_cfg.channels = input_cfg().channels*cfg->nbands();
    gain_actor_cfg.fragsize = 1;
    gain_actor_cfg.srate = input_cfg().srate/input_cfg().fragsize;
    gain_actor_cfg.domain = MHA_WAVEFORM;
    mhaconfig_t gain_actor_cfg_out(gain_actor_cfg);
    plug.prepare(gain_actor_cfg);
    try{
        PluginLoader::mhaconfig_compare(gain_actor_cfg_out,gain_actor_cfg,"Multibandcompressor:gain actor: ");
        plug_sigs = new plugin_signals_t(input_cfg().channels,cfg->nbands());
    }
    catch(...){
        plug.release();
        throw;
    }
}

void interface_t::release()
{
    plug.release();
    if( plug_sigs ){
        delete plug_sigs;
        plug_sigs = NULL;
    }
    burn.unset_fb_pars();
}

void interface_t::update_cfg()
{
    fftfb_plug_t* fb(new fftfb_plug_t(static_cast<MHAOvlFilter::fftfb_vars_t&>(*this),input_cfg(),ac,algo));
    push_config(fb);
    burn.set_fb_pars(fb->get_cf_hz(),input_cfg().channels,input_cfg().srate/input_cfg().fragsize);
}

mha_spec_t* interface_t::process(mha_spec_t* s)
{
    poll_config();
    num_channels.insert();
    if( plug_sigs ){
        burn.update_burner();
        plug_sigs->update_levels(cfg,s);
        plug.process(plug_sigs->plug_output,&(plug_sigs->plug_output));
        plug_sigs->apply_gains(cfg,burn,s);
    }
    return s;
}

}

MHAPLUGIN_CALLBACKS(multibandcompressor,multibandcompressor::interface_t,spec,spec)
MHAPLUGIN_DOCUMENTATION\
(multibandcompressor,
 "filterbank compression feature-extraction level-modification level-meter",
 "multibandcompressor provides a complete framework\n"
 "for dynamic range compression in multiple frequency\n"
 "bands.\n"
 "\n"
 "It contains the same filterbank as the\n"
 "fftfilterbank plugin (see there for documentation of\n"
 "the filterbank) and combines the frequency bands\n"
 "again after the compression.\n"
 "\n"
 "For the actual dynamic range compression,\n"
 "multibandcompressor can load any other plugin using\n"
 "the field plugin\\_name.  Common choices for this\n"
 "plugin would be dc\\_simple or dc.\n"
 "\n"
 "Note that the dynamic range compression receives a\n"
 "pseudo time signal where the sampling rate is the\n"
 "rate of the block processing, i.e. in each channel\n"
 "and band, there is exactly 1 signal sample for\n"
 "every block.\n"
 "These samples are a sparse, non-negative\n"
 "representation of the actual signal in the\n"
 "respective frequency band: The magnitude of each\n"
 "sample is chosen by multibandcompressor so that\n"
 "the level computed from this sparse signal is the\n"
 "same as the level computed from the full signal for\n"
 "this frequency band.\n"
 "\n"
 "The dynamic range compression will then apply gain\n"
 "(or attenuation) to the sparse signal in each\n"
 "frequency band.\n"
 "The gain applied to the sparse\n"
 "signal is measured by multibandcompressor\n"
 "and eventually applied to the\n"
 "respective full signal.\n"
 "\n"
 "Before the compressor gain is applied to\n"
 "the full signal, it may be\n"
 "modified by the after-burner built\n"
 "into the multibandcompressor plugin\n"
 "(sub-parser 'burn'): The purpose of\n"
 "the after-burner is to enforce a\n"
 "configurable Maximum Power Output\n"
 "(MPO) for each frequency band, and to\n"
 "compensate for drains and confluxes\n"
 "of sound energy through vents and\n"
 "open fittings.  Note that compensating\n"
 "for drains in this way can easily\n"
 "lead to feedback howling and should\n"
 "be done with caution.  The\n"
 "after-burner can be disabled\n"
 "by setting burn.bypass=yes.\n"
 )

// Local Variables:
// compile-command: "make"
// c-basic-offset: 4
// End:
<|endoftext|>


<commit_before>/*
** Author(s):
**  - Cedric GESTES <gestes@aldebaran-robotics.com>
**
** Copyright (C) 2012 Aldebaran Robotics
*/

#include <gtest/gtest.h>
#include <qi/signal.hpp>
#include <qi/future.hpp>
#include <qi/signalspy.hpp>
#include <qi/anyobject.hpp>
#include <qi/application.hpp>
#include <qi/type/objecttypebuilder.hpp>
#include <qi/type/dynamicobjectbuilder.hpp>

qiLogCategory("test");

class Foo
{
public:
  void func(int)               { }
  void func1(qi::Atomic<int>* r, int)
  {
    // hackish sleep so that asynchronous trigger detection is safer
    qi::os::msleep(1); ++*r;
  }
  void func2(qi::Atomic<int>* r, int, int) { ++*r; }
};
void foo(qi::Atomic<int>* r, int, int)     { ++*r; }
void foo2(qi::Atomic<int>* r, char, char)  { ++*r; }
void foo3(qi::Atomic<int>* r, Foo *)       { ++*r; }
void foolast(int, qi::Promise<void> prom, qi::Atomic<int>* r) { prom.setValue(0); ++*r; }

TEST(TestSignal, TestCompilation)
{
  qi::Atomic<int>        res{0};
  qi::Signal<int> s;
  Foo*                   f = (Foo*)1;
  qi::Promise<void>      prom;

  //do not count
  s.connect(qi::AnyFunction::from(&Foo::func, f));
  s.connect(boost::bind<void>(&Foo::func, f, _1));

  s.connect(boost::bind(&foo, &res, 12, _1));
  s.connect(boost::bind(&foo2, &res, 12, _1));
  s.connect(boost::bind<void>(&Foo::func1, f, &res, _1));
  s.connect(boost::bind<void>(&Foo::func2, f, &res, 5, _1));
  s.connect(boost::bind(&foo3, &res, f));
  s.connect(boost::bind(&foolast, _1, prom, &res));

  s(42);

  while (res.load() != 6)
    qi::os::msleep(10);

  ASSERT_EQ(6, res.load());
  ASSERT_TRUE(prom.future().isFinished());
  ASSERT_FALSE(prom.future().hasError());
}

void write42(boost::shared_ptr<int> ptr)
{
  *ptr = 42;
}

TEST(TestSignal, SharedPtr)
{
  // Redundant with Copy test, but just to be sure, check that shared_ptr
  // is correctly transmited.
  qi::Signal<boost::shared_ptr<int> > sig;
  sig.connect(qi::AnyFunction::from(&write42)).setCallType(qi::MetaCallType_Queued);
  {
    boost::shared_ptr<int> ptr(new int(12));
    sig(ptr);
  };
}

void byRef(int& i, bool* done)
{
  qiLogDebug() <<"byRef " << &i << ' ' << done;
  i = 12;
  *done = true;
}

TEST(TestSignal, AutoDisconnect)
{
  // Test automatic disconnection when passing shared_ptrs
  qi::Atomic<int> r{0};
  boost::shared_ptr<Foo> foo(new Foo());
  qi::Signal<qi::Atomic<int>*, int> sig;
  sig.connect(&Foo::func1, boost::weak_ptr<Foo>(foo), _1, _2).setCallType(qi::MetaCallType_Direct);
  sig(&r, 0);
  ASSERT_EQ(1, r.load());
  foo.reset();
  sig(&r, 0);
  ASSERT_EQ(1, r.load());
}

void waitFuture(qi::Atomic<int>& cnt, qi::Promise<void> start, qi::Future<void> f)
{
  if (++cnt == 2)
    start.setValue(0);
  f.wait();

  // force disconnection, may trigger a segfault
  throw qi::PointerLockException();
}

TEST(TestSignal, NonBlockingDestroy)
{
  // disconnect is blocking, but signal destruction should not be

  qi::Promise<void> start;
  qi::Promise<void> finish;
  qi::Atomic<int> cnt{0};

  {
    qi::Signal<void> sig;
    sig.connect(boost::bind(waitFuture, boost::ref(cnt), start, finish.future()));

    sig();
    sig();

    start.future().wait();
    // now that the callback is running, destroy the signal
  }

  // all went well, unblock the callback
  finish.setValue(0);
}

TEST(TestSignal, BadArity)
{
  // Test runtime detection of arity errors
  qi::Signal<> s;
  // avoid using s.connect() which will catch the problem at compile-time
  EXPECT_ANY_THROW(s.connect(qi::SignalSubscriber(qi::AnyFunction::from(&foo))));
  EXPECT_ANY_THROW(s.connect(qi::SignalSubscriber(qi::AnyFunction::from(&foo2))));
  EXPECT_ANY_THROW(s.connect(qi::AnyFunction::from((boost::function<void(qi::Atomic<int>*, int)>)boost::bind(&Foo::func1, (Foo*)0, _1, _2))));
}

void lol(int v, int& target)
{
  target = v;
}
TEST(TestSignal, SignalSignal)
{
  qi::SignalF<void (int)> sig1;
  qi::SignalF<void (int)> *sig2 = new  qi::SignalF<void (int)>();
  sig1.connect(*sig2);
  qi::SignalSpy spy(*sig2);
  sig1(10);
  ASSERT_TRUE(spy.waitUntil(1, qi::MilliSeconds(300)));
  ASSERT_EQ(10, spy.record(0).arg<int>(0));

  // Test autodisconnect
  delete sig2;
  sig1(20);
  ASSERT_FALSE(spy.waitUntil(2, qi::MilliSeconds(300)));
}

class SignalTest {
public:
  qi::Signal<int> sig;
  qi::Signal<int> sig2;
};

TEST(TestSignal, SignalSignal2)
{
  SignalTest st;
  qi::SignalSpy spy(st.sig2);
  st.sig.connect(st.sig2);
  qiLogDebug() << "sigptrs are " << &st.sig << " " << &st.sig2;
  st.sig(4242);
  ASSERT_TRUE(spy.waitUntil(1, qi::MilliSeconds(300)));
  assert(spy.recordCount() == 1u);
  EXPECT_EQ(4242, spy.lastRecord().arg<int>(0));
}

TEST(TestSignal, SignalN)
{
  qi::Signal<int> sig;
  qi::SignalSpy spy(sig);
  sig(5);
  ASSERT_TRUE(spy.waitUntil(1, qi::MilliSeconds(300)));
  ASSERT_EQ(5, spy.lastRecord().arg<int>(0));
}

class SigHolder
{
public:
  qi::Signal<> s0;
  qi::Signal<int> s1;
  qi::Signal<int, int> s2;
  void fire0() { s0();}
  void fire1(int i) { s1(i); }
  void fire2(int i, int j) { s2(i, j);}
};

QI_REGISTER_OBJECT(SigHolder, s0, s1, s2, fire0, fire1, fire2)

TEST(TestSignal, SignalNBind)
{
  int res = 0;
  boost::shared_ptr<SigHolder> so(new SigHolder);
  qi::AnyObject op = qi::AnyReference::from(so).to<qi::AnyObject>();
  qi::detail::printMetaObject(std::cerr, op.metaObject());
  op.connect("s1", (boost::function<void(int)>)boost::bind<void>(&lol, _1, boost::ref(res)));
  op.post("s1", 2);
  for (unsigned i=0; i<30 && res!=2; ++i) qi::os::msleep(10);
  ASSERT_EQ(2, res);
  so->s1(3);
  for (unsigned i=0; i<30 && res!=3; ++i) qi::os::msleep(10);
  ASSERT_EQ(3, res);
  so->s0.connect( boost::bind<void>(&lol, 42, boost::ref(res)));
  op.post("s0");
  for (unsigned i=0; i<30 && res!=42; ++i) qi::os::msleep(10);
  ASSERT_EQ(42, res);
}

void foothrow() {
  throw 42;
}

TEST(TestSignal, SignalThrow)
{
  qi::Signal<> sig;
  sig.connect(boost::bind<void>(&foothrow));
  ASSERT_NO_THROW(sig());
  qi::os::msleep(50);
}

void dynTest0(int& tgt)                     { tgt |= 1;}
void dynTest1(int& tgt, int)                { tgt |= 2;}
void dynTest2(int& tgt, int, int)           { tgt |= 4;}
void dynTest3(int& tgt, int, std::string)   { tgt |= 8;}
void dynTestN(int& tgt, const qi::AnyArguments& a)   { tgt |= 32;}
qi::AnyReference dynTestN2(int& tgt, const std::vector<qi::AnyReference>& a)   { tgt |= 16; return qi::AnyReference();}

TEST(TestSignal, Dynamic)
{
  qi::SignalBase s(qi::Signature("m"));
  int trig = 0;
  s.setCallType(qi::MetaCallType_Direct);
  s.connect((boost::function<void()>) boost::bind(&dynTest0, boost::ref(trig)));
  s.connect((boost::function<void(int)>) boost::bind(&dynTest1, boost::ref(trig), _1));
  s.connect((boost::function<void(int, int)>) boost::bind(&dynTest2, boost::ref(trig), _1, _2));
  s.connect((boost::function<void(int, std::string)>) boost::bind(&dynTest3, boost::ref(trig), _1, _2));
  s.connect(qi::AnyFunction::fromDynamicFunction(boost::bind(&dynTestN2, boost::ref(trig), _1)));
  s.connect((boost::function<void(const qi::AnyArguments&)>)(boost::bind(&dynTestN, boost::ref(trig), _1)));
  int a, b;
  std::string c;
  qi::GenericFunctionParameters params;
  s.trigger(params);
  EXPECT_EQ(49, trig);
  params.push_back(qi::AnyReference::from(a));
  trig = 0;
  s.trigger(params);
  EXPECT_EQ(50, trig);
  params.push_back(qi::AnyReference::from(b));
  trig = 0;
  s.trigger(params);
  EXPECT_EQ(52, trig);
  params[1] = qi::AnyReference::from(c);
  trig = 0;
  s.trigger(params);
  EXPECT_EQ(56, trig);
}

void onSubs(std::atomic<bool>& var, bool subs)
{
  var = subs;
}

void callback(int i)
{}

TEST(TestSignal, OnSubscriber)
{
  std::atomic<bool> subscribers(false);

  qi::Signal<int> sig(boost::bind(onSubs, boost::ref(subscribers), _1));
  ASSERT_FALSE(subscribers);
  qi::SignalLink l = sig.connect(&callback);
  ASSERT_TRUE(subscribers);
  ASSERT_TRUE(sig.disconnect(l));
  ASSERT_FALSE(subscribers);
}

void store2(qi::Promise<int> variable1, qi::Promise<int> variable2, int value1, int value2)
{
  variable1.setValue(value1);
  variable2.setValue(value2);
}

TEST(TestSignal, SignalToSignalWithExtraArgument)
{
  qi::Promise<int> target1, target2;
  qi::Signal<int> signal1;
  qi::Signal<int, int> signal2;
  signal1.connect(boost::bind(boost::ref(signal2), _1, 42));
  signal2.connect(store2, target1, target2, _1, _2);
  signal1(12);
  EXPECT_EQ(12, target1.future().value());
  EXPECT_EQ(42, target2.future().value());
}

TEST(TestSignalSpy, Counter)
{
  qi::Signal<int> sig;
  qi::SignalSpy sp(sig);
  QI_EMIT sig(1);
  QI_EMIT sig(1);
  ASSERT_TRUE(sp.waitUntil(2, qi::MilliSeconds(300)));
  ASSERT_EQ(sp.recordCount(), 2u);

  qi::DynamicObjectBuilder ob;
  ob.advertiseSignal("signal", &sig);
  qi::AnyObject obj(ob.object());
  qi::SignalSpy sp2(obj, "signal");
  QI_EMIT sig(1);
  QI_EMIT sig(1);
  ASSERT_TRUE(sp2.waitUntil(2, qi::MilliSeconds(300)));
  ASSERT_EQ(sp2.recordCount(), 2u);
}

TEST(TestSignalSpy, Async)
{
  qi::Signal<int> sig;
  qi::SignalSpy sp(sig);
  qi::async(boost::bind(boost::ref(sig), 1));
  qi::async(boost::bind(boost::ref(sig), 1));
  ASSERT_TRUE(sp.waitUntil(2, qi::Seconds(1)));
  ASSERT_EQ(sp.recordCount(), 2u);
}

TEST(TestSignalSpy, StoringTypedValueRecords)
{
  const std::vector<int> ints{ 1, 42, 13, 2016 };
  const std::vector<std::string> strings{ "poil", "slip", "banane", "pancréas" };
  qi::Signal<int, std::string> signal;
  qi::SignalSpy spy(signal);

  signal(ints[0], strings[0]);
  ASSERT_EQ(1u, spy.recordCount());
  auto record = spy.record(0);
  ASSERT_EQ(ints[0], record.arg<int>(0));
  ASSERT_EQ(strings[0], record.arg<std::string>(1));

  for(auto i = 1u; i < ints.size(); ++i)
  {
    signal(ints[i], strings[i]);
  }
  spy.waitUntil(ints.size(), qi::MilliSeconds(300));

  for(auto i = 1u; i < ints.size(); ++i)
  {
    record = spy.record(i);
    EXPECT_EQ(ints[i], record.arg<int>(0));
    EXPECT_EQ(strings[i], record.arg<std::string>(1));
  }

  auto records = spy.allRecords();
  ASSERT_EQ(ints.size(), records.size());
  for(auto i = 0u; i < records.size(); ++i)
  {
    EXPECT_EQ(ints[i], records[i].arg<int>(0));
    EXPECT_EQ(strings[i], records[i].arg<std::string>(1));
  }
}

int main(int argc, char **argv) {
  qi::Application app(argc, argv);
  ::testing::InitGoogleTest(&argc, argv);
  qi::log::addFilter("*", qi::LogLevel_Debug);
  return RUN_ALL_TESTS();
}
<commit_msg>Signal Spy: store signal emission records for more awesomeness<commit_after>/*
** Author(s):
**  - Cedric GESTES <gestes@aldebaran-robotics.com>
**
** Copyright (C) 2012 Aldebaran Robotics
*/

#include <gtest/gtest.h>
#include <qi/signal.hpp>
#include <qi/future.hpp>
#include <qi/signalspy.hpp>
#include <qi/anyobject.hpp>
#include <qi/application.hpp>
#include <qi/type/objecttypebuilder.hpp>
#include <qi/type/dynamicobjectbuilder.hpp>

qiLogCategory("test");

class Foo
{
public:
  void func(int)               { }
  void func1(qi::Atomic<int>* r, int)
  {
    // hackish sleep so that asynchronous trigger detection is safer
    qi::os::msleep(1); ++*r;
  }
  void func2(qi::Atomic<int>* r, int, int) { ++*r; }
};
void foo(qi::Atomic<int>* r, int, int)     { ++*r; }
void foo2(qi::Atomic<int>* r, char, char)  { ++*r; }
void foo3(qi::Atomic<int>* r, Foo *)       { ++*r; }
void foolast(int, qi::Promise<void> prom, qi::Atomic<int>* r) { prom.setValue(0); ++*r; }

TEST(TestSignal, TestCompilation)
{
  qi::Atomic<int>        res{0};
  qi::Signal<int> s;
  Foo*                   f = (Foo*)1;
  qi::Promise<void>      prom;

  //do not count
  s.connect(qi::AnyFunction::from(&Foo::func, f));
  s.connect(boost::bind<void>(&Foo::func, f, _1));

  s.connect(boost::bind(&foo, &res, 12, _1));
  s.connect(boost::bind(&foo2, &res, 12, _1));
  s.connect(boost::bind<void>(&Foo::func1, f, &res, _1));
  s.connect(boost::bind<void>(&Foo::func2, f, &res, 5, _1));
  s.connect(boost::bind(&foo3, &res, f));
  s.connect(boost::bind(&foolast, _1, prom, &res));

  s(42);

  while (res.load() != 6)
    qi::os::msleep(10);

  ASSERT_EQ(6, res.load());
  ASSERT_TRUE(prom.future().isFinished());
  ASSERT_FALSE(prom.future().hasError());
}

void write42(boost::shared_ptr<int> ptr)
{
  *ptr = 42;
}

TEST(TestSignal, SharedPtr)
{
  // Redundant with Copy test, but just to be sure, check that shared_ptr
  // is correctly transmited.
  qi::Signal<boost::shared_ptr<int> > sig;
  sig.connect(qi::AnyFunction::from(&write42)).setCallType(qi::MetaCallType_Queued);
  {
    boost::shared_ptr<int> ptr(new int(12));
    sig(ptr);
  };
}

void byRef(int& i, bool* done)
{
  qiLogDebug() <<"byRef " << &i << ' ' << done;
  i = 12;
  *done = true;
}

TEST(TestSignal, AutoDisconnect)
{
  // Test automatic disconnection when passing shared_ptrs
  qi::Atomic<int> r{0};
  boost::shared_ptr<Foo> foo(new Foo());
  qi::Signal<qi::Atomic<int>*, int> sig;
  sig.connect(&Foo::func1, boost::weak_ptr<Foo>(foo), _1, _2).setCallType(qi::MetaCallType_Direct);
  sig(&r, 0);
  ASSERT_EQ(1, r.load());
  foo.reset();
  sig(&r, 0);
  ASSERT_EQ(1, r.load());
}

void waitFuture(qi::Atomic<int>& cnt, qi::Promise<void> start, qi::Future<void> f)
{
  if (++cnt == 2)
    start.setValue(0);
  f.wait();

  // force disconnection, may trigger a segfault
  throw qi::PointerLockException();
}

TEST(TestSignal, NonBlockingDestroy)
{
  // disconnect is blocking, but signal destruction should not be

  qi::Promise<void> start;
  qi::Promise<void> finish;
  qi::Atomic<int> cnt{0};

  {
    qi::Signal<void> sig;
    sig.connect(boost::bind(waitFuture, boost::ref(cnt), start, finish.future()));

    sig();
    sig();

    start.future().wait();
    // now that the callback is running, destroy the signal
  }

  // all went well, unblock the callback
  finish.setValue(0);
}

TEST(TestSignal, BadArity)
{
  // Test runtime detection of arity errors
  qi::Signal<> s;
  // avoid using s.connect() which will catch the problem at compile-time
  EXPECT_ANY_THROW(s.connect(qi::SignalSubscriber(qi::AnyFunction::from(&foo))));
  EXPECT_ANY_THROW(s.connect(qi::SignalSubscriber(qi::AnyFunction::from(&foo2))));
  EXPECT_ANY_THROW(s.connect(qi::AnyFunction::from((boost::function<void(qi::Atomic<int>*, int)>)boost::bind(&Foo::func1, (Foo*)0, _1, _2))));
}

void lol(int v, int& target)
{
  target = v;
}
TEST(TestSignal, SignalSignal)
{
  qi::SignalF<void (int)> sig1;
  qi::SignalF<void (int)> *sig2 = new  qi::SignalF<void (int)>();
  sig1.connect(*sig2);
  qi::SignalSpy spy(*sig2);
  sig1(10);
  ASSERT_TRUE(spy.waitUntil(1, qi::MilliSeconds(300)));
  ASSERT_EQ(10, spy.record(0).arg<int>(0));

  // Test autodisconnect
  delete sig2;
  sig1(20);
  ASSERT_FALSE(spy.waitUntil(2, qi::MilliSeconds(300)));
}

class SignalTest {
public:
  qi::Signal<int> sig;
  qi::Signal<int> sig2;
};

TEST(TestSignal, SignalSignal2)
{
  SignalTest st;
  qi::SignalSpy spy(st.sig2);
  st.sig.connect(st.sig2);
  qiLogDebug() << "sigptrs are " << &st.sig << " " << &st.sig2;
  st.sig(4242);
  ASSERT_TRUE(spy.waitUntil(1, qi::MilliSeconds(300)));
  assert(spy.recordCount() == 1u);
  EXPECT_EQ(4242, spy.lastRecord().arg<int>(0));
}

TEST(TestSignal, SignalN)
{
  qi::Signal<int> sig;
  qi::SignalSpy spy(sig);
  sig(5);
  ASSERT_TRUE(spy.waitUntil(1, qi::MilliSeconds(300)));
  ASSERT_EQ(5, spy.lastRecord().arg<int>(0));
}

class SigHolder
{
public:
  qi::Signal<> s0;
  qi::Signal<int> s1;
  qi::Signal<int, int> s2;
  void fire0() { s0();}
  void fire1(int i) { s1(i); }
  void fire2(int i, int j) { s2(i, j);}
};

QI_REGISTER_OBJECT(SigHolder, s0, s1, s2, fire0, fire1, fire2)

TEST(TestSignal, SignalNBind)
{
  int res = 0;
  boost::shared_ptr<SigHolder> so(new SigHolder);
  qi::AnyObject op = qi::AnyReference::from(so).to<qi::AnyObject>();
  qi::detail::printMetaObject(std::cerr, op.metaObject());
  op.connect("s1", (boost::function<void(int)>)boost::bind<void>(&lol, _1, boost::ref(res)));
  op.post("s1", 2);
  for (unsigned i=0; i<30 && res!=2; ++i) qi::os::msleep(10);
  ASSERT_EQ(2, res);
  so->s1(3);
  for (unsigned i=0; i<30 && res!=3; ++i) qi::os::msleep(10);
  ASSERT_EQ(3, res);
  so->s0.connect( boost::bind<void>(&lol, 42, boost::ref(res)));
  op.post("s0");
  for (unsigned i=0; i<30 && res!=42; ++i) qi::os::msleep(10);
  ASSERT_EQ(42, res);
}

void foothrow() {
  throw 42;
}

TEST(TestSignal, SignalThrow)
{
  qi::Signal<> sig;
  sig.connect(boost::bind<void>(&foothrow));
  ASSERT_NO_THROW(sig());
  qi::os::msleep(50);
}

void dynTest0(int& tgt)                     { tgt |= 1;}
void dynTest1(int& tgt, int)                { tgt |= 2;}
void dynTest2(int& tgt, int, int)           { tgt |= 4;}
void dynTest3(int& tgt, int, std::string)   { tgt |= 8;}
void dynTestN(int& tgt, const qi::AnyArguments& a)   { tgt |= 32;}
qi::AnyReference dynTestN2(int& tgt, const std::vector<qi::AnyReference>& a)   { tgt |= 16; return qi::AnyReference();}

TEST(TestSignal, Dynamic)
{
  qi::SignalBase s(qi::Signature("m"));
  int trig = 0;
  s.setCallType(qi::MetaCallType_Direct);
  s.connect((boost::function<void()>) boost::bind(&dynTest0, boost::ref(trig)));
  s.connect((boost::function<void(int)>) boost::bind(&dynTest1, boost::ref(trig), _1));
  s.connect((boost::function<void(int, int)>) boost::bind(&dynTest2, boost::ref(trig), _1, _2));
  s.connect((boost::function<void(int, std::string)>) boost::bind(&dynTest3, boost::ref(trig), _1, _2));
  s.connect(qi::AnyFunction::fromDynamicFunction(boost::bind(&dynTestN2, boost::ref(trig), _1)));
  s.connect((boost::function<void(const qi::AnyArguments&)>)(boost::bind(&dynTestN, boost::ref(trig), _1)));
  int a, b;
  std::string c;
  qi::GenericFunctionParameters params;
  s.trigger(params);
  EXPECT_EQ(49, trig);
  params.push_back(qi::AnyReference::from(a));
  trig = 0;
  s.trigger(params);
  EXPECT_EQ(50, trig);
  params.push_back(qi::AnyReference::from(b));
  trig = 0;
  s.trigger(params);
  EXPECT_EQ(52, trig);
  params[1] = qi::AnyReference::from(c);
  trig = 0;
  s.trigger(params);
  EXPECT_EQ(56, trig);
}

void onSubs(std::atomic<bool>& var, bool subs)
{
  var = subs;
}

void callback(int i)
{}

TEST(TestSignal, OnSubscriber)
{
  std::atomic<bool> subscribers(false);

  qi::Signal<int> sig(boost::bind(onSubs, boost::ref(subscribers), _1));
  ASSERT_FALSE(subscribers);
  qi::SignalLink l = sig.connect(&callback);
  ASSERT_TRUE(subscribers);
  ASSERT_TRUE(sig.disconnect(l));
  ASSERT_FALSE(subscribers);
}

void store2(qi::Promise<int> variable1, qi::Promise<int> variable2, int value1, int value2)
{
  variable1.setValue(value1);
  variable2.setValue(value2);
}

TEST(TestSignal, SignalToSignalWithExtraArgument)
{
  qi::Promise<int> target1, target2;
  qi::Signal<int> signal1;
  qi::Signal<int, int> signal2;
  signal1.connect(boost::bind(boost::ref(signal2), _1, 42));
  signal2.connect(store2, target1, target2, _1, _2);
  signal1(12);
  EXPECT_EQ(12, target1.future().value());
  EXPECT_EQ(42, target2.future().value());
}

TEST(TestSignalSpy, Counter)
{
  qi::Signal<int> sig;
  qi::SignalSpy sp(sig);
  QI_EMIT sig(1);
  QI_EMIT sig(1);
  ASSERT_TRUE(sp.waitUntil(2, qi::MilliSeconds(300)));
  ASSERT_EQ(sp.recordCount(), 2u);

  qi::DynamicObjectBuilder ob;
  ob.advertiseSignal("signal", &sig);
  qi::AnyObject obj(ob.object());
  qi::SignalSpy sp2(obj, "signal");
  QI_EMIT sig(1);
  QI_EMIT sig(1);
  ASSERT_TRUE(sp2.waitUntil(2, qi::MilliSeconds(300)));
  ASSERT_EQ(sp2.recordCount(), 2u);
}

TEST(TestSignalSpy, Async)
{
  qi::Signal<int> sig;
  qi::SignalSpy sp(sig);
  qi::async(boost::bind(boost::ref(sig), 1));
  qi::async(boost::bind(boost::ref(sig), 1));
  ASSERT_TRUE(sp.waitUntil(2, qi::Seconds(1)));
  ASSERT_EQ(sp.recordCount(), 2u);
}

TEST(TestSignalSpy, StoringTypedValueRecords)
{
  const std::vector<int> ints{ 1, 42, 13, 2016 };
  const std::vector<std::string> strings{ "poil", "slip", "banane", "pancréas" };
  qi::Signal<int, std::string> signal;
  qi::SignalSpy spy(signal);

  signal(ints[0], strings[0]);
  ASSERT_EQ(1u, spy.recordCount());
  auto record = spy.record(0);
  ASSERT_EQ(ints[0], record.arg<int>(0));
  ASSERT_EQ(strings[0], record.arg<std::string>(1));

  for(auto i = 1u; i < ints.size(); ++i)
  {
    signal(ints[i], strings[i]);
  }
  spy.waitUntil(ints.size(), qi::MilliSeconds(300));

  for(auto i = 1u; i < ints.size(); ++i)
  {
    record = spy.record(i);
    EXPECT_EQ(ints[i], record.arg<int>(0));
    EXPECT_EQ(strings[i], record.arg<std::string>(1));
  }

  auto records = spy.allRecords();
  ASSERT_EQ(ints.size(), records.size());
  for(auto i = 0u; i < records.size(); ++i)
  {
    EXPECT_EQ(ints[i], records[i].arg<int>(0));
    EXPECT_EQ(strings[i], records[i].arg<std::string>(1));
  }
}

TEST(TestSignalSpy, StoringTypedValueRecords)
{
  const std::vector<int> ints{ 1, 42, 13, 2016 };
  const std::vector<std::string> strings{ "poil", "slip", "banane", "pancréas" };
  qi::Signal<int, std::string> signal;
  qi::SignalSpy spy(signal);

  signal(ints[0], strings[0]);
  ASSERT_EQ(1u, spy.recordCount());
  auto record = spy.record(0);
  ASSERT_EQ(ints[0], record.arg<int>(0));
  ASSERT_EQ(strings[0], record.arg<std::string>(1));

  for(auto i = 1u; i < ints.size(); ++i)
  {
    signal(ints[i], strings[i]);
  }
  spy.waitUntil(ints.size(), qi::MilliSeconds(300));

  for(auto i = 1u; i < ints.size(); ++i)
  {
    record = spy.record(i);
    EXPECT_EQ(ints[i], record.arg<int>(0));
    EXPECT_EQ(strings[i], record.arg<std::string>(1));
  }

  auto records = spy.allRecords();
  ASSERT_EQ(ints.size(), records.size());
  for(auto i = 0u; i < records.size(); ++i)
  {
    EXPECT_EQ(ints[i], records[i].arg<int>(0));
    EXPECT_EQ(strings[i], records[i].arg<std::string>(1));
  }
}

int main(int argc, char **argv) {
  qi::Application app(argc, argv);
  ::testing::InitGoogleTest(&argc, argv);
  qi::log::addFilter("*", qi::LogLevel_Debug);
  return RUN_ALL_TESTS();
}
<|endoftext|>


<commit_before>#include "catch.hpp"

#include "tile/tileID.h"
#include <set>

using namespace Tangram;

TEST_CASE( "Create TileIDs and check that they are correctly ordered", "[Core][TileID]" ) {

    TileID a = TileID(1, 1, 1);

    REQUIRE(a.x == 1);
    REQUIRE(a.y == 1);
    REQUIRE(a.z == 1);

    TileID b = TileID(2, 1, 1);
    TileID c = TileID(1, 2, 1);
    TileID d = TileID(1, 1, 2);

    REQUIRE(a < b);
    REQUIRE(b > a);

    REQUIRE(a < c);
    REQUIRE(c > a);

    REQUIRE(a > d);
    REQUIRE(d < a);

    REQUIRE(c < b);
    REQUIRE(b > c);

    REQUIRE(b > d);
    REQUIRE(d < b);

    REQUIRE(c > d);
    REQUIRE(d < c);

}

TEST_CASE( "Create TileIDs and find their parents and children", "[Core][TileID]" ) {

    TileID a = TileID(1, 2, 3);

    TileID parent = a.getParent();

    REQUIRE(parent == TileID(0, 1, 2));

    std::set<TileID> children;

    for (int i = 0; i < 4; i++) {
        children.insert(a.getChild(i));
    }

    std::set<TileID> requiredChildren = { TileID(2, 4, 4), TileID(3, 4, 4), TileID(2, 5, 4), TileID(3, 5, 4) };

    REQUIRE(children == requiredChildren);

    for (const TileID& t : requiredChildren) {
        REQUIRE(t.getParent() == a);
    }

}

TEST_CASE( "Ensure TileIDs correctly evaluate their validity", "[Core][TileID]") {

    REQUIRE(TileID(1, 1, 1).isValid());
    REQUIRE(TileID(1, 1, 1).isValid(1));
    REQUIRE(!TileID(1, 1, 2).isValid(1));

    REQUIRE(!TileID(-1,  0,  0).isValid());
    REQUIRE(!TileID( 0, -1,  0).isValid());
    REQUIRE(!TileID( 0,  0, -1).isValid());
    REQUIRE(!TileID( 1,  0,  0).isValid());
    REQUIRE(!TileID( 0,  1,  0).isValid());

    REQUIRE(!NOT_A_TILE.isValid());

}
<commit_msg>Add tests for TileID behavior with source zoom limits<commit_after>#include "catch.hpp"

#include "tile/tileID.h"
#include <set>

using namespace Tangram;

TEST_CASE( "Create TileIDs and check that they are correctly ordered", "[Core][TileID]" ) {

    TileID a = TileID(1, 1, 1);

    REQUIRE(a.x == 1);
    REQUIRE(a.y == 1);
    REQUIRE(a.z == 1);

    TileID b = TileID(2, 1, 1);
    TileID c = TileID(1, 2, 1);
    TileID d = TileID(1, 1, 2);

    REQUIRE(a < b);
    REQUIRE(b > a);

    REQUIRE(a < c);
    REQUIRE(c > a);

    REQUIRE(a > d);
    REQUIRE(d < a);

    REQUIRE(c < b);
    REQUIRE(b > c);

    REQUIRE(b > d);
    REQUIRE(d < b);

    REQUIRE(c > d);
    REQUIRE(d < c);

}

TEST_CASE( "Create TileIDs and find their parents and children", "[Core][TileID]" ) {

    TileID a = TileID(1, 2, 3);

    TileID parent = a.getParent();

    REQUIRE(parent == TileID(0, 1, 2));

    std::set<TileID> children;

    for (int i = 0; i < 4; i++) {
        children.insert(a.getChild(i));
    }

    std::set<TileID> requiredChildren = { TileID(2, 4, 4), TileID(3, 4, 4), TileID(2, 5, 4), TileID(3, 5, 4) };

    REQUIRE(children == requiredChildren);

    for (const TileID& t : requiredChildren) {
        REQUIRE(t.getParent() == a);
    }

}

TEST_CASE( "Ensure TileIDs correctly evaluate their validity", "[Core][TileID]") {

    REQUIRE(TileID(1, 1, 1).isValid());
    REQUIRE(TileID(1, 1, 1).isValid(1));
    REQUIRE(!TileID(1, 1, 2).isValid(1));

    REQUIRE(!TileID(-1,  0,  0).isValid());
    REQUIRE(!TileID( 0, -1,  0).isValid());
    REQUIRE(!TileID( 0,  0, -1).isValid());
    REQUIRE(!TileID( 1,  0,  0).isValid());
    REQUIRE(!TileID( 0,  1,  0).isValid());

    REQUIRE(!NOT_A_TILE.isValid());

}

TEST_CASE("TileID correctly applies source zoom limits", "[Core][TileID]") {

    auto a = TileID(1, 2, 4);
    REQUIRE(a.withMaxSourceZoom(4) == a);
    REQUIRE(a.withMaxSourceZoom(3) == TileID(0, 1, 3, 4, 0));
    REQUIRE(a.withMaxSourceZoom(2) == TileID(0, 0, 2, 4, 0));

    auto b = TileID(2, 1, 4, 6, 0); // Over-zoomed to zoom 6 with a limit of 4
    auto c = b.getParent();
    auto d = c.getParent();
    auto e = d.getParent();
    REQUIRE(c == TileID(2, 1, 4, 5, 0));
    REQUIRE(d == TileID(2, 1, 4, 4, 0));
    REQUIRE(e == TileID(1, 0, 3, 3, 0));

}
<|endoftext|>


<commit_before>/* RWrap
 *
 * Copyright (C) 2012 Damien "bl0b" Leroux
 *
 * Permission is hereby granted, free of charge, to any person obtaining a
 * copy of this software and associated documentation files (the "Software"),
 * to deal in the Software without restriction, including without limitation
 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
 * and/or sell copies of the Software, and to permit persons to whom the
 * Software is furnished to do so, subject to the following conditions:
 *
 * The above copyright notice and this permission notice shall be included
 * in all copies or substantial portions of the Software.
 *
 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
 * DEALINGS IN THE SOFTWARE.
 */

#include <RWrap/RWrap.h>

bool testlgl(bool x) { return !x; }
int testint(int x) { return -x; }
double testdouble(double x) { return -x; }

#define REV(name, _type_) \
    _type_ test##name(_type_ x) { return _type_(x.rbegin(), x.rend()); }

REV(string, std::string);
REV(veci, std::vector<int>);
REV(vecd, std::vector<double>);
REV(vecb, std::vector<bool>);
REV(vecs, std::vector<std::string>);
REV(vecs2, std::vector<const char*>);

double mul(double x, double y) {
    return x*y;
}

bool isTrue(bool x) {
    return !!x;
}

std::vector<int> hazcheez() {
    std::vector<int> ret;
    ret.push_back(23);
    ret.push_back(42);
    return ret;
}

std::vector<int> sqr(std::vector<int> vec) {
    for (int i = 0; i < vec.size(); ++i) {
        std::cout << vec[i] << ' ';
        vec[i] *= vec[i];
        std::cout << vec[i] << std::endl;
    }
    return vec;
}

Rwrap::List mydf() {
    Rwrap::List ret;
    std::vector<int> a, b;
    a.push_back(1);
    a.push_back(3);
    a.push_back(5);
    b.push_back(10);
    b.push_back(20);
    b.push_back(30);
    ret.add("a", a);
    ret.add("b", b);
    return ret;
}

int mylist(Rwrap::List l) {
    const char* coin = l["str"];
    int plop = l["int"];
    return 42 + 100 * plop;
}

double mul10(double ten, double y) {
    return ten * y;
}

std::vector<const char*> cvec() {
    std::vector<const char*> ret;
    ret.push_back("toto");
    ret.push_back("plop");
    ret.push_back("coin");
    return ret;
}

void testvoid(const char* x) {
    std::cout << x << std::endl;
}

/*SEXP testdf() {*/
Rwrap::DataFrame testdf() {
    std::vector<int> a;
    std::vector<const char*> b;
    std::vector<double> c;
    /*Rwrap::List ret;*/
    Rwrap::DataFrame ret;
    a.push_back(10);
    a.push_back(11);
    a.push_back(12);
    b.push_back("pouet");
    b.push_back("plop");
    b.push_back("coin");
    c.push_back(23.0);
    c.push_back(42.0);
    c.push_back(61.06);
    ret.add("hop", a);
    ret.add("zoup", b);
    ret.add("flourp", c);
    return ret;
    /*SEXP expr;*/
    /*PROTECT(expr = allocList(2));*/
    /*SET_TYPEOF(expr, LANGSXP);*/
    /*SETCAR(expr, install("as.data.frame"));*/
    /*SETCADR(expr, ret.toR());*/
    /*UNPROTECT(1);*/
    /*return eval(expr, R_GlobalEnv);*/
    /*return expr;*/
}

std::vector<int> getColi(Rwrap::List df, int i) {
    return df[i];
}

std::vector<const char*> getCols(Rwrap::List df, int i) {
    return df[i];
}

MODULE(testshlib)
    .reg(getColi).arg("df").arg("i").auto_glue()
    .reg(getCols).arg("df").arg("i").auto_glue()
    .reg(testdf).auto_glue()
    .reg(testvoid).auto_glue()
    .reg(testlgl).arg("x").auto_glue()
    .reg(testint).arg("x").auto_glue()
    .reg(testdouble).arg("x").auto_glue()
    .reg(teststring).arg("x").auto_glue()
    .reg(testveci).arg("x").auto_glue()
    .reg(testvecd).arg("x").auto_glue()
    .reg(testvecb).arg("x").auto_glue()
    .reg(testvecs).arg("x").auto_glue()
    .reg(testvecs2).arg("x").auto_glue()
    .reg(mul).arg("x").arg("y", "50").auto_glue()
    .reg(isTrue).arg("plop").auto_glue()
    .reg(hazcheez).auto_glue()
    .reg(mydf).wrap_result("as.data.frame(", ")").auto_glue()
    .reg(sqr).arg("vec").auto_glue()
    .reg(mylist).arg("l").auto_glue()
    .reg(mul10).implicit_arg("10").arg("y").auto_glue()
    .reg(cvec).auto_glue()
    ;

<commit_msg>Cleaned up after DataFrame is a success.<commit_after>/* RWrap
 *
 * Copyright (C) 2012 Damien "bl0b" Leroux
 *
 * Permission is hereby granted, free of charge, to any person obtaining a
 * copy of this software and associated documentation files (the "Software"),
 * to deal in the Software without restriction, including without limitation
 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
 * and/or sell copies of the Software, and to permit persons to whom the
 * Software is furnished to do so, subject to the following conditions:
 *
 * The above copyright notice and this permission notice shall be included
 * in all copies or substantial portions of the Software.
 *
 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
 * DEALINGS IN THE SOFTWARE.
 */

#include <RWrap/RWrap.h>

bool testlgl(bool x) { return !x; }
int testint(int x) { return -x; }
double testdouble(double x) { return -x; }

#define REV(name, _type_) \
    _type_ test##name(_type_ x) { return _type_(x.rbegin(), x.rend()); }

REV(string, std::string);
REV(veci, std::vector<int>);
REV(vecd, std::vector<double>);
REV(vecb, std::vector<bool>);
REV(vecs, std::vector<std::string>);
REV(vecs2, std::vector<const char*>);

double mul(double x, double y) {
    return x*y;
}

bool isTrue(bool x) {
    return !!x;
}

std::vector<int> hazcheez() {
    std::vector<int> ret;
    ret.push_back(23);
    ret.push_back(42);
    return ret;
}

std::vector<int> sqr(std::vector<int> vec) {
    for (int i = 0; i < vec.size(); ++i) {
        std::cout << vec[i] << ' ';
        vec[i] *= vec[i];
        std::cout << vec[i] << std::endl;
    }
    return vec;
}

Rwrap::List mydf() {
    Rwrap::List ret;
    std::vector<int> a, b;
    a.push_back(1);
    a.push_back(3);
    a.push_back(5);
    b.push_back(10);
    b.push_back(20);
    b.push_back(30);
    ret.add("a", a);
    ret.add("b", b);
    return ret;
}

int mylist(Rwrap::List l) {
    const char* coin = l["str"];
    int plop = l["int"];
    return 42 + 100 * plop;
}

double mul10(double ten, double y) {
    return ten * y;
}

std::vector<const char*> cvec() {
    std::vector<const char*> ret;
    ret.push_back("toto");
    ret.push_back("plop");
    ret.push_back("coin");
    return ret;
}

void testvoid(const char* x) {
    std::cout << x << std::endl;
}

Rwrap::DataFrame testdf() {
    std::vector<int> a;
    std::vector<const char*> b;
    std::vector<double> c;

    a.push_back(10);
    a.push_back(11);
    a.push_back(12);

    b.push_back("pouet");
    b.push_back("plop");
    b.push_back("coin");

    c.push_back(23.0);
    c.push_back(42.0);
    c.push_back(61.06);

    Rwrap::DataFrame ret;
    ret.add("hop", a);
    ret.add("zoup", b);
    ret.add("flourp", c);
    return ret;
}

std::vector<int> getColi(Rwrap::List df, int i) {
    return df[i];
}

std::vector<const char*> getCols(Rwrap::List df, int i) {
    return df[i];
}

MODULE(testshlib)
    .reg(getColi).arg("df").arg("i").auto_glue()
    .reg(getCols).arg("df").arg("i").auto_glue()
    .reg(testdf).auto_glue()
    .reg(testvoid).auto_glue()
    .reg(testlgl).arg("x").auto_glue()
    .reg(testint).arg("x").auto_glue()
    .reg(testdouble).arg("x").auto_glue()
    .reg(teststring).arg("x").auto_glue()
    .reg(testveci).arg("x").auto_glue()
    .reg(testvecd).arg("x").auto_glue()
    .reg(testvecb).arg("x").auto_glue()
    .reg(testvecs).arg("x").auto_glue()
    .reg(testvecs2).arg("x").auto_glue()
    .reg(mul).arg("x").arg("y", "50").auto_glue()
    .reg(isTrue).arg("plop").auto_glue()
    .reg(hazcheez).auto_glue()
    .reg(mydf).wrap_result("as.data.frame(", ")").auto_glue()
    .reg(sqr).arg("vec").auto_glue()
    .reg(mylist).arg("l").auto_glue()
    .reg(mul10).implicit_arg("10").arg("y").auto_glue()
    .reg(cvec).auto_glue()
    ;

<|endoftext|>


<commit_before>/*******************************************************************************
 * thrill/api/read_binary.hpp
 *
 * Part of Project Thrill.
 *
 * Copyright (C) 2015 Alexander Noe <aleexnoe@gmail.com>
 * Copyright (C) 2015 Timo Bingmann <tb@panthema.net>
 *
 * This file has no license. Only Chuck Norris can compile it.
 ******************************************************************************/

#pragma once
#ifndef THRILL_API_READ_BINARY_HEADER
#define THRILL_API_READ_BINARY_HEADER

#include <thrill/api/dia.hpp>
#include <thrill/api/dop_node.hpp>
#include <thrill/common/item_serialization_tools.hpp>
#include <thrill/common/logger.hpp>
#include <thrill/common/math.hpp>
#include <thrill/core/file_io.hpp>
#include <thrill/data/block.hpp>
#include <thrill/data/block_reader.hpp>
#include <thrill/net/buffer_builder.hpp>

#include <algorithm>
#include <string>

namespace thrill {
namespace api {

//! \addtogroup api Interface
//! \{

/*!
 * A DIANode which performs a line-based Read operation. Read reads a file from
 * the file system and emits it as a DIA.
 */
template <typename ValueType>
class ReadBinaryNode : public DOpNode<ValueType>
{
public:
    using Super = DOpNode<ValueType>;
    using Super::context_;
    using Super::result_file_;

    using FileSizePair = std::pair<std::string, size_t>;

    static const bool debug = false;

    /*!
     * Constructor for a ReadLinesNode. Sets the Context
     * and file path.
     *
     * \param ctx Reference to Context, which holds references to data and network.
     * \param filepath Path of the input file
     */
    ReadBinaryNode(Context& ctx,
                   const std::string& filepath,
                   StatsNode* stats_node)
        : Super(ctx, { }, "ReadBinary", stats_node),
          filepath_(filepath)
    {
        filelist_ = core::GlobFileSizePrefixSum(filepath_);

        size_t my_start, my_end;
        std::tie(my_start, my_end) =
            common::CalculateLocalRange(filelist_[filelist_.size() - 1].second,
                                        context_.num_workers(),
                                        context_.my_rank());
        size_t first_file = 0;
        size_t last_file = 0;

        while (filelist_[first_file + 1].second <= my_start) {
            first_file++;
            last_file++;
        }

        while (last_file < filelist_.size() - 1 && filelist_[last_file + 1].second <= my_end) {
            last_file++;
        }

        my_files_ = std::vector<FileSizePair>(
            filelist_.begin() + first_file,
            filelist_.begin() + last_file);

        LOG << my_files_.size() << " files from " << my_start << " to " << my_end;
    }

    virtual ~ReadBinaryNode() { }

    //! Executes the read operation. Reads a file line by line
    //! and emmits it after applyung the read function.
    void Execute() final { }

    void PushData() final {
        static const bool debug = false;
        LOG << "READING data " << result_file_.ToString();

        // Hook Read
        for (const FileSizePair& file : my_files_) {
            LOG << "OPENING FILE " << file.first;
            data::BlockReader<SysFileBlockSource> br(SysFileBlockSource(file.first, context_));
            while (br.HasNext()) {
                ValueType item = br.template NextNoSelfVerify<ValueType>();
                for (auto func : Super::callbacks_) {
                    func(item);
                }
            }
        }
        LOG << "DONE!";
    }

    void Dispose() final { }

    /*!
     * Produces an 'empty' function stack, which only contains the identity
     * emitter function.
     *
     * \return Empty function stack
     */
    auto ProduceStack() {
        return FunctionStack<ValueType>();
    }

    /*!
     * Returns "[ReadLinesNode]" as a string.
     * \return "[ReadLinesNode]"
     */
    std::string ToString() final {
        return "[ReadBinaryNode] Id: " + result_file_.ToString();
    }

private:
    //! Path of the input file.
    std::string filepath_;

    std::vector<FileSizePair> filelist_;
    std::vector<FileSizePair> my_files_;

    class SysFileBlockSource
    {
    public:
        const size_t read_size = 2 * 1024 * 1024;

        SysFileBlockSource(std::string path, Context& ctx) :
            context_(ctx), c_file_(core::SysFile::OpenForRead(path)) { }

        data::Block NextBlock() {
            if (done_) return data::Block();

            data::ByteBlockPtr bytes_ = data::ByteBlock::Allocate(read_size, context_.block_pool());

            ssize_t size = c_file_.read(bytes_->data(), read_size);
            if (size > 0) {
                return data::Block(bytes_, 0, size, 0, read_size);
            }
            else if (size < 0) {
                throw common::SystemException("File reading error", errno);
            }
            else {               //size == 0 -> read finished
                c_file_.close();
                done_ = true;
                return data::Block();
            }
        }

        bool closed() const {
            return done_;
        }

    protected:
        Context& context_;
        core::SysFile c_file_;
        bool done_ = false;
    };
};

/*!
 * ReadBinary is a DOp, which reads a file written by WriteBinary from the file
 * system and  creates an ordered DIA according to a given read function.
 *
 * \param ctx Reference to the context object
 * \param filepath Path of the file in the file system
 */
template <typename ValueType>
DIARef<ValueType> ReadBinary(Context& ctx, const std::string& filepath) {

    StatsNode* stats_node =
        ctx.stats_graph().AddNode("ReadBinary", NodeType::DOP);

    auto shared_node =
        std::make_shared<ReadBinaryNode<ValueType> >(
            ctx, filepath, stats_node);

    auto read_stack = shared_node->ProduceStack();

    return DIARef<ValueType, decltype(read_stack)>(
        shared_node, read_stack, { stats_node });
}

//! \}

} // namespace api
} // namespace thrill

#endif // !THRILL_API_READ_BINARY_HEADER

/******************************************************************************/
<commit_msg>Cleaning up ReadBinaryNode.<commit_after>/*******************************************************************************
 * thrill/api/read_binary.hpp
 *
 * Part of Project Thrill.
 *
 * Copyright (C) 2015 Alexander Noe <aleexnoe@gmail.com>
 * Copyright (C) 2015 Timo Bingmann <tb@panthema.net>
 *
 * This file has no license. Only Chuck Norris can compile it.
 ******************************************************************************/

#pragma once
#ifndef THRILL_API_READ_BINARY_HEADER
#define THRILL_API_READ_BINARY_HEADER

#include <thrill/api/dia.hpp>
#include <thrill/api/dop_node.hpp>
#include <thrill/common/item_serialization_tools.hpp>
#include <thrill/common/logger.hpp>
#include <thrill/common/math.hpp>
#include <thrill/core/file_io.hpp>
#include <thrill/data/block.hpp>
#include <thrill/data/block_reader.hpp>
#include <thrill/net/buffer_builder.hpp>

#include <algorithm>
#include <string>

namespace thrill {
namespace api {

//! \addtogroup api Interface
//! \{

/*!
 * A DIANode which performs a line-based Read operation. Read reads a file from
 * the file system and emits it as a DIA.
 */
template <typename ValueType>
class ReadBinaryNode : public DOpNode<ValueType>
{
    static const bool debug = false;

public:
    using Super = DOpNode<ValueType>;
    using Super::context_;
    using Super::result_file_;

    using FileSizePair = std::pair<std::string, size_t>;

    /*!
     * Constructor for a ReadLinesNode. Sets the Context
     * and file path.
     *
     * \param ctx Reference to Context, which holds references to data and network.
     * \param filepath Path of the input file
     */
    ReadBinaryNode(Context& ctx,
                   const std::string& filepath,
                   StatsNode* stats_node)
        : Super(ctx, { }, "ReadBinary", stats_node),
          filepath_(filepath)
    {
        filelist_ = core::GlobFileSizePrefixSum(filepath_);

        size_t my_start, my_end;
        std::tie(my_start, my_end) =
            common::CalculateLocalRange(filelist_[filelist_.size() - 1].second,
                                        context_.num_workers(),
                                        context_.my_rank());
        size_t first_file = 0;
        size_t last_file = 0;

        while (filelist_[first_file + 1].second <= my_start) {
            first_file++;
            last_file++;
        }

        while (last_file < filelist_.size() - 1 && filelist_[last_file + 1].second <= my_end) {
            last_file++;
        }

        my_files_ = std::vector<FileSizePair>(
            filelist_.begin() + first_file,
            filelist_.begin() + last_file);

        LOG << my_files_.size() << " files from " << my_start << " to " << my_end;
    }

    virtual ~ReadBinaryNode() { }

    //! Executes the read operation. Reads a file line by line
    //! and emmits it after applyung the read function.
    void Execute() final { }

    void PushData() final {
        static const bool debug = false;
        LOG << "READING data " << result_file_.ToString();

        // Hook Read
        for (const FileSizePair& file : my_files_) {
            LOG << "OPENING FILE " << file.first;

            data::BlockReader<SysFileBlockSource> br(
                SysFileBlockSource(file.first, context_));

            while (br.HasNext()) {
                ValueType item = br.template NextNoSelfVerify<ValueType>();
                for (auto func : Super::callbacks_) {
                    func(item);
                }
            }
        }
        LOG << "DONE!";
    }

    void Dispose() final { }

    /*!
     * Produces an 'empty' function stack, which only contains the identity
     * emitter function.
     *
     * \return Empty function stack
     */
    auto ProduceStack() {
        return FunctionStack<ValueType>();
    }

    std::string ToString() final {
        return "[ReadBinaryNode] Id: " + result_file_.ToString();
    }

private:
    //! Path of the input file.
    std::string filepath_;

    std::vector<FileSizePair> filelist_;
    std::vector<FileSizePair> my_files_;

    class SysFileBlockSource
    {
    public:
        const size_t block_size = data::default_block_size;

        SysFileBlockSource(std::string path, Context& ctx)
            : context_(ctx),
              sysfile_(core::SysFile::OpenForRead(path)) { }

        data::Block NextBlock() {
            if (done_) return data::Block();

            data::ByteBlockPtr bytes
                = data::ByteBlock::Allocate(block_size, context_.block_pool());

            ssize_t size = sysfile_.read(bytes->data(), block_size);
            if (size > 0) {
                return data::Block(bytes, 0, size, 0, 0);
            }
            else if (size < 0) {
                throw common::SystemException("File reading error", errno);
            }
            else {
                // size == 0 -> read finished
                sysfile_.close();
                done_ = true;
                return data::Block();
            }
        }

    protected:
        Context& context_;
        core::SysFile sysfile_;
        bool done_ = false;
    };
};

/*!
 * ReadBinary is a DOp, which reads a file written by WriteBinary from the file
 * system and  creates an ordered DIA according to a given read function.
 *
 * \param ctx Reference to the context object
 * \param filepath Path of the file in the file system
 */
template <typename ValueType>
DIARef<ValueType> ReadBinary(Context& ctx, const std::string& filepath) {

    StatsNode* stats_node =
        ctx.stats_graph().AddNode("ReadBinary", NodeType::DOP);

    auto shared_node =
        std::make_shared<ReadBinaryNode<ValueType> >(
            ctx, filepath, stats_node);

    auto read_stack = shared_node->ProduceStack();

    return DIARef<ValueType, decltype(read_stack)>(
        shared_node, read_stack, { stats_node });
}

//! \}

} // namespace api
} // namespace thrill

#endif // !THRILL_API_READ_BINARY_HEADER

/******************************************************************************/
<|endoftext|>


<commit_before>/*******************************************************************************
 * thrill/mem/page_mapper.hpp
 *
 * Part of Project Thrill.
 *
 * Copyright (C) 2015 Tobias Sturm <mail@tobiassturm.de>
 *
 * This file has no license. Only Chuck Norris can compile it.
 ******************************************************************************/

#pragma once
#ifndef THRILL_MEM_PAGE_MAPPER_HEADER
#define THRILL_MEM_PAGE_MAPPER_HEADER

//aliases - MAC supports 64bit files naturally
#ifdef __APPLE__
#define lseek64 lseek
#define open64 open
#define O_LARGEFILE 0
#define MAP_POPULATE 0
#endif

#include <fcntl.h>           //open
#include <sys/mman.h>        //mappings + advice
#include <sys/stat.h>        //open
#include <sys/types.h>       //open
#include <unistd.h>          //sysconfig
#include <stdio.h>            //remove

#ifdef USE_EXPLAIN
#include <libexplain/lseek.h>
#include <libexplain/mmap.h> // explain mmap errors
#endif

#include <queue>
#include <mutex>

#include <thrill/common/concurrent_queue.hpp>
#include <thrill/common/logger.hpp>
#include <thrill/data/byte_block.hpp>

namespace thrill {
namespace mem {

/*! The PageMapper maps objects onto disk using mmap and madvise and acts as
 * a wrapper for the c syscalls.
 *
 * OBJECT_SIZE must be divideable by page size
 */
template <size_t OBJECT_SIZE>
class PageMapper
{
public:
    //! when swap file is streched, it will be streched
    //! for min(1, min_growth_delta) objects
    static const uint32_t min_growth_delta = 0;

    //! Creates a PageMapper for objects of given size.
    //! Removes and creates a temporal file (PageMapper::swap_file_name)
    //! Checks if the object size is valid
    PageMapper(std::string swap_file_name) : swap_file_name_(swap_file_name) {
        // runtime check if OBJECT_SIZE is correct
        die_unless(OBJECT_SIZE % page_size() == 0);

        // create our swap file
        //- read + write access
        //- create the file
        //- delete content if file exists
        //- this is gonna be a large file-> use 64bit ptrs
        //- don't update access time
        static const int flags = O_RDWR | O_CREAT | O_TRUNC | O_LARGEFILE | O_NOATIME;

        // user can read+write, group may read
        static const int permission = S_IRUSR | S_IWUSR | S_IRGRP;

        sLOG << "creating swapfile" << swap_file_name;
        fd_ = open64(swap_file_name.c_str(), flags, permission);
        die_unless(fd_ != -1);
    }

    //! Removes the swap file
    ~PageMapper() {
        remove(swap_file_name_.c_str());
    }

    //! Allocates a memory region of OBJECT_SIZE with a
    //! file-backing. Returns the memory address of this region and a
    //! result_token that can be used to address the memory region.
    uint8_t * Allocate(uint32_t & result_token) {
        sLOG << "allocate memory w/ disk backing";
        result_token = next_free_token();
        return SwapIn(result_token, false /*prefetch*/);
    }

    //! Releases an allocated token.
    //!
    //! Does NOT write memory content back to disk. Use PageMapper::SwapOut
    //! instead. Make sure to call this only when the matching memory region
    //! has been writen back before (PageMapper::SwapOut).
    //!
    //! \param token that was returned from Allocate before
    void ReleaseToken(uint32_t token) {
        sLOG << "free swap token" << token;
        free_tokens_.push(token);
    }

    //! Swaps out a given memory region. The memory region will be invalidated and
    //! is not accessible afterwards. Only memory regions that have been
    //! allocated via PageMapper::Allocate can be swapped out.
    //!
    //! \param addr that was returned by PageMapper::Allocate before
    //! \param write_back set to false if memory region's content may be
    //!        dismissed
    void SwapOut(uint8_t* addr, bool write_back = true) {
        // we might sometimes not write back, if we want to unmap a block but
        // don't care about the content (block for net send operation)
        if (write_back) {
            sLOG << "writing back" << static_cast<void*>(addr);
            die_unless(msync(static_cast<void*>(addr), OBJECT_SIZE, MS_SYNC) == 0);
        }
        sLOG << "unmapping " << static_cast<void*>(addr);
        die_unless(munmap(static_cast<void*>(addr), OBJECT_SIZE) == 0);
    }

    //! Swaps in a given memory region.
    //! \param token that was returned from Allocate before
    //! \returns pointer to memory region
    uint8_t * SwapIn(uint32_t token, bool prefetch = true) const {
        // Flags exaplained:
        //- readable
        //- writeable
        static const int protection_flags = PROT_READ | PROT_WRITE;
        //- not shared with other processes -> no automatic writebacks
        //- no swap space reserved (we like to live dangerously)
        // TODO we want MAP_HUGE_2MB
        int flags = MAP_SHARED | MAP_NORESERVE;

        // we don't want to prefetch when we allocate a fresh mapping
        if (prefetch)
            flags |= MAP_POPULATE;

        static void* addr_hint = nullptr; //we give no hint - kernel decides alone
        off64_t offset = token * OBJECT_SIZE;

        void* result = mmap64(addr_hint, OBJECT_SIZE, protection_flags, flags, fd_, offset);
        sLOG << "swapping in token" << token << "to address" << result << "into offset" << offset << "prefetch?" << prefetch;
        die_unless(result != MAP_FAILED);
        return static_cast<uint8_t*>(result);
    }

    //! Hint that the object at the specified memory region is likely to be
    //! accessed in sequential order
    void WillNeed(uint8_t* addr) const {
        madvise(static_cast<void*>(addr), OBJECT_SIZE, MADV_SEQUENTIAL | MADV_WILLNEED);
    }

    //! Hint that the object at the specified memory region is likely not to
    //! be used.
    void WillNotNeed(uint8_t* addr) const {
        madvise(static_cast<void*>(addr), OBJECT_SIZE, MADV_DONTNEED);
    }

    //! Returns the page size. OBJECT_SIZE must be a multiple of the page size.
    static size_t page_size() {
        return sysconf(_SC_PAGESIZE);
    }

private:
    static const bool debug = false;
    std::string swap_file_name_;
    int fd_;
    size_t next_token_ = { 0 };
    std::mutex mutex_;
    common::ConcurrentQueue<uint32_t, std::allocator<uint32_t> > free_tokens_;

    //! Returns the next free token and eventually streches the swap file if
    //! required
    uint32_t next_free_token() {
        uint32_t result = 0;
        if (free_tokens_.try_pop(result)) {
            sLOG << "reuse swap token" << result;
            return result;
        }
        std::lock_guard<std::mutex>lock(mutex_);

        // remember result
        result = next_token_;

        // +1 since result is 0-based
        size_t file_size = (1 + min_growth_delta + result) * OBJECT_SIZE;
        sLOG << "streching swap file to" << file_size;

        // seek to end - 1 of file and write one zero-byte to 'strech' file
        die_unless(lseek64(fd_, file_size - 1, SEEK_SET) != -1);
        die_unless(write(fd_, "\0", 1) == 1); //expect 1byte written

        // push remaining allocated ids into free-queue
        for (uint32_t token = result + 1; token <= next_token_ + min_growth_delta; token++) {
            sLOG << "create new swap token" << token;
            free_tokens_.push(token);
        }

        next_token_++;
        sLOG << "use swap token" << result;
        return result;
    }
};

} // namespace mem
} // namespace thrill

#endif // !THRILL_MEM_PAGE_MAPPER_HEADER

/******************************************************************************/
<commit_msg>2nd attempt to fix Apple compatibility issues<commit_after>/*******************************************************************************
 * thrill/mem/page_mapper.hpp
 *
 * Part of Project Thrill.
 *
 * Copyright (C) 2015 Tobias Sturm <mail@tobiassturm.de>
 *
 * This file has no license. Only Chuck Norris can compile it.
 ******************************************************************************/

#pragma once
#ifndef THRILL_MEM_PAGE_MAPPER_HEADER
#define THRILL_MEM_PAGE_MAPPER_HEADER

//aliases - MAC supports 64bit files naturally
#ifdef __APPLE__
#define lseek64 lseek
#define open64 open
#define mmap64 mmap
#define off64_t off_t
#define O_NOATIME 0
#define O_LARGEFILE 0
#define MAP_POPULATE 0
#endif

#include <fcntl.h>           //open
#include <sys/mman.h>        //mappings + advice
#include <sys/stat.h>        //open
#include <sys/types.h>       //open
#include <unistd.h>          //sysconfig
#include <stdio.h>            //remove

#ifdef USE_EXPLAIN
#include <libexplain/lseek.h>
#include <libexplain/mmap.h> // explain mmap errors
#endif

#include <queue>
#include <mutex>

#include <thrill/common/concurrent_queue.hpp>
#include <thrill/common/logger.hpp>
#include <thrill/data/byte_block.hpp>

namespace thrill {
namespace mem {

/*! The PageMapper maps objects onto disk using mmap and madvise and acts as
 * a wrapper for the c syscalls.
 *
 * OBJECT_SIZE must be divideable by page size
 */
template <size_t OBJECT_SIZE>
class PageMapper
{
public:
    //! when swap file is streched, it will be streched
    //! for min(1, min_growth_delta) objects
    static const uint32_t min_growth_delta = 0;

    //! Creates a PageMapper for objects of given size.
    //! Removes and creates a temporal file (PageMapper::swap_file_name)
    //! Checks if the object size is valid
    PageMapper(std::string swap_file_name) : swap_file_name_(swap_file_name) {
        // runtime check if OBJECT_SIZE is correct
        die_unless(OBJECT_SIZE % page_size() == 0);

        // create our swap file
        //- read + write access
        //- create the file
        //- delete content if file exists
        //- this is gonna be a large file-> use 64bit ptrs
        //- don't update access time
        static int flags = O_RDWR | O_CREAT | O_TRUNC | O_LARGEFILE | O_NOATIME;

        // user can read+write, group may read
        static const int permission = S_IRUSR | S_IWUSR | S_IRGRP;

        sLOG << "creating swapfile" << swap_file_name;
        fd_ = open64(swap_file_name.c_str(), flags, permission);
        die_unless(fd_ != -1);
    }

    //! Removes the swap file
    ~PageMapper() {
        remove(swap_file_name_.c_str());
    }

    //! Allocates a memory region of OBJECT_SIZE with a
    //! file-backing. Returns the memory address of this region and a
    //! result_token that can be used to address the memory region.
    uint8_t * Allocate(uint32_t & result_token) {
        sLOG << "allocate memory w/ disk backing";
        result_token = next_free_token();
        return SwapIn(result_token, false /*prefetch*/);
    }

    //! Releases an allocated token.
    //!
    //! Does NOT write memory content back to disk. Use PageMapper::SwapOut
    //! instead. Make sure to call this only when the matching memory region
    //! has been writen back before (PageMapper::SwapOut).
    //!
    //! \param token that was returned from Allocate before
    void ReleaseToken(uint32_t token) {
        sLOG << "free swap token" << token;
        free_tokens_.push(token);
    }

    //! Swaps out a given memory region. The memory region will be invalidated and
    //! is not accessible afterwards. Only memory regions that have been
    //! allocated via PageMapper::Allocate can be swapped out.
    //!
    //! \param addr that was returned by PageMapper::Allocate before
    //! \param write_back set to false if memory region's content may be
    //!        dismissed
    void SwapOut(uint8_t* addr, bool write_back = true) {
        // we might sometimes not write back, if we want to unmap a block but
        // don't care about the content (block for net send operation)
        if (write_back) {
            sLOG << "writing back" << static_cast<void*>(addr);
            die_unless(msync(static_cast<void*>(addr), OBJECT_SIZE, MS_SYNC) == 0);
        }
        sLOG << "unmapping " << static_cast<void*>(addr);
        die_unless(munmap(static_cast<void*>(addr), OBJECT_SIZE) == 0);
    }

    //! Swaps in a given memory region.
    //! \param token that was returned from Allocate before
    //! \returns pointer to memory region
    uint8_t * SwapIn(uint32_t token, bool prefetch = true) const {
        // Flags exaplained:
        //- readable
        //- writeable
        static const int protection_flags = PROT_READ | PROT_WRITE;
        //- not shared with other processes -> no automatic writebacks
        //- no swap space reserved (we like to live dangerously)
        // TODO we want MAP_HUGE_2MB
        int flags = MAP_SHARED | MAP_NORESERVE;

        // we don't want to prefetch when we allocate a fresh mapping
        if (prefetch)
            flags |= MAP_POPULATE;

        static void* addr_hint = nullptr; //we give no hint - kernel decides alone
        off64_t offset = token * OBJECT_SIZE;

        void* result = mmap64(addr_hint, OBJECT_SIZE, protection_flags, flags, fd_, offset);
        sLOG << "swapping in token" << token << "to address" << result << "into offset" << offset << "prefetch?" << prefetch;
        die_unless(result != MAP_FAILED);
        return static_cast<uint8_t*>(result);
    }

    //! Hint that the object at the specified memory region is likely to be
    //! accessed in sequential order
    void WillNeed(uint8_t* addr) const {
        madvise(static_cast<void*>(addr), OBJECT_SIZE, MADV_SEQUENTIAL | MADV_WILLNEED);
    }

    //! Hint that the object at the specified memory region is likely not to
    //! be used.
    void WillNotNeed(uint8_t* addr) const {
        madvise(static_cast<void*>(addr), OBJECT_SIZE, MADV_DONTNEED);
    }

    //! Returns the page size. OBJECT_SIZE must be a multiple of the page size.
    static size_t page_size() {
        return sysconf(_SC_PAGESIZE);
    }

private:
    static const bool debug = false;
    std::string swap_file_name_;
    int fd_;
    size_t next_token_ = { 0 };
    std::mutex mutex_;
    common::ConcurrentQueue<uint32_t, std::allocator<uint32_t> > free_tokens_;

    //! Returns the next free token and eventually streches the swap file if
    //! required
    uint32_t next_free_token() {
        uint32_t result = 0;
        if (free_tokens_.try_pop(result)) {
            sLOG << "reuse swap token" << result;
            return result;
        }
        std::lock_guard<std::mutex>lock(mutex_);

        // remember result
        result = next_token_;

        // +1 since result is 0-based
        size_t file_size = (1 + min_growth_delta + result) * OBJECT_SIZE;
        sLOG << "streching swap file to" << file_size;

        // seek to end - 1 of file and write one zero-byte to 'strech' file
        die_unless(lseek64(fd_, file_size - 1, SEEK_SET) != -1);
        die_unless(write(fd_, "\0", 1) == 1); //expect 1byte written

        // push remaining allocated ids into free-queue
        for (uint32_t token = result + 1; token <= next_token_ + min_growth_delta; token++) {
            sLOG << "create new swap token" << token;
            free_tokens_.push(token);
        }

        next_token_++;
        sLOG << "use swap token" << result;
        return result;
    }
};

} // namespace mem
} // namespace thrill

#endif // !THRILL_MEM_PAGE_MAPPER_HEADER

/******************************************************************************/
<|endoftext|>