<commit_before>// Copyright (c) 2020 ASMlover. All rights reserved.
//
// Redistribution and use in source and binary forms, with or without
// modification, are permitted provided that the following conditions
// are met:
//
//  * Redistributions of source code must retain the above copyright
//    notice, this list ofconditions and the following disclaimer.
//
//  * Redistributions in binary form must reproduce the above copyright
//    notice, this list of conditions and the following disclaimer in
//    the documentation and/or other materialsprovided with the
//    distribution.
//
// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS
// FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE
// COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT,
// INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING,
// BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
// LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
// CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
// LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN
// ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
// POSSIBILITY OF SUCH DAMAGE.
#include <cstdarg>
#include <algorithm>
#include <iostream>
#include "chunk.hh"
#include "vm.hh"

namespace tadpole {

class CallFrame final : public Copyable {
  ClosureObject* closure_{};
  const u8_t* ip_{};
  sz_t stack_begpos_{};
};

VM::VM() noexcept {}
VM::~VM() {}
void VM::define_native(const str_t& name, TadpoleCFun&& fn) {}

void VM::append_object(BaseObject* o) {
  if (all_objects_.size() >= kGCThreshold)
    collect();
  all_objects_.push_back(o);
}

void VM::mark_object(BaseObject* o) {
  if (o == nullptr || o->is_marked())
    return;

#if defined(_TADPOLE_DEBUG_GC)
  std::cout << "[" << o << "] mark object: `" << o->stringify() << "`" << std::endl;
#endif

  o->set_marked(true);
  worklist_.push_back(o);
}

void VM::mark_value(const Value& v) {
  if (v.is_object())
    mark_object(v.as_object());
}

void VM::collect() {}
void VM::reclaim_object(BaseObject* o) {}

}
<commit_msg>:construction: chore(call-frame): finished the implemtation of call frame<commit_after>// Copyright (c) 2020 ASMlover. All rights reserved.
//
// Redistribution and use in source and binary forms, with or without
// modification, are permitted provided that the following conditions
// are met:
//
//  * Redistributions of source code must retain the above copyright
//    notice, this list ofconditions and the following disclaimer.
//
//  * Redistributions in binary form must reproduce the above copyright
//    notice, this list of conditions and the following disclaimer in
//    the documentation and/or other materialsprovided with the
//    distribution.
//
// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS
// FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE
// COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT,
// INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING,
// BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
// LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
// CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
// LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN
// ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
// POSSIBILITY OF SUCH DAMAGE.
#include <cstdarg>
#include <algorithm>
#include <iostream>
#include "chunk.hh"
#include "function_object.hh"
#include "closure_object.hh"
#include "vm.hh"

namespace tadpole {

class CallFrame final : public Copyable {
  ClosureObject* closure_{};
  const u8_t* ip_{};
  sz_t stack_begpos_{};
public:
  CallFrame() noexcept {}
  CallFrame(ClosureObject* c, const u8_t* i, sz_t begpos = 0) noexcept
    : closure_(c), ip_(i), stack_begpos_(begpos) {
  }

  inline ClosureObject* closure() const noexcept { return closure_; }
  inline FunctionObject* frame_fn() const noexcept { return closure_->fn(); }
  inline Chunk* frame_chunk() const noexcept { return frame_fn()->chunk(); }
  inline const u8_t* ip() const noexcept { return ip_; }
  inline u8_t get_ip(sz_t i) const noexcept { return ip_[i]; }
  inline u8_t inc_ip() noexcept { return *ip_++; }
  inline u8_t dec_ip() noexcept { return *ip_--; }
  inline sz_t stack_begpos() const noexcept { return stack_begpos_; }
};

VM::VM() noexcept {}
VM::~VM() {}
void VM::define_native(const str_t& name, TadpoleCFun&& fn) {}

void VM::append_object(BaseObject* o) {
  if (all_objects_.size() >= kGCThreshold)
    collect();
  all_objects_.push_back(o);
}

void VM::mark_object(BaseObject* o) {
  if (o == nullptr || o->is_marked())
    return;

#if defined(_TADPOLE_DEBUG_GC)
  std::cout << "[" << o << "] mark object: `" << o->stringify() << "`" << std::endl;
#endif

  o->set_marked(true);
  worklist_.push_back(o);
}

void VM::mark_value(const Value& v) {
  if (v.is_object())
    mark_object(v.as_object());
}

void VM::collect() {}
void VM::reclaim_object(BaseObject* o) {}

}
<|endoftext|>


<commit_before>#include "utest.h"
#include "thread/loopi.h"
#include "math/epsilon.h"
#include <numeric>

namespace
{
        // single-threaded
        template <typename tscalar, typename toperator>
        tscalar test_st(const size_t size, const toperator op)
        {
                std::vector<tscalar> results(size);
                for (size_t i = 0; i < results.size(); ++ i)
                {
                        results[i] = op(i);
                }

                return std::accumulate(results.begin(), results.end(), tscalar(0));
        }

        // multi-threaded
        template <typename tscalar, typename toperator>
        tscalar test_mt(const size_t size, const size_t chunk, const toperator op)
        {
                std::vector<tscalar> results(size);
                nano::loopi(size, chunk, [&results = results, size = size, op = op] (const size_t begin, const size_t end)
                {
                        assert(begin < end);
                        assert(0u <= begin);
                        assert(end <= size);
                        NANO_UNUSED1_RELEASE(size);
                        for (size_t i = begin; i < end; ++ i)
                        {
                                results[i] = op(i);
                        }
                });

                return std::accumulate(results.begin(), results.end(), tscalar(0));
        }
}

NANO_BEGIN_MODULE(test_thread_loop)

NANO_CASE(evaluate)
{
        const size_t min_size = 7;
        const size_t max_size = 3 * 3 * 7;

        using scalar_t = double;

        // operator to test
        const auto op = [](const size_t i)
        {
                const auto ii = static_cast<scalar_t>(i);
                return ii * ii + 1 - ii;
        };

        // test for different problems size
        for (size_t size = min_size; size <= max_size; size *= 3)
        {
                // single-threaded
                const auto st = test_st<scalar_t>(size, op);

                // multi-threaded
                for (size_t nthreads = 1; nthreads <= nano::logical_cpus(); nthreads += 2)
                {
                        nano::thread_pool_t::instance().activate(nthreads);

                        for (size_t chunk = 1; chunk < 8; ++ chunk)
                        {
                                const auto mt = test_mt<scalar_t>(size, chunk, op);
                                NANO_CHECK_CLOSE(st, mt, nano::epsilon1<scalar_t>());
                        }
                }
        }
}

NANO_END_MODULE()
<commit_msg>extend unit test<commit_after>#include "utest.h"
#include "thread/loopi.h"
#include "math/epsilon.h"
#include <numeric>

namespace
{
        // single-threaded
        template <typename tscalar, typename toperator>
        tscalar test_st(const size_t size, const toperator op)
        {
                std::vector<tscalar> results(size);
                for (size_t i = 0; i < results.size(); ++ i)
                {
                        results[i] = op(i);
                }

                return std::accumulate(results.begin(), results.end(), tscalar(0));
        }

        // multi-threaded
        template <typename tscalar, typename toperator>
        tscalar test_mt(const size_t size, const size_t chunk, const toperator op)
        {
                std::vector<tscalar> results(size);
                nano::loopi(size, chunk, [&results = results, size = size, op = op] (const size_t begin, const size_t end)
                {
                        assert(begin < end);
                        assert(0u <= begin);
                        assert(end <= size);
                        NANO_UNUSED1_RELEASE(size);
                        for (size_t i = begin; i < end; ++ i)
                        {
                                results[i] = op(i);
                        }
                });

                return std::accumulate(results.begin(), results.end(), tscalar(0));
        }
}

NANO_BEGIN_MODULE(test_thread_loop)

NANO_CASE(evaluate)
{
        const size_t min_size = 1;
        const size_t max_size = 9 * 9 * 9 * 1;

        using scalar_t = double;

        // operator to test
        const auto op = [](const size_t i)
        {
                const auto ii = static_cast<scalar_t>(i);
                return ii * ii + 1 - ii;
        };

        // test for different problems size
        for (size_t size = min_size; size <= max_size; size *= 3)
        {
                // single-threaded
                const auto st = test_st<scalar_t>(size, op);

                // multi-threaded
                for (size_t nthreads = 1; nthreads <= nano::logical_cpus(); nthreads += 2)
                {
                        nano::thread_pool_t::instance().activate(nthreads);

                        for (size_t chunk = 1; chunk < 8; ++ chunk)
                        {
                                const auto mt = test_mt<scalar_t>(size, chunk, op);
                                NANO_CHECK_CLOSE(st, mt, nano::epsilon1<scalar_t>());
                        }
                }
        }
}

NANO_END_MODULE()
<|endoftext|>


<commit_before>#include "vlc_video_source.h"
#include <cstdio>
#include <cstdlib>
#include <chrono>
#include <thread>

namespace gg
{

VideoSourceVLC::VideoSourceVLC(const std::string path)
    : _vlc_inst(nullptr)
    , _vlc_mp(nullptr)
    , _video_buffer(nullptr)
    , _data_length(0)
    , _cols(0)
    , _rows(0)
    , _sub(nullptr)
    , _path(path)
{
    this->init_vlc();
    this->run_vlc();
}

VideoSourceVLC::~VideoSourceVLC()
{
    stop_vlc();
    release_vlc();
}

bool VideoSourceVLC::get_frame_dimensions(int & width, int & height)
{
    ///\todo mutex

    if(this->_cols == 0 || this->_rows == 0)
        return false;

    width = this->_cols;
    height = this->_rows;
    return true;
}

bool VideoSourceVLC::get_frame(VideoFrame_BGRA & frame)
{
    throw VideoSourceError("VLC video source supports only I420 colour space");

    // TODO

    ///\todo mutex

    if (_cols == 0 || _rows == 0 || _video_buffer == nullptr)
        return false;

    // allocate and fill the image
    if (_cols * _rows * 4 == this->_data_length)
        frame = VideoFrame_BGRA(this->_video_buffer, this->_cols, this->_rows);
    else
        throw VideoSourceError("VLC video source does not support padded images");

    return true;
}

bool VideoSourceVLC::get_frame(VideoFrame_I420 & frame)
{
    if (_data_length > 0)
    {
        // TODO manage data?
        frame = VideoFrame_I420(_video_buffer, _data_length, _cols, _rows, false);
        return true;
    }
    else
        return false;

    // TODO #86
}

double VideoSourceVLC::get_frame_rate()
{
    //return libvlc_media_player_get_rate( m_mp );
    return libvlc_media_player_get_fps(_vlc_mp);
    //throw std::runtime_error("get_frame_rate to be implemented");
    //return 0.0;
}

void VideoSourceVLC::set_sub_frame(int x, int y, int width, int height)
{
    // TODO mutex?

    if (x >= _full.x and x + width <= _full.x + _full.width and
        y >= _full.y and y + height <= _full.y + _full.height)
    {
        stop_vlc();
        release_vlc();
        if (_sub == nullptr) _sub = new FrameBox;
        _sub->x = x;
        _sub->y = y;
        _sub->width = width;
        _sub->height = height;
        init_vlc();
        run_vlc();
    }
}

void VideoSourceVLC::get_full_frame()
{
    // TODO mutex?
    stop_vlc();
    release_vlc();
    if (_sub)
    {
        delete _sub;
        _sub = nullptr;
    }
    init_vlc();
    run_vlc();
}

void VideoSourceVLC::init_vlc()
{
    // VLC pointers
    libvlc_media_t * vlc_media = nullptr;

    // VLC options
    char smem_options[512];

    sprintf(smem_options, "#");
    if (_sub != nullptr)
    {
        unsigned int croptop = _sub->y,
                     cropbottom = _full.height - (_sub->y + _sub->height),
                     cropleft = _sub->x,
                     cropright = _full.width - (_sub->x + _sub->width);
        sprintf(smem_options,
                "%stranscode{vcodec=I420,vfilter=croppadd{",
                smem_options);
        sprintf(smem_options,
                "%scroptop=%u,cropbottom=%u,cropleft=%u,cropright=%u}}:",
                smem_options,
                croptop, cropbottom, cropleft, cropright);
    }
    sprintf(smem_options,
            "%ssmem{video-data=%lld,video-prerender-callback=%lld,video-postrender-callback=%lld}",
            smem_options,
            (long long int)(intptr_t)(void*) this,
            (long long int)(intptr_t)(void*) &VideoSourceVLC::prepareRender,
            (long long int)(intptr_t)(void*) &VideoSourceVLC::handleStream );

    const char * const vlc_args[] = {
        "-I", "dummy", // Don't use any interface
        "--ignore-config", // Don't use VLC's config
        "--extraintf=logger", // Log anything
        // TODO - what about the options below?
        //"--verbose=2", // Be much more verbose then normal for debugging purpose
        //"--clock-jitter=0",
        //"--file-caching=150",
        "--no-audio",
        "--sout", smem_options // Stream to memory
    };

    // We launch VLC
    _vlc_inst = libvlc_new(sizeof(vlc_args) / sizeof(vlc_args[0]), vlc_args);
    if (_vlc_inst == nullptr)
        throw VideoSourceError("Could not create VLC engine");

    // If path contains a colon (:), it will be treated as a
    // URL. Else, it will be considered as a local path.
    if (_path.find(":") == std::string::npos)
        vlc_media = libvlc_media_new_path(_vlc_inst, _path.c_str());
    else
        vlc_media = libvlc_media_new_location(_vlc_inst, _path.c_str());
    if (vlc_media == nullptr)
        throw VideoSourceError(std::string("Could not open ").append(_path));

    libvlc_media_add_option(vlc_media, ":noaudio");
    libvlc_media_add_option(vlc_media, ":no-video-title-show");

    // Create a media player playing environement
    _vlc_mp = libvlc_media_player_new_from_media(vlc_media);
    if (_vlc_mp == nullptr)
        throw VideoSourceError("Could not create VLC media player");

    // No need to keep the media now
    libvlc_media_release( vlc_media );
}

void VideoSourceVLC::run_vlc()
{
    // play the media_player
    if (libvlc_media_player_play(_vlc_mp) != 0)
        throw VideoSourceError("Could not start VLC media player");

    // empirically determined value that allows for initialisation
    // to succeed before any API functions are called on this object
    std::this_thread::sleep_for(std::chrono::milliseconds(350));
    unsigned int width, height;
    if (libvlc_video_get_size(_vlc_mp, 0, &width, &height) != 0)
        throw VideoSourceError("Could not get video dimensions");

    _full.x = 0;
    _full.y = 0;
    _full.width = width;
    _full.height = height;
}

void VideoSourceVLC::stop_vlc()
{
    if (libvlc_media_player_is_playing(_vlc_mp) == 1)
        // stop playing
        libvlc_media_player_stop(_vlc_mp);
}

void VideoSourceVLC::release_vlc()
{
    // free media player
    if (_vlc_mp)
        libvlc_media_player_release(_vlc_mp);

    // free engine
    if (_vlc_inst)
        libvlc_release(_vlc_inst);

    // free buffer
    if (_video_buffer != nullptr)
        delete[] _video_buffer;
    _data_length = 0;
    _cols = 0;
    _rows = 0;

    // free sub-frame
    if (_sub != nullptr)
    {
        delete _sub;
        _sub = nullptr;
    }

    // delete VLC pointers
    if (_vlc_mp)
    {
        delete _vlc_mp;
        _vlc_mp = nullptr;
    }
    if (_vlc_inst)
    {
        delete _vlc_inst;
        _vlc_inst = nullptr;
    }
}

void VideoSourceVLC::prepareRender(VideoSourceVLC * p_video_data,
                                   uint8_t ** pp_pixel_buffer,
                                   size_t size)
{
    ///\todo create mutex guard

    unsigned int width, height;
    if (libvlc_video_get_size(p_video_data->_vlc_mp, 0, &width, &height) != 0)
        return;
    p_video_data->_cols = width;
    p_video_data->_rows = height;

    if (size > p_video_data->_data_length)
        p_video_data->_video_buffer = reinterpret_cast<uint8_t *>(
                    realloc(p_video_data->_video_buffer, size * sizeof(uint8_t))
                    );
    p_video_data->_data_length = size;

    *pp_pixel_buffer = p_video_data->_video_buffer;
}

void VideoSourceVLC::handleStream(VideoSourceVLC * p_video_data,
                                  uint8_t * p_pixel_buffer,
                                  size_t cols,
                                  size_t rows,
                                  size_t colour_depth,
                                  size_t size)
{
    // TODO: explain how data should be handled (see #86)
    // TODO: Unlock the mutex
}

std::string VideoSourceVLC::encode_psz_geometry(int x, int y, int width, int height)
{
    std::string psz_geometry;
    psz_geometry.append(std::to_string(width)).append("x")
                .append(std::to_string(height))
                .append("+").append(std::to_string(x))
                .append("+").append(std::to_string(y));
    return psz_geometry;
}

}
<commit_msg>Issue #83: removed delete'ing of VLC pointers<commit_after>#include "vlc_video_source.h"
#include <cstdio>
#include <cstdlib>
#include <chrono>
#include <thread>

namespace gg
{

VideoSourceVLC::VideoSourceVLC(const std::string path)
    : _vlc_inst(nullptr)
    , _vlc_mp(nullptr)
    , _video_buffer(nullptr)
    , _data_length(0)
    , _cols(0)
    , _rows(0)
    , _sub(nullptr)
    , _path(path)
{
    this->init_vlc();
    this->run_vlc();
}

VideoSourceVLC::~VideoSourceVLC()
{
    stop_vlc();
    release_vlc();
}

bool VideoSourceVLC::get_frame_dimensions(int & width, int & height)
{
    ///\todo mutex

    if(this->_cols == 0 || this->_rows == 0)
        return false;

    width = this->_cols;
    height = this->_rows;
    return true;
}

bool VideoSourceVLC::get_frame(VideoFrame_BGRA & frame)
{
    throw VideoSourceError("VLC video source supports only I420 colour space");

    // TODO

    ///\todo mutex

    if (_cols == 0 || _rows == 0 || _video_buffer == nullptr)
        return false;

    // allocate and fill the image
    if (_cols * _rows * 4 == this->_data_length)
        frame = VideoFrame_BGRA(this->_video_buffer, this->_cols, this->_rows);
    else
        throw VideoSourceError("VLC video source does not support padded images");

    return true;
}

bool VideoSourceVLC::get_frame(VideoFrame_I420 & frame)
{
    if (_data_length > 0)
    {
        // TODO manage data?
        frame = VideoFrame_I420(_video_buffer, _data_length, _cols, _rows, false);
        return true;
    }
    else
        return false;

    // TODO #86
}

double VideoSourceVLC::get_frame_rate()
{
    //return libvlc_media_player_get_rate( m_mp );
    return libvlc_media_player_get_fps(_vlc_mp);
    //throw std::runtime_error("get_frame_rate to be implemented");
    //return 0.0;
}

void VideoSourceVLC::set_sub_frame(int x, int y, int width, int height)
{
    // TODO mutex?

    if (x >= _full.x and x + width <= _full.x + _full.width and
        y >= _full.y and y + height <= _full.y + _full.height)
    {
        stop_vlc();
        release_vlc();
        if (_sub == nullptr) _sub = new FrameBox;
        _sub->x = x;
        _sub->y = y;
        _sub->width = width;
        _sub->height = height;
        init_vlc();
        run_vlc();
    }
}

void VideoSourceVLC::get_full_frame()
{
    // TODO mutex?
    stop_vlc();
    release_vlc();
    if (_sub)
    {
        delete _sub;
        _sub = nullptr;
    }
    init_vlc();
    run_vlc();
}

void VideoSourceVLC::init_vlc()
{
    // VLC pointers
    libvlc_media_t * vlc_media = nullptr;

    // VLC options
    char smem_options[512];

    sprintf(smem_options, "#");
    if (_sub != nullptr)
    {
        unsigned int croptop = _sub->y,
                     cropbottom = _full.height - (_sub->y + _sub->height),
                     cropleft = _sub->x,
                     cropright = _full.width - (_sub->x + _sub->width);
        sprintf(smem_options,
                "%stranscode{vcodec=I420,vfilter=croppadd{",
                smem_options);
        sprintf(smem_options,
                "%scroptop=%u,cropbottom=%u,cropleft=%u,cropright=%u}}:",
                smem_options,
                croptop, cropbottom, cropleft, cropright);
    }
    sprintf(smem_options,
            "%ssmem{video-data=%lld,video-prerender-callback=%lld,video-postrender-callback=%lld}",
            smem_options,
            (long long int)(intptr_t)(void*) this,
            (long long int)(intptr_t)(void*) &VideoSourceVLC::prepareRender,
            (long long int)(intptr_t)(void*) &VideoSourceVLC::handleStream );

    const char * const vlc_args[] = {
        "-I", "dummy", // Don't use any interface
        "--ignore-config", // Don't use VLC's config
        "--extraintf=logger", // Log anything
        // TODO - what about the options below?
        //"--verbose=2", // Be much more verbose then normal for debugging purpose
        //"--clock-jitter=0",
        //"--file-caching=150",
        "--no-audio",
        "--sout", smem_options // Stream to memory
    };

    // We launch VLC
    _vlc_inst = libvlc_new(sizeof(vlc_args) / sizeof(vlc_args[0]), vlc_args);
    if (_vlc_inst == nullptr)
        throw VideoSourceError("Could not create VLC engine");

    // If path contains a colon (:), it will be treated as a
    // URL. Else, it will be considered as a local path.
    if (_path.find(":") == std::string::npos)
        vlc_media = libvlc_media_new_path(_vlc_inst, _path.c_str());
    else
        vlc_media = libvlc_media_new_location(_vlc_inst, _path.c_str());
    if (vlc_media == nullptr)
        throw VideoSourceError(std::string("Could not open ").append(_path));

    libvlc_media_add_option(vlc_media, ":noaudio");
    libvlc_media_add_option(vlc_media, ":no-video-title-show");

    // Create a media player playing environement
    _vlc_mp = libvlc_media_player_new_from_media(vlc_media);
    if (_vlc_mp == nullptr)
        throw VideoSourceError("Could not create VLC media player");

    // No need to keep the media now
    libvlc_media_release( vlc_media );
}

void VideoSourceVLC::run_vlc()
{
    // play the media_player
    if (libvlc_media_player_play(_vlc_mp) != 0)
        throw VideoSourceError("Could not start VLC media player");

    // empirically determined value that allows for initialisation
    // to succeed before any API functions are called on this object
    std::this_thread::sleep_for(std::chrono::milliseconds(350));
    unsigned int width, height;
    if (libvlc_video_get_size(_vlc_mp, 0, &width, &height) != 0)
        throw VideoSourceError("Could not get video dimensions");

    _full.x = 0;
    _full.y = 0;
    _full.width = width;
    _full.height = height;
}

void VideoSourceVLC::stop_vlc()
{
    if (libvlc_media_player_is_playing(_vlc_mp) == 1)
        // stop playing
        libvlc_media_player_stop(_vlc_mp);
}

void VideoSourceVLC::release_vlc()
{
    // free media player
    if (_vlc_mp)
        libvlc_media_player_release(_vlc_mp);

    // free engine
    if (_vlc_inst)
        libvlc_release(_vlc_inst);

    // free buffer
    if (_video_buffer != nullptr)
        delete[] _video_buffer;
    _data_length = 0;
    _cols = 0;
    _rows = 0;

    // free sub-frame
    if (_sub != nullptr)
    {
        delete _sub;
        _sub = nullptr;
    }
}

void VideoSourceVLC::prepareRender(VideoSourceVLC * p_video_data,
                                   uint8_t ** pp_pixel_buffer,
                                   size_t size)
{
    ///\todo create mutex guard

    unsigned int width, height;
    if (libvlc_video_get_size(p_video_data->_vlc_mp, 0, &width, &height) != 0)
        return;
    p_video_data->_cols = width;
    p_video_data->_rows = height;

    if (size > p_video_data->_data_length)
        p_video_data->_video_buffer = reinterpret_cast<uint8_t *>(
                    realloc(p_video_data->_video_buffer, size * sizeof(uint8_t))
                    );
    p_video_data->_data_length = size;

    *pp_pixel_buffer = p_video_data->_video_buffer;
}

void VideoSourceVLC::handleStream(VideoSourceVLC * p_video_data,
                                  uint8_t * p_pixel_buffer,
                                  size_t cols,
                                  size_t rows,
                                  size_t colour_depth,
                                  size_t size)
{
    // TODO: explain how data should be handled (see #86)
    // TODO: Unlock the mutex
}

std::string VideoSourceVLC::encode_psz_geometry(int x, int y, int width, int height)
{
    std::string psz_geometry;
    psz_geometry.append(std::to_string(width)).append("x")
                .append(std::to_string(height))
                .append("+").append(std::to_string(x))
                .append("+").append(std::to_string(y));
    return psz_geometry;
}

}
<|endoftext|>


<commit_before>// Copyright (c) 2017 The Bitcoin Core developers
// Distributed under the MIT software license, see the accompanying
// file COPYING or http://www.opensource.org/licenses/mit-license.php.

#include <wallet/coinselection.h>
#include <util.h>
#include <utilmoneystr.h>

// Descending order comparator
struct {
    bool operator()(const OutputGroup& a, const OutputGroup& b) const
    {
        return a.effective_value > b.effective_value;
    }
} descending;

/*
 * This is the Branch and Bound Coin Selection algorithm designed by Murch. It searches for an input
 * set that can pay for the spending target and does not exceed the spending target by more than the
 * cost of creating and spending a change output. The algorithm uses a depth-first search on a binary
 * tree. In the binary tree, each node corresponds to the inclusion or the omission of a UTXO. UTXOs
 * are sorted by their effective values and the trees is explored deterministically per the inclusion
 * branch first. At each node, the algorithm checks whether the selection is within the target range.
 * While the selection has not reached the target range, more UTXOs are included. When a selection's
 * value exceeds the target range, the complete subtree deriving from this selection can be omitted.
 * At that point, the last included UTXO is deselected and the corresponding omission branch explored
 * instead. The search ends after the complete tree has been searched or after a limited number of tries.
 *
 * The search continues to search for better solutions after one solution has been found. The best
 * solution is chosen by minimizing the waste metric. The waste metric is defined as the cost to
 * spend the current inputs at the given fee rate minus the long term expected cost to spend the
 * inputs, plus the amount the selection exceeds the spending target:
 *
 * waste = selectionTotal - target + inputs × (currentFeeRate - longTermFeeRate)
 *
 * The algorithm uses two additional optimizations. A lookahead keeps track of the total value of
 * the unexplored UTXOs. A subtree is not explored if the lookahead indicates that the target range
 * cannot be reached. Further, it is unnecessary to test equivalent combinations. This allows us
 * to skip testing the inclusion of UTXOs that match the effective value and waste of an omitted
 * predecessor.
 *
 * The Branch and Bound algorithm is described in detail in Murch's Master Thesis:
 * https://murch.one/wp-content/uploads/2016/11/erhardt2016coinselection.pdf
 *
 * @param const std::vector<CInputCoin>& utxo_pool The set of UTXOs that we are choosing from.
 *        These UTXOs will be sorted in descending order by effective value and the CInputCoins'
 *        values are their effective values.
 * @param const CAmount& target_value This is the value that we want to select. It is the lower
 *        bound of the range.
 * @param const CAmount& cost_of_change This is the cost of creating and spending a change output.
 *        This plus target_value is the upper bound of the range.
 * @param std::set<CInputCoin>& out_set -> This is an output parameter for the set of CInputCoins
 *        that have been selected.
 * @param CAmount& value_ret -> This is an output parameter for the total value of the CInputCoins
 *        that were selected.
 * @param CAmount not_input_fees -> The fees that need to be paid for the outputs and fixed size
 *        overhead (version, locktime, marker and flag)
 */

static const size_t TOTAL_TRIES = 100000;

bool SelectCoinsBnB(std::vector<OutputGroup>& utxo_pool, const CAmount& target_value, const CAmount& cost_of_change, std::set<CInputCoin>& out_set, CAmount& value_ret, CAmount not_input_fees)
{
    out_set.clear();
    CAmount curr_value = 0;

    std::vector<bool> curr_selection; // select the utxo at this index
    curr_selection.reserve(utxo_pool.size());
    CAmount actual_target = not_input_fees + target_value;

    // Calculate curr_available_value
    CAmount curr_available_value = 0;
    for (const OutputGroup& utxo : utxo_pool) {
        // Assert that this utxo is not negative. It should never be negative, effective value calculation should have removed it
        assert(utxo.effective_value > 0);
        curr_available_value += utxo.effective_value;
    }
    if (curr_available_value < actual_target) {
        return false;
    }

    // Sort the utxo_pool
    std::sort(utxo_pool.begin(), utxo_pool.end(), descending);

    CAmount curr_waste = 0;
    std::vector<bool> best_selection;
    CAmount best_waste = MAX_MONEY;

    // Depth First search loop for choosing the UTXOs
    for (size_t i = 0; i < TOTAL_TRIES; ++i) {
        // Conditions for starting a backtrack
        bool backtrack = false;
        if (curr_value + curr_available_value < actual_target ||                // Cannot possibly reach target with the amount remaining in the curr_available_value.
            curr_value > actual_target + cost_of_change ||    // Selected value is out of range, go back and try other branch
            (curr_waste > best_waste && (utxo_pool.at(0).fee - utxo_pool.at(0).long_term_fee) > 0)) { // Don't select things which we know will be more wasteful if the waste is increasing
            backtrack = true;
        } else if (curr_value >= actual_target) {       // Selected value is within range
            curr_waste += (curr_value - actual_target); // This is the excess value which is added to the waste for the below comparison
            // Adding another UTXO after this check could bring the waste down if the long term fee is higher than the current fee.
            // However we are not going to explore that because this optimization for the waste is only done when we have hit our target
            // value. Adding any more UTXOs will be just burning the UTXO; it will go entirely to fees. Thus we aren't going to
            // explore any more UTXOs to avoid burning money like that.
            if (curr_waste <= best_waste) {
                best_selection = curr_selection;
                best_selection.resize(utxo_pool.size());
                best_waste = curr_waste;
            }
            curr_waste -= (curr_value - actual_target); // Remove the excess value as we will be selecting different coins now
            backtrack = true;
        }

        // Backtracking, moving backwards
        if (backtrack) {
            // Walk backwards to find the last included UTXO that still needs to have its omission branch traversed.
            while (!curr_selection.empty() && !curr_selection.back()) {
                curr_selection.pop_back();
                curr_available_value += utxo_pool.at(curr_selection.size()).effective_value;
            }

            if (curr_selection.empty()) { // We have walked back to the first utxo and no branch is untraversed. All solutions searched
                break;
            }

            // Output was included on previous iterations, try excluding now.
            curr_selection.back() = false;
            OutputGroup& utxo = utxo_pool.at(curr_selection.size() - 1);
            curr_value -= utxo.effective_value;
            curr_waste -= utxo.fee - utxo.long_term_fee;
        } else { // Moving forwards, continuing down this branch
            OutputGroup& utxo = utxo_pool.at(curr_selection.size());

            // Remove this utxo from the curr_available_value utxo amount
            curr_available_value -= utxo.effective_value;

            // Avoid searching a branch if the previous UTXO has the same value and same waste and was excluded. Since the ratio of fee to
            // long term fee is the same, we only need to check if one of those values match in order to know that the waste is the same.
            if (!curr_selection.empty() && !curr_selection.back() &&
                utxo.effective_value == utxo_pool.at(curr_selection.size() - 1).effective_value &&
                utxo.fee == utxo_pool.at(curr_selection.size() - 1).fee) {
                curr_selection.push_back(false);
            } else {
                // Inclusion branch first (Largest First Exploration)
                curr_selection.push_back(true);
                curr_value += utxo.effective_value;
                curr_waste += utxo.fee - utxo.long_term_fee;
            }
        }
    }

    // Check for solution
    if (best_selection.empty()) {
        return false;
    }

    // Set output set
    value_ret = 0;
    for (size_t i = 0; i < best_selection.size(); ++i) {
        if (best_selection.at(i)) {
            util::insert(out_set, utxo_pool.at(i).m_outputs);
            value_ret += utxo_pool.at(i).m_value;
        }
    }

    return true;
}

static void ApproximateBestSubset(const std::vector<OutputGroup>& groups, const CAmount& nTotalLower, const CAmount& nTargetValue,
                                  std::vector<char>& vfBest, CAmount& nBest, int iterations = 1000)
{
    std::vector<char> vfIncluded;

    vfBest.assign(groups.size(), true);
    nBest = nTotalLower;

    FastRandomContext insecure_rand;

    for (int nRep = 0; nRep < iterations && nBest != nTargetValue; nRep++)
    {
        vfIncluded.assign(groups.size(), false);
        CAmount nTotal = 0;
        bool fReachedTarget = false;
        for (int nPass = 0; nPass < 2 && !fReachedTarget; nPass++)
        {
            for (unsigned int i = 0; i < groups.size(); i++)
            {
                //The solver here uses a randomized algorithm,
                //the randomness serves no real security purpose but is just
                //needed to prevent degenerate behavior and it is important
                //that the rng is fast. We do not use a constant random sequence,
                //because there may be some privacy improvement by making
                //the selection random.
                if (nPass == 0 ? insecure_rand.randbool() : !vfIncluded[i])
                {
                    nTotal += groups[i].m_value;
                    vfIncluded[i] = true;
                    if (nTotal >= nTargetValue)
                    {
                        fReachedTarget = true;
                        if (nTotal < nBest)
                        {
                            nBest = nTotal;
                            vfBest = vfIncluded;
                        }
                        nTotal -= groups[i].m_value;
                        vfIncluded[i] = false;
                    }
                }
            }
        }
    }
}

bool KnapsackSolver(const CAmount& nTargetValue, std::vector<OutputGroup>& groups, std::set<CInputCoin>& setCoinsRet, CAmount& nValueRet)
{
    setCoinsRet.clear();
    nValueRet = 0;

    // List of values less than target
    boost::optional<OutputGroup> lowest_larger;
    std::vector<OutputGroup> applicable_groups;
    CAmount nTotalLower = 0;

    random_shuffle(groups.begin(), groups.end(), GetRandInt);

    for (const OutputGroup& group : groups) {
        if (group.m_value == nTargetValue) {
            util::insert(setCoinsRet, group.m_outputs);
            nValueRet += group.m_value;
            return true;
        } else if (group.m_value < nTargetValue + MIN_CHANGE) {
            applicable_groups.push_back(group);
            nTotalLower += group.m_value;
        } else if (!lowest_larger || group.m_value < lowest_larger->m_value) {
            lowest_larger = group;
        }
    }

    if (nTotalLower == nTargetValue) {
        for (const auto& group : applicable_groups) {
            util::insert(setCoinsRet, group.m_outputs);
            nValueRet += group.m_value;
        }
        return true;
    }

    if (nTotalLower < nTargetValue) {
        if (!lowest_larger) return false;
        util::insert(setCoinsRet, lowest_larger->m_outputs);
        nValueRet += lowest_larger->m_value;
        return true;
    }

    // Solve subset sum by stochastic approximation
    std::sort(applicable_groups.begin(), applicable_groups.end(), descending);
    std::vector<char> vfBest;
    CAmount nBest;

    ApproximateBestSubset(applicable_groups, nTotalLower, nTargetValue, vfBest, nBest);
    if (nBest != nTargetValue && nTotalLower >= nTargetValue + MIN_CHANGE) {
        ApproximateBestSubset(applicable_groups, nTotalLower, nTargetValue + MIN_CHANGE, vfBest, nBest);
    }

    // If we have a bigger coin and (either the stochastic approximation didn't find a good solution,
    //                                   or the next bigger coin is closer), return the bigger coin
    if (lowest_larger &&
        ((nBest != nTargetValue && nBest < nTargetValue + MIN_CHANGE) || lowest_larger->m_value <= nBest)) {
        util::insert(setCoinsRet, lowest_larger->m_outputs);
        nValueRet += lowest_larger->m_value;
    } else {
        for (unsigned int i = 0; i < applicable_groups.size(); i++) {
            if (vfBest[i]) {
                util::insert(setCoinsRet, applicable_groups[i].m_outputs);
                nValueRet += applicable_groups[i].m_value;
            }
        }

        if (LogAcceptCategory(BCLog::SELECTCOINS)) {
            LogPrint(BCLog::SELECTCOINS, "SelectCoins() best subset: "); /* Continued */
            for (unsigned int i = 0; i < applicable_groups.size(); i++) {
                if (vfBest[i]) {
                    LogPrint(BCLog::SELECTCOINS, "%s ", FormatMoney(applicable_groups[i].m_value)); /* Continued */
                }
            }
            LogPrint(BCLog::SELECTCOINS, "total %s\n", FormatMoney(nBest));
        }
    }

    return true;
}

/******************************************************************************

 OutputGroup

 ******************************************************************************/

void OutputGroup::Insert(const CInputCoin& output, int depth, bool from_me, size_t ancestors, size_t descendants) {
    m_outputs.push_back(output);
    m_from_me &= from_me;
    m_value += output.effective_value;
    m_depth = std::min(m_depth, depth);
    // m_ancestors is currently the max ancestor count for all coins in the group; however, this is
    // not ideal, as a wallet will consider e.g. thirty 2-ancestor coins as having two ancestors,
    // when in reality it has 60 ancestors.
    m_ancestors = std::max(m_ancestors, ancestors);
    // m_descendants is the count as seen from the top ancestor, not the descendants as seen from the
    // coin itself; thus, this value is accurate
    m_descendants = std::max(m_descendants, descendants);
    effective_value = m_value;
}

std::vector<CInputCoin>::iterator OutputGroup::Discard(const CInputCoin& output) {
    auto it = m_outputs.begin();
    while (it != m_outputs.end() && it->outpoint != output.outpoint) ++it;
    if (it == m_outputs.end()) return it;
    m_value -= output.effective_value;
    effective_value -= output.effective_value;
    return m_outputs.erase(it);
}

bool OutputGroup::EligibleForSpending(const CoinEligibilityFilter& eligibility_filter) const
{
    return m_depth >= (m_from_me ? eligibility_filter.conf_mine : eligibility_filter.conf_theirs)
        && m_ancestors <= eligibility_filter.max_ancestors
        && m_descendants <= eligibility_filter.max_descendants;
}
<commit_msg>wallet: sum ancestors rather than taking max in output groups<commit_after>// Copyright (c) 2017 The Bitcoin Core developers
// Distributed under the MIT software license, see the accompanying
// file COPYING or http://www.opensource.org/licenses/mit-license.php.

#include <wallet/coinselection.h>
#include <util.h>
#include <utilmoneystr.h>

// Descending order comparator
struct {
    bool operator()(const OutputGroup& a, const OutputGroup& b) const
    {
        return a.effective_value > b.effective_value;
    }
} descending;

/*
 * This is the Branch and Bound Coin Selection algorithm designed by Murch. It searches for an input
 * set that can pay for the spending target and does not exceed the spending target by more than the
 * cost of creating and spending a change output. The algorithm uses a depth-first search on a binary
 * tree. In the binary tree, each node corresponds to the inclusion or the omission of a UTXO. UTXOs
 * are sorted by their effective values and the trees is explored deterministically per the inclusion
 * branch first. At each node, the algorithm checks whether the selection is within the target range.
 * While the selection has not reached the target range, more UTXOs are included. When a selection's
 * value exceeds the target range, the complete subtree deriving from this selection can be omitted.
 * At that point, the last included UTXO is deselected and the corresponding omission branch explored
 * instead. The search ends after the complete tree has been searched or after a limited number of tries.
 *
 * The search continues to search for better solutions after one solution has been found. The best
 * solution is chosen by minimizing the waste metric. The waste metric is defined as the cost to
 * spend the current inputs at the given fee rate minus the long term expected cost to spend the
 * inputs, plus the amount the selection exceeds the spending target:
 *
 * waste = selectionTotal - target + inputs × (currentFeeRate - longTermFeeRate)
 *
 * The algorithm uses two additional optimizations. A lookahead keeps track of the total value of
 * the unexplored UTXOs. A subtree is not explored if the lookahead indicates that the target range
 * cannot be reached. Further, it is unnecessary to test equivalent combinations. This allows us
 * to skip testing the inclusion of UTXOs that match the effective value and waste of an omitted
 * predecessor.
 *
 * The Branch and Bound algorithm is described in detail in Murch's Master Thesis:
 * https://murch.one/wp-content/uploads/2016/11/erhardt2016coinselection.pdf
 *
 * @param const std::vector<CInputCoin>& utxo_pool The set of UTXOs that we are choosing from.
 *        These UTXOs will be sorted in descending order by effective value and the CInputCoins'
 *        values are their effective values.
 * @param const CAmount& target_value This is the value that we want to select. It is the lower
 *        bound of the range.
 * @param const CAmount& cost_of_change This is the cost of creating and spending a change output.
 *        This plus target_value is the upper bound of the range.
 * @param std::set<CInputCoin>& out_set -> This is an output parameter for the set of CInputCoins
 *        that have been selected.
 * @param CAmount& value_ret -> This is an output parameter for the total value of the CInputCoins
 *        that were selected.
 * @param CAmount not_input_fees -> The fees that need to be paid for the outputs and fixed size
 *        overhead (version, locktime, marker and flag)
 */

static const size_t TOTAL_TRIES = 100000;

bool SelectCoinsBnB(std::vector<OutputGroup>& utxo_pool, const CAmount& target_value, const CAmount& cost_of_change, std::set<CInputCoin>& out_set, CAmount& value_ret, CAmount not_input_fees)
{
    out_set.clear();
    CAmount curr_value = 0;

    std::vector<bool> curr_selection; // select the utxo at this index
    curr_selection.reserve(utxo_pool.size());
    CAmount actual_target = not_input_fees + target_value;

    // Calculate curr_available_value
    CAmount curr_available_value = 0;
    for (const OutputGroup& utxo : utxo_pool) {
        // Assert that this utxo is not negative. It should never be negative, effective value calculation should have removed it
        assert(utxo.effective_value > 0);
        curr_available_value += utxo.effective_value;
    }
    if (curr_available_value < actual_target) {
        return false;
    }

    // Sort the utxo_pool
    std::sort(utxo_pool.begin(), utxo_pool.end(), descending);

    CAmount curr_waste = 0;
    std::vector<bool> best_selection;
    CAmount best_waste = MAX_MONEY;

    // Depth First search loop for choosing the UTXOs
    for (size_t i = 0; i < TOTAL_TRIES; ++i) {
        // Conditions for starting a backtrack
        bool backtrack = false;
        if (curr_value + curr_available_value < actual_target ||                // Cannot possibly reach target with the amount remaining in the curr_available_value.
            curr_value > actual_target + cost_of_change ||    // Selected value is out of range, go back and try other branch
            (curr_waste > best_waste && (utxo_pool.at(0).fee - utxo_pool.at(0).long_term_fee) > 0)) { // Don't select things which we know will be more wasteful if the waste is increasing
            backtrack = true;
        } else if (curr_value >= actual_target) {       // Selected value is within range
            curr_waste += (curr_value - actual_target); // This is the excess value which is added to the waste for the below comparison
            // Adding another UTXO after this check could bring the waste down if the long term fee is higher than the current fee.
            // However we are not going to explore that because this optimization for the waste is only done when we have hit our target
            // value. Adding any more UTXOs will be just burning the UTXO; it will go entirely to fees. Thus we aren't going to
            // explore any more UTXOs to avoid burning money like that.
            if (curr_waste <= best_waste) {
                best_selection = curr_selection;
                best_selection.resize(utxo_pool.size());
                best_waste = curr_waste;
            }
            curr_waste -= (curr_value - actual_target); // Remove the excess value as we will be selecting different coins now
            backtrack = true;
        }

        // Backtracking, moving backwards
        if (backtrack) {
            // Walk backwards to find the last included UTXO that still needs to have its omission branch traversed.
            while (!curr_selection.empty() && !curr_selection.back()) {
                curr_selection.pop_back();
                curr_available_value += utxo_pool.at(curr_selection.size()).effective_value;
            }

            if (curr_selection.empty()) { // We have walked back to the first utxo and no branch is untraversed. All solutions searched
                break;
            }

            // Output was included on previous iterations, try excluding now.
            curr_selection.back() = false;
            OutputGroup& utxo = utxo_pool.at(curr_selection.size() - 1);
            curr_value -= utxo.effective_value;
            curr_waste -= utxo.fee - utxo.long_term_fee;
        } else { // Moving forwards, continuing down this branch
            OutputGroup& utxo = utxo_pool.at(curr_selection.size());

            // Remove this utxo from the curr_available_value utxo amount
            curr_available_value -= utxo.effective_value;

            // Avoid searching a branch if the previous UTXO has the same value and same waste and was excluded. Since the ratio of fee to
            // long term fee is the same, we only need to check if one of those values match in order to know that the waste is the same.
            if (!curr_selection.empty() && !curr_selection.back() &&
                utxo.effective_value == utxo_pool.at(curr_selection.size() - 1).effective_value &&
                utxo.fee == utxo_pool.at(curr_selection.size() - 1).fee) {
                curr_selection.push_back(false);
            } else {
                // Inclusion branch first (Largest First Exploration)
                curr_selection.push_back(true);
                curr_value += utxo.effective_value;
                curr_waste += utxo.fee - utxo.long_term_fee;
            }
        }
    }

    // Check for solution
    if (best_selection.empty()) {
        return false;
    }

    // Set output set
    value_ret = 0;
    for (size_t i = 0; i < best_selection.size(); ++i) {
        if (best_selection.at(i)) {
            util::insert(out_set, utxo_pool.at(i).m_outputs);
            value_ret += utxo_pool.at(i).m_value;
        }
    }

    return true;
}

static void ApproximateBestSubset(const std::vector<OutputGroup>& groups, const CAmount& nTotalLower, const CAmount& nTargetValue,
                                  std::vector<char>& vfBest, CAmount& nBest, int iterations = 1000)
{
    std::vector<char> vfIncluded;

    vfBest.assign(groups.size(), true);
    nBest = nTotalLower;

    FastRandomContext insecure_rand;

    for (int nRep = 0; nRep < iterations && nBest != nTargetValue; nRep++)
    {
        vfIncluded.assign(groups.size(), false);
        CAmount nTotal = 0;
        bool fReachedTarget = false;
        for (int nPass = 0; nPass < 2 && !fReachedTarget; nPass++)
        {
            for (unsigned int i = 0; i < groups.size(); i++)
            {
                //The solver here uses a randomized algorithm,
                //the randomness serves no real security purpose but is just
                //needed to prevent degenerate behavior and it is important
                //that the rng is fast. We do not use a constant random sequence,
                //because there may be some privacy improvement by making
                //the selection random.
                if (nPass == 0 ? insecure_rand.randbool() : !vfIncluded[i])
                {
                    nTotal += groups[i].m_value;
                    vfIncluded[i] = true;
                    if (nTotal >= nTargetValue)
                    {
                        fReachedTarget = true;
                        if (nTotal < nBest)
                        {
                            nBest = nTotal;
                            vfBest = vfIncluded;
                        }
                        nTotal -= groups[i].m_value;
                        vfIncluded[i] = false;
                    }
                }
            }
        }
    }
}

bool KnapsackSolver(const CAmount& nTargetValue, std::vector<OutputGroup>& groups, std::set<CInputCoin>& setCoinsRet, CAmount& nValueRet)
{
    setCoinsRet.clear();
    nValueRet = 0;

    // List of values less than target
    boost::optional<OutputGroup> lowest_larger;
    std::vector<OutputGroup> applicable_groups;
    CAmount nTotalLower = 0;

    random_shuffle(groups.begin(), groups.end(), GetRandInt);

    for (const OutputGroup& group : groups) {
        if (group.m_value == nTargetValue) {
            util::insert(setCoinsRet, group.m_outputs);
            nValueRet += group.m_value;
            return true;
        } else if (group.m_value < nTargetValue + MIN_CHANGE) {
            applicable_groups.push_back(group);
            nTotalLower += group.m_value;
        } else if (!lowest_larger || group.m_value < lowest_larger->m_value) {
            lowest_larger = group;
        }
    }

    if (nTotalLower == nTargetValue) {
        for (const auto& group : applicable_groups) {
            util::insert(setCoinsRet, group.m_outputs);
            nValueRet += group.m_value;
        }
        return true;
    }

    if (nTotalLower < nTargetValue) {
        if (!lowest_larger) return false;
        util::insert(setCoinsRet, lowest_larger->m_outputs);
        nValueRet += lowest_larger->m_value;
        return true;
    }

    // Solve subset sum by stochastic approximation
    std::sort(applicable_groups.begin(), applicable_groups.end(), descending);
    std::vector<char> vfBest;
    CAmount nBest;

    ApproximateBestSubset(applicable_groups, nTotalLower, nTargetValue, vfBest, nBest);
    if (nBest != nTargetValue && nTotalLower >= nTargetValue + MIN_CHANGE) {
        ApproximateBestSubset(applicable_groups, nTotalLower, nTargetValue + MIN_CHANGE, vfBest, nBest);
    }

    // If we have a bigger coin and (either the stochastic approximation didn't find a good solution,
    //                                   or the next bigger coin is closer), return the bigger coin
    if (lowest_larger &&
        ((nBest != nTargetValue && nBest < nTargetValue + MIN_CHANGE) || lowest_larger->m_value <= nBest)) {
        util::insert(setCoinsRet, lowest_larger->m_outputs);
        nValueRet += lowest_larger->m_value;
    } else {
        for (unsigned int i = 0; i < applicable_groups.size(); i++) {
            if (vfBest[i]) {
                util::insert(setCoinsRet, applicable_groups[i].m_outputs);
                nValueRet += applicable_groups[i].m_value;
            }
        }

        if (LogAcceptCategory(BCLog::SELECTCOINS)) {
            LogPrint(BCLog::SELECTCOINS, "SelectCoins() best subset: "); /* Continued */
            for (unsigned int i = 0; i < applicable_groups.size(); i++) {
                if (vfBest[i]) {
                    LogPrint(BCLog::SELECTCOINS, "%s ", FormatMoney(applicable_groups[i].m_value)); /* Continued */
                }
            }
            LogPrint(BCLog::SELECTCOINS, "total %s\n", FormatMoney(nBest));
        }
    }

    return true;
}

/******************************************************************************

 OutputGroup

 ******************************************************************************/

void OutputGroup::Insert(const CInputCoin& output, int depth, bool from_me, size_t ancestors, size_t descendants) {
    m_outputs.push_back(output);
    m_from_me &= from_me;
    m_value += output.effective_value;
    m_depth = std::min(m_depth, depth);
    // ancestors here express the number of ancestors the new coin will end up having, which is
    // the sum, rather than the max; this will overestimate in the cases where multiple inputs
    // have common ancestors
    m_ancestors += ancestors;
    // descendants is the count as seen from the top ancestor, not the descendants as seen from the
    // coin itself; thus, this value is counted as the max, not the sum
    m_descendants = std::max(m_descendants, descendants);
    effective_value = m_value;
}

std::vector<CInputCoin>::iterator OutputGroup::Discard(const CInputCoin& output) {
    auto it = m_outputs.begin();
    while (it != m_outputs.end() && it->outpoint != output.outpoint) ++it;
    if (it == m_outputs.end()) return it;
    m_value -= output.effective_value;
    effective_value -= output.effective_value;
    return m_outputs.erase(it);
}

bool OutputGroup::EligibleForSpending(const CoinEligibilityFilter& eligibility_filter) const
{
    return m_depth >= (m_from_me ? eligibility_filter.conf_mine : eligibility_filter.conf_theirs)
        && m_ancestors <= eligibility_filter.max_ancestors
        && m_descendants <= eligibility_filter.max_descendants;
}
<|endoftext|>


<commit_before>//===--------------------------- new.cpp ----------------------------------===//
//
//                     The LLVM Compiler Infrastructure
//
// This file is dual licensed under the MIT and the University of Illinois Open
// Source Licenses. See LICENSE.TXT for details.
//
//===----------------------------------------------------------------------===//

#define _LIBCPP_BUILDING_NEW

#include <stdlib.h>

#include "new"

#if defined(_LIBCPP_ABI_MICROSOFT)
// nothing todo
#elif defined(LIBCXX_BUILDING_LIBCXXABI)
#include <cxxabi.h>
#elif defined(LIBCXXRT)
#include <cxxabi.h>
#include "support/runtime/new_handler_fallback.ipp"
#elif defined(__GLIBCXX__)
// nothing todo
#else
#include "support/runtime/new_handler_fallback.ipp"
#endif

namespace std
{

#ifndef __GLIBCXX__
const nothrow_t nothrow = {};
#endif

#ifndef LIBSTDCXX

void
__throw_bad_alloc()
{
#ifndef _LIBCPP_NO_EXCEPTIONS
    throw bad_alloc();
#else
    _VSTD::abort();
#endif
}

#endif // !LIBSTDCXX

}  // std

#if !defined(__GLIBCXX__) && !defined(_LIBCPP_ABI_MICROSOFT)

// Implement all new and delete operators as weak definitions
// in this shared library, so that they can be overridden by programs
// that define non-weak copies of the functions.

_LIBCPP_WEAK
void *
operator new(std::size_t size) _THROW_BAD_ALLOC
{
    if (size == 0)
        size = 1;
    void* p;
    while ((p = ::malloc(size)) == 0)
    {
        // If malloc fails and there is a new_handler,
        // call it to try free up memory.
        std::new_handler nh = std::get_new_handler();
        if (nh)
            nh();
        else
#ifndef _LIBCPP_NO_EXCEPTIONS
            throw std::bad_alloc();
#else
            break;
#endif
    }
    return p;
}

_LIBCPP_WEAK
void*
operator new(size_t size, const std::nothrow_t&) _NOEXCEPT
{
    void* p = 0;
#ifndef _LIBCPP_NO_EXCEPTIONS
    try
    {
#endif  // _LIBCPP_NO_EXCEPTIONS
        p = ::operator new(size);
#ifndef _LIBCPP_NO_EXCEPTIONS
    }
    catch (...)
    {
    }
#endif  // _LIBCPP_NO_EXCEPTIONS
    return p;
}

_LIBCPP_WEAK
void*
operator new[](size_t size) _THROW_BAD_ALLOC
{
    return ::operator new(size);
}

_LIBCPP_WEAK
void*
operator new[](size_t size, const std::nothrow_t&) _NOEXCEPT
{
    void* p = 0;
#ifndef _LIBCPP_NO_EXCEPTIONS
    try
    {
#endif  // _LIBCPP_NO_EXCEPTIONS
        p = ::operator new[](size);
#ifndef _LIBCPP_NO_EXCEPTIONS
    }
    catch (...)
    {
    }
#endif  // _LIBCPP_NO_EXCEPTIONS
    return p;
}

_LIBCPP_WEAK
void
operator delete(void* ptr) _NOEXCEPT
{
    if (ptr)
        ::free(ptr);
}

_LIBCPP_WEAK
void
operator delete(void* ptr, const std::nothrow_t&) _NOEXCEPT
{
    ::operator delete(ptr);
}

_LIBCPP_WEAK
void
operator delete(void* ptr, size_t) _NOEXCEPT
{
    ::operator delete(ptr);
}

_LIBCPP_WEAK
void
operator delete[] (void* ptr) _NOEXCEPT
{
    ::operator delete(ptr);
}

_LIBCPP_WEAK
void
operator delete[] (void* ptr, const std::nothrow_t&) _NOEXCEPT
{
    ::operator delete[](ptr);
}

_LIBCPP_WEAK
void
operator delete[] (void* ptr, size_t) _NOEXCEPT
{
    ::operator delete[](ptr);
}

#if !defined(_LIBCPP_HAS_NO_ALIGNED_ALLOCATION)

_LIBCPP_WEAK
void *
operator new(std::size_t size, std::align_val_t alignment) _THROW_BAD_ALLOC
{
    if (size == 0)
        size = 1;
    if (static_cast<size_t>(alignment) < sizeof(void*))
      alignment = std::align_val_t(sizeof(void*));
    void* p;
#if defined(_LIBCPP_MSVCRT)
    while ((p = _aligned_malloc(size, static_cast<size_t>(alignment))) == nullptr)
#else
    while (::posix_memalign(&p, static_cast<size_t>(alignment), size) != 0)
#endif
    {
        // If posix_memalign fails and there is a new_handler,
        // call it to try free up memory.
        std::new_handler nh = std::get_new_handler();
        if (nh)
            nh();
        else {
#ifndef _LIBCPP_NO_EXCEPTIONS
            throw std::bad_alloc();
#else
            p = nullptr; // posix_memalign doesn't initialize 'p' on failure
            break;
#endif
        }
    }
    return p;
}

_LIBCPP_WEAK
void*
operator new(size_t size, std::align_val_t alignment, const std::nothrow_t&) _NOEXCEPT
{
    void* p = 0;
#ifndef _LIBCPP_NO_EXCEPTIONS
    try
    {
#endif  // _LIBCPP_NO_EXCEPTIONS
        p = ::operator new(size, alignment);
#ifndef _LIBCPP_NO_EXCEPTIONS
    }
    catch (...)
    {
    }
#endif  // _LIBCPP_NO_EXCEPTIONS
    return p;
}

_LIBCPP_WEAK
void*
operator new[](size_t size, std::align_val_t alignment) _THROW_BAD_ALLOC
{
    return ::operator new(size, alignment);
}

_LIBCPP_WEAK
void*
operator new[](size_t size, std::align_val_t alignment, const std::nothrow_t&) _NOEXCEPT
{
    void* p = 0;
#ifndef _LIBCPP_NO_EXCEPTIONS
    try
    {
#endif  // _LIBCPP_NO_EXCEPTIONS
        p = ::operator new[](size, alignment);
#ifndef _LIBCPP_NO_EXCEPTIONS
    }
    catch (...)
    {
    }
#endif  // _LIBCPP_NO_EXCEPTIONS
    return p;
}

_LIBCPP_WEAK
void
operator delete(void* ptr, std::align_val_t) _NOEXCEPT
{
    if (ptr)
#if defined(_LIBCPP_MSVCRT)
        ::_aligned_free(ptr);
#else
        ::free(ptr);
#endif
}

_LIBCPP_WEAK
void
operator delete(void* ptr, std::align_val_t alignment, const std::nothrow_t&) _NOEXCEPT
{
    ::operator delete(ptr, alignment);
}

_LIBCPP_WEAK
void
operator delete(void* ptr, size_t, std::align_val_t alignment) _NOEXCEPT
{
    ::operator delete(ptr, alignment);
}

_LIBCPP_WEAK
void
operator delete[] (void* ptr, std::align_val_t alignment) _NOEXCEPT
{
    ::operator delete(ptr, alignment);
}

_LIBCPP_WEAK
void
operator delete[] (void* ptr, std::align_val_t alignment, const std::nothrow_t&) _NOEXCEPT
{
    ::operator delete[](ptr, alignment);
}

_LIBCPP_WEAK
void
operator delete[] (void* ptr, size_t, std::align_val_t alignment) _NOEXCEPT
{
    ::operator delete[](ptr, alignment);
}

#endif // !_LIBCPP_HAS_NO_ALIGNED_ALLOCATION
#endif // !__GLIBCXX__ && !_LIBCPP_ABI_MICROSOFT
<commit_msg>Attempt to fix Apple buildit bots<commit_after>//===--------------------------- new.cpp ----------------------------------===//
//
//                     The LLVM Compiler Infrastructure
//
// This file is dual licensed under the MIT and the University of Illinois Open
// Source Licenses. See LICENSE.TXT for details.
//
//===----------------------------------------------------------------------===//

#define _LIBCPP_BUILDING_NEW

#include <stdlib.h>

#include "new"

#if defined(_LIBCPP_ABI_MICROSOFT)
// nothing todo
#elif defined(LIBCXX_BUILDING_LIBCXXABI)
#include <cxxabi.h>
#elif defined(LIBCXXRT)
#include <cxxabi.h>
#include "support/runtime/new_handler_fallback.ipp"
#elif defined(__GLIBCXX__)
// nothing todo
#else
# if defined(__APPLE__) && !defined(_LIBCPP_BUILDING_HAS_NO_ABI_LIBRARY)
#   include <cxxabi.h> // FIXME: remove this once buildit is gone.
# else
#   include "support/runtime/new_handler_fallback.ipp"
# endif
#endif

namespace std
{

#ifndef __GLIBCXX__
const nothrow_t nothrow = {};
#endif

#ifndef LIBSTDCXX

void
__throw_bad_alloc()
{
#ifndef _LIBCPP_NO_EXCEPTIONS
    throw bad_alloc();
#else
    _VSTD::abort();
#endif
}

#endif // !LIBSTDCXX

}  // std

#if !defined(__GLIBCXX__) && !defined(_LIBCPP_ABI_MICROSOFT)

// Implement all new and delete operators as weak definitions
// in this shared library, so that they can be overridden by programs
// that define non-weak copies of the functions.

_LIBCPP_WEAK
void *
operator new(std::size_t size) _THROW_BAD_ALLOC
{
    if (size == 0)
        size = 1;
    void* p;
    while ((p = ::malloc(size)) == 0)
    {
        // If malloc fails and there is a new_handler,
        // call it to try free up memory.
        std::new_handler nh = std::get_new_handler();
        if (nh)
            nh();
        else
#ifndef _LIBCPP_NO_EXCEPTIONS
            throw std::bad_alloc();
#else
            break;
#endif
    }
    return p;
}

_LIBCPP_WEAK
void*
operator new(size_t size, const std::nothrow_t&) _NOEXCEPT
{
    void* p = 0;
#ifndef _LIBCPP_NO_EXCEPTIONS
    try
    {
#endif  // _LIBCPP_NO_EXCEPTIONS
        p = ::operator new(size);
#ifndef _LIBCPP_NO_EXCEPTIONS
    }
    catch (...)
    {
    }
#endif  // _LIBCPP_NO_EXCEPTIONS
    return p;
}

_LIBCPP_WEAK
void*
operator new[](size_t size) _THROW_BAD_ALLOC
{
    return ::operator new(size);
}

_LIBCPP_WEAK
void*
operator new[](size_t size, const std::nothrow_t&) _NOEXCEPT
{
    void* p = 0;
#ifndef _LIBCPP_NO_EXCEPTIONS
    try
    {
#endif  // _LIBCPP_NO_EXCEPTIONS
        p = ::operator new[](size);
#ifndef _LIBCPP_NO_EXCEPTIONS
    }
    catch (...)
    {
    }
#endif  // _LIBCPP_NO_EXCEPTIONS
    return p;
}

_LIBCPP_WEAK
void
operator delete(void* ptr) _NOEXCEPT
{
    if (ptr)
        ::free(ptr);
}

_LIBCPP_WEAK
void
operator delete(void* ptr, const std::nothrow_t&) _NOEXCEPT
{
    ::operator delete(ptr);
}

_LIBCPP_WEAK
void
operator delete(void* ptr, size_t) _NOEXCEPT
{
    ::operator delete(ptr);
}

_LIBCPP_WEAK
void
operator delete[] (void* ptr) _NOEXCEPT
{
    ::operator delete(ptr);
}

_LIBCPP_WEAK
void
operator delete[] (void* ptr, const std::nothrow_t&) _NOEXCEPT
{
    ::operator delete[](ptr);
}

_LIBCPP_WEAK
void
operator delete[] (void* ptr, size_t) _NOEXCEPT
{
    ::operator delete[](ptr);
}

#if !defined(_LIBCPP_HAS_NO_ALIGNED_ALLOCATION)

_LIBCPP_WEAK
void *
operator new(std::size_t size, std::align_val_t alignment) _THROW_BAD_ALLOC
{
    if (size == 0)
        size = 1;
    if (static_cast<size_t>(alignment) < sizeof(void*))
      alignment = std::align_val_t(sizeof(void*));
    void* p;
#if defined(_LIBCPP_MSVCRT)
    while ((p = _aligned_malloc(size, static_cast<size_t>(alignment))) == nullptr)
#else
    while (::posix_memalign(&p, static_cast<size_t>(alignment), size) != 0)
#endif
    {
        // If posix_memalign fails and there is a new_handler,
        // call it to try free up memory.
        std::new_handler nh = std::get_new_handler();
        if (nh)
            nh();
        else {
#ifndef _LIBCPP_NO_EXCEPTIONS
            throw std::bad_alloc();
#else
            p = nullptr; // posix_memalign doesn't initialize 'p' on failure
            break;
#endif
        }
    }
    return p;
}

_LIBCPP_WEAK
void*
operator new(size_t size, std::align_val_t alignment, const std::nothrow_t&) _NOEXCEPT
{
    void* p = 0;
#ifndef _LIBCPP_NO_EXCEPTIONS
    try
    {
#endif  // _LIBCPP_NO_EXCEPTIONS
        p = ::operator new(size, alignment);
#ifndef _LIBCPP_NO_EXCEPTIONS
    }
    catch (...)
    {
    }
#endif  // _LIBCPP_NO_EXCEPTIONS
    return p;
}

_LIBCPP_WEAK
void*
operator new[](size_t size, std::align_val_t alignment) _THROW_BAD_ALLOC
{
    return ::operator new(size, alignment);
}

_LIBCPP_WEAK
void*
operator new[](size_t size, std::align_val_t alignment, const std::nothrow_t&) _NOEXCEPT
{
    void* p = 0;
#ifndef _LIBCPP_NO_EXCEPTIONS
    try
    {
#endif  // _LIBCPP_NO_EXCEPTIONS
        p = ::operator new[](size, alignment);
#ifndef _LIBCPP_NO_EXCEPTIONS
    }
    catch (...)
    {
    }
#endif  // _LIBCPP_NO_EXCEPTIONS
    return p;
}

_LIBCPP_WEAK
void
operator delete(void* ptr, std::align_val_t) _NOEXCEPT
{
    if (ptr)
#if defined(_LIBCPP_MSVCRT)
        ::_aligned_free(ptr);
#else
        ::free(ptr);
#endif
}

_LIBCPP_WEAK
void
operator delete(void* ptr, std::align_val_t alignment, const std::nothrow_t&) _NOEXCEPT
{
    ::operator delete(ptr, alignment);
}

_LIBCPP_WEAK
void
operator delete(void* ptr, size_t, std::align_val_t alignment) _NOEXCEPT
{
    ::operator delete(ptr, alignment);
}

_LIBCPP_WEAK
void
operator delete[] (void* ptr, std::align_val_t alignment) _NOEXCEPT
{
    ::operator delete(ptr, alignment);
}

_LIBCPP_WEAK
void
operator delete[] (void* ptr, std::align_val_t alignment, const std::nothrow_t&) _NOEXCEPT
{
    ::operator delete[](ptr, alignment);
}

_LIBCPP_WEAK
void
operator delete[] (void* ptr, size_t, std::align_val_t alignment) _NOEXCEPT
{
    ::operator delete[](ptr, alignment);
}

#endif // !_LIBCPP_HAS_NO_ALIGNED_ALLOCATION
#endif // !__GLIBCXX__ && !_LIBCPP_ABI_MICROSOFT
<|endoftext|>


<commit_before>// construct.cpp: define the "vg construct" subcommand.

#include <omp.h>
#include <unistd.h>
#include <getopt.h>
#include <memory>

#include "subcommand.hpp"

#include "../stream.hpp"
#include "../constructor.hpp"
#include "../msa_converter.hpp"
#include "../region.hpp"

using namespace std;
using namespace vg;
using namespace vg::subcommand;

void help_construct(char** argv) {
    cerr << "usage: " << argv[0] << " construct [options] >new.vg" << endl
         << "options:" << endl
         << "construct from a reference and variant calls:" << endl
         << "    -r, --reference FILE   input FASTA reference (may repeat)" << endl
         << "    -v, --vcf FILE         input VCF (may repeat)" << endl
         << "    -n, --rename V=F       rename contig V in the VCFs to contig F in the FASTAs (may repeat)" << endl
         << "    -a, --alt-paths        save paths for alts of variants by variant ID" << endl
         << "    -R, --region REGION    specify a particular chromosome or 1-based inclusive region" << endl
         << "    -C, --region-is-chrom  don't attempt to parse the region (use when the reference" << endl
         << "                           sequence name could be inadvertently parsed as a region)" << endl
         << "    -z, --region-size N    variants per region to parallelize (default: 1024)" << endl
         << "    -t, --threads N        use N threads to construct graph (defaults to numCPUs)" << endl
         << "    -S, --handle-sv        include structural variants in construction of graph." << endl
         << "    -I, --insertions FILE  a FASTA file containing insertion sequences "<< endl
         << "                           (referred to in VCF) to add to graph." << endl
         << "    -f, --flat-alts N      don't chop up alternate alleles from input VCF" << endl
         << "construct from a multiple sequence alignment:" << endl
         << "    -M, --msa FILE         input multiple sequence alignment" << endl
         << "    -F, --msa-format       format of the MSA file (options: fasta, maf, clustal; default fasta)" << endl
         << "    -d, --drop-msa-paths   don't add paths for the MSA sequences into the graph" << endl
         << "shared construction options:" << endl
         << "    -m, --node-max N       limit the maximum allowable node sequence size (defaults to 1000)" << endl
         << "                           nodes greater than this threshold will be divided" << endl
         << "                           Note: nodes larger than ~1024 bp can't be GCSA2-indexed" << endl
         << "    -p, --progress         show progress" << endl;

}

int main_construct(int argc, char** argv) {

    if (argc == 2) {
        help_construct(argv);
        return 1;
    }

    // Make a constructor to fill in
    Constructor constructor;

    // We also parse some arguments separately.
    vector<string> fasta_filenames;
    vector<string> vcf_filenames;
    vector<string> insertion_filenames;
    string region;
    bool region_is_chrom = false;
    string msa_filename;
    int max_node_size = 1000;
    bool keep_paths = true;
    string msa_format = "fasta";
    bool show_progress = false;

    int c;
    optind = 2; // force optind past command positional argument
    while (true) {
        static struct option long_options[] =
            {
                /* These options set a flag. */
                //{"verbose", no_argument,       &verbose_flag, 1},
                {"vcf", required_argument, 0, 'v'},
                {"reference", required_argument, 0, 'r'},
                {"msa", required_argument, 0, 'M'},
                {"msa-format", required_argument, 0, 'F'},
                {"drop-msa-paths", no_argument, 0, 'd'},
                {"rename", required_argument, 0, 'n'},
                {"alt-paths", no_argument, 0, 'a'},
                {"handle-sv", no_argument, 0, 'S'},
                {"insertions", required_argument, 0, 'I'},
                {"progress",  no_argument, 0, 'p'},
                {"region-size", required_argument, 0, 'z'},
                {"threads", required_argument, 0, 't'},
                {"region", required_argument, 0, 'R'},
                {"region-is-chrom", no_argument, 0, 'C'},
                {"node-max", required_argument, 0, 'm'},\
                {"flat-alts", no_argument, 0, 'f'},
                {0, 0, 0, 0}
            };

        int option_index = 0;
        c = getopt_long (argc, argv, "v:r:n:ph?z:t:R:m:as:CfSI:M:dF:",
                         long_options, &option_index);

        /* Detect the end of the options. */
        if (c == -1)
            break;

        switch (c)
        {
        case 'v':
            vcf_filenames.push_back(optarg);
            break;

        case 'M':
            msa_filename = optarg;
            break;

        case 'F':
            msa_format = optarg;
            break;

        case 'd':
            keep_paths = false;
            break;

        case 'r':
            fasta_filenames.push_back(optarg);
            break;

        case 'S':
            constructor.do_svs = true;
            break;

        case 'I':
            insertion_filenames.push_back(optarg);
            break;

        case 'n':
            {
                // Parse the rename old=new
                string key_value(optarg);
                auto found = key_value.find('=');
                if (found == string::npos || found == 0 || found + 1 == key_value.size()) {
                    cerr << "error:[vg construct] could not parse rename " << key_value << endl;
                    exit(1);
                }
                // Parse out the two parts
                string vcf_contig = key_value.substr(0, found);
                string fasta_contig = key_value.substr(found + 1);
                // Add the name mapping
                constructor.add_name_mapping(vcf_contig, fasta_contig);
            }
            break;

        case 'a':
            constructor.alt_paths = true;
            break;

        case 'p':
            show_progress = true;
            break;

        case 'z':
            constructor.vars_per_chunk = parse<int>(optarg);
            break;

        case 'R':
            region = optarg;
            break;

        case 'C':
            region_is_chrom = true;
            break;

        case 't':
            omp_set_num_threads(parse<int>(optarg));
            break;

        case 'm':
            max_node_size = parse<int>(optarg);
            break;

        case 'f':
            constructor.flat = true;
            break;

        case 'h':
        case '?':
            /* getopt_long already printed an error message. */
            help_construct(argv);
            exit(1);
            break;

        default:
            abort ();

        }
    }

    if (max_node_size == 0) {
        // Make sure we can actually make nodes
        cerr << "error:[vg construct] max node size cannot be 0" << endl;
        exit(1);
    }

    if (!msa_filename.empty() && !fasta_filenames.empty()) {
        cerr << "error:[vg construct] cannot construct from a reference/VCF and an MSA simultaneously" << endl;
        exit(1);
    }

    if (!fasta_filenames.empty()) {
        // Actually use the Constructor.
        // TODO: If we aren't always going to use the Constructor, refactor the subcommand to not always create and configure it.

        // Make an emitter that serializes the actual Graph objects, with buffering.
        // But just serialize one graph at a time in each group.
        stream::ProtobufEmitter<Graph> emitter(cout, 1);

        // We need a callback to handle pieces of graph as they are produced.
        auto callback = [&](Graph& big_chunk) {
            // Sort the nodes by ID so that the serialized chunks come out in sorted order
            // TODO: We still interleave chunks from different threads working on different contigs
            std::sort(big_chunk.mutable_node()->begin(), big_chunk.mutable_node()->end(), [](const Node& a, const Node& b) -> bool {
                // Return true if a comes before b
                return a.id() < b.id();
            });

            // Wrap the chunk in a vg object that can properly divide it into
            // reasonably sized serialized chunks.
            VG* g = new VG(big_chunk, false, true);

            // Check our work. Never output an invalid graph.
            // But allow for edges where one node isn't there, because we need those to connect segments.
            assert(g->is_valid(true, false, true, true));
            // One thread at a time can write to the emitter and the output stream
#pragma omp critical (emitter)
            g->serialize_to_emitter(emitter);
        };

        // Copy shared parameters into the constructor
        constructor.max_node_size = max_node_size;
        constructor.show_progress = show_progress;

        if (!region.empty()) {
            // We want to limit to a certain region
            if (!region_is_chrom) {
                // We are allowed to parse the region.
                // Break out sequence name and region bounds
                string seq_name;
                int64_t start_pos = -1, stop_pos = -1;
                parse_region(region,
                             seq_name,
                             start_pos,
                             stop_pos);

                if (start_pos > 0 && stop_pos > 0) {
                    // These are 0-based, so if both are nonzero we got a real set of coordinates
                    if (constructor.show_progress) {
                        cerr << "Restricting to " << seq_name << " from " << start_pos << " to " << stop_pos << endl;
                    }
                    constructor.allowed_vcf_names.insert(seq_name);
                    // Make sure to correct the coordinates to 0-based exclusive-end, from 1-based inclusive-end
                    constructor.allowed_vcf_regions[seq_name] = make_pair(start_pos - 1, stop_pos);
                } else if (start_pos < 0 && stop_pos < 0) {
                    // We just got a name
                    cerr << "Restricting to " << seq_name << " from 1 to end" << endl;
                    constructor.allowed_vcf_names.insert(seq_name);
                } else {
                    // This doesn't make sense. Does it have like one coordinate?
                    cerr << "error:[vg construct] could not parse " << region << endl;
                    exit(1);
                }
            } else {
                // We have been told not to parse the region
                cerr << "Restricting to " << region << " from 1 to end" << endl;
                constructor.allowed_vcf_names.insert(region);
            }
        }

        // This will own all the VCF files
        vector<unique_ptr<vcflib::VariantCallFile>> variant_files;
        for (auto& vcf_filename : vcf_filenames) {
            // Make sure each VCF file exists. Otherwise Tabix++ may exit with a non-
            // helpful message.

            // We can't invoke stat woithout a place for it to write. But all we
            // really want is its return value.
            struct stat temp;
            if(stat(vcf_filename.c_str(), &temp)) {
                cerr << "error:[vg construct] file \"" << vcf_filename << "\" not found" << endl;
                return 1;
            }
            vcflib::VariantCallFile* variant_file = new vcflib::VariantCallFile();
            variant_file->parseSamples = false; // Major speedup if there are many samples.
            variant_files.emplace_back(variant_file);
            variant_file->open(vcf_filename);
            if (!variant_file->is_open()) {
                cerr << "error:[vg construct] could not open" << vcf_filename << endl;
                return 1;
            }
        }

        if (fasta_filenames.empty()) {
            cerr << "error:[vg construct] a reference is required for graph construction" << endl;
            return 1;
        }
        vector<unique_ptr<FastaReference>> references;
        for (auto& fasta_filename : fasta_filenames) {
            // Open each FASTA file
            FastaReference* reference = new FastaReference();
            references.emplace_back(reference);
            reference->open(fasta_filename);
        }

        vector<unique_ptr<FastaReference> > insertions;
        for (auto& insertion_filename : insertion_filenames){
            // Open up those insertion files
            FastaReference* insertion = new FastaReference();
            insertions.emplace_back(insertion);
            insertion->open(insertion_filename);
        }

        // Make vectors of just bare pointers
        vector<vcflib::VariantCallFile*> vcf_pointers;
        for(auto& vcf : variant_files) {
            vcf_pointers.push_back(vcf.get());
        }
        vector<FastaReference*> fasta_pointers;
        for(auto& fasta : references) {
            fasta_pointers.push_back(fasta.get());
        }
        vector<FastaReference*> ins_pointers;
        for (auto& ins : insertions){
            ins_pointers.push_back(ins.get());
        }

        if (ins_pointers.size() > 1){
            cerr << "Error: only one insertion file may be provided." << endl;
            exit(1);
        }

        // Construct the graph.
        constructor.construct_graph(fasta_pointers, vcf_pointers,
                                    ins_pointers, callback);

        // The output will be flushed when the ProtobufEmitter we use in the callback goes away.
        // Don't add an extra EOF marker or anything.

        // NB: If you worry about "still reachable but possibly lost" warnings in valgrind,
        // this would free all the memory used by protobuf:
        //ShutdownProtobufLibrary();
    }
    else if (!msa_filename.empty()) {

        ifstream msa_file(msa_filename);
        if (!msa_file) {
            cerr << "error:[vg construct] could not open MSA file " << msa_filename << endl;
            exit(1);
        }

        MSAConverter msa_converter;
        msa_converter.show_progress = show_progress;

        msa_converter.load_alignments(msa_file, msa_format);
        VG msa_graph = msa_converter.make_graph(keep_paths, max_node_size);

        msa_graph.serialize_to_ostream(cout);
    }
    else {
        cerr << "error:[vg construct] a reference or an MSA is required for construct" << endl;
        exit(1);
    }

    return 0;
}

// Register subcommand
static Subcommand vg_construct("construct", "graph construction", PIPELINE, 1, main_construct);

<commit_msg>reduce default node size from 1000 to 32<commit_after>// construct.cpp: define the "vg construct" subcommand.

#include <omp.h>
#include <unistd.h>
#include <getopt.h>
#include <memory>

#include "subcommand.hpp"

#include "../stream.hpp"
#include "../constructor.hpp"
#include "../msa_converter.hpp"
#include "../region.hpp"

using namespace std;
using namespace vg;
using namespace vg::subcommand;

void help_construct(char** argv) {
    cerr << "usage: " << argv[0] << " construct [options] >new.vg" << endl
         << "options:" << endl
         << "construct from a reference and variant calls:" << endl
         << "    -r, --reference FILE   input FASTA reference (may repeat)" << endl
         << "    -v, --vcf FILE         input VCF (may repeat)" << endl
         << "    -n, --rename V=F       rename contig V in the VCFs to contig F in the FASTAs (may repeat)" << endl
         << "    -a, --alt-paths        save paths for alts of variants by variant ID" << endl
         << "    -R, --region REGION    specify a particular chromosome or 1-based inclusive region" << endl
         << "    -C, --region-is-chrom  don't attempt to parse the region (use when the reference" << endl
         << "                           sequence name could be inadvertently parsed as a region)" << endl
         << "    -z, --region-size N    variants per region to parallelize (default: 1024)" << endl
         << "    -t, --threads N        use N threads to construct graph (defaults to numCPUs)" << endl
         << "    -S, --handle-sv        include structural variants in construction of graph." << endl
         << "    -I, --insertions FILE  a FASTA file containing insertion sequences "<< endl
         << "                           (referred to in VCF) to add to graph." << endl
         << "    -f, --flat-alts N      don't chop up alternate alleles from input VCF" << endl
         << "construct from a multiple sequence alignment:" << endl
         << "    -M, --msa FILE         input multiple sequence alignment" << endl
         << "    -F, --msa-format       format of the MSA file (options: fasta, maf, clustal; default fasta)" << endl
         << "    -d, --drop-msa-paths   don't add paths for the MSA sequences into the graph" << endl
         << "shared construction options:" << endl
         << "    -m, --node-max N       limit the maximum allowable node sequence size (defaults to 32)" << endl
         << "                           nodes greater than this threshold will be divided" << endl
         << "                           Note: nodes larger than ~1024 bp can't be GCSA2-indexed" << endl
         << "    -p, --progress         show progress" << endl;

}

int main_construct(int argc, char** argv) {

    if (argc == 2) {
        help_construct(argv);
        return 1;
    }

    // Make a constructor to fill in
    Constructor constructor;

    // We also parse some arguments separately.
    vector<string> fasta_filenames;
    vector<string> vcf_filenames;
    vector<string> insertion_filenames;
    string region;
    bool region_is_chrom = false;
    string msa_filename;
    int max_node_size = 32;
    bool keep_paths = true;
    string msa_format = "fasta";
    bool show_progress = false;

    int c;
    optind = 2; // force optind past command positional argument
    while (true) {
        static struct option long_options[] =
            {
                /* These options set a flag. */
                //{"verbose", no_argument,       &verbose_flag, 1},
                {"vcf", required_argument, 0, 'v'},
                {"reference", required_argument, 0, 'r'},
                {"msa", required_argument, 0, 'M'},
                {"msa-format", required_argument, 0, 'F'},
                {"drop-msa-paths", no_argument, 0, 'd'},
                {"rename", required_argument, 0, 'n'},
                {"alt-paths", no_argument, 0, 'a'},
                {"handle-sv", no_argument, 0, 'S'},
                {"insertions", required_argument, 0, 'I'},
                {"progress",  no_argument, 0, 'p'},
                {"region-size", required_argument, 0, 'z'},
                {"threads", required_argument, 0, 't'},
                {"region", required_argument, 0, 'R'},
                {"region-is-chrom", no_argument, 0, 'C'},
                {"node-max", required_argument, 0, 'm'},\
                {"flat-alts", no_argument, 0, 'f'},
                {0, 0, 0, 0}
            };

        int option_index = 0;
        c = getopt_long (argc, argv, "v:r:n:ph?z:t:R:m:as:CfSI:M:dF:",
                         long_options, &option_index);

        /* Detect the end of the options. */
        if (c == -1)
            break;

        switch (c)
        {
        case 'v':
            vcf_filenames.push_back(optarg);
            break;

        case 'M':
            msa_filename = optarg;
            break;

        case 'F':
            msa_format = optarg;
            break;

        case 'd':
            keep_paths = false;
            break;

        case 'r':
            fasta_filenames.push_back(optarg);
            break;

        case 'S':
            constructor.do_svs = true;
            break;

        case 'I':
            insertion_filenames.push_back(optarg);
            break;

        case 'n':
            {
                // Parse the rename old=new
                string key_value(optarg);
                auto found = key_value.find('=');
                if (found == string::npos || found == 0 || found + 1 == key_value.size()) {
                    cerr << "error:[vg construct] could not parse rename " << key_value << endl;
                    exit(1);
                }
                // Parse out the two parts
                string vcf_contig = key_value.substr(0, found);
                string fasta_contig = key_value.substr(found + 1);
                // Add the name mapping
                constructor.add_name_mapping(vcf_contig, fasta_contig);
            }
            break;

        case 'a':
            constructor.alt_paths = true;
            break;

        case 'p':
            show_progress = true;
            break;

        case 'z':
            constructor.vars_per_chunk = parse<int>(optarg);
            break;

        case 'R':
            region = optarg;
            break;

        case 'C':
            region_is_chrom = true;
            break;

        case 't':
            omp_set_num_threads(parse<int>(optarg));
            break;

        case 'm':
            max_node_size = parse<int>(optarg);
            break;

        case 'f':
            constructor.flat = true;
            break;

        case 'h':
        case '?':
            /* getopt_long already printed an error message. */
            help_construct(argv);
            exit(1);
            break;

        default:
            abort ();

        }
    }

    if (max_node_size == 0) {
        // Make sure we can actually make nodes
        cerr << "error:[vg construct] max node size cannot be 0" << endl;
        exit(1);
    }

    if (!msa_filename.empty() && !fasta_filenames.empty()) {
        cerr << "error:[vg construct] cannot construct from a reference/VCF and an MSA simultaneously" << endl;
        exit(1);
    }

    if (!fasta_filenames.empty()) {
        // Actually use the Constructor.
        // TODO: If we aren't always going to use the Constructor, refactor the subcommand to not always create and configure it.

        // Make an emitter that serializes the actual Graph objects, with buffering.
        // But just serialize one graph at a time in each group.
        stream::ProtobufEmitter<Graph> emitter(cout, 1);

        // We need a callback to handle pieces of graph as they are produced.
        auto callback = [&](Graph& big_chunk) {
            // Sort the nodes by ID so that the serialized chunks come out in sorted order
            // TODO: We still interleave chunks from different threads working on different contigs
            std::sort(big_chunk.mutable_node()->begin(), big_chunk.mutable_node()->end(), [](const Node& a, const Node& b) -> bool {
                // Return true if a comes before b
                return a.id() < b.id();
            });

            // Wrap the chunk in a vg object that can properly divide it into
            // reasonably sized serialized chunks.
            VG* g = new VG(big_chunk, false, true);

            // Check our work. Never output an invalid graph.
            // But allow for edges where one node isn't there, because we need those to connect segments.
            assert(g->is_valid(true, false, true, true));
            // One thread at a time can write to the emitter and the output stream
#pragma omp critical (emitter)
            g->serialize_to_emitter(emitter);
        };

        // Copy shared parameters into the constructor
        constructor.max_node_size = max_node_size;
        constructor.show_progress = show_progress;

        if (!region.empty()) {
            // We want to limit to a certain region
            if (!region_is_chrom) {
                // We are allowed to parse the region.
                // Break out sequence name and region bounds
                string seq_name;
                int64_t start_pos = -1, stop_pos = -1;
                parse_region(region,
                             seq_name,
                             start_pos,
                             stop_pos);

                if (start_pos > 0 && stop_pos > 0) {
                    // These are 0-based, so if both are nonzero we got a real set of coordinates
                    if (constructor.show_progress) {
                        cerr << "Restricting to " << seq_name << " from " << start_pos << " to " << stop_pos << endl;
                    }
                    constructor.allowed_vcf_names.insert(seq_name);
                    // Make sure to correct the coordinates to 0-based exclusive-end, from 1-based inclusive-end
                    constructor.allowed_vcf_regions[seq_name] = make_pair(start_pos - 1, stop_pos);
                } else if (start_pos < 0 && stop_pos < 0) {
                    // We just got a name
                    cerr << "Restricting to " << seq_name << " from 1 to end" << endl;
                    constructor.allowed_vcf_names.insert(seq_name);
                } else {
                    // This doesn't make sense. Does it have like one coordinate?
                    cerr << "error:[vg construct] could not parse " << region << endl;
                    exit(1);
                }
            } else {
                // We have been told not to parse the region
                cerr << "Restricting to " << region << " from 1 to end" << endl;
                constructor.allowed_vcf_names.insert(region);
            }
        }

        // This will own all the VCF files
        vector<unique_ptr<vcflib::VariantCallFile>> variant_files;
        for (auto& vcf_filename : vcf_filenames) {
            // Make sure each VCF file exists. Otherwise Tabix++ may exit with a non-
            // helpful message.

            // We can't invoke stat woithout a place for it to write. But all we
            // really want is its return value.
            struct stat temp;
            if(stat(vcf_filename.c_str(), &temp)) {
                cerr << "error:[vg construct] file \"" << vcf_filename << "\" not found" << endl;
                return 1;
            }
            vcflib::VariantCallFile* variant_file = new vcflib::VariantCallFile();
            variant_file->parseSamples = false; // Major speedup if there are many samples.
            variant_files.emplace_back(variant_file);
            variant_file->open(vcf_filename);
            if (!variant_file->is_open()) {
                cerr << "error:[vg construct] could not open" << vcf_filename << endl;
                return 1;
            }
        }

        if (fasta_filenames.empty()) {
            cerr << "error:[vg construct] a reference is required for graph construction" << endl;
            return 1;
        }
        vector<unique_ptr<FastaReference>> references;
        for (auto& fasta_filename : fasta_filenames) {
            // Open each FASTA file
            FastaReference* reference = new FastaReference();
            references.emplace_back(reference);
            reference->open(fasta_filename);
        }

        vector<unique_ptr<FastaReference> > insertions;
        for (auto& insertion_filename : insertion_filenames){
            // Open up those insertion files
            FastaReference* insertion = new FastaReference();
            insertions.emplace_back(insertion);
            insertion->open(insertion_filename);
        }

        // Make vectors of just bare pointers
        vector<vcflib::VariantCallFile*> vcf_pointers;
        for(auto& vcf : variant_files) {
            vcf_pointers.push_back(vcf.get());
        }
        vector<FastaReference*> fasta_pointers;
        for(auto& fasta : references) {
            fasta_pointers.push_back(fasta.get());
        }
        vector<FastaReference*> ins_pointers;
        for (auto& ins : insertions){
            ins_pointers.push_back(ins.get());
        }

        if (ins_pointers.size() > 1){
            cerr << "Error: only one insertion file may be provided." << endl;
            exit(1);
        }

        // Construct the graph.
        constructor.construct_graph(fasta_pointers, vcf_pointers,
                                    ins_pointers, callback);

        // The output will be flushed when the ProtobufEmitter we use in the callback goes away.
        // Don't add an extra EOF marker or anything.

        // NB: If you worry about "still reachable but possibly lost" warnings in valgrind,
        // this would free all the memory used by protobuf:
        //ShutdownProtobufLibrary();
    }
    else if (!msa_filename.empty()) {

        ifstream msa_file(msa_filename);
        if (!msa_file) {
            cerr << "error:[vg construct] could not open MSA file " << msa_filename << endl;
            exit(1);
        }

        MSAConverter msa_converter;
        msa_converter.show_progress = show_progress;

        msa_converter.load_alignments(msa_file, msa_format);
        VG msa_graph = msa_converter.make_graph(keep_paths, max_node_size);

        msa_graph.serialize_to_ostream(cout);
    }
    else {
        cerr << "error:[vg construct] a reference or an MSA is required for construct" << endl;
        exit(1);
    }

    return 0;
}

// Register subcommand
static Subcommand vg_construct("construct", "graph construction", PIPELINE, 1, main_construct);

<|endoftext|>