<commit_before>#ifndef MY_LITTLE_ARG_PARSER_H
#define MY_LITTLE_ARG_PARSER_H

#include <unordered_map>

namespace mylittleparser
{
    class ParseEntry
    {
    public:

        ParseEntry () : data_ (nullptr), size_ (0), nargs_ (0) { }

        ~ParseEntry () { if (data_) delete[] data_; data_ = nullptr; }

        template<typename T>
        inline T operator[](int idx) { return get<T> (idx); }

        template<typename T>
        inline T get (int idx)
        {
            if (sizeof (T) != size_ || 0 > idx || (int) nargs_ <= idx)
                throw this;
            return reinterpret_cast<T*>(data_)[idx];
        }

    private:

        friend class ArgumentParser;

        template<typename T>
        inline void resize (size_t nargs)
        {
            size_ = sizeof (T);
            nargs_ = nargs;
            if (data_) delete[] data_;
            data_ = new unsigned char[size_ * nargs_];
        }

        template<typename T>
        inline void set (int idx, const T& val)
        {
            reinterpret_cast<T*>(data_)[idx] = val;
        }

        unsigned char* data_;
        size_t size_;
        size_t nargs_;
    };

    typedef std::unordered_map<std::string, ParseEntry> ParseArgs;

    class ArgumentParser
    {
    public:

        ArgumentParser (int argc, char** argv)
            : argc_ (argc), argv_ (argv)
        { }

        ~ArgumentParser ()
        {

        }

        template <typename T, int nargs = 0, bool required = false>
        void add_argument (const char* name, const char* help,
                           const char* abbr = nullptr,
                           typename std::decay<decltype(
                               ParseType<std::remove_cv<T>::type,
                               nargs>::parse)>::type* = nullptr)
        {
            typedef std::remove_cv<T>::type U;

            int idx = search<required> (name);
            parse<U, nargs> (name, idx);
        }

        ParseArgs parse_args () { return std::move (parse_args_); }

    private:

        template<bool required> int search (const char* name)
        {
            int idx = 1;
            while (idx < argc_ && strcoll (name, argv_[idx]) != 0) idx++;
            return idx;
        }

        template<> int search<true> (const char* name)
        {
            int idx = search<false> (name);
            if (argc_ == idx) { throw this; }
            return idx;
        }

        template<typename U, int nargs> void parse (const char* name, int idx)
        {
            parse_args_[name].resize<U> (nargs);
            for (int i = 1; i <= nargs && idx + i < argc_; ++i)
                parse_args_[name].set<U> (
                    i - 1, ParseType<U, nargs>::parse (argv_[idx + i]));
        }

        template<> void parse<bool, 0> (const char* name, int idx)
        {
            parse_args_[name].resize<bool> (1);
            parse_args_[name].set<bool> (0, idx < argc_);
        }

        template<typename T, int nargs> struct ParseType { };

        template<int nargs> struct ParseType<bool, nargs>
        {
            static typename std::enable_if<nargs == 0, bool>::type parse () { }
        };

        template<int nargs> struct ParseType<int, nargs>
        {
            static typename std::enable_if<nargs >= 1, int>::type
                parse (const char* str)
            {
                return std::atoi (str);
            }
        };

        template<int nargs> struct ParseType<char*, nargs>
        {
            static typename std::enable_if<nargs >= 1, char*>::type
                parse (const char* str)
            {
                return const_cast<char*>(str);
            }
        };

        template<int nargs> struct ParseType<const char*, nargs>
        {
            static typename std::enable_if<nargs >= 1, const char*>::type
                parse (const char* str)
            {
                return str;
            }
        };

        int argc_;
        char** argv_;
        std::unordered_map<std::string, ParseEntry> parse_args_;
    };
}

#endif
<commit_msg>Add helping<commit_after>#ifndef MY_LITTLE_ARG_PARSER_H
#define MY_LITTLE_ARG_PARSER_H

#include <unordered_map>

namespace mylittleparser
{
    class ParseEntry
    {
    public:

        ParseEntry () : data_ (nullptr), size_ (0), nargs_ (0) { }

        ~ParseEntry () { if (data_) delete[] data_; data_ = nullptr; }

        template<typename T>
        inline T operator[](int idx) { return get<T> (idx); }

        template<typename T>
        inline T get (int idx)
        {
            if (sizeof (T) != size_ || 0 > idx || (int) nargs_ <= idx)
                throw this;
            return reinterpret_cast<T*>(data_)[idx];
        }

    private:

        friend class ArgumentParser;

        template<typename T>
        inline void resize (size_t nargs)
        {
            size_ = sizeof (T);
            nargs_ = nargs;
            if (data_) delete[] data_;
            data_ = new unsigned char[size_ * nargs_];
        }

        template<typename T>
        inline void set (int idx, const T& val)
        {
            reinterpret_cast<T*>(data_)[idx] = val;
        }

        unsigned char* data_;
        size_t size_;
        size_t nargs_;
    };

    typedef std::unordered_map<std::string, ParseEntry> ParseArgs;

    class ArgumentParser
    {
    public:

        ArgumentParser (int argc, char** argv)
            : argc_ (argc), argv_ (argv), help_(false)
        {
            help_ = search<false> ("--help", "-h") < argc_;
        }

        ~ArgumentParser ()
        {

        }

        template <typename T, int nargs = 0, bool required = false>
        void add_argument (const char* name, const char* help,
                           const char* abbr = nullptr,
                           typename std::decay<decltype(
                               ParseType<std::remove_cv<T>::type,
                               nargs>::parse)>::type* = nullptr)
        {
            typedef std::remove_cv<T>::type U;

            int idx = search<required> (name, abbr);
            parse<U, nargs> (name, idx);
        }

        ParseArgs parse_args ()
        {
            if (help_) exit (0);
            return parse_args_;
        }

    private:

        template<bool required> int search (const char* name, const char* abbr)
        {
            int idx = 1;
            while (idx < argc_ && strcoll (name, argv_[idx]) != 0) idx++;
            return idx;
        }

        template<> int search<true> (const char* name, const char* abbr)
        {
            int idx = search<false> (name, abbr);
            if (argc_ == idx) { throw this; }
            return idx;
        }

        template<typename U, int nargs> void parse (const char* name, int idx)
        {
            parse_args_[name].resize<U> (nargs);
            for (int i = 1; i <= nargs && idx + i < argc_; ++i)
                parse_args_[name].set<U> (
                    i - 1, ParseType<U, nargs>::parse (argv_[idx + i]));
        }

        template<> void parse<bool, 0> (const char* name, int idx)
        {
            parse_args_[name].resize<bool> (1);
            parse_args_[name].set<bool> (0, idx < argc_);
        }

        template<typename T, int nargs> struct ParseType { };

        template<int nargs> struct ParseType<bool, nargs>
        {
            static typename std::enable_if<nargs == 0, bool>::type parse () { }
        };

        template<int nargs> struct ParseType<int, nargs>
        {
            static typename std::enable_if<nargs >= 1, int>::type
                parse (const char* str)
            {
                return std::atoi (str);
            }
        };

        template<int nargs> struct ParseType<char*, nargs>
        {
            static typename std::enable_if<nargs >= 1, char*>::type
                parse (const char* str)
            {
                return const_cast<char*>(str);
            }
        };

        template<int nargs> struct ParseType<const char*, nargs>
        {
            static typename std::enable_if<nargs >= 1, const char*>::type
                parse (const char* str)
            {
                return str;
            }
        };

        int argc_;
        char** argv_;
        bool help_;
        ParseArgs parse_args_;
    };
}

#endif
<|endoftext|>


<commit_before>/*
 * Copyright 2018 Google, Inc.
 *
 * Redistribution and use in source and binary forms, with or without
 * modification, are permitted provided that the following conditions are
 * met: redistributions of source code must retain the above copyright
 * notice, this list of conditions and the following disclaimer;
 * redistributions in binary form must reproduce the above copyright
 * notice, this list of conditions and the following disclaimer in the
 * documentation and/or other materials provided with the distribution;
 * neither the name of the copyright holders nor the names of its
 * contributors may be used to endorse or promote products derived from
 * this software without specific prior written permission.
 *
 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
 * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
 * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
 * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
 * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
 * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
 * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
 * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 *
 * Authors: Gabe Black
 */

#ifndef __BASE_FIBER_HH__
#define __BASE_FIBER_HH__

#include <ucontext.h>

#include <cstddef>
#include <cstdint>

/**
 * This class represents a fiber, which is a light weight sort of thread which
 * is cooperatively scheduled and runs sequentially with other fibers, swapping
 * in and out of a single actual thread of execution.
 *
 * To define your own threads, create a subclass of Fiber and override its
 * main() function to do what you want your fiber to do. You can start it by
 * calling its run() method which will stop your execution and start the other
 * fiber in your place.
 *
 * If your main() function ends, that fiber will automatically switch to either
 * the primary fiber, or to a particular fiber you specified at construction
 * time, and your fiber is considered finished.
 */

class Fiber
{
  public:
    const static size_t DefaultStackSize = 0x50000;

    /// stack_size is the size of the stack available to this fiber.
    /// link points to another fiber which will start executing when this
    /// fiber's main function returns.
    Fiber(size_t stack_size=DefaultStackSize);
    Fiber(Fiber *link, size_t stack_size=DefaultStackSize);

    virtual ~Fiber();

    /// Start executing the fiber represented by this object. This function
    /// will "return" when the current fiber is switched back to later on.
    void run();

    /// Returns whether the "main" function of this fiber has finished.
    ///
    bool finished() const { return _finished; };

    /// Get a pointer to the current running Fiber.
    ///
    static Fiber *currentFiber();
    /// Get a pointer to the primary Fiber.
    /// This Fiber represents the thread of execution started by the OS, and
    /// which has a Fiber attached to it after the fact.
    static Fiber *primaryFiber();

  protected:
    /// This method is called when this fiber is first run. Override it to
    /// give your fiber something to do. When main returns, the fiber will
    /// mark itself as finished and switch to its link fiber.
    virtual void main() = 0;

    void setStarted() { started = true; }

  private:
    static void entryTrampoline();
    void start();

    ucontext_t ctx;
    Fiber *link;

    // The stack for this context, or a nullptr if allocated elsewhere.
    uint8_t *stack;
    size_t stackSize;

    bool started;
    bool _finished;
    void createContext();
};

#endif // __BASE_FIBER_HH__
<commit_msg>base: Fix ucontext compilation error for macOS<commit_after>/*
 * Copyright 2018 Google, Inc.
 *
 * Redistribution and use in source and binary forms, with or without
 * modification, are permitted provided that the following conditions are
 * met: redistributions of source code must retain the above copyright
 * notice, this list of conditions and the following disclaimer;
 * redistributions in binary form must reproduce the above copyright
 * notice, this list of conditions and the following disclaimer in the
 * documentation and/or other materials provided with the distribution;
 * neither the name of the copyright holders nor the names of its
 * contributors may be used to endorse or promote products derived from
 * this software without specific prior written permission.
 *
 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
 * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
 * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
 * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
 * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
 * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
 * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
 * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 *
 * Authors: Gabe Black
 */

#ifndef __BASE_FIBER_HH__
#define __BASE_FIBER_HH__

// ucontext functions (like getcontext, setcontext etc) have been marked
// as deprecated and are hence hidden in latest macOS releases.
// By defining _XOPEN_SOURCE we make them available at compilation time.
#if defined(__APPLE__) && defined(__MACH__)
#define _XOPEN_SOURCE 600
#include <ucontext.h>
#undef _XOPEN_SOURCE
#else
#include <ucontext.h>
#endif

#include <cstddef>
#include <cstdint>

/**
 * This class represents a fiber, which is a light weight sort of thread which
 * is cooperatively scheduled and runs sequentially with other fibers, swapping
 * in and out of a single actual thread of execution.
 *
 * To define your own threads, create a subclass of Fiber and override its
 * main() function to do what you want your fiber to do. You can start it by
 * calling its run() method which will stop your execution and start the other
 * fiber in your place.
 *
 * If your main() function ends, that fiber will automatically switch to either
 * the primary fiber, or to a particular fiber you specified at construction
 * time, and your fiber is considered finished.
 */

class Fiber
{
  public:
    const static size_t DefaultStackSize = 0x50000;

    /// stack_size is the size of the stack available to this fiber.
    /// link points to another fiber which will start executing when this
    /// fiber's main function returns.
    Fiber(size_t stack_size=DefaultStackSize);
    Fiber(Fiber *link, size_t stack_size=DefaultStackSize);

    virtual ~Fiber();

    /// Start executing the fiber represented by this object. This function
    /// will "return" when the current fiber is switched back to later on.
    void run();

    /// Returns whether the "main" function of this fiber has finished.
    ///
    bool finished() const { return _finished; };

    /// Get a pointer to the current running Fiber.
    ///
    static Fiber *currentFiber();
    /// Get a pointer to the primary Fiber.
    /// This Fiber represents the thread of execution started by the OS, and
    /// which has a Fiber attached to it after the fact.
    static Fiber *primaryFiber();

  protected:
    /// This method is called when this fiber is first run. Override it to
    /// give your fiber something to do. When main returns, the fiber will
    /// mark itself as finished and switch to its link fiber.
    virtual void main() = 0;

    void setStarted() { started = true; }

  private:
    static void entryTrampoline();
    void start();

    ucontext_t ctx;
    Fiber *link;

    // The stack for this context, or a nullptr if allocated elsewhere.
    uint8_t *stack;
    size_t stackSize;

    bool started;
    bool _finished;
    void createContext();
};

#endif // __BASE_FIBER_HH__
<|endoftext|>


<commit_before>#include <bf/bitvector.hpp>

#include <cassert>

namespace bf {

typedef bitvector::size_type size_type;
typedef bitvector::block_type block_type;

namespace {

uint8_t count_table[] = {
  0, 1, 1, 2, 1, 2, 2, 3, 1, 2, 2, 3, 2, 3, 3, 4, 1, 2, 2, 3, 2, 3, 3, 4, 2, 3,
  3, 4, 3, 4, 4, 5, 1, 2, 2, 3, 2, 3, 3, 4, 2, 3, 3, 4, 3, 4, 4, 5, 2, 3, 3, 4,
  3, 4, 4, 5, 3, 4, 4, 5, 4, 5, 5, 6, 1, 2, 2, 3, 2, 3, 3, 4, 2, 3, 3, 4, 3, 4,
  4, 5, 2, 3, 3, 4, 3, 4, 4, 5, 3, 4, 4, 5, 4, 5, 5, 6, 2, 3, 3, 4, 3, 4, 4, 5,
  3, 4, 4, 5, 4, 5, 5, 6, 3, 4, 4, 5, 4, 5, 5, 6, 4, 5, 5, 6, 5, 6, 6, 7, 1, 2,
  2, 3, 2, 3, 3, 4, 2, 3, 3, 4, 3, 4, 4, 5, 2, 3, 3, 4, 3, 4, 4, 5, 3, 4, 4, 5,
  4, 5, 5, 6, 2, 3, 3, 4, 3, 4, 4, 5, 3, 4, 4, 5, 4, 5, 5, 6, 3, 4, 4, 5, 4, 5,
  5, 6, 4, 5, 5, 6, 5, 6, 6, 7, 2, 3, 3, 4, 3, 4, 4, 5, 3, 4, 4, 5, 4, 5, 5, 6,
  3, 4, 4, 5, 4, 5, 5, 6, 4, 5, 5, 6, 5, 6, 6, 7, 3, 4, 4, 5, 4, 5, 5, 6, 4, 5,
  5, 6, 5, 6, 6, 7, 4, 5, 5, 6, 5, 6, 6, 7, 5, 6, 6, 7, 6, 7, 7, 8};

} // namespace <anonymous>

bitvector::reference::reference(block_type& block, block_type i)
    : block_(block), mask_(block_type(1) << i) {
  assert(i < bits_per_block);
}

bitvector::reference& bitvector::reference::flip() {
  block_ ^= mask_;
  return *this;
}

bitvector::reference::operator bool() const {
  return (block_ & mask_) != 0;
}

bool bitvector::reference::operator~() const {
  return (block_ & mask_) == 0;
}

bitvector::reference& bitvector::reference::operator=(bool x) {
  x ? block_ |= mask_ : block_ &= ~mask_;
  return *this;
}

bitvector::reference& bitvector::reference::operator=(reference const& other) {
  other ? block_ |= mask_ : block_ &= ~mask_;
  return *this;
}

bitvector::reference& bitvector::reference::operator|=(bool x) {
  if (x)
    block_ |= mask_;
  return *this;
}

bitvector::reference& bitvector::reference::operator&=(bool x) {
  if (!x)
    block_ &= ~mask_;
  return *this;
}

bitvector::reference& bitvector::reference::operator^=(bool x) {
  if (x)
    block_ ^= mask_;
  return *this;
}

bitvector::reference& bitvector::reference::operator-=(bool x) {
  if (x)
    block_ &= ~mask_;
  return *this;
}

bitvector::bitvector() : num_bits_(0) {
}

bitvector::bitvector(size_type size, bool value)
    : bits_(bits_to_blocks(size), value ? ~block_type(0) : 0), num_bits_(size) {
}

bitvector::bitvector(bitvector const& other)
    : bits_(other.bits_), num_bits_(other.num_bits_) {
}

bitvector::bitvector(bitvector&& other)
    : bits_(std::move(other.bits_)), num_bits_(other.num_bits_) {
  other.num_bits_ = 0;
}

bitvector bitvector::operator~() const {
  bitvector b(*this);
  b.flip();
  return b;
}

bitvector& bitvector::operator=(bitvector other) {
  swap(*this, other);
  return *this;
}

void swap(bitvector& x, bitvector& y) {
  using std::swap;
  swap(x.bits_, y.bits_);
  swap(x.num_bits_, y.num_bits_);
}

bitvector bitvector::operator<<(size_type n) const {
  bitvector b(*this);
  return b <<= n;
}

bitvector bitvector::operator>>(size_type n) const {
  bitvector b(*this);
  return b >>= n;
}

bitvector& bitvector::operator<<=(size_type n) {
  if (n >= num_bits_)
    return reset();

  if (n > 0) {
    auto last = blocks() - 1;
    auto div = n / bits_per_block;
    auto r = bit_index(n);
    auto b = &bits_[0];
    assert(blocks() >= 1);
    assert(div <= last);

    if (r != 0) {
      for (size_type i = last - div; i > 0; --i)
        b[i + div] = (b[i] << r) | (b[i - 1] >> (bits_per_block - r));
      b[div] = b[0] << r;
    } else {
      for (size_type i = last - div; i > 0; --i)
        b[i + div] = b[i];
      b[div] = b[0];
    }

    std::fill_n(b, div, block_type(0));
    zero_unused_bits();
  }

  return *this;
}

bitvector& bitvector::operator>>=(size_type n) {
  if (n >= num_bits_)
    return reset();

  if (n > 0) {
    auto last = blocks() - 1;
    auto div = n / bits_per_block;
    auto r = bit_index(n);
    auto b = &bits_[0];
    assert(blocks() >= 1);
    assert(div <= last);

    if (r != 0) {
      for (size_type i = last - div; i > 0; --i)
        b[i - div] = (b[i] >> r) | (b[i + 1] << (bits_per_block - r));
      b[last - div] = b[last] >> r;
    } else {
      for (size_type i = div; i <= last; ++i)
        b[i - div] = b[i];
    }

    std::fill_n(b + (blocks() - div), div, block_type(0));
  }

  return *this;
}

bitvector& bitvector::operator&=(bitvector const& other) {
  assert(size() >= other.size());
  for (size_type i = 0; i < blocks(); ++i)
    bits_[i] &= other.bits_[i];
  return *this;
}

bitvector& bitvector::operator|=(bitvector const& other) {
  assert(size() >= other.size());
  for (size_type i = 0; i < blocks(); ++i)
    bits_[i] |= other.bits_[i];
  return *this;
}

bitvector& bitvector::operator^=(bitvector const& other) {
  assert(size() >= other.size());
  for (size_type i = 0; i < blocks(); ++i)
    bits_[i] ^= other.bits_[i];
  return *this;
}

bitvector& bitvector::operator-=(bitvector const& other) {
  assert(size() >= other.size());
  for (size_type i = 0; i < blocks(); ++i)
    bits_[i] &= ~other.bits_[i];
  return *this;
}

bitvector operator&(bitvector const& x, bitvector const& y) {
  bitvector b(x);
  return b &= y;
}

bitvector operator|(bitvector const& x, bitvector const& y) {
  bitvector b(x);
  return b |= y;
}

bitvector operator^(bitvector const& x, bitvector const& y) {
  bitvector b(x);
  return b ^= y;
}

bitvector operator-(bitvector const& x, bitvector const& y) {
  bitvector b(x);
  return b -= y;
}

bool operator==(bitvector const& x, bitvector const& y) {
  return x.num_bits_ == y.num_bits_ && x.bits_ == y.bits_;
}

bool operator!=(bitvector const& x, bitvector const& y) {
  return !(x == y);
}

bool operator<(bitvector const& x, bitvector const& y) {
  assert(x.size() == y.size());
  for (size_type r = x.blocks(); r > 0; --r) {
    auto i = r - 1;
    if (x.bits_[i] < y.bits_[i])
      return true;
    else if (x.bits_[i] > y.bits_[i])
      return false;
  }
  return false;
}

void bitvector::resize(size_type n, bool value) {
  auto old = blocks();
  auto required = bits_to_blocks(n);
  auto block_value = value ? ~block_type(0) : block_type(0);

  if (required != old)
    bits_.resize(required, block_value);

  if (value && (n > num_bits_) && extra_bits())
    bits_[old - 1] |= (block_value << extra_bits());

  num_bits_ = n;
  zero_unused_bits();
}

void bitvector::clear() noexcept {
  bits_.clear();
  num_bits_ = 0;
}

void bitvector::push_back(bool bit) {
  auto s = size();
  resize(s + 1);
  set(s, bit);
}

void bitvector::append(block_type block) {
  auto excess = extra_bits();
  if (excess) {
    assert(!bits_.empty());
    bits_.push_back(block >> (bits_per_block - excess));
    bits_[bits_.size() - 2] |= (block << excess);
  } else {
    bits_.push_back(block);
  }
  num_bits_ += bits_per_block;
}

bitvector& bitvector::set(size_type i, bool bit) {
  assert(i < num_bits_);

  if (bit)
    bits_[block_index(i)] |= bit_mask(i);
  else
    reset(i);

  return *this;
}

bitvector& bitvector::set() {
  std::fill(bits_.begin(), bits_.end(), ~block_type(0));
  zero_unused_bits();
  return *this;
}

bitvector& bitvector::reset(size_type i) {
  assert(i < num_bits_);
  bits_[block_index(i)] &= ~bit_mask(i);
  return *this;
}

bitvector& bitvector::reset() {
  std::fill(bits_.begin(), bits_.end(), block_type(0));
  return *this;
}

bitvector& bitvector::flip(size_type i) {
  assert(i < num_bits_);
  bits_[block_index(i)] ^= bit_mask(i);
  return *this;
}

bitvector& bitvector::flip() {
  for (size_type i = 0; i < blocks(); ++i)
    bits_[i] = ~bits_[i];
  zero_unused_bits();
  return *this;
}

bool bitvector::operator[](size_type i) const {
  assert(i < num_bits_);
  return (bits_[block_index(i)] & bit_mask(i)) != 0;
}

bitvector::reference bitvector::operator[](size_type i) {
  assert(i < num_bits_);
  return {bits_[block_index(i)], bit_index(i)};
}

size_type bitvector::count() const {
  auto first = bits_.begin();
  size_t n = 0;
  auto length = blocks();
  while (length) {
    auto block = *first;
    while (block) {
      // TODO: use __popcnt if available.
      n += count_table[block & ((1u << 8) - 1)];
      block >>= 8;
    }
    ++first;
    --length;
  }
  return n;
}

size_type bitvector::blocks() const {
  return bits_.size();
}

size_type bitvector::size() const {
  return num_bits_;
}

bool bitvector::empty() const {
  return bits_.empty();
}

size_type bitvector::find_first() const {
  return find_from(0);
}

size_type bitvector::find_next(size_type i) const {
  if (i >= (size() - 1) || size() == 0)
    return npos;
  ++i;
  auto bi = block_index(i);
  auto block = bits_[bi] & (~block_type(0) << bit_index(i));
  return block ? bi * bits_per_block + lowest_bit(block) : find_from(bi + 1);
}

size_type bitvector::lowest_bit(block_type block) {
  auto x = block - (block & (block - 1)); // Extract right-most 1-bit.
  size_type log = 0;
  while (x >>= 1)
    ++log;
  return log;
}

block_type bitvector::extra_bits() const {
  return bit_index(size());
}

void bitvector::zero_unused_bits() {
  if (extra_bits())
    bits_.back() &= ~(~block_type(0) << extra_bits());
}

size_type bitvector::find_from(size_type i) const {
  while (i < blocks() && bits_[i] == 0)
    ++i;
  if (i >= blocks())
    return npos;
  return i * bits_per_block + lowest_bit(bits_[i]);
}

std::string to_string(bitvector const& b, bool msb_to_lsb, bool all,
                      size_t cut_off) {
  std::string str;
  auto str_size = all ? bitvector::bits_per_block * b.blocks() : b.size();
  if (cut_off == 0 || str_size <= cut_off) {
    str.assign(str_size, '0');
  } else {
    str.assign(cut_off + 2, '0');
    str[cut_off + 0] = '.';
    str[cut_off + 1] = '.';
    str_size = cut_off;
  }

  for (bitvector::size_type i = 0; i < std::min(str_size, b.size()); ++i)
    if (b[i])
      str[msb_to_lsb ? str_size - i - 1 : i] = '1';

  return str;
}

} // namespace bf
<commit_msg>include algorithm for std::min<commit_after>#include <bf/bitvector.hpp>

#include <algorithm>
#include <cassert>

namespace bf {

typedef bitvector::size_type size_type;
typedef bitvector::block_type block_type;

namespace {

uint8_t count_table[] = {
  0, 1, 1, 2, 1, 2, 2, 3, 1, 2, 2, 3, 2, 3, 3, 4, 1, 2, 2, 3, 2, 3, 3, 4, 2, 3,
  3, 4, 3, 4, 4, 5, 1, 2, 2, 3, 2, 3, 3, 4, 2, 3, 3, 4, 3, 4, 4, 5, 2, 3, 3, 4,
  3, 4, 4, 5, 3, 4, 4, 5, 4, 5, 5, 6, 1, 2, 2, 3, 2, 3, 3, 4, 2, 3, 3, 4, 3, 4,
  4, 5, 2, 3, 3, 4, 3, 4, 4, 5, 3, 4, 4, 5, 4, 5, 5, 6, 2, 3, 3, 4, 3, 4, 4, 5,
  3, 4, 4, 5, 4, 5, 5, 6, 3, 4, 4, 5, 4, 5, 5, 6, 4, 5, 5, 6, 5, 6, 6, 7, 1, 2,
  2, 3, 2, 3, 3, 4, 2, 3, 3, 4, 3, 4, 4, 5, 2, 3, 3, 4, 3, 4, 4, 5, 3, 4, 4, 5,
  4, 5, 5, 6, 2, 3, 3, 4, 3, 4, 4, 5, 3, 4, 4, 5, 4, 5, 5, 6, 3, 4, 4, 5, 4, 5,
  5, 6, 4, 5, 5, 6, 5, 6, 6, 7, 2, 3, 3, 4, 3, 4, 4, 5, 3, 4, 4, 5, 4, 5, 5, 6,
  3, 4, 4, 5, 4, 5, 5, 6, 4, 5, 5, 6, 5, 6, 6, 7, 3, 4, 4, 5, 4, 5, 5, 6, 4, 5,
  5, 6, 5, 6, 6, 7, 4, 5, 5, 6, 5, 6, 6, 7, 5, 6, 6, 7, 6, 7, 7, 8};

} // namespace <anonymous>

bitvector::reference::reference(block_type& block, block_type i)
    : block_(block), mask_(block_type(1) << i) {
  assert(i < bits_per_block);
}

bitvector::reference& bitvector::reference::flip() {
  block_ ^= mask_;
  return *this;
}

bitvector::reference::operator bool() const {
  return (block_ & mask_) != 0;
}

bool bitvector::reference::operator~() const {
  return (block_ & mask_) == 0;
}

bitvector::reference& bitvector::reference::operator=(bool x) {
  x ? block_ |= mask_ : block_ &= ~mask_;
  return *this;
}

bitvector::reference& bitvector::reference::operator=(reference const& other) {
  other ? block_ |= mask_ : block_ &= ~mask_;
  return *this;
}

bitvector::reference& bitvector::reference::operator|=(bool x) {
  if (x)
    block_ |= mask_;
  return *this;
}

bitvector::reference& bitvector::reference::operator&=(bool x) {
  if (!x)
    block_ &= ~mask_;
  return *this;
}

bitvector::reference& bitvector::reference::operator^=(bool x) {
  if (x)
    block_ ^= mask_;
  return *this;
}

bitvector::reference& bitvector::reference::operator-=(bool x) {
  if (x)
    block_ &= ~mask_;
  return *this;
}

bitvector::bitvector() : num_bits_(0) {
}

bitvector::bitvector(size_type size, bool value)
    : bits_(bits_to_blocks(size), value ? ~block_type(0) : 0), num_bits_(size) {
}

bitvector::bitvector(bitvector const& other)
    : bits_(other.bits_), num_bits_(other.num_bits_) {
}

bitvector::bitvector(bitvector&& other)
    : bits_(std::move(other.bits_)), num_bits_(other.num_bits_) {
  other.num_bits_ = 0;
}

bitvector bitvector::operator~() const {
  bitvector b(*this);
  b.flip();
  return b;
}

bitvector& bitvector::operator=(bitvector other) {
  swap(*this, other);
  return *this;
}

void swap(bitvector& x, bitvector& y) {
  using std::swap;
  swap(x.bits_, y.bits_);
  swap(x.num_bits_, y.num_bits_);
}

bitvector bitvector::operator<<(size_type n) const {
  bitvector b(*this);
  return b <<= n;
}

bitvector bitvector::operator>>(size_type n) const {
  bitvector b(*this);
  return b >>= n;
}

bitvector& bitvector::operator<<=(size_type n) {
  if (n >= num_bits_)
    return reset();

  if (n > 0) {
    auto last = blocks() - 1;
    auto div = n / bits_per_block;
    auto r = bit_index(n);
    auto b = &bits_[0];
    assert(blocks() >= 1);
    assert(div <= last);

    if (r != 0) {
      for (size_type i = last - div; i > 0; --i)
        b[i + div] = (b[i] << r) | (b[i - 1] >> (bits_per_block - r));
      b[div] = b[0] << r;
    } else {
      for (size_type i = last - div; i > 0; --i)
        b[i + div] = b[i];
      b[div] = b[0];
    }

    std::fill_n(b, div, block_type(0));
    zero_unused_bits();
  }

  return *this;
}

bitvector& bitvector::operator>>=(size_type n) {
  if (n >= num_bits_)
    return reset();

  if (n > 0) {
    auto last = blocks() - 1;
    auto div = n / bits_per_block;
    auto r = bit_index(n);
    auto b = &bits_[0];
    assert(blocks() >= 1);
    assert(div <= last);

    if (r != 0) {
      for (size_type i = last - div; i > 0; --i)
        b[i - div] = (b[i] >> r) | (b[i + 1] << (bits_per_block - r));
      b[last - div] = b[last] >> r;
    } else {
      for (size_type i = div; i <= last; ++i)
        b[i - div] = b[i];
    }

    std::fill_n(b + (blocks() - div), div, block_type(0));
  }

  return *this;
}

bitvector& bitvector::operator&=(bitvector const& other) {
  assert(size() >= other.size());
  for (size_type i = 0; i < blocks(); ++i)
    bits_[i] &= other.bits_[i];
  return *this;
}

bitvector& bitvector::operator|=(bitvector const& other) {
  assert(size() >= other.size());
  for (size_type i = 0; i < blocks(); ++i)
    bits_[i] |= other.bits_[i];
  return *this;
}

bitvector& bitvector::operator^=(bitvector const& other) {
  assert(size() >= other.size());
  for (size_type i = 0; i < blocks(); ++i)
    bits_[i] ^= other.bits_[i];
  return *this;
}

bitvector& bitvector::operator-=(bitvector const& other) {
  assert(size() >= other.size());
  for (size_type i = 0; i < blocks(); ++i)
    bits_[i] &= ~other.bits_[i];
  return *this;
}

bitvector operator&(bitvector const& x, bitvector const& y) {
  bitvector b(x);
  return b &= y;
}

bitvector operator|(bitvector const& x, bitvector const& y) {
  bitvector b(x);
  return b |= y;
}

bitvector operator^(bitvector const& x, bitvector const& y) {
  bitvector b(x);
  return b ^= y;
}

bitvector operator-(bitvector const& x, bitvector const& y) {
  bitvector b(x);
  return b -= y;
}

bool operator==(bitvector const& x, bitvector const& y) {
  return x.num_bits_ == y.num_bits_ && x.bits_ == y.bits_;
}

bool operator!=(bitvector const& x, bitvector const& y) {
  return !(x == y);
}

bool operator<(bitvector const& x, bitvector const& y) {
  assert(x.size() == y.size());
  for (size_type r = x.blocks(); r > 0; --r) {
    auto i = r - 1;
    if (x.bits_[i] < y.bits_[i])
      return true;
    else if (x.bits_[i] > y.bits_[i])
      return false;
  }
  return false;
}

void bitvector::resize(size_type n, bool value) {
  auto old = blocks();
  auto required = bits_to_blocks(n);
  auto block_value = value ? ~block_type(0) : block_type(0);

  if (required != old)
    bits_.resize(required, block_value);

  if (value && (n > num_bits_) && extra_bits())
    bits_[old - 1] |= (block_value << extra_bits());

  num_bits_ = n;
  zero_unused_bits();
}

void bitvector::clear() noexcept {
  bits_.clear();
  num_bits_ = 0;
}

void bitvector::push_back(bool bit) {
  auto s = size();
  resize(s + 1);
  set(s, bit);
}

void bitvector::append(block_type block) {
  auto excess = extra_bits();
  if (excess) {
    assert(!bits_.empty());
    bits_.push_back(block >> (bits_per_block - excess));
    bits_[bits_.size() - 2] |= (block << excess);
  } else {
    bits_.push_back(block);
  }
  num_bits_ += bits_per_block;
}

bitvector& bitvector::set(size_type i, bool bit) {
  assert(i < num_bits_);

  if (bit)
    bits_[block_index(i)] |= bit_mask(i);
  else
    reset(i);

  return *this;
}

bitvector& bitvector::set() {
  std::fill(bits_.begin(), bits_.end(), ~block_type(0));
  zero_unused_bits();
  return *this;
}

bitvector& bitvector::reset(size_type i) {
  assert(i < num_bits_);
  bits_[block_index(i)] &= ~bit_mask(i);
  return *this;
}

bitvector& bitvector::reset() {
  std::fill(bits_.begin(), bits_.end(), block_type(0));
  return *this;
}

bitvector& bitvector::flip(size_type i) {
  assert(i < num_bits_);
  bits_[block_index(i)] ^= bit_mask(i);
  return *this;
}

bitvector& bitvector::flip() {
  for (size_type i = 0; i < blocks(); ++i)
    bits_[i] = ~bits_[i];
  zero_unused_bits();
  return *this;
}

bool bitvector::operator[](size_type i) const {
  assert(i < num_bits_);
  return (bits_[block_index(i)] & bit_mask(i)) != 0;
}

bitvector::reference bitvector::operator[](size_type i) {
  assert(i < num_bits_);
  return {bits_[block_index(i)], bit_index(i)};
}

size_type bitvector::count() const {
  auto first = bits_.begin();
  size_t n = 0;
  auto length = blocks();
  while (length) {
    auto block = *first;
    while (block) {
      // TODO: use __popcnt if available.
      n += count_table[block & ((1u << 8) - 1)];
      block >>= 8;
    }
    ++first;
    --length;
  }
  return n;
}

size_type bitvector::blocks() const {
  return bits_.size();
}

size_type bitvector::size() const {
  return num_bits_;
}

bool bitvector::empty() const {
  return bits_.empty();
}

size_type bitvector::find_first() const {
  return find_from(0);
}

size_type bitvector::find_next(size_type i) const {
  if (i >= (size() - 1) || size() == 0)
    return npos;
  ++i;
  auto bi = block_index(i);
  auto block = bits_[bi] & (~block_type(0) << bit_index(i));
  return block ? bi * bits_per_block + lowest_bit(block) : find_from(bi + 1);
}

size_type bitvector::lowest_bit(block_type block) {
  auto x = block - (block & (block - 1)); // Extract right-most 1-bit.
  size_type log = 0;
  while (x >>= 1)
    ++log;
  return log;
}

block_type bitvector::extra_bits() const {
  return bit_index(size());
}

void bitvector::zero_unused_bits() {
  if (extra_bits())
    bits_.back() &= ~(~block_type(0) << extra_bits());
}

size_type bitvector::find_from(size_type i) const {
  while (i < blocks() && bits_[i] == 0)
    ++i;
  if (i >= blocks())
    return npos;
  return i * bits_per_block + lowest_bit(bits_[i]);
}

std::string to_string(bitvector const& b, bool msb_to_lsb, bool all,
                      size_t cut_off) {
  std::string str;
  auto str_size = all ? bitvector::bits_per_block * b.blocks() : b.size();
  if (cut_off == 0 || str_size <= cut_off) {
    str.assign(str_size, '0');
  } else {
    str.assign(cut_off + 2, '0');
    str[cut_off + 0] = '.';
    str[cut_off + 1] = '.';
    str_size = cut_off;
  }

  for (bitvector::size_type i = 0; i < std::min(str_size, b.size()); ++i)
    if (b[i])
      str[msb_to_lsb ? str_size - i - 1 : i] = '1';

  return str;
}

} // namespace bf
<|endoftext|>


<commit_before>#include <glibmm/i18n.h>
#include <iostream>

#include "page.h"
using namespace AhoViewer::Booru;

#include "curler.h"
#include "image.h"
#include "settings.h"

Page::Page(Gtk::Menu *menu)
  : Gtk::ScrolledWindow(),
    m_PopupMenu(menu),
    m_ImageFetcher(std::unique_ptr<ImageFetcher>(new ImageFetcher())),
    m_IconView(Gtk::manage(new Gtk::IconView())),
    m_Tab(Gtk::manage(new Gtk::EventBox())),
    m_TabIcon(Gtk::manage(new Gtk::Image(Gtk::Stock::NEW, Gtk::ICON_SIZE_MENU))),
    m_TabLabel(Gtk::manage(new Gtk::Label(_("New Tab")))),
    m_TabButton(Gtk::manage(new Gtk::Button())),
    m_ImageList(std::make_shared<ImageList>(this)),
    m_Page(0),
    m_NumPosts(0),
    m_LastPage(false),
    m_Saving(false),
    m_SaveCancel(Gio::Cancellable::create()),
    m_GetPostsThread(nullptr),
    m_SaveImagesThread(nullptr)
{
    set_policy(Gtk::POLICY_NEVER, Gtk::POLICY_AUTOMATIC);
    set_shadow_type(Gtk::SHADOW_ETCHED_IN);

    // Create page tab {{{
    Glib::RefPtr<Gtk::RcStyle> style = m_TabButton->get_modifier_style();
    style->set_ythickness(0);
    style->set_xthickness(0);

    m_TabButton->modify_style(style);
    m_TabButton->add(*(Gtk::manage(new Gtk::Image(Gtk::Stock::CLOSE, Gtk::ICON_SIZE_MENU))));
    m_TabButton->property_relief() = Gtk::RELIEF_NONE;
    m_TabButton->set_focus_on_click(false);
    m_TabButton->set_tooltip_text(_("Close Tab"));
    m_TabButton->signal_clicked().connect([ this ]() { m_SignalClosed(this); });

    m_TabLabel->set_alignment(0.0, 0.5);
    m_TabLabel->set_ellipsize(Pango::ELLIPSIZE_END);

    Gtk::HBox *hbox = Gtk::manage(new Gtk::HBox());
    hbox->pack_start(*m_TabIcon, false, false);
    hbox->pack_start(*m_TabLabel, true, true, 2);
    hbox->pack_start(*m_TabButton, false, false);
    hbox->show_all();

    m_Tab->set_visible_window(false);
    m_Tab->add(*hbox);
    m_Tab->signal_button_release_event().connect(sigc::mem_fun(*this, &Page::on_tab_button_release_event));
    // }}}

    get_vadjustment()->signal_value_changed().connect(sigc::mem_fun(*this, &Page::on_value_changed));

    ModelColumns columns;
    m_ListStore = Gtk::ListStore::create(columns);
    m_IconView->set_model(m_ListStore);
    m_IconView->set_selection_mode(Gtk::SELECTION_BROWSE);
    m_IconView->set_item_width(Image::BooruThumbnailSize - m_IconView->get_margin()
                                                         - m_IconView->property_item_padding().get_value());
    m_IconView->signal_selection_changed().connect(sigc::mem_fun(*this, &Page::on_selection_changed));
    m_IconView->signal_button_press_event().connect(sigc::mem_fun(*this, &Page::on_button_press_event));

    // Workaround to have fully centered pixbufs
    Gtk::CellRendererPixbuf *cell = Gtk::manage(new Gtk::CellRendererPixbuf());
    gtk_cell_layout_pack_start(GTK_CELL_LAYOUT(m_IconView->gobj()),
                               GTK_CELL_RENDERER(cell->gobj()), TRUE);
    gtk_cell_layout_add_attribute(GTK_CELL_LAYOUT(m_IconView->gobj()),
                                  GTK_CELL_RENDERER(cell->gobj()), "pixbuf", 0);

    m_SignalPostsDownloaded.connect(sigc::mem_fun(*this, &Page::on_posts_downloaded));
    m_SignalSaveProgressDisp.connect([ this ]()
    {
        m_SignalSaveProgress(m_SaveImagesCurrent, m_SaveImagesTotal);
    });

    add(*m_IconView);
    show_all();
}

Page::~Page()
{
    m_Curler.cancel();
    m_CountsCurler.cancel();

    if (m_GetPostsThread)
    {
        m_GetPostsThread->join();
        m_GetPostsThread = nullptr;
    }

    cancel_save();
}

void Page::set_selected(const size_t index)
{
    get_window()->freeze_updates();

    Gtk::TreePath path(std::to_string(index));
    m_IconView->select_path(path);
    scroll_to_selected();

    get_window()->thaw_updates();
}

void Page::scroll_to_selected()
{
    std::vector<Gtk::TreePath> paths = m_IconView->get_selected_items();

    if (!paths.empty())
    {
        Gtk::TreePath path = paths[0];
        if (path)
            m_IconView->scroll_to_path(path, false, 0, 0);
    }
}

void Page::search(const std::shared_ptr<Site> &site)
{
    if (!ask_cancel_save())
        return;

    m_Curler.cancel();
    m_CountsCurler.cancel();

    cancel_save();
    m_ImageList->clear();

    if (m_GetPostsThread)
        m_GetPostsThread->join();

    m_Site = site;
    m_Page = 1;
    m_LastPage = false;
    m_SearchTags = m_Tags;

    // Trim whitespace for the tab label text
    if (!m_SearchTags.empty())
    {
        size_t f = m_SearchTags.find_first_not_of(' '),
               l = m_SearchTags.find_last_not_of(' ');
        m_SearchTags = f == std::string::npos ? "" : " - " + m_SearchTags.substr(f, l - f + 1);
    }
    else
    {
        m_SearchTags = "*";
    }

    m_TabLabel->set_text(m_Site->get_name() + m_SearchTags);
    m_TabIcon->set(m_Site->get_icon_pixbuf());

    m_Curler.set_referer(m_Site->get_url());
    m_CountsCurler.set_referer(m_Site->get_url());

    get_posts();
}

void Page::save_image(const std::string &path, const std::shared_ptr<Image> &img)
{
    m_SaveCancel->reset();

    m_Saving            = true;
    m_SaveImagesThread  = Glib::Threads::Thread::create([ this, path, img ]()
    {
        img->save(path);
        m_Saving = false;
    });
}

void Page::save_images(const std::string &path)
{
    m_SaveCancel->reset();

    m_Saving            = true;
    m_SaveImagesCurrent = 0;
    m_SaveImagesTotal   = m_ImageList->get_vector_size();
    m_SaveImagesThread  = Glib::Threads::Thread::create([ this, path ]()
    {
        // 2 if cachesize is 0, 8 if cachesize > 2
        Glib::ThreadPool pool(std::max(std::min(Settings.get_int("CacheSize") * 4, 8), 2));
        for (const std::shared_ptr<AhoViewer::Image> &img : *m_ImageList)
        {
            pool.push([ this, path, img ]()
            {
                if (m_SaveCancel->is_cancelled())
                    return;

                std::shared_ptr<Image> bimage = std::static_pointer_cast<Image>(img);
                bimage->save(Glib::build_filename(path, Glib::path_get_basename(bimage->get_filename())));
                ++m_SaveImagesCurrent;

                if (!m_SaveCancel->is_cancelled())
                    m_SignalSaveProgressDisp();
            });
        }

        pool.shutdown(m_SaveCancel->is_cancelled());
        m_Saving = false;
    });
}

/**
 * Returns true if we want to cancel or we're not saving
 **/
bool Page::ask_cancel_save()
{
    if (!m_Saving)
        return true;

    Gtk::Window *window = static_cast<Gtk::Window*>(get_toplevel());
    Gtk::MessageDialog dialog(*window, _("Are you sure that you want to stop saving images?"),
                              false, Gtk::MESSAGE_QUESTION, Gtk::BUTTONS_YES_NO, true);
    dialog.set_secondary_text(_("Closing this tab will stop the save operation."));

    return dialog.run() == Gtk::RESPONSE_YES;
}

void Page::cancel_save()
{
    m_SaveCancel->cancel();

    for (const std::shared_ptr<AhoViewer::Image> &img : *m_ImageList)
    {
        std::shared_ptr<Image> bimage = std::static_pointer_cast<Image>(img);
        bimage->cancel_download();
    }

    if (m_SaveImagesThread)
    {
        m_SaveImagesThread->join();
        m_SaveImagesThread = nullptr;
    }
}

void Page::get_posts()
{
    std::string tags = m_SearchTags;

    if (m_Tags.find("rating:") == std::string::npos)
    {
        if (Settings.get_booru_max_rating() == Site::Rating::SAFE)
        {
            tags += " rating:safe";
        }
        else if (Settings.get_booru_max_rating() == Site::Rating::QUESTIONABLE)
        {
            tags += " -rating:explicit";
        }
    }

    tags = m_Curler.escape(tags);
    m_Curler.set_url(m_Site->get_posts_url(tags, m_Page));

    m_GetPostsThread = Glib::Threads::Thread::create([ this, tags ]()
    {
        size_t postsCount = 0;
        // Danbooru doesn't give the post count with the posts
        // Get it from thier counts api
        if (m_Page == 1 && m_Site->get_type() == Site::Type::DANBOORU)
        {
            m_CountsCurler.set_url(m_Site->get_url() + "/counts/posts.xml?tags=" + tags);
            if (m_CountsCurler.perform())
            {
                xmlDocument doc(reinterpret_cast<char*>(m_CountsCurler.get_data()), m_CountsCurler.get_data_size());
                postsCount = std::stoul(doc.get_children()[0].get_value());
            }
            else if (m_CountsCurler.is_cancelled())
            {
                return;
            }
        }

        if (m_Site->get_type() == Site::Type::GELBOORU)
            m_Curler.set_cookie_file(m_Site->get_cookie());
        else
            m_Curler.set_http_auth(m_Site->get_username(), m_Site->get_password());

        if (m_Curler.perform())
        {
            m_Posts = std::unique_ptr<xmlDocument>(
                    new xmlDocument(reinterpret_cast<char*>(m_Curler.get_data()),
                                    m_Curler.get_data_size()));
            m_NumPosts = m_Posts->get_n_nodes();

            if (postsCount)
                m_Posts->set_attribute("count", std::to_string(postsCount));
        }
        else if (!m_Curler.is_cancelled())
        {
            m_NumPosts = 0;
            std::cerr << "Error while downloading posts on " << m_Curler.get_url() << std::endl
                      << "  " << m_Curler.get_error() << std::endl;
        }

        if (!m_Curler.is_cancelled())
            m_SignalPostsDownloaded();
    });
}

bool Page::get_next_page()
{
    // Do not fetch the next page if this is the last
    // or the current page is still loading
    if (m_LastPage || m_GetPostsThread)
        return false;

    if (!m_Saving)
    {
        ++m_Page;
        get_posts();

        return false;
    }
    else if (!m_GetNextPageConn)
    {
        m_GetNextPageConn = Glib::signal_timeout().connect(sigc::mem_fun(*this, &Page::get_next_page), 1000);
    }

    return true;
}

/**
 * Adds the downloaded posts to the image list.
 **/
void Page::on_posts_downloaded()
{
    if (m_Posts && m_Posts->get_attribute("success") == "false" && !m_Posts->get_attribute("reason").empty())
    {
        m_SignalDownloadError(m_Posts->get_attribute("reason"));
    }
    else if (m_NumPosts > 0)
    {
        reserve(m_NumPosts);
        m_ImageList->load(*m_Posts, *this);
    }
    // 401 = Unauthorized
    else if (m_Curler.get_response_code() == 401)
    {
        Glib::ustring e = Glib::ustring::compose(_("Failed to login as %1 on %2"),
                                                 m_Site->get_username(),
                                                 m_Site->get_name());
        m_SignalDownloadError(e);
    }
    else if (m_Page == 1)
    {
        m_SignalDownloadError(_("No results found"));
    }

    if (m_NumPosts < static_cast<size_t>(Settings.get_int("BooruLimit")))
        m_LastPage = true;

    m_Posts = nullptr;
    m_GetPostsThread->join();
    m_GetPostsThread = nullptr;
}

void Page::on_selection_changed()
{
    std::vector<Gtk::TreePath> paths = m_IconView->get_selected_items();

    if (!paths.empty())
    {
        Gtk::TreePath path = paths[0];
        if (path)
        {
            size_t index = path[0];

            if (index + Settings.get_int("CacheSize") >= m_ImageList->get_vector_size() - 1)
                get_next_page();

            m_SignalSelectedChanged(index);
        }
    }
}

/**
 * Vertical scrollbar value changed
 **/
void Page::on_value_changed()
{
    double value = get_vadjustment()->get_value(),
           limit = get_vadjustment()->get_upper() -
                   get_vadjustment()->get_page_size() -
                   get_vadjustment()->get_step_increment();

    if (value >= limit)
        get_next_page();
}

bool Page::on_button_press_event(GdkEventButton *e)
{
    if (e->type == GDK_BUTTON_PRESS && e->button == 3)
    {
        Gtk::TreePath path = m_IconView->get_path_at_pos(e->x, e->y);

        if (path)
        {
            m_IconView->select_path(path);
            m_IconView->scroll_to_path(path, false, 0, 0);

            m_PopupMenu->popup(e->button, e->time);

            return true;
        }
    }

    return false;
}

bool Page::on_tab_button_release_event(GdkEventButton *e)
{
    if (e->type == GDK_BUTTON_RELEASE && e->button == 2)
        m_SignalClosed(this);

    return false;
}
<commit_msg>booru/page: Fix issues with tags introduced in e312099<commit_after>#include <glibmm/i18n.h>
#include <iostream>

#include "page.h"
using namespace AhoViewer::Booru;

#include "curler.h"
#include "image.h"
#include "settings.h"

Page::Page(Gtk::Menu *menu)
  : Gtk::ScrolledWindow(),
    m_PopupMenu(menu),
    m_ImageFetcher(std::unique_ptr<ImageFetcher>(new ImageFetcher())),
    m_IconView(Gtk::manage(new Gtk::IconView())),
    m_Tab(Gtk::manage(new Gtk::EventBox())),
    m_TabIcon(Gtk::manage(new Gtk::Image(Gtk::Stock::NEW, Gtk::ICON_SIZE_MENU))),
    m_TabLabel(Gtk::manage(new Gtk::Label(_("New Tab")))),
    m_TabButton(Gtk::manage(new Gtk::Button())),
    m_ImageList(std::make_shared<ImageList>(this)),
    m_Page(0),
    m_NumPosts(0),
    m_LastPage(false),
    m_Saving(false),
    m_SaveCancel(Gio::Cancellable::create()),
    m_GetPostsThread(nullptr),
    m_SaveImagesThread(nullptr)
{
    set_policy(Gtk::POLICY_NEVER, Gtk::POLICY_AUTOMATIC);
    set_shadow_type(Gtk::SHADOW_ETCHED_IN);

    // Create page tab {{{
    Glib::RefPtr<Gtk::RcStyle> style = m_TabButton->get_modifier_style();
    style->set_ythickness(0);
    style->set_xthickness(0);

    m_TabButton->modify_style(style);
    m_TabButton->add(*(Gtk::manage(new Gtk::Image(Gtk::Stock::CLOSE, Gtk::ICON_SIZE_MENU))));
    m_TabButton->property_relief() = Gtk::RELIEF_NONE;
    m_TabButton->set_focus_on_click(false);
    m_TabButton->set_tooltip_text(_("Close Tab"));
    m_TabButton->signal_clicked().connect([ this ]() { m_SignalClosed(this); });

    m_TabLabel->set_alignment(0.0, 0.5);
    m_TabLabel->set_ellipsize(Pango::ELLIPSIZE_END);

    Gtk::HBox *hbox = Gtk::manage(new Gtk::HBox());
    hbox->pack_start(*m_TabIcon, false, false);
    hbox->pack_start(*m_TabLabel, true, true, 2);
    hbox->pack_start(*m_TabButton, false, false);
    hbox->show_all();

    m_Tab->set_visible_window(false);
    m_Tab->add(*hbox);
    m_Tab->signal_button_release_event().connect(sigc::mem_fun(*this, &Page::on_tab_button_release_event));
    // }}}

    get_vadjustment()->signal_value_changed().connect(sigc::mem_fun(*this, &Page::on_value_changed));

    ModelColumns columns;
    m_ListStore = Gtk::ListStore::create(columns);
    m_IconView->set_model(m_ListStore);
    m_IconView->set_selection_mode(Gtk::SELECTION_BROWSE);
    m_IconView->set_item_width(Image::BooruThumbnailSize - m_IconView->get_margin()
                                                         - m_IconView->property_item_padding().get_value());
    m_IconView->signal_selection_changed().connect(sigc::mem_fun(*this, &Page::on_selection_changed));
    m_IconView->signal_button_press_event().connect(sigc::mem_fun(*this, &Page::on_button_press_event));

    // Workaround to have fully centered pixbufs
    Gtk::CellRendererPixbuf *cell = Gtk::manage(new Gtk::CellRendererPixbuf());
    gtk_cell_layout_pack_start(GTK_CELL_LAYOUT(m_IconView->gobj()),
                               GTK_CELL_RENDERER(cell->gobj()), TRUE);
    gtk_cell_layout_add_attribute(GTK_CELL_LAYOUT(m_IconView->gobj()),
                                  GTK_CELL_RENDERER(cell->gobj()), "pixbuf", 0);

    m_SignalPostsDownloaded.connect(sigc::mem_fun(*this, &Page::on_posts_downloaded));
    m_SignalSaveProgressDisp.connect([ this ]()
    {
        m_SignalSaveProgress(m_SaveImagesCurrent, m_SaveImagesTotal);
    });

    add(*m_IconView);
    show_all();
}

Page::~Page()
{
    m_Curler.cancel();
    m_CountsCurler.cancel();

    if (m_GetPostsThread)
    {
        m_GetPostsThread->join();
        m_GetPostsThread = nullptr;
    }

    cancel_save();
}

void Page::set_selected(const size_t index)
{
    get_window()->freeze_updates();

    Gtk::TreePath path(std::to_string(index));
    m_IconView->select_path(path);
    scroll_to_selected();

    get_window()->thaw_updates();
}

void Page::scroll_to_selected()
{
    std::vector<Gtk::TreePath> paths = m_IconView->get_selected_items();

    if (!paths.empty())
    {
        Gtk::TreePath path = paths[0];
        if (path)
            m_IconView->scroll_to_path(path, false, 0, 0);
    }
}

void Page::search(const std::shared_ptr<Site> &site)
{
    if (!ask_cancel_save())
        return;

    m_Curler.cancel();
    m_CountsCurler.cancel();

    cancel_save();
    m_ImageList->clear();

    if (m_GetPostsThread)
        m_GetPostsThread->join();

    m_Site = site;
    m_Page = 1;
    m_LastPage = false;
    m_SearchTags = m_Tags;

    std::string tags = m_Tags;

    // Trim whitespace for the tab label text
    if (!tags.empty())
    {
        size_t f = tags.find_first_not_of(' '),
               l = tags.find_last_not_of(' ');
        tags = f == std::string::npos ? "" : " - " + tags.substr(f, l - f + 1);
    }

    if (m_SearchTags.empty())
        m_SearchTags = "*";

    m_TabLabel->set_text(m_Site->get_name() + tags);
    m_TabIcon->set(m_Site->get_icon_pixbuf());

    m_Curler.set_referer(m_Site->get_url());
    m_CountsCurler.set_referer(m_Site->get_url());

    get_posts();
}

void Page::save_image(const std::string &path, const std::shared_ptr<Image> &img)
{
    m_SaveCancel->reset();

    m_Saving            = true;
    m_SaveImagesThread  = Glib::Threads::Thread::create([ this, path, img ]()
    {
        img->save(path);
        m_Saving = false;
    });
}

void Page::save_images(const std::string &path)
{
    m_SaveCancel->reset();

    m_Saving            = true;
    m_SaveImagesCurrent = 0;
    m_SaveImagesTotal   = m_ImageList->get_vector_size();
    m_SaveImagesThread  = Glib::Threads::Thread::create([ this, path ]()
    {
        // 2 if cachesize is 0, 8 if cachesize > 2
        Glib::ThreadPool pool(std::max(std::min(Settings.get_int("CacheSize") * 4, 8), 2));
        for (const std::shared_ptr<AhoViewer::Image> &img : *m_ImageList)
        {
            pool.push([ this, path, img ]()
            {
                if (m_SaveCancel->is_cancelled())
                    return;

                std::shared_ptr<Image> bimage = std::static_pointer_cast<Image>(img);
                bimage->save(Glib::build_filename(path, Glib::path_get_basename(bimage->get_filename())));
                ++m_SaveImagesCurrent;

                if (!m_SaveCancel->is_cancelled())
                    m_SignalSaveProgressDisp();
            });
        }

        pool.shutdown(m_SaveCancel->is_cancelled());
        m_Saving = false;
    });
}

/**
 * Returns true if we want to cancel or we're not saving
 **/
bool Page::ask_cancel_save()
{
    if (!m_Saving)
        return true;

    Gtk::Window *window = static_cast<Gtk::Window*>(get_toplevel());
    Gtk::MessageDialog dialog(*window, _("Are you sure that you want to stop saving images?"),
                              false, Gtk::MESSAGE_QUESTION, Gtk::BUTTONS_YES_NO, true);
    dialog.set_secondary_text(_("Closing this tab will stop the save operation."));

    return dialog.run() == Gtk::RESPONSE_YES;
}

void Page::cancel_save()
{
    m_SaveCancel->cancel();

    for (const std::shared_ptr<AhoViewer::Image> &img : *m_ImageList)
    {
        std::shared_ptr<Image> bimage = std::static_pointer_cast<Image>(img);
        bimage->cancel_download();
    }

    if (m_SaveImagesThread)
    {
        m_SaveImagesThread->join();
        m_SaveImagesThread = nullptr;
    }
}

void Page::get_posts()
{
    std::string tags = m_SearchTags;

    if (m_Tags.find("rating:") == std::string::npos)
    {
        if (Settings.get_booru_max_rating() == Site::Rating::SAFE)
        {
            tags += " rating:safe";
        }
        else if (Settings.get_booru_max_rating() == Site::Rating::QUESTIONABLE)
        {
            tags += " -rating:explicit";
        }
    }

    tags = m_Curler.escape(tags);
    m_Curler.set_url(m_Site->get_posts_url(tags, m_Page));

    m_GetPostsThread = Glib::Threads::Thread::create([ this, tags ]()
    {
        size_t postsCount = 0;
        // Danbooru doesn't give the post count with the posts
        // Get it from thier counts api
        if (m_Page == 1 && m_Site->get_type() == Site::Type::DANBOORU)
        {
            m_CountsCurler.set_url(m_Site->get_url() + "/counts/posts.xml?tags=" + tags);
            if (m_CountsCurler.perform())
            {
                xmlDocument doc(reinterpret_cast<char*>(m_CountsCurler.get_data()), m_CountsCurler.get_data_size());
                postsCount = std::stoul(doc.get_children()[0].get_value());
            }
            else if (m_CountsCurler.is_cancelled())
            {
                return;
            }
        }

        if (m_Site->get_type() == Site::Type::GELBOORU)
            m_Curler.set_cookie_file(m_Site->get_cookie());
        else
            m_Curler.set_http_auth(m_Site->get_username(), m_Site->get_password());

        if (m_Curler.perform())
        {
            m_Posts = std::unique_ptr<xmlDocument>(
                    new xmlDocument(reinterpret_cast<char*>(m_Curler.get_data()),
                                    m_Curler.get_data_size()));
            m_NumPosts = m_Posts->get_n_nodes();

            if (postsCount)
                m_Posts->set_attribute("count", std::to_string(postsCount));
        }
        else if (!m_Curler.is_cancelled())
        {
            m_NumPosts = 0;
            std::cerr << "Error while downloading posts on " << m_Curler.get_url() << std::endl
                      << "  " << m_Curler.get_error() << std::endl;
        }

        if (!m_Curler.is_cancelled())
            m_SignalPostsDownloaded();
    });
}

bool Page::get_next_page()
{
    // Do not fetch the next page if this is the last
    // or the current page is still loading
    if (m_LastPage || m_GetPostsThread)
        return false;

    if (!m_Saving)
    {
        ++m_Page;
        get_posts();

        return false;
    }
    else if (!m_GetNextPageConn)
    {
        m_GetNextPageConn = Glib::signal_timeout().connect(sigc::mem_fun(*this, &Page::get_next_page), 1000);
    }

    return true;
}

/**
 * Adds the downloaded posts to the image list.
 **/
void Page::on_posts_downloaded()
{
    if (m_Posts && m_Posts->get_attribute("success") == "false" && !m_Posts->get_attribute("reason").empty())
    {
        m_SignalDownloadError(m_Posts->get_attribute("reason"));
    }
    else if (m_NumPosts > 0)
    {
        reserve(m_NumPosts);
        m_ImageList->load(*m_Posts, *this);
    }
    // 401 = Unauthorized
    else if (m_Curler.get_response_code() == 401)
    {
        Glib::ustring e = Glib::ustring::compose(_("Failed to login as %1 on %2"),
                                                 m_Site->get_username(),
                                                 m_Site->get_name());
        m_SignalDownloadError(e);
    }
    else if (m_Page == 1)
    {
        m_SignalDownloadError(_("No results found"));
    }

    if (m_NumPosts < static_cast<size_t>(Settings.get_int("BooruLimit")))
        m_LastPage = true;

    m_Posts = nullptr;
    m_GetPostsThread->join();
    m_GetPostsThread = nullptr;
}

void Page::on_selection_changed()
{
    std::vector<Gtk::TreePath> paths = m_IconView->get_selected_items();

    if (!paths.empty())
    {
        Gtk::TreePath path = paths[0];
        if (path)
        {
            size_t index = path[0];

            if (index + Settings.get_int("CacheSize") >= m_ImageList->get_vector_size() - 1)
                get_next_page();

            m_SignalSelectedChanged(index);
        }
    }
}

/**
 * Vertical scrollbar value changed
 **/
void Page::on_value_changed()
{
    double value = get_vadjustment()->get_value(),
           limit = get_vadjustment()->get_upper() -
                   get_vadjustment()->get_page_size() -
                   get_vadjustment()->get_step_increment();

    if (value >= limit)
        get_next_page();
}

bool Page::on_button_press_event(GdkEventButton *e)
{
    if (e->type == GDK_BUTTON_PRESS && e->button == 3)
    {
        Gtk::TreePath path = m_IconView->get_path_at_pos(e->x, e->y);

        if (path)
        {
            m_IconView->select_path(path);
            m_IconView->scroll_to_path(path, false, 0, 0);

            m_PopupMenu->popup(e->button, e->time);

            return true;
        }
    }

    return false;
}

bool Page::on_tab_button_release_event(GdkEventButton *e)
{
    if (e->type == GDK_BUTTON_RELEASE && e->button == 2)
        m_SignalClosed(this);

    return false;
}
<|endoftext|>


<commit_before>// Copyright (c) 2014, Salesforce.com, Inc.  All rights reserved.
//
// Redistribution and use in source and binary forms, with or without
// modification, are permitted provided that the following conditions
// are met:
//
// - Redistributions of source code must retain the above copyright
//   notice, this list of conditions and the following disclaimer.
// - Redistributions in binary form must reproduce the above copyright
//   notice, this list of conditions and the following disclaimer in the
//   documentation and/or other materials provided with the distribution.
// - Neither the name of Salesforce.com nor the names of its contributors
//   may be used to endorse or promote products derived from this
//   software without specific prior written permission.
//
// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS
// FOR A PARTICULAR PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL THE
// COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT,
// INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING,
// BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS
// OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
// ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR
// TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE
// USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.

#include <algorithm>
#include <distributions/clustering.hpp>
#include <distributions/special.hpp>

namespace distributions
{

//----------------------------------------------------------------------------
// Assignments

template<class count_t>
std::vector<count_t> Clustering<count_t>::count_assignments (
        const Assignments & assignments)
{
    // Count group sizes in an assignment vector with the following properties:
    // 0 is the first group
    // there are no empty groups
    // the group IDs are contiguous.

    std::vector<count_t> counts;
    for (auto pair : assignments) {
        size_t gid = pair.second;
        if (DIST_UNLIKELY(gid >= counts.size())) {
            counts.resize(gid + 1, 0);
        }
        ++counts[gid];
    }

    if (DIST_DEBUG_LEVEL >= 2) {
        if (not counts.empty()) {
            count_t min_count =
                * std::min_element(counts.begin(), counts.end());
            DIST_ASSERT(min_count > 0, "groups are not contiguous");
        }
    }

    return counts;
}

//----------------------------------------------------------------------------
// Pitman-Yor Model

template<class count_t>
std::vector<count_t> Clustering<count_t>::PitmanYor::sample_assignments (
        count_t size,
        rng_t & rng) const
{
    // Note that we can ignore the constant shift of -log(size + alpha) in
    //
    //   float py.score_add_value(
    //          count_t this_group_size,
    //          count_t total_group_count,
    //          count_t size)
    //   {
    //     return this_group_size == 0
    //          ? fast_log( (alpha + d * total_group_count)
    //                     / (size + alpha) )
    //          : fast_log((this_group_size - d) / (size + alpha));
    //   }
    //
    // which then permits caching of each table's score.

    DIST_ASSERT(float(size) + 1.f > float(size), "underflow expected");

    std::vector<count_t> assignments(size);
    std::vector<float> likelihoods;
    likelihoods.reserve(100);  // just pick something safe

    // initialize empty table
    count_t table_count = 0;
    const float py_likelihood_new = 1 - d;
    const float py_likelihood_empty = alpha;
    likelihoods.push_back(py_likelihood_empty);

    // add first entry
    if (DIST_LIKELY(size)) {
        count_t i = 0;
        count_t assign = 0;
        assignments[i] = assign;

        table_count = 1;
        const float py_likelihood_empty = alpha + d * table_count;
        likelihoods.push_back(py_likelihood_empty);
        likelihoods[assign] = py_likelihood_new;
    }

    // add all remaining entries
    for (count_t i = 1; DIST_LIKELY(i < size); ++i) {

        // This is cool - for fixed alpha, d, the likelihood will roughly
        // exponentially decay along the likelihood vector.  And in sampling
        // we linearly scan from the front, so we only need to examine an
        // expected constant number of entries.  This results in a expected
        // runtime of the whole sampler linear in size.
        float total = i + alpha;
        count_t assign = sample_from_likelihoods(rng, likelihoods, total);
        assignments[i] = assign;

        if (DIST_UNLIKELY(assign == table_count)) {

            // new table
            table_count += 1;
            const float py_likelihood_empty = alpha + d * table_count;
            likelihoods.push_back(py_likelihood_empty);
            likelihoods[assign] = py_likelihood_new;

        } else {

            // existing table
            likelihoods[assign] += 1.0f;
        }
    }

    return assignments;
}

template<class count_t>
float Clustering<count_t>::PitmanYor::score_counts (
        const std::vector<count_t> & counts) const
{
    double score = 0.0;
    size_t sample_size = 0;
    size_t nonempty_group_count = 0;

    for (size_t count : counts) {
        if (count) {
            size_t group_size = 0;

            score += score_add_value(
                group_size++,
                nonempty_group_count++,
                sample_size++);

            while (group_size < count) {
                score += score_add_value(
                    group_size++,
                    nonempty_group_count,
                    sample_size++);
            }
        }
    }

    return score;
}

//----------------------------------------------------------------------------
// Low-Entropy Model

// this code was generated by derivations/clustering.py
static const float log_partition_function_table[48] =
{
    0.00000000, 0.00000000, 1.60943791, 3.68887945, 6.07993320,
    8.70549682, 11.51947398, 14.49108422, 17.59827611, 20.82445752,
    24.15668300, 27.58456586, 31.09958507, 34.69462231, 38.36364086,
    42.10145572, 45.90356476, 49.76602176, 53.68533918, 57.65841234,
    61.68245958, 65.75497413, 69.87368527, 74.03652635, 78.24160846,
    82.48719834, 86.77169993, 91.09363859, 95.45164780, 99.84445762,
    104.27088480, 108.72982416, 113.22024112, 117.74116515, 122.29168392,
    126.87093829, 131.47811772, 136.11245629, 140.77322911, 145.45974907,
    150.17136399, 154.90745399, 159.66742919, 164.45072752, 169.25681285,
    174.08517319, 178.93531914, 183.80678238
};

// this code was generated by derivations/clustering.py
template<class count_t>
float Clustering<count_t>::LowEntropy::log_partition_function (
        count_t sample_size) const
{
    // TODO incorporate dataset_size for higher accuracy
    count_t n = sample_size;
    if (n < 48) {
        return log_partition_function_table[n];
    } else {
        float coeff = 0.28269584f;
        float log_z_max = n * fast_log(n);
        return log_z_max * (1.f + coeff * powf(n, -0.75f));
    }
}

// ad hoc approximation,
// see `python derivations/clustering.py dataprob`
// see `python derivations/clustering.py approximations`
template<class count_t>
inline float Clustering<count_t>::LowEntropy::_approximate_dataprob_correction (
        count_t sample_size) const
{
    float n = fast_log(sample_size);
    float N = fast_log(dataset_size);
    return 0.061f * n * (n - N) * powf(n + N, 0.75f);
}

template<class count_t>
float Clustering<count_t>::LowEntropy::score_counts (
        const std::vector<count_t> & counts) const
{
    float score = 0.0;
    count_t sample_size = 0;
    for (count_t count : counts) {
        sample_size += count;
        if (count > 1) {
            score += count * fast_log(count);
        }
    }
    DIST_ASSERT_LE(sample_size, dataset_size);

    if (sample_size != dataset_size) {
        float log_factor = _approximate_postpred_correction(sample_size);
        score += log_factor * (counts.size() - 1);
        score += _approximate_dataprob_correction(sample_size);
    }
    score -= log_partition_function(sample_size);
    return score;
}

template<class count_t>
std::vector<count_t> Clustering<count_t>::LowEntropy::sample_assignments (
        count_t sample_size,
        rng_t & rng) const
{
    DIST_ASSERT_LE(sample_size, dataset_size);

    std::vector<count_t> assignments(sample_size);
    std::vector<count_t> counts;
    std::vector<float> likelihoods;
    counts.reserve(100);
    likelihoods.reserve(100);
    const count_t bogus = 0;
    count_t size = 0;

    for (count_t & assign : assignments) {

        float likelihood_empty = fast_exp(score_add_value(0, bogus, size));
        if (DIST_UNLIKELY(counts.empty()) or counts.back()) {
            counts.push_back(0);
            likelihoods.push_back(likelihood_empty);
        } else {
            likelihoods.back() = likelihood_empty;
        }

        assign = sample_from_likelihoods(rng, likelihoods);
        count_t & count = counts[assign];
        count += 1;
        size += 1;
        float & likelihood = likelihoods[assign];
        float new_likelihood = fast_exp(score_add_value(count, bogus, bogus));
        likelihood = new_likelihood;
    }

    return assignments;
}

//----------------------------------------------------------------------------
// Explicit template instantiation

template struct Clustering<int32_t>;
//template struct Clustering<int64_t>;
//template struct Clustering<uint32_t>;
//template struct Clustering<uint64_t>;

} // namespace distributions
<commit_msg>Do not be stupid in PitmanYor::score_counts<commit_after>// Copyright (c) 2014, Salesforce.com, Inc.  All rights reserved.
//
// Redistribution and use in source and binary forms, with or without
// modification, are permitted provided that the following conditions
// are met:
//
// - Redistributions of source code must retain the above copyright
//   notice, this list of conditions and the following disclaimer.
// - Redistributions in binary form must reproduce the above copyright
//   notice, this list of conditions and the following disclaimer in the
//   documentation and/or other materials provided with the distribution.
// - Neither the name of Salesforce.com nor the names of its contributors
//   may be used to endorse or promote products derived from this
//   software without specific prior written permission.
//
// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS
// FOR A PARTICULAR PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL THE
// COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT,
// INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING,
// BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS
// OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
// ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR
// TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE
// USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.

#include <algorithm>
#include <distributions/clustering.hpp>
#include <distributions/special.hpp>

namespace distributions
{

//----------------------------------------------------------------------------
// Assignments

template<class count_t>
std::vector<count_t> Clustering<count_t>::count_assignments (
        const Assignments & assignments)
{
    // Count group sizes in an assignment vector with the following properties:
    // 0 is the first group
    // there are no empty groups
    // the group IDs are contiguous.

    std::vector<count_t> counts;
    for (auto pair : assignments) {
        size_t gid = pair.second;
        if (DIST_UNLIKELY(gid >= counts.size())) {
            counts.resize(gid + 1, 0);
        }
        ++counts[gid];
    }

    if (DIST_DEBUG_LEVEL >= 2) {
        if (not counts.empty()) {
            count_t min_count =
                * std::min_element(counts.begin(), counts.end());
            DIST_ASSERT(min_count > 0, "groups are not contiguous");
        }
    }

    return counts;
}

//----------------------------------------------------------------------------
// Pitman-Yor Model

template<class count_t>
std::vector<count_t> Clustering<count_t>::PitmanYor::sample_assignments (
        count_t size,
        rng_t & rng) const
{
    // Note that we can ignore the constant shift of -log(size + alpha) in
    //
    //   float py.score_add_value(
    //          count_t this_group_size,
    //          count_t total_group_count,
    //          count_t size)
    //   {
    //     return this_group_size == 0
    //          ? fast_log( (alpha + d * total_group_count)
    //                     / (size + alpha) )
    //          : fast_log((this_group_size - d) / (size + alpha));
    //   }
    //
    // which then permits caching of each table's score.

    DIST_ASSERT(float(size) + 1.f > float(size), "underflow expected");

    std::vector<count_t> assignments(size);
    std::vector<float> likelihoods;
    likelihoods.reserve(100);  // just pick something safe

    // initialize empty table
    count_t table_count = 0;
    const float py_likelihood_new = 1 - d;
    const float py_likelihood_empty = alpha;
    likelihoods.push_back(py_likelihood_empty);

    // add first entry
    if (DIST_LIKELY(size)) {
        count_t i = 0;
        count_t assign = 0;
        assignments[i] = assign;

        table_count = 1;
        const float py_likelihood_empty = alpha + d * table_count;
        likelihoods.push_back(py_likelihood_empty);
        likelihoods[assign] = py_likelihood_new;
    }

    // add all remaining entries
    for (count_t i = 1; DIST_LIKELY(i < size); ++i) {

        // This is cool - for fixed alpha, d, the likelihood will roughly
        // exponentially decay along the likelihood vector.  And in sampling
        // we linearly scan from the front, so we only need to examine an
        // expected constant number of entries.  This results in a expected
        // runtime of the whole sampler linear in size.
        float total = i + alpha;
        count_t assign = sample_from_likelihoods(rng, likelihoods, total);
        assignments[i] = assign;

        if (DIST_UNLIKELY(assign == table_count)) {

            // new table
            table_count += 1;
            const float py_likelihood_empty = alpha + d * table_count;
            likelihoods.push_back(py_likelihood_empty);
            likelihoods[assign] = py_likelihood_new;

        } else {

            // existing table
            likelihoods[assign] += 1.0f;
        }
    }

    return assignments;
}

inline float fast_log_ratio (float numer, float denom)
{
    return fast_log(numer / denom);
}

inline float fast_lgamma_ratio (float start, size_t count)
{
    return fast_lgamma(start + count) - fast_lgamma(start);
}

template<class count_t>
float Clustering<count_t>::PitmanYor::score_counts (
        const std::vector<count_t> & counts) const
{
    double score = 0.0;
    size_t sample_size = 0;
    size_t nonempty_group_count = 0;

    for (size_t count : counts) {
        if (count) {
            if (count == 1) {

                score += fast_log_ratio(
                    alpha + d * nonempty_group_count,
                    alpha + sample_size);

            } else if (count == 2) {

                score += fast_log_ratio(
                    (alpha + d * nonempty_group_count) * (1 - d),
                    (alpha + sample_size) * (alpha + sample_size + 1));

            } else {

                score += fast_log(alpha + d * nonempty_group_count);
                score += fast_lgamma_ratio(1 - d, count - 1);
                score -= fast_lgamma_ratio(alpha + sample_size, count);
            }

            nonempty_group_count += 1;
            sample_size += count;
        }
    }

    return score;
}

//----------------------------------------------------------------------------
// Low-Entropy Model

// this code was generated by derivations/clustering.py
static const float log_partition_function_table[48] =
{
    0.00000000, 0.00000000, 1.60943791, 3.68887945, 6.07993320,
    8.70549682, 11.51947398, 14.49108422, 17.59827611, 20.82445752,
    24.15668300, 27.58456586, 31.09958507, 34.69462231, 38.36364086,
    42.10145572, 45.90356476, 49.76602176, 53.68533918, 57.65841234,
    61.68245958, 65.75497413, 69.87368527, 74.03652635, 78.24160846,
    82.48719834, 86.77169993, 91.09363859, 95.45164780, 99.84445762,
    104.27088480, 108.72982416, 113.22024112, 117.74116515, 122.29168392,
    126.87093829, 131.47811772, 136.11245629, 140.77322911, 145.45974907,
    150.17136399, 154.90745399, 159.66742919, 164.45072752, 169.25681285,
    174.08517319, 178.93531914, 183.80678238
};

// this code was generated by derivations/clustering.py
template<class count_t>
float Clustering<count_t>::LowEntropy::log_partition_function (
        count_t sample_size) const
{
    // TODO incorporate dataset_size for higher accuracy
    count_t n = sample_size;
    if (n < 48) {
        return log_partition_function_table[n];
    } else {
        float coeff = 0.28269584f;
        float log_z_max = n * fast_log(n);
        return log_z_max * (1.f + coeff * powf(n, -0.75f));
    }
}

// ad hoc approximation,
// see `python derivations/clustering.py dataprob`
// see `python derivations/clustering.py approximations`
template<class count_t>
inline float Clustering<count_t>::LowEntropy::_approximate_dataprob_correction (
        count_t sample_size) const
{
    float n = fast_log(sample_size);
    float N = fast_log(dataset_size);
    return 0.061f * n * (n - N) * powf(n + N, 0.75f);
}

template<class count_t>
float Clustering<count_t>::LowEntropy::score_counts (
        const std::vector<count_t> & counts) const
{
    float score = 0.0;
    count_t sample_size = 0;
    for (count_t count : counts) {
        sample_size += count;
        if (count > 1) {
            score += count * fast_log(count);
        }
    }
    DIST_ASSERT_LE(sample_size, dataset_size);

    if (sample_size != dataset_size) {
        float log_factor = _approximate_postpred_correction(sample_size);
        score += log_factor * (counts.size() - 1);
        score += _approximate_dataprob_correction(sample_size);
    }
    score -= log_partition_function(sample_size);
    return score;
}

template<class count_t>
std::vector<count_t> Clustering<count_t>::LowEntropy::sample_assignments (
        count_t sample_size,
        rng_t & rng) const
{
    DIST_ASSERT_LE(sample_size, dataset_size);

    std::vector<count_t> assignments(sample_size);
    std::vector<count_t> counts;
    std::vector<float> likelihoods;
    counts.reserve(100);
    likelihoods.reserve(100);
    const count_t bogus = 0;
    count_t size = 0;

    for (count_t & assign : assignments) {

        float likelihood_empty = fast_exp(score_add_value(0, bogus, size));
        if (DIST_UNLIKELY(counts.empty()) or counts.back()) {
            counts.push_back(0);
            likelihoods.push_back(likelihood_empty);
        } else {
            likelihoods.back() = likelihood_empty;
        }

        assign = sample_from_likelihoods(rng, likelihoods);
        count_t & count = counts[assign];
        count += 1;
        size += 1;
        float & likelihood = likelihoods[assign];
        float new_likelihood = fast_exp(score_add_value(count, bogus, bogus));
        likelihood = new_likelihood;
    }

    return assignments;
}

//----------------------------------------------------------------------------
// Explicit template instantiation

template struct Clustering<int32_t>;
//template struct Clustering<int64_t>;
//template struct Clustering<uint32_t>;
//template struct Clustering<uint64_t>;

} // namespace distributions
<|endoftext|>