<commit_before>#ifndef TUDOCOMP_VIEW_H
#define TUDOCOMP_VIEW_H

#include <algorithm>
#include <cmath>
#include <cstddef>
#include <fstream>
#include <iostream>
#include <memory>
#include <sstream>
#include <string>
#include <type_traits>
#include <utility>
#include <iomanip>
#include <cstring>
#include <glog/logging.h>

namespace tudocomp {

/// A view into a slice of memory.
///
/// This is an abstraction around a `const uint8_t*` pointer and a `size_t` size,
/// and represents N bytes of memory starting at that pointer.
///
/// Creating/Copying/Modifying a View will not copy any of the data it points at.
class View {
    const uint8_t* m_data;
    size_t   m_size;

    inline void bound_check(size_t pos) const {
        if (pos >= m_size) {
            std::stringstream ss;
            ss << "accessing view with bounds [0, ";
            ss << m_size;
            ss << ") at out-of-bounds index ";
            ss << pos;
            throw std::out_of_range(ss.str());
        }
    }

public:
    // Type members

    using value_type             = uint8_t;
    using size_type              = std::size_t;
    using difference_type        = std::ptrdiff_t;
    using const_reference        = const value_type&;
    using const_pointer          = const value_type*;
    using const_iterator         = const_pointer;
    using const_reverse_iterator = std::reverse_iterator<const_iterator>;

    // static value members

    static const size_type npos = -1;

    // Constructors

    /// Construct a View pointing at `len` elements starting from `data`
    inline View(const uint8_t* data, size_t len): m_data(data), m_size(len) {}

    /// Construct a View as a copy of `other`
    inline View(const View& other):   View(other.m_data, other.m_size) {}

    /// Construct a View pointing at the contents of a vector
    inline View(const std::vector<uint8_t>& other):
        View(other.data(), other.size()) {}

    /// Construct a View pointing at the contents of a string
    inline View(const std::string& other):
        View((const uint8_t*) other.data(), other.size()) {}

    /// Construct a View pointing at a C-style null terminated string.
    ///
    /// Can be used to construct from a string literal
    inline View(const char* other):
        View((const uint8_t*) other, strlen(other)) {}

    /// Construct a View pointing at `len` elements starting from `data`
    inline View(const char* data, size_t len):
        View((const uint8_t*) data, len) {}

    // Conversion

    /// Construct a string with the contents of this View
    inline operator std::string() const {
        return std::string(cbegin(), cend());
    }

    /// Construct a vector with the contents of this View
    inline operator std::vector<uint8_t>() const {
        return std::vector<uint8_t>(cbegin(), cend());
    }

    // Element access

    /// Access the element at `pos`
    ///
    /// This method is always bounds checked
    inline const_reference at(size_type pos) const {
        bound_check(pos);
        return m_data[pos];
    }

    /// Access the element at `pos`
    ///
    /// This method is bounds checked in debug builds
    inline const_reference operator[](size_type pos) const {
#ifdef DEBUG
        bound_check(pos);
#endif
        return m_data[pos];
    }

    /// Access the first element
    inline const_reference front() const {
        return (*this)[0];
    }

    /// Access the last element
    inline const_reference back() const {
        return (*this)[m_size - 1];
    }

    /// The backing memory location
    inline const uint8_t* data() const {
        return &front();
    }

    // Iterators

    /// Begin of iterator
    inline const_iterator begin() const {
        return &(*this)[0];
    }
    /// Begin of const iterator
    inline const_iterator cbegin() const {
        return begin();
    }

    /// End of iterator
    inline const_iterator end() const {
        return &(*this)[m_size];
    }
    /// End of const iterator
    inline const_iterator cend() const {
        return end();
    }

    /// Begin of reverse iterator
    inline const_reverse_iterator rbegin() const {
        return std::reverse_iterator<const_iterator>(end());
    }
    /// Begin of const reverse iterator
    inline const_reverse_iterator crbegin() const {
        return rbegin();
    }

    /// End of reverse iterator
    inline const_reverse_iterator rend() const {
        return std::reverse_iterator<const_iterator>(begin());
    }
    /// End of const reverse iterator
    inline const_reverse_iterator crend() const {
        return rend();
    }

    // Capacity

    /// Returns `true` if empty
    inline bool empty() const {
        return m_size == 0;
    }

    /// Returns size of the View
    inline size_type size() const {
        return m_size;
    }

    /// Returns max size of the View. Always the same as `size()`
    inline size_type max_size() const {
        return size();
    }

    // Slicing

    /// Construct a new View that is a sub view into the current one.
    ///
    /// Passing `npos` to `to` will create a substr until the end of the View
    ///
    /// # Example
    ///
    /// `View("abcd").substr(1, 3) == View("bc")`
    inline View substr(size_type from, size_type to = npos) const {
        if (to == npos) {
            to = m_size;
        }

        DCHECK(from <= to);
        DCHECK(from <= size());
        DCHECK(to <= size());

        return View(&(*this)[from], to - from);
    }

    // Modifiers

    /// Swap two Views
    inline void swap(View& other) {
        using std::swap;

        swap(m_data, other.m_data);
        swap(m_size, other.m_size);
    }

    /// Swap two Views
    inline friend void swap(View& a, View& b) {
        a.swap(b);
    }

    /// Sets the size to 0
    inline void clear() {
        m_size = 0;
    }

    /// Removes the first `n` elements from the View
    inline void remove_prefix(size_type n) {
        *this = substr(n);
    }

    /// Removes the last `n` elements from the View
    inline void remove_suffix(size_type n) {
        *this = substr(0, m_size - n);
    }

    // string predicates

    /// Returns `true` if the View starts with `c`
    inline bool starts_with(uint8_t c) const {
        return !empty() && (front() == c);
    }
    /// Returns `true` if the View starts with `c`
    inline bool starts_with(char c) const {
        return starts_with(uint8_t(c));
    }
    /// Returns `true` if the View starts with `x`
    inline bool starts_with(const View& x) const;
    /// Returns `true` if the View ends with `c`
    inline bool ends_with(uint8_t c) const {
        return !empty() && (back() == c);
    }
    /// Returns `true` if the View ends with `c`
    inline bool ends_with(char c) const {
        return ends_with(uint8_t(c));
    }
    /// Returns `true` if the View ends with `x`
    inline bool ends_with(const View& x) const;

};

inline bool operator==(const View& lhs, const View& rhs) {
    // TODO: memcmp!
    return (lhs.size() == rhs.size())
        && std::equal(lhs.cbegin(), lhs.cend(), rhs.cbegin());
}
inline bool operator!=(const View& lhs, const View& rhs) {
    return !(lhs == rhs);
}
inline bool operator<(const View& lhs, const View& rhs) {
    return std::lexicographical_compare(lhs.cbegin(), lhs.cend(),
                                        rhs.cbegin(), rhs.cend());
}
inline bool operator>(const View& lhs, const View& rhs) {
    return std::lexicographical_compare(lhs.cbegin(), lhs.cend(),
                                        rhs.cbegin(), rhs.cend(),
                                        [](const uint8_t& l, const uint8_t& r){
                                            return l > r;
                                        });
}
inline bool operator<=(const View& lhs, const View& rhs) {
    return !(lhs > rhs);
}
inline bool operator>=(const View& lhs, const View& rhs) {
    return !(lhs < rhs);
}

inline std::ostream& operator<<(std::ostream& os, const View& v) {
    os.write((const char*) v.data(), v.size());
    return os;
}

inline bool View::starts_with(const View& x) const {
    return (x.size() <= size())
        && (substr(0, x.size()) == x);
}

inline bool View::ends_with(const View& x) const {
    return (x.size() <= size())
        && (substr(size() - x.size()) == x);
}
}

#endif
<commit_msg>use memcmp for view comparisions<commit_after>#ifndef TUDOCOMP_VIEW_H
#define TUDOCOMP_VIEW_H

#include <algorithm>
#include <cmath>
#include <cstddef>
#include <fstream>
#include <iostream>
#include <memory>
#include <sstream>
#include <string>
#include <type_traits>
#include <utility>
#include <iomanip>
#include <cstring>
#include <glog/logging.h>

namespace tudocomp {

/// A view into a slice of memory.
///
/// This is an abstraction around a `const uint8_t*` pointer and a `size_t` size,
/// and represents N bytes of memory starting at that pointer.
///
/// Creating/Copying/Modifying a View will not copy any of the data it points at.
class View {
    const uint8_t* m_data;
    size_t   m_size;

    inline void bound_check(size_t pos) const {
        if (pos >= m_size) {
            std::stringstream ss;
            ss << "accessing view with bounds [0, ";
            ss << m_size;
            ss << ") at out-of-bounds index ";
            ss << pos;
            throw std::out_of_range(ss.str());
        }
    }

public:
    // Type members

    using value_type             = uint8_t;
    using size_type              = std::size_t;
    using difference_type        = std::ptrdiff_t;
    using const_reference        = const value_type&;
    using const_pointer          = const value_type*;
    using const_iterator         = const_pointer;
    using const_reverse_iterator = std::reverse_iterator<const_iterator>;

    // static value members

    static const size_type npos = -1;

    // Constructors

    /// Construct a View pointing at `len` elements starting from `data`
    inline View(const uint8_t* data, size_t len): m_data(data), m_size(len) {}

    /// Construct a View as a copy of `other`
    inline View(const View& other):   View(other.m_data, other.m_size) {}

    /// Construct a View pointing at the contents of a vector
    inline View(const std::vector<uint8_t>& other):
        View(other.data(), other.size()) {}

    /// Construct a View pointing at the contents of a string
    inline View(const std::string& other):
        View((const uint8_t*) other.data(), other.size()) {}

    /// Construct a View pointing at a C-style null terminated string.
    ///
    /// Can be used to construct from a string literal
    inline View(const char* other):
        View((const uint8_t*) other, strlen(other)) {}

    /// Construct a View pointing at `len` elements starting from `data`
    inline View(const char* data, size_t len):
        View((const uint8_t*) data, len) {}

    // Conversion

    /// Construct a string with the contents of this View
    inline operator std::string() const {
        return std::string(cbegin(), cend());
    }

    /// Construct a vector with the contents of this View
    inline operator std::vector<uint8_t>() const {
        return std::vector<uint8_t>(cbegin(), cend());
    }

    // Element access

    /// Access the element at `pos`
    ///
    /// This method is always bounds checked
    inline const_reference at(size_type pos) const {
        bound_check(pos);
        return m_data[pos];
    }

    /// Access the element at `pos`
    ///
    /// This method is bounds checked in debug builds
    inline const_reference operator[](size_type pos) const {
#ifdef DEBUG
        bound_check(pos);
#endif
        return m_data[pos];
    }

    /// Access the first element
    inline const_reference front() const {
        return (*this)[0];
    }

    /// Access the last element
    inline const_reference back() const {
        return (*this)[m_size - 1];
    }

    /// The backing memory location
    inline const uint8_t* data() const {
        return &front();
    }

    // Iterators

    /// Begin of iterator
    inline const_iterator begin() const {
        return &(*this)[0];
    }
    /// Begin of const iterator
    inline const_iterator cbegin() const {
        return begin();
    }

    /// End of iterator
    inline const_iterator end() const {
        return &(*this)[m_size];
    }
    /// End of const iterator
    inline const_iterator cend() const {
        return end();
    }

    /// Begin of reverse iterator
    inline const_reverse_iterator rbegin() const {
        return std::reverse_iterator<const_iterator>(end());
    }
    /// Begin of const reverse iterator
    inline const_reverse_iterator crbegin() const {
        return rbegin();
    }

    /// End of reverse iterator
    inline const_reverse_iterator rend() const {
        return std::reverse_iterator<const_iterator>(begin());
    }
    /// End of const reverse iterator
    inline const_reverse_iterator crend() const {
        return rend();
    }

    // Capacity

    /// Returns `true` if empty
    inline bool empty() const {
        return m_size == 0;
    }

    /// Returns size of the View
    inline size_type size() const {
        return m_size;
    }

    /// Returns max size of the View. Always the same as `size()`
    inline size_type max_size() const {
        return size();
    }

    // Slicing

    /// Construct a new View that is a sub view into the current one.
    ///
    /// Passing `npos` to `to` will create a substr until the end of the View
    ///
    /// # Example
    ///
    /// `View("abcd").substr(1, 3) == View("bc")`
    inline View substr(size_type from, size_type to = npos) const {
        if (to == npos) {
            to = m_size;
        }

        DCHECK(from <= to);
        DCHECK(from <= size());
        DCHECK(to <= size());

        return View(&(*this)[from], to - from);
    }

    // Modifiers

    /// Swap two Views
    inline void swap(View& other) {
        using std::swap;

        swap(m_data, other.m_data);
        swap(m_size, other.m_size);
    }

    /// Swap two Views
    inline friend void swap(View& a, View& b) {
        a.swap(b);
    }

    /// Sets the size to 0
    inline void clear() {
        m_size = 0;
    }

    /// Removes the first `n` elements from the View
    inline void remove_prefix(size_type n) {
        *this = substr(n);
    }

    /// Removes the last `n` elements from the View
    inline void remove_suffix(size_type n) {
        *this = substr(0, m_size - n);
    }

    // string predicates

    /// Returns `true` if the View starts with `c`
    inline bool starts_with(uint8_t c) const {
        return !empty() && (front() == c);
    }
    /// Returns `true` if the View starts with `c`
    inline bool starts_with(char c) const {
        return starts_with(uint8_t(c));
    }
    /// Returns `true` if the View starts with `x`
    inline bool starts_with(const View& x) const;
    /// Returns `true` if the View ends with `c`
    inline bool ends_with(uint8_t c) const {
        return !empty() && (back() == c);
    }
    /// Returns `true` if the View ends with `c`
    inline bool ends_with(char c) const {
        return ends_with(uint8_t(c));
    }
    /// Returns `true` if the View ends with `x`
    inline bool ends_with(const View& x) const;

};

inline bool operator==(const View& lhs, const View& rhs) {
    // TODO: memcmp!
    return (lhs.size() == rhs.size())
        && (std::memcmp(lhs.data(), rhs.data(), lhs.size()) == 0);
}
inline bool operator!=(const View& lhs, const View& rhs) {
    return !(lhs == rhs);
}
inline bool operator<(const View& lhs, const View& rhs) {
    return std::lexicographical_compare(lhs.cbegin(), lhs.cend(),
                                        rhs.cbegin(), rhs.cend());
}
inline bool operator>(const View& lhs, const View& rhs) {
    return std::lexicographical_compare(lhs.cbegin(), lhs.cend(),
                                        rhs.cbegin(), rhs.cend(),
                                        [](const uint8_t& l, const uint8_t& r){
                                            return l > r;
                                        });
}
inline bool operator<=(const View& lhs, const View& rhs) {
    return !(lhs > rhs);
}
inline bool operator>=(const View& lhs, const View& rhs) {
    return !(lhs < rhs);
}

inline std::ostream& operator<<(std::ostream& os, const View& v) {
    os.write((const char*) v.data(), v.size());
    return os;
}

inline bool View::starts_with(const View& x) const {
    return (x.size() <= size())
        && (substr(0, x.size()) == x);
}

inline bool View::ends_with(const View& x) const {
    return (x.size() <= size())
        && (substr(size() - x.size()) == x);
}
}

#endif
<|endoftext|>


<commit_before>//----------------------------------------------------------------------------
/// \file  basic_persist_array.hpp
//----------------------------------------------------------------------------
/// \brief Implementation of persistent array storage class.
//----------------------------------------------------------------------------
// Copyright (c) 2010 Omnibius, LLC
// Author:  Serge Aleynikov <saleyn@gmail.com>
// Created: 2010-10-25
//----------------------------------------------------------------------------
/*
***** BEGIN LICENSE BLOCK *****

This file is part of the util open-source project.

Copyright (C) 2010 Serge Aleynikov <saleyn@gmail.com>

This library is free software; you can redistribute it and/or
modify it under the terms of the GNU Lesser General Public
License as published by the Free Software Foundation; either
vsn 2.1 of the License, or (at your option) any later vsn.

This library is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
Lesser General Public License for more details.

You should have received a copy of the GNU Lesser General Public
License along with this library; if not, write to the Free Software
Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA

***** END LICENSE BLOCK *****
*/

#ifndef _UTIL_PERSISTENT_ARRAY_HPP_
#define _UTIL_PERSISTENT_ARRAY_HPP_

#include <boost/interprocess/sync/file_lock.hpp>
#include <boost/interprocess/sync/scoped_lock.hpp>
#include <boost/interprocess/file_mapping.hpp>
#include <boost/interprocess/mapped_region.hpp>
#include <boost/static_assert.hpp>
#include <boost/scope_exit.hpp>
#include <boost/thread.hpp>
#include <boost/filesystem.hpp>
#include <util/compiler_hints.hpp>
#include <util/atomic.hpp>
#include <util/error.hpp>
#include <stdexcept>
#include <fstream>
#include <string>
#include <vector>
#include <cstring>
#include <cstddef>
#include <cstdlib>
#include <unistd.h>

namespace util {

    namespace { namespace bip = boost::interprocess; }
    namespace detail { struct empty_data {}; }

    template <
        typename T,
        size_t NLocks = 32,
        typename Lock = boost::mutex,
        typename ExtraHeaderData = detail::empty_data>
    struct persist_array {
        struct header {
            static const uint32_t s_version = 0xa0b1c2d3;
            uint32_t        version;
            volatile long   rec_count;
            size_t          max_recs;
            size_t          rec_size;
            Lock            locks[NLocks];
            ExtraHeaderData extra_header_data;
            T               records[0];
        };

    protected:
        typedef persist_array<T, NLocks, Lock, ExtraHeaderData> self_type;

        BOOST_STATIC_ASSERT(!(NLocks & (NLocks - 1))); // must be power of 2

        static const size_t s_lock_mask = NLocks-1;
        // Shared memory file mapping
        bip::file_mapping  m_file;
        // Shared memory mapped region
        bip::mapped_region m_region;

        // Locks that guard access to internal record structures.
        header* m_header;
        T*      m_begin;
        T*      m_end;

        void check_range(size_t a_id) const throw (badarg_error) {
            size_t n = m_header->max_recs;
            if (likely(a_id < n))
                return;
            throw badarg_error("Invalid record id specified ", a_id, " (max=", n-1, ')');
        }

    public:
        typedef Lock                        lock_type;
        typedef typename Lock::scoped_lock  scoped_lock;
        typedef typename Lock::scoped_try_lock  scoped_try_lock;

        BOOST_STATIC_ASSERT((NLocks & (NLocks-1)) == 0); // Must be power of 2.

        persist_array()
            : m_header(NULL), m_begin(NULL), m_end(NULL)
        {}

        /// Default permission mask used for opening a file
        static int default_file_mode() { return S_IRUSR | S_IWUSR | S_IRGRP | S_IWGRP; }

        /// Initialize the storage
        /// @return true if the storage file didn't exist and was created
        bool init(const char* a_filename, size_t a_max_recs, bool a_read_only = false,
            int a_mode = default_file_mode()) throw (io_error);

        size_t count()    const { return static_cast<size_t>(m_header->rec_count); }
        size_t capacity() const { return m_header->max_recs; }

        ExtraHeaderData& extra_header_data() { return m_header->extra_header_data; }

        /// Allocate next record and return its ID.
        /// @return
        size_t allocate_rec() throw(std::runtime_error) {
            size_t n = static_cast<size_t>(atomic::add(&m_header->rec_count,1));
            if (n >= capacity()) {
                atomic::dec(&m_header->rec_count);
                throw std::runtime_error("Out of storage capacity!");
            }
            return n;
        }

        std::pair<T*, size_t> get_next() {
            size_t n = allocate_rec();
            return std::make_pair(get(n), n);
        }

        Lock& get_lock(size_t a_rec_id) {
            return m_header->locks[a_rec_id & s_lock_mask];
        }

        /// Add a record with given ID to the store
        void add(size_t a_id, const T& a_rec) {
            BOOST_ASSERT(a_id < m_header->rec_count);
            scoped_lock guard(get_lock(a_id));
            *get(a_id) = a_rec;
        }

        /// Add a record to the storage and return it's id
        size_t add(const T& a_rec) {
            size_t n = allocate_rec();
            scoped_lock guard(get_lock(n));
            *(m_begin+n) = a_rec;
            return n;
        }

        /// @return id of the given object in the storage
        size_t id_of(const T* a_rec) const { return a_rec - m_begin; }

        const T& operator[] (size_t a_id) const throw(badarg_error) {
            check_range(a_id); return *get(a_id);
        }
        T& operator[] (size_t a_id) throw(badarg_error) {
            check_range(a_id); return *get(a_id);
        }

        const T* get(size_t a_rec_id) const {
            return likely(a_rec_id < m_header->max_recs) ? m_begin+a_rec_id : NULL;
        }

        T* get(size_t a_rec_id) {
            return likely(a_rec_id < m_header->max_recs) ? m_begin+a_rec_id : NULL;
        }

        /// Flush header to disk
        bool flush_header() { return m_region.flush(0, sizeof(header)); }

        /// Flush region of cached records to disk
        bool flush(size_t a_from_rec = 0, size_t a_num_recs = 0) {
            return m_region.flush(a_from_rec, a_num_recs*sizeof(T));
        }

        /// Remove memory mapped file from disk
        void remove() {
            m_file.remove(m_file.get_name());
        }

        const T*    begin() const { return m_begin; }
        const T*    end()   const { return m_end; }
        T*          begin()       { return m_begin; }
        T*          end()         { return m_end; }

        template <class Visitor>
        void for_each(const Visitor& a_visitor) {
            size_t n = 0;
            for (const T* p = begin(), *e = p + count(); p != e; ++p, ++n)
                a_visitor(n, p);
        }

        std::ostream& dump(std::ostream& out, const std::string& a_prefix="") {
            for (const T* p = begin(), *e = p + count(); p != e; ++p)
                out << a_prefix << *p << std::endl;
            return out;
        }
    };

    //-------------------------------------------------------------------------
    // Implementation
    //-------------------------------------------------------------------------

    template <typename T, size_t NLocks, typename Lock, typename Ext>
    bool persist_array<T,NLocks,Lock,Ext>::
    init(const char* a_filename, size_t a_max_recs, bool a_read_only, int a_mode)
        throw (io_error)
    {
        static const size_t s_pack_size = getpagesize();

        size_t sz = sizeof(header) + (a_max_recs * sizeof(T));
        sz += (s_pack_size - sz % s_pack_size);

        bool l_exists;
        try {
            boost::filesystem::path l_name(a_filename);
            try {
                boost::filesystem::create_directories(l_name.parent_path());
            } catch (boost::system::system_error& e) {
                throw io_error("Cannot create directory: ",
                    l_name.parent_path(), ": ", e.what());
            }

            {
                std::filebuf f;
                f.open(a_filename, std::ios_base::in | std::ios_base::out
                    | std::ios_base::binary);
                l_exists = f.is_open();

                if (l_exists && !a_read_only) {
                    bip::file_lock flock(a_filename);
                    bip::scoped_lock<bip::file_lock> g_lock(flock);

                    header h;
                    f.sgetn(reinterpret_cast<char*>(&h), sizeof(header));

                    if (h.version != header::s_version) {
                        throw io_error("Invalid file format ", a_filename);
                    }
                    if (h.rec_size != sizeof(T)) {
                        throw io_error("Invalid item size in file ", a_filename,
                            " (expected ", sizeof(T), " got ", h.rec_size, ')');
                    }
                    // Increase the file size if instructed to do so.
                    if (h.max_recs < a_max_recs) {
                        h.max_recs = a_max_recs;
                        f.pubseekoff(0, std::ios_base::beg);
                        f.sputn(reinterpret_cast<char*>(&h), sizeof(header));
                        f.pubseekoff(sz-1, std::ios_base::beg);
                        f.sputc(0);
                    }
                }
            }

            if (!l_exists && !a_read_only) {
                int l_fd = ::open(a_filename, O_RDWR | O_CREAT | O_TRUNC, a_mode);
                if (l_fd < 0)
                    throw io_error(errno, "Error creating file ", a_filename);

                BOOST_SCOPE_EXIT_TPL( (&l_fd) ) {
                    ::close(l_fd);
                } BOOST_SCOPE_EXIT_END;

                bip::file_lock flock(a_filename);
                bip::scoped_lock<bip::file_lock> g_lock(flock);

                header h;
                h.version   = header::s_version;
                h.rec_count = 0;
                h.max_recs  = a_max_recs;
                h.rec_size  = sizeof(T);

                if (::write(l_fd, &h, sizeof(h)) < 0)
                    throw io_error(errno, "Error writing to file ", a_filename);

                if (::ftruncate(l_fd, sz) < 0)
                    throw io_error(errno, "Error setting file ",
                        a_filename, " to size ", sz);

                ::fsync(l_fd);
            }

            bip::file_mapping shmf(a_filename, a_read_only ? bip::read_only : bip::read_write);
            bip::mapped_region region(shmf, a_read_only ? bip::read_only : bip::read_write);

            //Get the address of the mapped region
            void*  addr  = region.get_address();
            size_t size  = region.get_size();

            m_file.swap(shmf);
            m_region.swap(region);

            //if (!exists && !a_read_only)
            //    memset(static_cast<char*>(addr) + sizeof(header), 0, size - sizeof(header));
            m_header = static_cast<header*>(addr);
            m_begin  = m_header->records;
            m_end    = m_begin + a_max_recs;
            BOOST_ASSERT(reinterpret_cast<char*>(m_end) <= static_cast<char*>(addr)+size);

            //if (!l_exists && !a_read_only) {
            if (!a_read_only) {
                bip::file_lock flock(a_filename);
                bip::scoped_lock<bip::file_lock> g_lock(flock);

                // If the file is open for writing, initialize the locks
                // since previous program crash might have left locks in inconsistent state
                for (Lock* l = m_header->locks, *e = l + NLocks; l != e; ++l)
                    new (l) Lock();
            }

        } catch (io_error& e) {
            throw;
        } catch (std::exception& e) {
            throw io_error(e.what());
        }

        return !l_exists;
    }

} // namespace util

#endif // _UTIL_PERSISTENT_ARRAY_HPP_
<commit_msg>Exposed internal header<commit_after>//----------------------------------------------------------------------------
/// \file  basic_persist_array.hpp
//----------------------------------------------------------------------------
/// \brief Implementation of persistent array storage class.
//----------------------------------------------------------------------------
// Copyright (c) 2010 Omnibius, LLC
// Author:  Serge Aleynikov <saleyn@gmail.com>
// Created: 2010-10-25
//----------------------------------------------------------------------------
/*
***** BEGIN LICENSE BLOCK *****

This file is part of the util open-source project.

Copyright (C) 2010 Serge Aleynikov <saleyn@gmail.com>

This library is free software; you can redistribute it and/or
modify it under the terms of the GNU Lesser General Public
License as published by the Free Software Foundation; either
vsn 2.1 of the License, or (at your option) any later vsn.

This library is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
Lesser General Public License for more details.

You should have received a copy of the GNU Lesser General Public
License along with this library; if not, write to the Free Software
Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA

***** END LICENSE BLOCK *****
*/

#ifndef _UTIL_PERSISTENT_ARRAY_HPP_
#define _UTIL_PERSISTENT_ARRAY_HPP_

#include <boost/interprocess/sync/file_lock.hpp>
#include <boost/interprocess/sync/scoped_lock.hpp>
#include <boost/interprocess/file_mapping.hpp>
#include <boost/interprocess/mapped_region.hpp>
#include <boost/static_assert.hpp>
#include <boost/scope_exit.hpp>
#include <boost/thread.hpp>
#include <boost/filesystem.hpp>
#include <util/compiler_hints.hpp>
#include <util/atomic.hpp>
#include <util/error.hpp>
#include <stdexcept>
#include <fstream>
#include <string>
#include <vector>
#include <cstring>
#include <cstddef>
#include <cstdlib>
#include <unistd.h>

namespace util {

    namespace { namespace bip = boost::interprocess; }
    namespace detail { struct empty_data {}; }

    template <
        typename T,
        size_t NLocks = 32,
        typename Lock = boost::mutex,
        typename ExtraHeaderData = detail::empty_data>
    struct persist_array {
        struct header {
            static const uint32_t s_version = 0xa0b1c2d3;
            uint32_t        version;
            volatile long   rec_count;
            size_t          max_recs;
            size_t          rec_size;
            Lock            locks[NLocks];
            ExtraHeaderData extra_header_data;
            T               records[0];
        };

    protected:
        typedef persist_array<T, NLocks, Lock, ExtraHeaderData> self_type;

        BOOST_STATIC_ASSERT(!(NLocks & (NLocks - 1))); // must be power of 2

        static const size_t s_lock_mask = NLocks-1;
        // Shared memory file mapping
        bip::file_mapping  m_file;
        // Shared memory mapped region
        bip::mapped_region m_region;

        // Locks that guard access to internal record structures.
        header* m_header;
        T*      m_begin;
        T*      m_end;

        void check_range(size_t a_id) const throw (badarg_error) {
            size_t n = m_header->max_recs;
            if (likely(a_id < n))
                return;
            throw badarg_error("Invalid record id specified ", a_id, " (max=", n-1, ')');
        }

    public:
        typedef Lock                        lock_type;
        typedef typename Lock::scoped_lock  scoped_lock;
        typedef typename Lock::scoped_try_lock  scoped_try_lock;

        BOOST_STATIC_ASSERT((NLocks & (NLocks-1)) == 0); // Must be power of 2.

        persist_array()
            : m_header(NULL), m_begin(NULL), m_end(NULL)
        {}

        /// Default permission mask used for opening a file
        static int default_file_mode() { return S_IRUSR | S_IWUSR | S_IRGRP | S_IWGRP; }

        /// Initialize the storage
        /// @return true if the storage file didn't exist and was created
        bool init(const char* a_filename, size_t a_max_recs, bool a_read_only = false,
            int a_mode = default_file_mode()) throw (io_error);

        size_t count()    const { return static_cast<size_t>(m_header->rec_count); }
        size_t capacity() const { return m_header->max_recs; }

        /// Return internal storage header.
        /// Use only for debugging
        const header&           header_data() const { BOOST_ASSERT(m_header); return *m_header; }

        /// Return user-defined custom header data.
        const ExtraHeaderData&  extra_header_data() const { return m_header->extra_header_data; }
        ExtraHeaderData&        extra_header_data() { return m_header->extra_header_data; }

        /// Allocate next record and return its ID.
        /// @return
        size_t allocate_rec() throw(std::runtime_error) {
            size_t n = static_cast<size_t>(atomic::add(&m_header->rec_count,1));
            if (n >= capacity()) {
                atomic::dec(&m_header->rec_count);
                throw std::runtime_error("Out of storage capacity!");
            }
            return n;
        }

        std::pair<T*, size_t> get_next() {
            size_t n = allocate_rec();
            return std::make_pair(get(n), n);
        }

        Lock& get_lock(size_t a_rec_id) {
            return m_header->locks[a_rec_id & s_lock_mask];
        }

        /// Add a record with given ID to the store
        void add(size_t a_id, const T& a_rec) {
            BOOST_ASSERT(a_id < m_header->rec_count);
            scoped_lock guard(get_lock(a_id));
            *get(a_id) = a_rec;
        }

        /// Add a record to the storage and return it's id
        size_t add(const T& a_rec) {
            size_t n = allocate_rec();
            scoped_lock guard(get_lock(n));
            *(m_begin+n) = a_rec;
            return n;
        }

        /// @return id of the given object in the storage
        size_t id_of(const T* a_rec) const { return a_rec - m_begin; }

        const T& operator[] (size_t a_id) const throw(badarg_error) {
            check_range(a_id); return *get(a_id);
        }
        T& operator[] (size_t a_id) throw(badarg_error) {
            check_range(a_id); return *get(a_id);
        }

        const T* get(size_t a_rec_id) const {
            return likely(a_rec_id < m_header->max_recs) ? m_begin+a_rec_id : NULL;
        }

        T* get(size_t a_rec_id) {
            return likely(a_rec_id < m_header->max_recs) ? m_begin+a_rec_id : NULL;
        }

        /// Flush header to disk
        bool flush_header() { return m_region.flush(0, sizeof(header)); }

        /// Flush region of cached records to disk
        bool flush(size_t a_from_rec = 0, size_t a_num_recs = 0) {
            return m_region.flush(a_from_rec, a_num_recs*sizeof(T));
        }

        /// Remove memory mapped file from disk
        void remove() {
            m_file.remove(m_file.get_name());
        }

        const T*    begin() const { return m_begin; }
        const T*    end()   const { return m_end; }
        T*          begin()       { return m_begin; }
        T*          end()         { return m_end; }

        template <class Visitor>
        void for_each(const Visitor& a_visitor) {
            size_t n = 0;
            for (const T* p = begin(), *e = p + count(); p != e; ++p, ++n)
                a_visitor(n, p);
        }

        std::ostream& dump(std::ostream& out, const std::string& a_prefix="") {
            for (const T* p = begin(), *e = p + count(); p != e; ++p)
                out << a_prefix << *p << std::endl;
            return out;
        }
    };

    //-------------------------------------------------------------------------
    // Implementation
    //-------------------------------------------------------------------------

    template <typename T, size_t NLocks, typename Lock, typename Ext>
    bool persist_array<T,NLocks,Lock,Ext>::
    init(const char* a_filename, size_t a_max_recs, bool a_read_only, int a_mode)
        throw (io_error)
    {
        static const size_t s_pack_size = getpagesize();

        size_t sz = sizeof(header) + (a_max_recs * sizeof(T));
        sz += (s_pack_size - sz % s_pack_size);

        bool l_exists;
        try {
            boost::filesystem::path l_name(a_filename);
            try {
                boost::filesystem::create_directories(l_name.parent_path());
            } catch (boost::system::system_error& e) {
                throw io_error("Cannot create directory: ",
                    l_name.parent_path(), ": ", e.what());
            }

            {
                std::filebuf f;
                f.open(a_filename, std::ios_base::in | std::ios_base::out
                    | std::ios_base::binary);
                l_exists = f.is_open();

                if (l_exists && !a_read_only) {
                    bip::file_lock flock(a_filename);
                    bip::scoped_lock<bip::file_lock> g_lock(flock);

                    header h;
                    f.sgetn(reinterpret_cast<char*>(&h), sizeof(header));

                    if (h.version != header::s_version) {
                        throw io_error("Invalid file format ", a_filename);
                    }
                    if (h.rec_size != sizeof(T)) {
                        throw io_error("Invalid item size in file ", a_filename,
                            " (expected ", sizeof(T), " got ", h.rec_size, ')');
                    }
                    // Increase the file size if instructed to do so.
                    if (h.max_recs < a_max_recs) {
                        h.max_recs = a_max_recs;
                        f.pubseekoff(0, std::ios_base::beg);
                        f.sputn(reinterpret_cast<char*>(&h), sizeof(header));
                        f.pubseekoff(sz-1, std::ios_base::beg);
                        f.sputc(0);
                    }
                }
            }

            if (!l_exists && !a_read_only) {
                int l_fd = ::open(a_filename, O_RDWR | O_CREAT | O_TRUNC, a_mode);
                if (l_fd < 0)
                    throw io_error(errno, "Error creating file ", a_filename);

                BOOST_SCOPE_EXIT_TPL( (&l_fd) ) {
                    ::close(l_fd);
                } BOOST_SCOPE_EXIT_END;

                bip::file_lock flock(a_filename);
                bip::scoped_lock<bip::file_lock> g_lock(flock);

                header h;
                h.version   = header::s_version;
                h.rec_count = 0;
                h.max_recs  = a_max_recs;
                h.rec_size  = sizeof(T);

                if (::write(l_fd, &h, sizeof(h)) < 0)
                    throw io_error(errno, "Error writing to file ", a_filename);

                if (::ftruncate(l_fd, sz) < 0)
                    throw io_error(errno, "Error setting file ",
                        a_filename, " to size ", sz);

                ::fsync(l_fd);
            }

            bip::file_mapping shmf(a_filename, a_read_only ? bip::read_only : bip::read_write);
            bip::mapped_region region(shmf, a_read_only ? bip::read_only : bip::read_write);

            //Get the address of the mapped region
            void*  addr  = region.get_address();
            size_t size  = region.get_size();

            m_file.swap(shmf);
            m_region.swap(region);

            //if (!exists && !a_read_only)
            //    memset(static_cast<char*>(addr) + sizeof(header), 0, size - sizeof(header));
            m_header = static_cast<header*>(addr);
            m_begin  = m_header->records;
            m_end    = m_begin + a_max_recs;
            BOOST_ASSERT(reinterpret_cast<char*>(m_end) <= static_cast<char*>(addr)+size);

            //if (!l_exists && !a_read_only) {
            if (!a_read_only) {
                bip::file_lock flock(a_filename);
                bip::scoped_lock<bip::file_lock> g_lock(flock);

                // If the file is open for writing, initialize the locks
                // since previous program crash might have left locks in inconsistent state
                for (Lock* l = m_header->locks, *e = l + NLocks; l != e; ++l)
                    new (l) Lock();
            }

        } catch (io_error& e) {
            throw;
        } catch (std::exception& e) {
            throw io_error(e.what());
        }

        return !l_exists;
    }

} // namespace util

#endif // _UTIL_PERSISTENT_ARRAY_HPP_
<|endoftext|>


<commit_before>#ifndef V_SMC_CORE_PARTICLE_HPP
#define V_SMC_CORE_PARTICLE_HPP

#include <algorithm>
#include <limits>
#include <cmath>
#include <cstddef>
#include <mkl_cblas.h>
#include <boost/function.hpp>
#include <boost/random/binomial_distribution.hpp>
#include <boost/random/mersenne_twister.hpp>
#include <vSMC/core/buffer.hpp>
#include <vSMC/core/rng.hpp>

namespace vSMC {

/// Resample scheme
enum ResampleScheme {MULTINOMIAL, RESIDUAL, STRATIFIED, SYSTEMATIC};

/// \brief Particle class
///
/// Particle class store the particle set and arrays of weights and log
/// weights. It provides access to particle values as well as weights. It
/// computes and manages resources for ESS, resampling, etc, tasks unique to
/// each iteration.
template <typename T>
class Particle
{
    public :

    typedef V_SMC_RNG_TYPE rng_type;

    /// \brief Construct a Particle object with given number of particles
    ///
    /// \param N The number of particles
    Particle (std::size_t N, rng_type::seed_type seed = V_SMC_RNG_SEED) :
        size_(N), value_(N),
        weight_(N), log_weight_(N), inc_weight_(N), replication_(N),
        ess_(0), resampled_(false), zconst_(0), estimate_zconst_(false),
        prng_(N)
    {
        rng_type rng(seed);
        rng.step_size(size_);
        for (std::size_t i = 0; i != size_; ++i) {
            rng.advance_ctr(i);
            prng_[i] = rng;
        }

        double equal_weight = 1.0 / size_;
        for (std::size_t i = 0; i != size_; ++i) {
            weight_[i] = equal_weight;
            log_weight_[i] = 0;
        }
    }

    /// \brief Size of the particle set
    ///
    /// \return The number of particles
    std::size_t size () const
    {
        return size_;
    }

    /// \brief Read and write access to particle values
    ///
    /// \return A reference to the particle values, type (T &)
    ///
    /// \note The Particle class guarantee that during the life type of the
    /// object, the reference returned by this member will no be a dangle
    /// handler.
    T &value ()
    {
        return value_;
    }

    /// \brief Read only access to particle values
    ///
    /// \return A const reference to the particle values
    const T &value () const
    {
        return value_;
    }

    /// \brief Read only access to the weights
    ///
    /// \return A const pointer to the weights
    ///
    /// \note The Particle class guarantee that during the life type of the
    /// object, the pointer returned by this always valid and point to the
    /// same address
    const double *weight_ptr () const
    {
        return weight_.data();
    }

    /// \brief Read only access to the log weights
    ///
    /// \return A const pointer to the log weights
    const double *log_weight_ptr () const
    {
        return log_weight_.data();
    }

    /// \brief Set the log weights
    ///
    /// \param [in] new_weight New log weights
    void set_log_weight (const double *new_weight)
    {
        cblas_dcopy(size_, new_weight, 1, log_weight_.data(), 1);
        set_weight();
    }

    /// \brief Add to the log weights
    ///
    /// \param [in] inc_weight Incremental log weights
    void add_log_weight (const double *inc_weight)
    {
        if (estimate_zconst_) {
            for (std::size_t i = 0; i != size_; ++i)
                inc_weight_[i] = std::exp(inc_weight[i]);
            zconst_ += std::log(cblas_ddot(size_, weight_.data(), 1,
                        inc_weight_.data(), 1));
        }

        for (std::size_t i = 0; i != size_; ++i)
            log_weight_[i] += inc_weight[i];
        set_weight();
    }

    /// \brief The ESS (Effective Sample Size)
    ///
    /// \return The value of ESS for current particle set
    double ess () const
    {
        return ess_;
    }

    /// \brief Get indicator of resampling
    ///
    /// \return A bool value, \b true if the current iteration was resampled
    bool resampled () const
    {
        return resampled_;
    }

    /// \brief Set indicator of resampling
    ///
    /// \param resampled \b true if the current iteration was resampled
    void resampled (bool resampled)
    {
        resampled_ = resampled;
    }

    /// \brief Toggle whether or not record SMC normalizing constant
    ///
    /// \param estimate_zconst Start estimating normalzing constant if true
    void zconst (bool estimate_zconst)
    {
        estimate_zconst_ = estimate_zconst;
    }

    /// \brief Get the value of SMC normalizing constant
    ///
    /// \return SMC normalizng constant estimate
    double zconst () const
    {
        return zconst_;
    }

    /// \brief Reset the value of SMC normalizing constant
    void reset_zconst ()
    {
        zconst_ = 0;
    }

    /// \brief Perform resampling
    ///
    /// \param scheme The resampling scheme, see ResamplingScheme
    void resample (ResampleScheme scheme)
    {
        switch (scheme) {
            case MULTINOMIAL :
                resample_multinomial();
                break;
            case RESIDUAL :
                resample_residual();
                break;
            case STRATIFIED :
                resample_stratified();
                break;
            case SYSTEMATIC :
                resample_systematic();
                break;
            default :
                throw std::runtime_error(
                        "ERROR: vSMC::Particle::resample: "
                        "Unknown Resample Scheme");
        }
    }

    rng_type *prng ()
    {
        return prng_.data();
    }

    rng_type &prng (std::size_t id)
    {
        return prng_[id];
    }

    private :

    std::size_t size_;
    T value_;

    internal::Buffer<double> weight_;
    internal::Buffer<double> log_weight_;
    internal::Buffer<double> inc_weight_;
    internal::Buffer<int> replication_;

    double ess_;
    bool resampled_;
    double zconst_;
    bool estimate_zconst_;

    internal::Buffer<rng_type> prng_;
    typedef boost::random::binomial_distribution<int, double> binom_type;
    binom_type binom_;

    void set_weight ()
    {
        double max_weight = -std::numeric_limits<double>::infinity();

        for (std::size_t i = 0; i != size_; ++i)
            max_weight = std::max(max_weight, log_weight_[i]);
        for (std::size_t i = 0; i != size_; ++i) {
            log_weight_[i] -= max_weight;
            weight_[i] = std::exp(log_weight_[i]);
        }
        double sum = cblas_dasum(size_, weight_.data(), 1);
        cblas_dscal(size_, 1 / sum, weight_.data(), 1);
        ess_ = 1 / cblas_ddot(size_, weight_.data(), 1, weight_.data(), 1);
    }

    void weight2replication ()
    {
        double tp = 0;
        for (std::size_t i = 0; i != size_; ++i)
            tp += weight_[i];

        double sum_p = 0;
        std::size_t sum_n = 0;
        for (std::size_t i = 0; i != size_; ++i) {
            replication_[i] = 0;
            if (sum_n < size_ && weight_[i] > 0) {
                binom_type::param_type
                    param(size_ - sum_n, weight_[i] / (tp - sum_p));
                replication_[i] = binom_(prng_[i], param);
            }
            sum_p += weight_[i];
            sum_n += replication_[i];
        }
    }

    void resample_multinomial ()
    {
        weight2replication();
        resample_do();
    }

    void resample_residual ()
    {
        // Reuse weight and log_weight. weight: act as the fractional part of
        // N * weight. log_weight: act as the integral part of N * weight.
        // They all will be reset to equal weights after resampling.  So it is
        // safe to modify them here.
        for (std::size_t i = 0; i != size_; ++i)
            weight_[i] = std::modf(size_ * weight_[i], log_weight_.data() + i);
        std::size_t size = size_;
        size -= cblas_dasum(size_, log_weight_.data(), 1);
        weight2replication();
        for (std::size_t i = 0; i != size_; ++i)
            replication_[i] += log_weight_[i];
        resample_do();
    }

    void resample_stratified () {}
    void resample_systematic () {}

    void resample_do ()
    {
        std::size_t from = 0;
        std::size_t time = 0;

        for (std::size_t to = 0; to != size_; ++to) {
            if (!replication_[to]) {
                // replication_[to] has zero child, copy from elsewhere
                if (replication_[from] - time <= 1) {
                    // only 1 child left on replication_[from]
                    time = 0;
                    do // move from to some position with at least 2 children
                        ++from;
                    while (replication_[from] < 2);
                }
                value_.copy(from, to);
                ++time;
            }
        }

        ess_ = size_;
        double equal_weight = 1.0 / size_;
        for (std::size_t i = 0; i != size_; ++i) {
            weight_[i] = equal_weight;
            log_weight_[i] = 0;
        }
    }
}; // class Particle

} // namespace vSMC

#endif // V_SMC_CORE_PARTICLE_HPP
<commit_msg>simplify finding max logic<commit_after>#ifndef V_SMC_CORE_PARTICLE_HPP
#define V_SMC_CORE_PARTICLE_HPP

#include <cmath>
#include <cstddef>
#include <mkl_cblas.h>
#include <boost/function.hpp>
#include <boost/random/binomial_distribution.hpp>
#include <boost/random/mersenne_twister.hpp>
#include <vSMC/core/buffer.hpp>
#include <vSMC/core/rng.hpp>

namespace vSMC {

/// Resample scheme
enum ResampleScheme {MULTINOMIAL, RESIDUAL, STRATIFIED, SYSTEMATIC};

/// \brief Particle class
///
/// Particle class store the particle set and arrays of weights and log
/// weights. It provides access to particle values as well as weights. It
/// computes and manages resources for ESS, resampling, etc, tasks unique to
/// each iteration.
template <typename T>
class Particle
{
    public :

    typedef V_SMC_RNG_TYPE rng_type;

    /// \brief Construct a Particle object with given number of particles
    ///
    /// \param N The number of particles
    Particle (std::size_t N, rng_type::seed_type seed = V_SMC_RNG_SEED) :
        size_(N), value_(N),
        weight_(N), log_weight_(N), inc_weight_(N), replication_(N),
        ess_(0), resampled_(false), zconst_(0), estimate_zconst_(false),
        prng_(N)
    {
        rng_type rng(seed);
        rng.step_size(size_);
        for (std::size_t i = 0; i != size_; ++i) {
            rng.advance_ctr(i);
            prng_[i] = rng;
        }

        double equal_weight = 1.0 / size_;
        for (std::size_t i = 0; i != size_; ++i) {
            weight_[i] = equal_weight;
            log_weight_[i] = 0;
        }
    }

    /// \brief Size of the particle set
    ///
    /// \return The number of particles
    std::size_t size () const
    {
        return size_;
    }

    /// \brief Read and write access to particle values
    ///
    /// \return A reference to the particle values, type (T &)
    ///
    /// \note The Particle class guarantee that during the life type of the
    /// object, the reference returned by this member will no be a dangle
    /// handler.
    T &value ()
    {
        return value_;
    }

    /// \brief Read only access to particle values
    ///
    /// \return A const reference to the particle values
    const T &value () const
    {
        return value_;
    }

    /// \brief Read only access to the weights
    ///
    /// \return A const pointer to the weights
    ///
    /// \note The Particle class guarantee that during the life type of the
    /// object, the pointer returned by this always valid and point to the
    /// same address
    const double *weight_ptr () const
    {
        return weight_.data();
    }

    /// \brief Read only access to the log weights
    ///
    /// \return A const pointer to the log weights
    const double *log_weight_ptr () const
    {
        return log_weight_.data();
    }

    /// \brief Set the log weights
    ///
    /// \param [in] new_weight New log weights
    void set_log_weight (const double *new_weight)
    {
        cblas_dcopy(size_, new_weight, 1, log_weight_.data(), 1);
        set_weight();
    }

    /// \brief Add to the log weights
    ///
    /// \param [in] inc_weight Incremental log weights
    void add_log_weight (const double *inc_weight)
    {
        if (estimate_zconst_) {
            for (std::size_t i = 0; i != size_; ++i)
                inc_weight_[i] = std::exp(inc_weight[i]);
            zconst_ += std::log(cblas_ddot(size_, weight_.data(), 1,
                        inc_weight_.data(), 1));
        }

        for (std::size_t i = 0; i != size_; ++i)
            log_weight_[i] += inc_weight[i];
        set_weight();
    }

    /// \brief The ESS (Effective Sample Size)
    ///
    /// \return The value of ESS for current particle set
    double ess () const
    {
        return ess_;
    }

    /// \brief Get indicator of resampling
    ///
    /// \return A bool value, \b true if the current iteration was resampled
    bool resampled () const
    {
        return resampled_;
    }

    /// \brief Set indicator of resampling
    ///
    /// \param resampled \b true if the current iteration was resampled
    void resampled (bool resampled)
    {
        resampled_ = resampled;
    }

    /// \brief Toggle whether or not record SMC normalizing constant
    ///
    /// \param estimate_zconst Start estimating normalzing constant if true
    void zconst (bool estimate_zconst)
    {
        estimate_zconst_ = estimate_zconst;
    }

    /// \brief Get the value of SMC normalizing constant
    ///
    /// \return SMC normalizng constant estimate
    double zconst () const
    {
        return zconst_;
    }

    /// \brief Reset the value of SMC normalizing constant
    void reset_zconst ()
    {
        zconst_ = 0;
    }

    /// \brief Perform resampling
    ///
    /// \param scheme The resampling scheme, see ResamplingScheme
    void resample (ResampleScheme scheme)
    {
        switch (scheme) {
            case MULTINOMIAL :
                resample_multinomial();
                break;
            case RESIDUAL :
                resample_residual();
                break;
            case STRATIFIED :
                resample_stratified();
                break;
            case SYSTEMATIC :
                resample_systematic();
                break;
            default :
                throw std::runtime_error(
                        "ERROR: vSMC::Particle::resample: "
                        "Unknown Resample Scheme");
        }
    }

    rng_type *prng ()
    {
        return prng_.data();
    }

    rng_type &prng (std::size_t id)
    {
        return prng_[id];
    }

    private :

    std::size_t size_;
    T value_;

    internal::Buffer<double> weight_;
    internal::Buffer<double> log_weight_;
    internal::Buffer<double> inc_weight_;
    internal::Buffer<int> replication_;

    double ess_;
    bool resampled_;
    double zconst_;
    bool estimate_zconst_;

    internal::Buffer<rng_type> prng_;
    typedef boost::random::binomial_distribution<int, double> binom_type;
    binom_type binom_;

    void set_weight ()
    {
        double max_weight = log_weight_[0];
        for (std::size_t i = 0; i != size_; ++i)
            if (log_weight_[i] > max_weight)
                max_weight = log_weight_[i];
        for (std::size_t i = 0; i != size_; ++i) {
            log_weight_[i] -= max_weight;
            weight_[i] = std::exp(log_weight_[i]);
        }
        double sum = cblas_dasum(size_, weight_.data(), 1);
        cblas_dscal(size_, 1 / sum, weight_.data(), 1);
        ess_ = 1 / cblas_ddot(size_, weight_.data(), 1, weight_.data(), 1);
    }

    void weight2replication ()
    {
        double tp = 0;
        for (std::size_t i = 0; i != size_; ++i)
            tp += weight_[i];

        double sum_p = 0;
        std::size_t sum_n = 0;
        for (std::size_t i = 0; i != size_; ++i) {
            replication_[i] = 0;
            if (sum_n < size_ && weight_[i] > 0) {
                binom_type::param_type
                    param(size_ - sum_n, weight_[i] / (tp - sum_p));
                replication_[i] = binom_(prng_[i], param);
            }
            sum_p += weight_[i];
            sum_n += replication_[i];
        }
    }

    void resample_multinomial ()
    {
        weight2replication();
        resample_do();
    }

    void resample_residual ()
    {
        // Reuse weight and log_weight. weight: act as the fractional part of
        // N * weight. log_weight: act as the integral part of N * weight.
        // They all will be reset to equal weights after resampling.  So it is
        // safe to modify them here.
        for (std::size_t i = 0; i != size_; ++i)
            weight_[i] = std::modf(size_ * weight_[i], log_weight_.data() + i);
        std::size_t size = size_;
        size -= cblas_dasum(size_, log_weight_.data(), 1);
        weight2replication();
        for (std::size_t i = 0; i != size_; ++i)
            replication_[i] += log_weight_[i];
        resample_do();
    }

    void resample_stratified () {}
    void resample_systematic () {}

    void resample_do ()
    {
        std::size_t from = 0;
        std::size_t time = 0;

        for (std::size_t to = 0; to != size_; ++to) {
            if (!replication_[to]) {
                // replication_[to] has zero child, copy from elsewhere
                if (replication_[from] - time <= 1) {
                    // only 1 child left on replication_[from]
                    time = 0;
                    do // move from to some position with at least 2 children
                        ++from;
                    while (replication_[from] < 2);
                }
                value_.copy(from, to);
                ++time;
            }
        }

        ess_ = size_;
        double equal_weight = 1.0 / size_;
        for (std::size_t i = 0; i != size_; ++i) {
            weight_[i] = equal_weight;
            log_weight_[i] = 0;
        }
    }
}; // class Particle

} // namespace vSMC

#endif // V_SMC_CORE_PARTICLE_HPP
<|endoftext|>


<commit_before>/*=========================================================================

  Program:   Visualization Toolkit
  Module:    vtkCell3D.cxx
  Language:  C++
  Date:      $Date$
  Version:   $Revision$

Copyright (c) 1993-2001 Ken Martin, Will Schroeder, Bill Lorensen
All rights reserved.

Redistribution and use in source and binary forms, with or without
modification, are permitted provided that the following conditions are met:

 * Redistributions of source code must retain the above copyright notice,
   this list of conditions and the following disclaimer.

 * Redistributions in binary form must reproduce the above copyright notice,
   this list of conditions and the following disclaimer in the documentation
   and/or other materials provided with the distribution.

 * Neither name of Ken Martin, Will Schroeder, or Bill Lorensen nor the names
   of any contributors may be used to endorse or promote products derived
   from this software without specific prior written permission.

 * Modified source versions must be plainly marked as such, and must not be
   misrepresented as being the original software.

THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS ``AS IS''
AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHORS OR CONTRIBUTORS BE LIABLE FOR
ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.

=========================================================================*/
#include "vtkCell3D.h"
#include "vtkOrderedTriangulator.h"
#include "vtkPointLocator.h"

vtkCell3D::~vtkCell3D()
{
  if ( this->Triangulator )
    {
    this->Triangulator->Delete();
    this->Triangulator = NULL;
    }
}

void vtkCell3D::Clip(float value, vtkScalars *cellScalars,
                     vtkPointLocator *locator, vtkCellArray *tets,
                     vtkPointData *inPD, vtkPointData *outPD,
                     vtkCellData *inCD, int cellId, vtkCellData *outCD,
                     int insideOut)
{
  vtkCell3D *cell3D = (vtkCell3D *)this; //has to be to be in this method
  int numPts=this->GetNumberOfPoints();
  int numEdges=this->GetNumberOfEdges();
  int *edges;
  int i;

  // Create one if necessary
  if ( ! this->Triangulator )
    {
    this->Triangulator = vtkOrderedTriangulator::New();
    }

  // Initialize Delaunay insertion process with voxel triangulation.
  // No more than (numPts + numEdges) points can be inserted.
  this->Triangulator->InitTriangulation(this->GetBounds(),
                                        (numPts + numEdges));

  // Inject ordered voxel corner points into triangulation. Recall
  // that the PreSortedOn() flag was set in the triangulator.
  int type, ptId, id;
  int internalId[VTK_CELL_SIZE];
  float s1, s2, *xPtr, t, p1[3], p2[3], x[3];
  for (i=0; i<numPts; i++)
    {
    ptId = this->PointIds->GetId(i);

    // Currently all points are injected because of the possibility
    // of intersection point merging.
    s1 = cellScalars->GetScalar(ptId);
    if ( (s1 >= value && !insideOut) || (s1 < value && insideOut) )
      {
      type = 0; //inside
      }
    else
      {
      type = 1; //outside, but to be inserted anyway
      }

    xPtr = this->Points->GetPoint(ptId);
    if ( locator->InsertUniquePoint(xPtr, id) )
      {
      outPD->CopyData(inPD,this->PointIds->GetId(ptId), id);
      }
    internalId[ptId] = this->Triangulator->InsertPoint(id, xPtr, type);
    }//for eight voxel corner points

  // For each edge intersection point, insert into triangulation. Edge
  // intersections come from clipping value. Have to be careful of
  // intersections near exisiting points (causes bad Delaunay behavior).
  for (int edgeNum=0; edgeNum < numEdges; edgeNum++)
    {
    cell3D->GetEdgePoints(edgeNum, edges);
    s1 = cellScalars->GetScalar(edges[0]);
    s2 = cellScalars->GetScalar(edges[1]);
    if ( (s1 < value && s2 >= value) || (s1 >= value && s2 < value) )
      {
      t = (value - s1) / (s2 - s1);

      // Check to see whether near the intersection is near a voxel corner.
      // If so,have to merge requiring a change of type to type=boundary.
      if ( t < 0.01 )
        {
        this->Triangulator->UpdatePointType(internalId[edges[0]], 2);
        continue;
        }
      else if ( t > 0.99 )
        {
        this->Triangulator->UpdatePointType(internalId[edges[1]], 2);
        continue;
        }

      // generate edge intersection point
      this->Points->GetPoint(edges[0],p1);
      this->Points->GetPoint(edges[1],p2);
      for (i=0; i<3; i++)
        {
        x[i] = p1[i] + t * (p2[i] - p1[i]);
        }

      // Incorporate point into output and interpolate edge data as necessary
      if ( locator->InsertUniquePoint(x, ptId) )
        {
        outPD->InterpolateEdge(inPD, ptId, this->PointIds->GetId(edges[0]),
                               this->PointIds->GetId(edges[1]), t);
        }

      //Insert into Delaunay triangulation
      this->Triangulator->InsertPoint(ptId,x,2);

      }//if edge intersects value
    }//for all edges

  // triangulate the points
  this->Triangulator->Triangulate();

  // Add the triangulation to the mesh
//  this->Triangulator->AddTetras(0,output);
//  if ( this->GenerateClippedOutput )
//    {
//    this->Triangulator->AddTetras(1,clippedOutput);
//    }
}

void vtkCell3D::PrintSelf(ostream& os, vtkIndent indent)
{
  vtkCell::PrintSelf(os,indent);
}
<commit_msg>ENH:First crack at 3D cell clipping...needs some more debugging<commit_after>/*=========================================================================

  Program:   Visualization Toolkit
  Module:    vtkCell3D.cxx
  Language:  C++
  Date:      $Date$
  Version:   $Revision$

Copyright (c) 1993-2001 Ken Martin, Will Schroeder, Bill Lorensen
All rights reserved.

Redistribution and use in source and binary forms, with or without
modification, are permitted provided that the following conditions are met:

 * Redistributions of source code must retain the above copyright notice,
   this list of conditions and the following disclaimer.

 * Redistributions in binary form must reproduce the above copyright notice,
   this list of conditions and the following disclaimer in the documentation
   and/or other materials provided with the distribution.

 * Neither name of Ken Martin, Will Schroeder, or Bill Lorensen nor the names
   of any contributors may be used to endorse or promote products derived
   from this software without specific prior written permission.

 * Modified source versions must be plainly marked as such, and must not be
   misrepresented as being the original software.

THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS ``AS IS''
AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHORS OR CONTRIBUTORS BE LIABLE FOR
ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.

=========================================================================*/
#include "vtkCell3D.h"
#include "vtkOrderedTriangulator.h"
#include "vtkPointLocator.h"

vtkCell3D::~vtkCell3D()
{
  if ( this->Triangulator )
    {
    this->Triangulator->Delete();
    this->Triangulator = NULL;
    }
}

void vtkCell3D::Clip(float value, vtkScalars *cellScalars,
                     vtkPointLocator *locator, vtkCellArray *tets,
                     vtkPointData *inPD, vtkPointData *outPD,
                     vtkCellData *inCD, int cellId, vtkCellData *outCD,
                     int insideOut)
{
  vtkCell3D *cell3D = (vtkCell3D *)this; //has to be to be in this method
  int numPts=this->GetNumberOfPoints();
  int numEdges=this->GetNumberOfEdges();
  int *edges;
  int i;

  // Create one if necessary
  if ( ! this->Triangulator )
    {
    this->Triangulator = vtkOrderedTriangulator::New();
    }

  // Initialize Delaunay insertion process with voxel triangulation.
  // No more than (numPts + numEdges) points can be inserted.
  this->Triangulator->InitTriangulation(this->GetBounds(),
                                        (numPts + numEdges));

  // Inject ordered voxel corner points into triangulation. Recall
  // that the PreSortedOn() flag was set in the triangulator.
  int type, ptId, id;
  int internalId[VTK_CELL_SIZE];
  float s1, s2, *xPtr, t, p1[3], p2[3], x[3];
  for (i=0; i<numPts; i++)
    {
    ptId = this->PointIds->GetId(i);

    // Currently all points are injected because of the possibility
    // of intersection point merging.
    s1 = cellScalars->GetScalar(ptId);
    if ( (s1 >= value && !insideOut) || (s1 < value && insideOut) )
      {
      type = 0; //inside
      }
    else
      {
      type = 1; //outside, but to be inserted anyway
      }

    xPtr = this->Points->GetPoint(ptId);
    if ( locator->InsertUniquePoint(xPtr, id) )
      {
      outPD->CopyData(inPD,this->PointIds->GetId(ptId), id);
      }
    internalId[ptId] = this->Triangulator->InsertPoint(id, xPtr, type);
    }//for eight voxel corner points

  // For each edge intersection point, insert into triangulation. Edge
  // intersections come from clipping value. Have to be careful of
  // intersections near exisiting points (causes bad Delaunay behavior).
  for (int edgeNum=0; edgeNum < numEdges; edgeNum++)
    {
    cell3D->GetEdgePoints(edgeNum, edges);
    s1 = cellScalars->GetScalar(edges[0]);
    s2 = cellScalars->GetScalar(edges[1]);
    if ( (s1 < value && s2 >= value) || (s1 >= value && s2 < value) )
      {
      t = (value - s1) / (s2 - s1);

      // Check to see whether near the intersection is near a voxel corner.
      // If so,have to merge requiring a change of type to type=boundary.
      if ( t < 0.01 )
        {
        this->Triangulator->UpdatePointType(internalId[edges[0]], 2);
        continue;
        }
      else if ( t > 0.99 )
        {
        this->Triangulator->UpdatePointType(internalId[edges[1]], 2);
        continue;
        }

      // generate edge intersection point
      this->Points->GetPoint(edges[0],p1);
      this->Points->GetPoint(edges[1],p2);
      for (i=0; i<3; i++)
        {
        x[i] = p1[i] + t * (p2[i] - p1[i]);
        }

      // Incorporate point into output and interpolate edge data as necessary
      if ( locator->InsertUniquePoint(x, ptId) )
        {
        outPD->InterpolateEdge(inPD, ptId, this->PointIds->GetId(edges[0]),
                               this->PointIds->GetId(edges[1]), t);
        }

      //Insert into Delaunay triangulation
      this->Triangulator->InsertPoint(ptId,x,2);

      }//if edge intersects value
    }//for all edges

  // triangulate the points
  this->Triangulator->Triangulate();

  // Add the triangulation to the mesh
  this->Triangulator->AddTetras(0,tets);
}

void vtkCell3D::PrintSelf(ostream& os, vtkIndent indent)
{
  vtkCell::PrintSelf(os,indent);
}
<|endoftext|>


<commit_before>/*=========================================================================
*
* Copyright Marius Staring, Stefan Klein, David Doria. 2011.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0.txt
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*
*=========================================================================*/
#ifndef __computeoverlapsummary_hxx
#define __computeoverlapsummary_hxx

#include "itkImageFileReader.h"
#include "itkImageFileWriter.h"
#include "itkLabelOverlapMeasuresImageFilter.h"

template< class TComponentType, unsigned int VDimension >
void
ITKToolsComputeOverlapSummary< TComponentType, VDimension >
::ComputeOverlapSummary(
 const std::string & inputImage1,
 const std::string & inputImage2,
 const std::string & outputFileName,
 const std::string & seperator )
{
  typedef itk::ImageFileReader<InputImageType>  ReaderType;
  typename ReaderType::Pointer reader1 = ReaderType::New();
  reader1->SetFileName( inputImage1.c_str() );
  typename ReaderType::Pointer reader2 = ReaderType::New();
  reader2->SetFileName( inputImage2.c_str() );

  typedef itk::LabelOverlapMeasuresImageFilter<InputImageType> FilterType;
  typename FilterType::Pointer filter = FilterType::New();
  filter->SetSourceImage( reader1->GetOutput() );
  filter->SetTargetImage( reader2->GetOutput() );
  filter->Update();

  FILE * pFile;
  pFile = fopen ( outputFileName.c_str(), "w");

  if (pFile==NULL) {
    throw invalidfilexception();
  }

  fprintf ( pFile, "%s%s%s%s%s%s%s%s%s%s%s\n", "Label", seperator, "Union (jaccard)", seperator, "Mean (dice)", seperator, "Volume sim.", seperator, "False negative", seperator, "False positive");
  fprintf ( pFile, "%s%s%f%s%f%s%f%s%f%s%f\n", "Total", seperator, filter->GetUnionOverlap(), seperator, filter->GetMeanOverlap(), seperator, filter->GetVolumeSimilarity(), seperator, filter->GetFalseNegativeError(), seperator, filter->GetFalsePositiveError() );

  typename FilterType::MapType labelMap = filter->GetLabelSetMeasures();
  typename FilterType::MapType::const_iterator it;
  for( it = labelMap.begin(); it != labelMap.end(); ++it )
  {
    if( (*it).first == 0 )
    {
      continue;
    }

    int label = (*it).first;
    fprintf ( pFile, "%i%s%f%s%f%s%f%s%f%s%f\n", label, seperator, filter->GetUnionOverlap( label ), seperator, filter->GetMeanOverlap( label ), seperator, filter->GetVolumeSimilarity( label ), seperator, filter->GetFalseNegativeError( label ), seperator, filter->GetFalsePositiveError( label ) );
  }
  fclose (pFile);
} // end ComputeOverlapSummary()

#endif // end #ifndef __computeoverlapsummary_hxx
<commit_msg>- Fixed a stupid bug (conversion of string to char * for fprintf)<commit_after>/*=========================================================================
*
* Copyright Marius Staring, Stefan Klein, David Doria. 2011.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0.txt
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*
*=========================================================================*/
#ifndef __computeoverlapsummary_hxx
#define __computeoverlapsummary_hxx

#include "itkImageFileReader.h"
#include "itkImageFileWriter.h"
#include "itkLabelOverlapMeasuresImageFilter.h"

template< class TComponentType, unsigned int VDimension >
void
ITKToolsComputeOverlapSummary< TComponentType, VDimension >
::ComputeOverlapSummary(
 const std::string & inputImage1,
 const std::string & inputImage2,
 const std::string & outputFileName,
 const std::string & seperator )
{
  typedef itk::ImageFileReader<InputImageType>  ReaderType;
  typename ReaderType::Pointer reader1 = ReaderType::New();
  reader1->SetFileName( inputImage1.c_str() );
  typename ReaderType::Pointer reader2 = ReaderType::New();
  reader2->SetFileName( inputImage2.c_str() );

  typedef itk::LabelOverlapMeasuresImageFilter<InputImageType> FilterType;
  typename FilterType::Pointer filter = FilterType::New();
  filter->SetSourceImage( reader1->GetOutput() );
  filter->SetTargetImage( reader2->GetOutput() );
  filter->Update();

  FILE * pFile;
  pFile = fopen ( outputFileName.c_str(), "w");

  if (pFile==NULL) {
    throw invalidfilexception();
  }

  fprintf ( pFile, "%s%s%s%s%s%s%s%s%s%s%s\n", "Label", seperator.c_str(), "Union (jaccard)", seperator.c_str(), "Mean (dice)", seperator.c_str(), "Volume sim.", seperator.c_str(), "False negative", seperator.c_str(), "False positive");
  fprintf ( pFile, "%s%s%f%s%f%s%f%s%f%s%f\n", "Total", seperator.c_str(), filter->GetUnionOverlap(), seperator.c_str(), filter->GetMeanOverlap(), seperator.c_str(), filter->GetVolumeSimilarity(), seperator.c_str(), filter->GetFalseNegativeError(), seperator.c_str(), filter->GetFalsePositiveError() );

  typename FilterType::MapType labelMap = filter->GetLabelSetMeasures();
  typename FilterType::MapType::const_iterator it;
  for( it = labelMap.begin(); it != labelMap.end(); ++it )
  {
    if( (*it).first == 0 )
    {
      continue;
    }

    int label = (*it).first;
    fprintf ( pFile, "%i%s%f%s%f%s%f%s%f%s%f\n", label, seperator.c_str(), filter->GetUnionOverlap( label ), seperator.c_str(), filter->GetMeanOverlap( label ), seperator.c_str(), filter->GetVolumeSimilarity( label ), seperator.c_str(), filter->GetFalseNegativeError( label ), seperator.c_str(), filter->GetFalsePositiveError( label ) );
  }
  fclose (pFile);
} // end ComputeOverlapSummary()

#endif // end #ifndef __computeoverlapsummary_hxx
<|endoftext|>


<commit_before>#include "stdafx.h"
#include "Core/SymbolTable.h"
#include "Util/FileClasses.h"
#include "Util/Util.h"
#include "Common.h"

const wchar_t validSymbolCharacters[] = L"ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789_.";

bool operator<(SymbolKey const& lhs, SymbolKey const& rhs)
{
	if (lhs.file != rhs.file)
		return lhs.file < rhs.file;
	if (lhs.section != rhs.section)
		return lhs.section < rhs.section;
	return lhs.name.compare(rhs.name) < 0;
}

SymbolTable::SymbolTable()
{
	uniqueCount = 0;
}

SymbolTable::~SymbolTable()
{
	clear();
}

void SymbolTable::clear()
{
	for (size_t i = 0; i < labels.size(); i++)
	{
		delete labels[i];
	}

	symbols.clear();
	labels.clear();
	equationsCount = 0;
	uniqueCount = 0;
}

void SymbolTable::setFileSectionValues(const std::wstring& symbol, unsigned int& file, unsigned int& section)
{
	if (symbol[0] == '@')
	{
		if (symbol[1] != '@')
		{
			// static label, @. the section doesn't matter
			section = -1;
		} else {
			// local label, @@. the file doesn't matter
			file = -1;
		}
	} else {
		// global label. neither file nor section matters
		file = section = -1;
	}
}

Label* SymbolTable::getLabel(const std::wstring& symbol, unsigned int file, unsigned int section)
{
	if (isValidSymbolName(symbol) == false)
		return NULL;

	int actualSection = section;
	setFileSectionValues(symbol,file,section);
	SymbolKey key = { symbol, file, section };

	// find label, create new one if it doesn't exist
	auto it = symbols.find(key);
	if (it == symbols.end())
	{
		SymbolInfo value = { LabelSymbol, labels.size() };
		symbols[key] = value;

		Label* result = new Label(symbol);
		if (section == actualSection)
			result->setSection(section);			// local, set section of parent
		else
			result->setSection(actualSection+1);	// global, set section of children
		labels.push_back(result);
		return result;
	}

	// make sure not to match symbols that aren't labels
	if (it->second.type != LabelSymbol)
		return NULL;

	return labels[it->second.index];
}

bool SymbolTable::symbolExists(const std::wstring& symbol, unsigned int file, unsigned int section)
{
	if (isValidSymbolName(symbol) == false)
		return false;

	setFileSectionValues(symbol,file,section);

	SymbolKey key = { symbol, file, section };
	auto it = symbols.find(key);
	return it != symbols.end();
}

bool SymbolTable::isValidSymbolName(const std::wstring& symbol)
{
	size_t size = symbol.size();
	size_t start = 0;

	// don't match empty names
	if (size == 0 || symbol.compare(L"@") == 0 || symbol.compare(L"@@") == 0)
		return false;

	if (symbol[0] == '@')
	{
		start++;
		if (size > 1 && symbol[1] == '@')
			start++;
	}

	if (symbol[start] >= '0' && symbol[start] <= '9')
		return false;

	for (size_t i = start; i < size; i++)
	{
		if (wcschr(validSymbolCharacters,symbol[i]) == NULL)
			return false;
	}

	return true;
}

bool SymbolTable::isValidSymbolCharacter(wchar_t character, bool first)
{
	if ((character >= 'a' && character <= 'z') || character >= 'A' && character <= 'Z') return true;
	if (!first && character >= '0' && character <= '9') return true;
	if (character == '_' || character == '.') return true;
	if (character == '@') return true;
	return false;
}

bool SymbolTable::addEquation(const std::wstring& name, unsigned int file, unsigned int section, size_t referenceIndex)
{
	if (isValidSymbolName(name) == false)
		return false;

	if (symbolExists(name,file,section))
		return false;

	setFileSectionValues(name,file,section);

	SymbolKey key = { name, file, section };
	SymbolInfo value = { EquationSymbol, referenceIndex };
	symbols[key] = value;

	equationsCount++;
	return true;
}

bool SymbolTable::findEquation(const std::wstring& name, unsigned int file, unsigned int section, size_t& dest)
{
	setFileSectionValues(name,file,section);

	SymbolKey key = { name, file, section };
	auto it = symbols.find(key);
	if (it == symbols.end() || it->second.type != EquationSymbol)
		return false;

	dest = it->second.index;
	return true;
}

// TODO: better
std::wstring SymbolTable::getUniqueLabelName()
{
	return formatString(L"__armips_label_%08X__",uniqueCount++);
}

void SymbolTable::addLabels(const std::vector<LabelDefinition>& labels)
{
	int lastSection = 0;
	for (const LabelDefinition& def: labels)
	{
		if (!isValidSymbolName(def.name))
			continue;

		Label* label = getLabel(def.name,(unsigned int)Global.FileInfo.FileNum,Global.Section);
		if (label == NULL)
			continue;

		if (isLocalSymbol(def.name) == false)
			Global.Section++;

		label->setDefined(true);
		label->setValue(def.value);
	}
}

int SymbolTable::findSection(u64 address)
{
	int smallestBefore = -1;
	int smallestDiff = 0x7FFFFFFF;

	for (auto& lab: labels)
	{
		int diff = (int)(address-lab->getValue());
		if (diff >= 0 && diff < smallestDiff)
		{
			smallestDiff = diff;
			smallestBefore = lab->getSection();
		}
	}

	return smallestBefore;
}<commit_msg>When generating label name, keep it lower case<commit_after>#include "stdafx.h"
#include "Core/SymbolTable.h"
#include "Util/FileClasses.h"
#include "Util/Util.h"
#include "Common.h"

const wchar_t validSymbolCharacters[] = L"ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789_.";

bool operator<(SymbolKey const& lhs, SymbolKey const& rhs)
{
	if (lhs.file != rhs.file)
		return lhs.file < rhs.file;
	if (lhs.section != rhs.section)
		return lhs.section < rhs.section;
	return lhs.name.compare(rhs.name) < 0;
}

SymbolTable::SymbolTable()
{
	uniqueCount = 0;
}

SymbolTable::~SymbolTable()
{
	clear();
}

void SymbolTable::clear()
{
	for (size_t i = 0; i < labels.size(); i++)
	{
		delete labels[i];
	}

	symbols.clear();
	labels.clear();
	equationsCount = 0;
	uniqueCount = 0;
}

void SymbolTable::setFileSectionValues(const std::wstring& symbol, unsigned int& file, unsigned int& section)
{
	if (symbol[0] == '@')
	{
		if (symbol[1] != '@')
		{
			// static label, @. the section doesn't matter
			section = -1;
		} else {
			// local label, @@. the file doesn't matter
			file = -1;
		}
	} else {
		// global label. neither file nor section matters
		file = section = -1;
	}
}

Label* SymbolTable::getLabel(const std::wstring& symbol, unsigned int file, unsigned int section)
{
	if (isValidSymbolName(symbol) == false)
		return NULL;

	int actualSection = section;
	setFileSectionValues(symbol,file,section);
	SymbolKey key = { symbol, file, section };

	// find label, create new one if it doesn't exist
	auto it = symbols.find(key);
	if (it == symbols.end())
	{
		SymbolInfo value = { LabelSymbol, labels.size() };
		symbols[key] = value;

		Label* result = new Label(symbol);
		if (section == actualSection)
			result->setSection(section);			// local, set section of parent
		else
			result->setSection(actualSection+1);	// global, set section of children
		labels.push_back(result);
		return result;
	}

	// make sure not to match symbols that aren't labels
	if (it->second.type != LabelSymbol)
		return NULL;

	return labels[it->second.index];
}

bool SymbolTable::symbolExists(const std::wstring& symbol, unsigned int file, unsigned int section)
{
	if (isValidSymbolName(symbol) == false)
		return false;

	setFileSectionValues(symbol,file,section);

	SymbolKey key = { symbol, file, section };
	auto it = symbols.find(key);
	return it != symbols.end();
}

bool SymbolTable::isValidSymbolName(const std::wstring& symbol)
{
	size_t size = symbol.size();
	size_t start = 0;

	// don't match empty names
	if (size == 0 || symbol.compare(L"@") == 0 || symbol.compare(L"@@") == 0)
		return false;

	if (symbol[0] == '@')
	{
		start++;
		if (size > 1 && symbol[1] == '@')
			start++;
	}

	if (symbol[start] >= '0' && symbol[start] <= '9')
		return false;

	for (size_t i = start; i < size; i++)
	{
		if (wcschr(validSymbolCharacters,symbol[i]) == NULL)
			return false;
	}

	return true;
}

bool SymbolTable::isValidSymbolCharacter(wchar_t character, bool first)
{
	if ((character >= 'a' && character <= 'z') || character >= 'A' && character <= 'Z') return true;
	if (!first && character >= '0' && character <= '9') return true;
	if (character == '_' || character == '.') return true;
	if (character == '@') return true;
	return false;
}

bool SymbolTable::addEquation(const std::wstring& name, unsigned int file, unsigned int section, size_t referenceIndex)
{
	if (isValidSymbolName(name) == false)
		return false;

	if (symbolExists(name,file,section))
		return false;

	setFileSectionValues(name,file,section);

	SymbolKey key = { name, file, section };
	SymbolInfo value = { EquationSymbol, referenceIndex };
	symbols[key] = value;

	equationsCount++;
	return true;
}

bool SymbolTable::findEquation(const std::wstring& name, unsigned int file, unsigned int section, size_t& dest)
{
	setFileSectionValues(name,file,section);

	SymbolKey key = { name, file, section };
	auto it = symbols.find(key);
	if (it == symbols.end() || it->second.type != EquationSymbol)
		return false;

	dest = it->second.index;
	return true;
}

// TODO: better
std::wstring SymbolTable::getUniqueLabelName()
{
	return formatString(L"__armips_label_%08x__",uniqueCount++);
}

void SymbolTable::addLabels(const std::vector<LabelDefinition>& labels)
{
	int lastSection = 0;
	for (const LabelDefinition& def: labels)
	{
		if (!isValidSymbolName(def.name))
			continue;

		Label* label = getLabel(def.name,(unsigned int)Global.FileInfo.FileNum,Global.Section);
		if (label == NULL)
			continue;

		if (isLocalSymbol(def.name) == false)
			Global.Section++;

		label->setDefined(true);
		label->setValue(def.value);
	}
}

int SymbolTable::findSection(u64 address)
{
	int smallestBefore = -1;
	int smallestDiff = 0x7FFFFFFF;

	for (auto& lab: labels)
	{
		int diff = (int)(address-lab->getValue());
		if (diff >= 0 && diff < smallestDiff)
		{
			smallestDiff = diff;
			smallestBefore = lab->getSection();
		}
	}

	return smallestBefore;
}<|endoftext|>