<commit_before>// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.

#include "datastore.h"

using vespalib::GenerationHeldBase;

namespace search {
namespace datastore {

namespace {

/*
 * Minimum dead bytes in active write buffer before switching to new
 * active write buffer even if another active buffer has more dead
 * bytes due to considering the active write buffer as too dead.
 */
constexpr size_t TOODEAD_SLACK = 0x4000u;

/*
 * Check if active write buffer is too dead for further use, i.e. if it
 * is likely to be the worst buffer at next compaction.  If so, filling it
 * up completely will be wasted work, as data will have to be moved again
 * rather soon.
 */
bool activeWriteBufferTooDead(const BufferState &state)
{
    size_t deadElems = state.getDeadElems();
    size_t deadBytes = deadElems * state.getClusterSize();
    return ((deadBytes >= TOODEAD_SLACK) && (deadElems * 2 >= state.size()));
}

}

DataStoreBase::FallbackHold::FallbackHold(size_t size,
        BufferState::Alloc &&buffer,
        size_t usedElems,
        BufferTypeBase *typeHandler,
        uint32_t typeId)
    : GenerationHeldBase(size),
      _buffer(std::move(buffer)),
      _usedElems(usedElems),
      _typeHandler(typeHandler),
      _typeId(typeId)
{
}

DataStoreBase::FallbackHold::~FallbackHold(void)
{
    _typeHandler->destroyElements(_buffer.get(), _usedElems);
}

class DataStoreBase::BufferHold : public GenerationHeldBase
{
    DataStoreBase &_dsb;
    uint32_t _bufferId;

public:
    BufferHold(size_t size,
               DataStoreBase &dsb,
               uint32_t bufferId)
        : GenerationHeldBase(size),
          _dsb(dsb),
          _bufferId(bufferId)
    {
    }

    virtual
    ~BufferHold(void)
    {
        _dsb.doneHoldBuffer(_bufferId);
    }
};

DataStoreBase::DataStoreBase(uint32_t numBuffers,
                             size_t maxClusters)
    : _buffers(numBuffers),
      _activeBufferIds(),
      _states(numBuffers),
      _typeHandlers(),
      _freeListLists(),
      _freeListsEnabled(false),
      _initializing(false),
      _elemHold1List(),
      _elemHold2List(),
      _numBuffers(numBuffers),
      _maxClusters(maxClusters),
      _genHolder()
{
}

DataStoreBase::~DataStoreBase(void)
{
    disableFreeLists();

    assert(_elemHold1List.empty());
    assert(_elemHold2List.empty());
}

void
DataStoreBase::switchActiveBuffer(uint32_t typeId, size_t sizeNeeded)
{
    size_t activeBufferId = _activeBufferIds[typeId];
    do {
        // start using next buffer
        activeBufferId = nextBufferId(activeBufferId);
    } while (!_states[activeBufferId].isFree());
    onActive(activeBufferId, typeId, sizeNeeded);
    _activeBufferIds[typeId] = activeBufferId;
}

void
DataStoreBase::switchOrGrowActiveBuffer(uint32_t typeId, size_t sizeNeeded)
{
    auto typeHandler = _typeHandlers[typeId];
    uint32_t clusterSize = typeHandler->getClusterSize();
    size_t numClustersForNewBuffer = typeHandler->getNumClustersForNewBuffer();
    size_t numEntriesForNewBuffer = numClustersForNewBuffer * clusterSize;
    uint32_t bufferId = _activeBufferIds[typeId];
    if (sizeNeeded + _states[bufferId].size() >= numEntriesForNewBuffer) {
        // Don't try to resize existing buffer, new buffer will be large enough
        switchActiveBuffer(typeId, sizeNeeded);
    } else {
        fallbackResize(bufferId, sizeNeeded);
    }
}

void
DataStoreBase::initActiveBuffers(void)
{
    uint32_t numTypes = _activeBufferIds.size();
    for (uint32_t typeId = 0; typeId < numTypes; ++typeId) {
        size_t activeBufferId = 0;
        while (!_states[activeBufferId].isFree()) {
            // start using next buffer
            activeBufferId = nextBufferId(activeBufferId);
        }
        onActive(activeBufferId, typeId, 0u);
        _activeBufferIds[typeId] = activeBufferId;
    }
}

uint32_t
DataStoreBase::addType(BufferTypeBase *typeHandler)
{
    uint32_t typeId = _activeBufferIds.size();
    assert(typeId == _typeHandlers.size());
    typeHandler->clampMaxClusters(_maxClusters);
    _activeBufferIds.push_back(0);
    _typeHandlers.push_back(typeHandler);
    _freeListLists.push_back(BufferState::FreeListList());
    return typeId;
}

uint32_t
DataStoreBase::getNumActiveBuffers() const
{
    uint32_t result = 0;
    for (const auto &state : _states) {
        if (state.isActive()) {
            ++result;
        }
    }
    return result;
}

void
DataStoreBase::transferElemHoldList(generation_t generation)
{
    ElemHold2List &elemHold2List = _elemHold2List;
    for (const ElemHold1ListElem & elemHold1 : _elemHold1List) {
        elemHold2List.push_back(ElemHold2ListElem(elemHold1, generation));
    }
    _elemHold1List.clear();
}

void
DataStoreBase::transferHoldLists(generation_t generation)
{
    _genHolder.transferHoldLists(generation);
    if (hasElemHold1())
        transferElemHoldList(generation);
}

void
DataStoreBase::doneHoldBuffer(uint32_t bufferId)
{
    _states[bufferId].onFree(_buffers[bufferId]);
}

void
DataStoreBase::trimHoldLists(generation_t usedGen)
{
    trimElemHoldList(usedGen);	// Trim entries before trimming buffers

    _genHolder.trimHoldLists(usedGen);
}

void
DataStoreBase::clearHoldLists(void)
{
    transferElemHoldList(0);
    clearElemHoldList();
    _genHolder.clearHoldLists();
}

void
DataStoreBase::dropBuffers(void)
{
    uint32_t numBuffers = _buffers.size();
    for (uint32_t bufferId = 0; bufferId < numBuffers; ++bufferId) {
        _states[bufferId].dropBuffer(_buffers[bufferId]);
    }
    _genHolder.clearHoldLists();
}

MemoryUsage
DataStoreBase::getMemoryUsage(void) const
{
    MemStats stats = getMemStats();
    MemoryUsage usage;
    usage.setAllocatedBytes(stats._allocBytes);
    usage.setUsedBytes(stats._usedBytes);
    usage.setDeadBytes(stats._deadBytes);
    usage.setAllocatedBytesOnHold(stats._holdBytes);
    return usage;
}

void
DataStoreBase::holdBuffer(uint32_t bufferId)
{
    _states[bufferId].onHold();
    size_t holdBytes = 0u;	// getMemStats() still accounts held buffers
    GenerationHeldBase::UP hold(new BufferHold(holdBytes, *this, bufferId));
    _genHolder.hold(std::move(hold));
}

void
DataStoreBase::enableFreeLists(void)
{
    for (BufferState & bState : _states) {
        if (!bState.isActive() || bState.getCompacting()) {
            continue;
        }
        bState.setFreeListList(&_freeListLists[bState.getTypeId()]);
    }
    _freeListsEnabled = true;
}

void
DataStoreBase::disableFreeLists(void)
{
    for (BufferState & bState : _states) {
        bState.setFreeListList(nullptr);
    }
    _freeListsEnabled = false;
}

void
DataStoreBase::enableFreeList(uint32_t bufferId)
{
    BufferState &state = _states[bufferId];
    if (_freeListsEnabled &&
        state.isActive() &&
        !state.getCompacting()) {
        state.setFreeListList(&_freeListLists[state.getTypeId()]);
    }
}

void
DataStoreBase::disableFreeList(uint32_t bufferId)
{
    _states[bufferId].setFreeListList(NULL);
}

void
DataStoreBase::disableElemHoldList(void)
{
    for (auto &state : _states) {
        if (!state.isFree()) {
            state.disableElemHoldList();
        }
    }
}

DataStoreBase::MemStats
DataStoreBase::getMemStats(void) const
{
    MemStats stats;

    for (const BufferState & bState: _states) {
        auto typeHandler = bState.getTypeHandler();
        BufferState::State state = bState.getState();
        if ((state == BufferState::FREE) || (typeHandler == nullptr)) {
            ++stats._freeBuffers;
        } else if (state == BufferState::ACTIVE) {
            size_t elementSize = typeHandler->elementSize();
            ++stats._activeBuffers;
            stats._allocElems += bState.capacity();
            stats._usedElems += bState.size();
            stats._deadElems += bState.getDeadElems();
            stats._holdElems += bState.getHoldElems();
            stats._allocBytes += bState.capacity() * elementSize;
            stats._usedBytes += (bState.size() * elementSize) + bState.getExtraUsedBytes();
            stats._deadBytes += bState.getDeadElems() * elementSize;
            stats._holdBytes += (bState.getHoldElems() * elementSize) + bState.getExtraHoldBytes();
        } else if (state == BufferState::HOLD) {
            size_t elementSize = typeHandler->elementSize();
            ++stats._holdBuffers;
            stats._allocElems += bState.capacity();
            stats._usedElems += bState.size();
            stats._deadElems += bState.getDeadElems();
            stats._holdElems += bState.getHoldElems();
            stats._allocBytes += bState.capacity() * elementSize;
            stats._usedBytes += (bState.size() * elementSize) + bState.getExtraUsedBytes();
            stats._deadBytes += bState.getDeadElems() * elementSize;
            stats._holdBytes += (bState.getHoldElems() * elementSize) + bState.getExtraHoldBytes();
        } else {
            abort();
        }
    }
    size_t genHolderHeldBytes = _genHolder.getHeldBytes();
    stats._holdBytes += genHolderHeldBytes;
    stats._allocBytes += genHolderHeldBytes;
    stats._usedBytes += genHolderHeldBytes;
    return stats;
}

AddressSpace
DataStoreBase::getAddressSpaceUsage(void) const
{
    size_t usedClusters = 0;
    size_t deadClusters = 0;
    size_t limitClusters = 0;
    for (const BufferState & bState: _states) {
        if (bState.isActive()) {
            uint32_t clusterSize = bState.getClusterSize();
            usedClusters += bState.size() / clusterSize;
            deadClusters += bState.getDeadElems() / clusterSize;
            limitClusters += bState.capacity() / clusterSize;
        } else if (bState.isOnHold()) {
        } else if (bState.isFree()) {
            limitClusters += _maxClusters;
        } else {
            abort();
        }
    }
    return AddressSpace(usedClusters, deadClusters, limitClusters);
}

void
DataStoreBase::onActive(uint32_t bufferId, uint32_t typeId,
                        size_t sizeNeeded)
{
    assert(typeId < _typeHandlers.size());
    assert(bufferId < _numBuffers);
    BufferState &state = _states[bufferId];
    state.onActive(bufferId, typeId,
                   _typeHandlers[typeId],
                   sizeNeeded,
                   _buffers[bufferId]);
    enableFreeList(bufferId);
}

std::vector<uint32_t>
DataStoreBase::startCompact(uint32_t typeId)
{
    std::vector<uint32_t> toHold;

    for (uint32_t bufferId = 0; bufferId < _numBuffers; ++bufferId) {
        BufferState &state = getBufferState(bufferId);
        if (state.isActive() &&
            state.getTypeId() == typeId &&
            !state.getCompacting()) {
            state.setCompacting();
            toHold.push_back(bufferId);
            disableFreeList(bufferId);
        }
    }
    switchActiveBuffer(typeId, 0u);
    return toHold;
}

void
DataStoreBase::finishCompact(const std::vector<uint32_t> &toHold)
{
    for (uint32_t bufferId : toHold) {
        holdBuffer(bufferId);
    }
}

void
DataStoreBase::fallbackResize(uint32_t bufferId, uint64_t sizeNeeded)
{
    BufferState &state = getBufferState(bufferId);
    BufferState::Alloc toHoldBuffer;
    size_t oldUsedElems = state.size();
    size_t oldAllocElems = state.capacity();
    size_t elementSize = state.getTypeHandler()->elementSize();
    state.fallbackResize(bufferId,
                         sizeNeeded,
                         _buffers[bufferId],
                         toHoldBuffer);
    GenerationHeldBase::UP
        hold(new FallbackHold(oldAllocElems * elementSize,
                              std::move(toHoldBuffer),
                              oldUsedElems,
                              state.getTypeHandler(),
                              state.getTypeId()));
    if (!_initializing) {
        _genHolder.hold(std::move(hold));
    }
}

uint32_t
DataStoreBase::startCompactWorstBuffer(uint32_t typeId) {
    uint32_t activeBufferId = getActiveBufferId(typeId);
    const BufferTypeBase *typeHandler = _typeHandlers[typeId];
    assert(typeHandler->getActiveBuffers() >= 1u);
    if (typeHandler->getActiveBuffers() == 1u) {
        // Single active buffer for type, no need for scan
        _states[activeBufferId].setCompacting();
        _states[activeBufferId].disableElemHoldList();
        disableFreeList(activeBufferId);
        switchActiveBuffer(typeId, 0u);
        return activeBufferId;
    }
    // Multiple active buffers for type, must perform full scan
    return startCompactWorstBuffer(activeBufferId,
                                   [=](const BufferState &state) { return state.isActive(typeId); });
}

uint32_t
DataStoreBase::startCompactWorstBuffer()
{
    uint32_t activeBufferId = getActiveBufferId(0);
   return startCompactWorstBuffer(activeBufferId, [](const BufferState &state){ return state.isActive(); });
}

template <typename BufferStateActiveFilter>
uint32_t
DataStoreBase::startCompactWorstBuffer(uint32_t initWorstBufferId, BufferStateActiveFilter &&filterFunc)
{
    uint32_t worstBufferId = initWorstBufferId;
    size_t worstDeadElems = 0;
    for (uint32_t bufferId = 0; bufferId < _numBuffers; ++bufferId) {
        const auto &state = getBufferState(bufferId);
        if (filterFunc(state)) {
            size_t deadElems = state.getDeadElems() - state.getTypeHandler()->getReservedElements(bufferId);
            if (deadElems > worstDeadElems) {
                worstBufferId = bufferId;
                worstDeadElems = deadElems;
            }
        }
    }
    auto &worstBufferState = getBufferState(worstBufferId);
    uint32_t activeBufferId = getActiveBufferId(worstBufferState.getTypeId());
    if ((worstBufferId == activeBufferId) ||
        activeWriteBufferTooDead(getBufferState(activeBufferId)))
    {
        switchActiveBuffer(worstBufferState.getTypeId(), 0u);
    }
    worstBufferState.setCompacting();
    worstBufferState.disableElemHoldList();
    disableFreeList(worstBufferId);
    return worstBufferId;
}

} // namespace datastore
} // namespace search
<commit_msg>Buffers on hold contributes to used and limit, but not to dead, during calculation of address space usage.<commit_after>// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.

#include "datastore.h"

using vespalib::GenerationHeldBase;

namespace search {
namespace datastore {

namespace {

/*
 * Minimum dead bytes in active write buffer before switching to new
 * active write buffer even if another active buffer has more dead
 * bytes due to considering the active write buffer as too dead.
 */
constexpr size_t TOODEAD_SLACK = 0x4000u;

/*
 * Check if active write buffer is too dead for further use, i.e. if it
 * is likely to be the worst buffer at next compaction.  If so, filling it
 * up completely will be wasted work, as data will have to be moved again
 * rather soon.
 */
bool activeWriteBufferTooDead(const BufferState &state)
{
    size_t deadElems = state.getDeadElems();
    size_t deadBytes = deadElems * state.getClusterSize();
    return ((deadBytes >= TOODEAD_SLACK) && (deadElems * 2 >= state.size()));
}

}

DataStoreBase::FallbackHold::FallbackHold(size_t size,
        BufferState::Alloc &&buffer,
        size_t usedElems,
        BufferTypeBase *typeHandler,
        uint32_t typeId)
    : GenerationHeldBase(size),
      _buffer(std::move(buffer)),
      _usedElems(usedElems),
      _typeHandler(typeHandler),
      _typeId(typeId)
{
}

DataStoreBase::FallbackHold::~FallbackHold(void)
{
    _typeHandler->destroyElements(_buffer.get(), _usedElems);
}

class DataStoreBase::BufferHold : public GenerationHeldBase
{
    DataStoreBase &_dsb;
    uint32_t _bufferId;

public:
    BufferHold(size_t size,
               DataStoreBase &dsb,
               uint32_t bufferId)
        : GenerationHeldBase(size),
          _dsb(dsb),
          _bufferId(bufferId)
    {
    }

    virtual
    ~BufferHold(void)
    {
        _dsb.doneHoldBuffer(_bufferId);
    }
};

DataStoreBase::DataStoreBase(uint32_t numBuffers,
                             size_t maxClusters)
    : _buffers(numBuffers),
      _activeBufferIds(),
      _states(numBuffers),
      _typeHandlers(),
      _freeListLists(),
      _freeListsEnabled(false),
      _initializing(false),
      _elemHold1List(),
      _elemHold2List(),
      _numBuffers(numBuffers),
      _maxClusters(maxClusters),
      _genHolder()
{
}

DataStoreBase::~DataStoreBase(void)
{
    disableFreeLists();

    assert(_elemHold1List.empty());
    assert(_elemHold2List.empty());
}

void
DataStoreBase::switchActiveBuffer(uint32_t typeId, size_t sizeNeeded)
{
    size_t activeBufferId = _activeBufferIds[typeId];
    do {
        // start using next buffer
        activeBufferId = nextBufferId(activeBufferId);
    } while (!_states[activeBufferId].isFree());
    onActive(activeBufferId, typeId, sizeNeeded);
    _activeBufferIds[typeId] = activeBufferId;
}

void
DataStoreBase::switchOrGrowActiveBuffer(uint32_t typeId, size_t sizeNeeded)
{
    auto typeHandler = _typeHandlers[typeId];
    uint32_t clusterSize = typeHandler->getClusterSize();
    size_t numClustersForNewBuffer = typeHandler->getNumClustersForNewBuffer();
    size_t numEntriesForNewBuffer = numClustersForNewBuffer * clusterSize;
    uint32_t bufferId = _activeBufferIds[typeId];
    if (sizeNeeded + _states[bufferId].size() >= numEntriesForNewBuffer) {
        // Don't try to resize existing buffer, new buffer will be large enough
        switchActiveBuffer(typeId, sizeNeeded);
    } else {
        fallbackResize(bufferId, sizeNeeded);
    }
}

void
DataStoreBase::initActiveBuffers(void)
{
    uint32_t numTypes = _activeBufferIds.size();
    for (uint32_t typeId = 0; typeId < numTypes; ++typeId) {
        size_t activeBufferId = 0;
        while (!_states[activeBufferId].isFree()) {
            // start using next buffer
            activeBufferId = nextBufferId(activeBufferId);
        }
        onActive(activeBufferId, typeId, 0u);
        _activeBufferIds[typeId] = activeBufferId;
    }
}

uint32_t
DataStoreBase::addType(BufferTypeBase *typeHandler)
{
    uint32_t typeId = _activeBufferIds.size();
    assert(typeId == _typeHandlers.size());
    typeHandler->clampMaxClusters(_maxClusters);
    _activeBufferIds.push_back(0);
    _typeHandlers.push_back(typeHandler);
    _freeListLists.push_back(BufferState::FreeListList());
    return typeId;
}

uint32_t
DataStoreBase::getNumActiveBuffers() const
{
    uint32_t result = 0;
    for (const auto &state : _states) {
        if (state.isActive()) {
            ++result;
        }
    }
    return result;
}

void
DataStoreBase::transferElemHoldList(generation_t generation)
{
    ElemHold2List &elemHold2List = _elemHold2List;
    for (const ElemHold1ListElem & elemHold1 : _elemHold1List) {
        elemHold2List.push_back(ElemHold2ListElem(elemHold1, generation));
    }
    _elemHold1List.clear();
}

void
DataStoreBase::transferHoldLists(generation_t generation)
{
    _genHolder.transferHoldLists(generation);
    if (hasElemHold1())
        transferElemHoldList(generation);
}

void
DataStoreBase::doneHoldBuffer(uint32_t bufferId)
{
    _states[bufferId].onFree(_buffers[bufferId]);
}

void
DataStoreBase::trimHoldLists(generation_t usedGen)
{
    trimElemHoldList(usedGen);	// Trim entries before trimming buffers

    _genHolder.trimHoldLists(usedGen);
}

void
DataStoreBase::clearHoldLists(void)
{
    transferElemHoldList(0);
    clearElemHoldList();
    _genHolder.clearHoldLists();
}

void
DataStoreBase::dropBuffers(void)
{
    uint32_t numBuffers = _buffers.size();
    for (uint32_t bufferId = 0; bufferId < numBuffers; ++bufferId) {
        _states[bufferId].dropBuffer(_buffers[bufferId]);
    }
    _genHolder.clearHoldLists();
}

MemoryUsage
DataStoreBase::getMemoryUsage(void) const
{
    MemStats stats = getMemStats();
    MemoryUsage usage;
    usage.setAllocatedBytes(stats._allocBytes);
    usage.setUsedBytes(stats._usedBytes);
    usage.setDeadBytes(stats._deadBytes);
    usage.setAllocatedBytesOnHold(stats._holdBytes);
    return usage;
}

void
DataStoreBase::holdBuffer(uint32_t bufferId)
{
    _states[bufferId].onHold();
    size_t holdBytes = 0u;	// getMemStats() still accounts held buffers
    GenerationHeldBase::UP hold(new BufferHold(holdBytes, *this, bufferId));
    _genHolder.hold(std::move(hold));
}

void
DataStoreBase::enableFreeLists(void)
{
    for (BufferState & bState : _states) {
        if (!bState.isActive() || bState.getCompacting()) {
            continue;
        }
        bState.setFreeListList(&_freeListLists[bState.getTypeId()]);
    }
    _freeListsEnabled = true;
}

void
DataStoreBase::disableFreeLists(void)
{
    for (BufferState & bState : _states) {
        bState.setFreeListList(nullptr);
    }
    _freeListsEnabled = false;
}

void
DataStoreBase::enableFreeList(uint32_t bufferId)
{
    BufferState &state = _states[bufferId];
    if (_freeListsEnabled &&
        state.isActive() &&
        !state.getCompacting()) {
        state.setFreeListList(&_freeListLists[state.getTypeId()]);
    }
}

void
DataStoreBase::disableFreeList(uint32_t bufferId)
{
    _states[bufferId].setFreeListList(NULL);
}

void
DataStoreBase::disableElemHoldList(void)
{
    for (auto &state : _states) {
        if (!state.isFree()) {
            state.disableElemHoldList();
        }
    }
}

DataStoreBase::MemStats
DataStoreBase::getMemStats(void) const
{
    MemStats stats;

    for (const BufferState & bState: _states) {
        auto typeHandler = bState.getTypeHandler();
        BufferState::State state = bState.getState();
        if ((state == BufferState::FREE) || (typeHandler == nullptr)) {
            ++stats._freeBuffers;
        } else if (state == BufferState::ACTIVE) {
            size_t elementSize = typeHandler->elementSize();
            ++stats._activeBuffers;
            stats._allocElems += bState.capacity();
            stats._usedElems += bState.size();
            stats._deadElems += bState.getDeadElems();
            stats._holdElems += bState.getHoldElems();
            stats._allocBytes += bState.capacity() * elementSize;
            stats._usedBytes += (bState.size() * elementSize) + bState.getExtraUsedBytes();
            stats._deadBytes += bState.getDeadElems() * elementSize;
            stats._holdBytes += (bState.getHoldElems() * elementSize) + bState.getExtraHoldBytes();
        } else if (state == BufferState::HOLD) {
            size_t elementSize = typeHandler->elementSize();
            ++stats._holdBuffers;
            stats._allocElems += bState.capacity();
            stats._usedElems += bState.size();
            stats._deadElems += bState.getDeadElems();
            stats._holdElems += bState.getHoldElems();
            stats._allocBytes += bState.capacity() * elementSize;
            stats._usedBytes += (bState.size() * elementSize) + bState.getExtraUsedBytes();
            stats._deadBytes += bState.getDeadElems() * elementSize;
            stats._holdBytes += (bState.getHoldElems() * elementSize) + bState.getExtraHoldBytes();
        } else {
            abort();
        }
    }
    size_t genHolderHeldBytes = _genHolder.getHeldBytes();
    stats._holdBytes += genHolderHeldBytes;
    stats._allocBytes += genHolderHeldBytes;
    stats._usedBytes += genHolderHeldBytes;
    return stats;
}

AddressSpace
DataStoreBase::getAddressSpaceUsage(void) const
{
    size_t usedClusters = 0;
    size_t deadClusters = 0;
    size_t limitClusters = 0;
    for (const BufferState & bState: _states) {
        if (bState.isActive()) {
            uint32_t clusterSize = bState.getClusterSize();
            usedClusters += bState.size() / clusterSize;
            deadClusters += bState.getDeadElems() / clusterSize;
            limitClusters += bState.capacity() / clusterSize;
        } else if (bState.isOnHold()) {
            uint32_t clusterSize = bState.getClusterSize();
            usedClusters += bState.size() / clusterSize;
            limitClusters += bState.capacity() / clusterSize;
        } else if (bState.isFree()) {
            limitClusters += _maxClusters;
        } else {
            abort();
        }
    }
    return AddressSpace(usedClusters, deadClusters, limitClusters);
}

void
DataStoreBase::onActive(uint32_t bufferId, uint32_t typeId,
                        size_t sizeNeeded)
{
    assert(typeId < _typeHandlers.size());
    assert(bufferId < _numBuffers);
    BufferState &state = _states[bufferId];
    state.onActive(bufferId, typeId,
                   _typeHandlers[typeId],
                   sizeNeeded,
                   _buffers[bufferId]);
    enableFreeList(bufferId);
}

std::vector<uint32_t>
DataStoreBase::startCompact(uint32_t typeId)
{
    std::vector<uint32_t> toHold;

    for (uint32_t bufferId = 0; bufferId < _numBuffers; ++bufferId) {
        BufferState &state = getBufferState(bufferId);
        if (state.isActive() &&
            state.getTypeId() == typeId &&
            !state.getCompacting()) {
            state.setCompacting();
            toHold.push_back(bufferId);
            disableFreeList(bufferId);
        }
    }
    switchActiveBuffer(typeId, 0u);
    return toHold;
}

void
DataStoreBase::finishCompact(const std::vector<uint32_t> &toHold)
{
    for (uint32_t bufferId : toHold) {
        holdBuffer(bufferId);
    }
}

void
DataStoreBase::fallbackResize(uint32_t bufferId, uint64_t sizeNeeded)
{
    BufferState &state = getBufferState(bufferId);
    BufferState::Alloc toHoldBuffer;
    size_t oldUsedElems = state.size();
    size_t oldAllocElems = state.capacity();
    size_t elementSize = state.getTypeHandler()->elementSize();
    state.fallbackResize(bufferId,
                         sizeNeeded,
                         _buffers[bufferId],
                         toHoldBuffer);
    GenerationHeldBase::UP
        hold(new FallbackHold(oldAllocElems * elementSize,
                              std::move(toHoldBuffer),
                              oldUsedElems,
                              state.getTypeHandler(),
                              state.getTypeId()));
    if (!_initializing) {
        _genHolder.hold(std::move(hold));
    }
}

uint32_t
DataStoreBase::startCompactWorstBuffer(uint32_t typeId) {
    uint32_t activeBufferId = getActiveBufferId(typeId);
    const BufferTypeBase *typeHandler = _typeHandlers[typeId];
    assert(typeHandler->getActiveBuffers() >= 1u);
    if (typeHandler->getActiveBuffers() == 1u) {
        // Single active buffer for type, no need for scan
        _states[activeBufferId].setCompacting();
        _states[activeBufferId].disableElemHoldList();
        disableFreeList(activeBufferId);
        switchActiveBuffer(typeId, 0u);
        return activeBufferId;
    }
    // Multiple active buffers for type, must perform full scan
    return startCompactWorstBuffer(activeBufferId,
                                   [=](const BufferState &state) { return state.isActive(typeId); });
}

uint32_t
DataStoreBase::startCompactWorstBuffer()
{
    uint32_t activeBufferId = getActiveBufferId(0);
   return startCompactWorstBuffer(activeBufferId, [](const BufferState &state){ return state.isActive(); });
}

template <typename BufferStateActiveFilter>
uint32_t
DataStoreBase::startCompactWorstBuffer(uint32_t initWorstBufferId, BufferStateActiveFilter &&filterFunc)
{
    uint32_t worstBufferId = initWorstBufferId;
    size_t worstDeadElems = 0;
    for (uint32_t bufferId = 0; bufferId < _numBuffers; ++bufferId) {
        const auto &state = getBufferState(bufferId);
        if (filterFunc(state)) {
            size_t deadElems = state.getDeadElems() - state.getTypeHandler()->getReservedElements(bufferId);
            if (deadElems > worstDeadElems) {
                worstBufferId = bufferId;
                worstDeadElems = deadElems;
            }
        }
    }
    auto &worstBufferState = getBufferState(worstBufferId);
    uint32_t activeBufferId = getActiveBufferId(worstBufferState.getTypeId());
    if ((worstBufferId == activeBufferId) ||
        activeWriteBufferTooDead(getBufferState(activeBufferId)))
    {
        switchActiveBuffer(worstBufferState.getTypeId(), 0u);
    }
    worstBufferState.setCompacting();
    worstBufferState.disableElemHoldList();
    disableFreeList(worstBufferId);
    return worstBufferId;
}

} // namespace datastore
} // namespace search
<|endoftext|>


<commit_before>#include <repeat.hpp>

#include "helpers.hpp"

#include <vector>
#include <string>
#include <iterator>

#include "catch.hpp"

using iter::repeat;

TEST_CASE("repeat: one argument keeps giving value back", "[repeat]") {
    auto r = repeat('a');
    auto it = std::begin(r);
    REQUIRE( *it == 'a' );
    ++it;
    REQUIRE( *it == 'a' );
    ++it;
    REQUIRE( *it == 'a' );
    ++it;
    REQUIRE( *it == 'a' );
    ++it;
    REQUIRE( *it == 'a' );
}

TEST_CASE("repeat: two argument repeats a number of times", "[repeat]") {
    auto r = repeat('a', 3);
    std::string s(std::begin(r), std::end(r));
    REQUIRE( s == "aaa" );
}
<commit_msg>tests repeat with 0 count<commit_after>#include <repeat.hpp>

#include "helpers.hpp"

#include <vector>
#include <string>
#include <iterator>

#include "catch.hpp"

using iter::repeat;

TEST_CASE("repeat: one argument keeps giving value back", "[repeat]") {
    auto r = repeat('a');
    auto it = std::begin(r);
    REQUIRE( *it == 'a' );
    ++it;
    REQUIRE( *it == 'a' );
    ++it;
    REQUIRE( *it == 'a' );
    ++it;
    REQUIRE( *it == 'a' );
    ++it;
    REQUIRE( *it == 'a' );
}

TEST_CASE("repeat: two argument repeats a number of times", "[repeat]") {
    auto r = repeat('a', 3);
    std::string s(std::begin(r), std::end(r));
    REQUIRE( s == "aaa" );
}

TEST_CASE("repeat: 0 count gives empty sequence", "[repeat]") {
    auto r = repeat('a', 0);
    REQUIRE( std::begin(r) == std::end(r) );
}
<|endoftext|>


<commit_before>#include <iostream>
#include <string>
#include <vector>
#include <tuple>

#include "acmacs-base/argv.hh"
#include "acmacs-base/filesystem.hh"
#include "acmacs-base/named-type.hh"
#include "acmacs-chart-2/factory-import.hh"
#include "acmacs-chart-2/chart.hh"

// ----------------------------------------------------------------------

// using namespace acmacs::argv;

// struct Options : public argv
// {
//     Options(int a_argc, const char* const a_argv[], on_error on_err = on_error::exit) : argv() { parse(a_argc, a_argv, on_err); }

//     argument<str> period{*this, arg_name{"monthly|yearly|weekly"}, mandatory};
//     argument<Date> start{*this, arg_name{"start-date"}, mandatory};
//     argument<Date> end{*this, arg_name{"end-date"}, mandatory};
//     argument<str> output{*this, arg_name{"output.json"}, dflt{""}};
// };

// ----------------------------------------------------------------------

inline bool is_acmacs_file(const fs::path& path)
{
  if (path.extension() == ".ace")
    return true;
  if (path.extension() == ".bz2") {
    if (path.stem().extension() == ".acd1")
      return true;
  }
  return false;
}

// ----------------------------------------------------------------------

class SerumIds
{
 public:
    using Name = acmacs::chart::Name;
    using Passage = acmacs::chart::Passage;
    using Reassortant = acmacs::chart::Reassortant;
    using Annotations = acmacs::chart::Annotations;
    using SerumId = acmacs::chart::SerumId;
    using SerumIdRoot = acmacs::named_t<std::string, struct SerumIdRootTag>;
    using SerumEntry = std::tuple<Name, Reassortant, Annotations, SerumId, Passage>;
    using TableEntry = std::tuple<acmacs::chart::Lab, acmacs::chart::Assay, acmacs::chart::RbcSpecies, acmacs::chart::TableDate>;

    SerumIds() = default;

      // void add(const Name& name, const Reassortant& reassortant, const Annotations& annotations, const SerumId& serum_id, const Passage& passage);
    void add(const SerumEntry& serum, const TableEntry& table);

 private:
    std::map<SerumIdRoot, std::map<SerumId, std::vector<std::tuple<SerumEntry, TableEntry>>>> data_;

    SerumIdRoot serum_id_root(const SerumEntry& serum, const TableEntry& table) const
        {
            return SerumIdRoot(std::get<SerumId>(serum));
        }

};

struct NameId
{
  NameId(std::string n, std::string si, std::string d) : name(n), serum_id(si), date(d) {}
  bool operator<(const NameId& rhs) const { if (name == rhs.name) { if (serum_id == rhs.serum_id) return date < rhs.date; else return serum_id < rhs.serum_id; } else return name < rhs.name; }
  bool operator==(const NameId& rhs) const { return name == rhs.name && serum_id == rhs.serum_id && date == rhs.date; }
  bool operator!=(const NameId& rhs) const { return !operator==(rhs); }
  std::string name;
  std::string serum_id;
  std::string date;
};

struct NameIds
{
  NameIds(std::string n) : name(n) {}
  std::string name;
  std::vector<std::pair<std::string, size_t>> id_count;
};

// ----------------------------------------------------------------------

int main(int /*argc*/, const char* /*argv*/[])
{
    int exit_code = 0;
    try {
        // Options opt(argc, argv);
        std::vector<NameId> name_id;
        size_t charts_processed = 0;
        for (auto& entry : fs::directory_iterator(".")) {
            if (entry.is_regular_file() && is_acmacs_file(entry.path())) {
                // std::cout << entry.path() << '\n';
                auto chart = acmacs::chart::import_from_file(entry.path());
                const std::string date = chart->info()->date();
                auto sera = chart->sera();
                for (auto serum : *sera)
                    name_id.emplace_back(serum->designation_without_serum_id(), serum->serum_id(), date);
                ++charts_processed;
            }
        }
        std::cout << charts_processed << " charts processed\n";
        std::sort(name_id.begin(), name_id.end());
        // std::cout << name_id.size() << " sera found\n";
        // name_id.erase(std::unique(name_id.begin(), name_id.end()),
        // name_id.end());
        std::cout << name_id.size() << " sera found\n";

        // std::vector<std::pair<std::string, std::vector<std::string>>> name_ids;
        // for (const auto& entry : name_id) {
        //     if (name_ids.empty() || name_ids.back().first != entry.first)
        //         name_ids.emplace_back(entry.first, std::vector<std::string>{entry.second});
        //     else
        //         name_ids.back().second.push_back(entry.second);
        // }
        // std::cout << name_ids.size() << " serum names found\n";

        // for (const auto& entry : name_ids) {
        //     std::cout << entry.first << '\n';
        //     for (const auto& serum_id : entry.second) {
        //         std::cout << "  " << serum_id;
        //         if (serum_id[0] == 'F' || serum_id[0] == 'R') {
        //             if (serum_id.back() != 'D')
        //                 std::cout << "  Warning: FIX!";
        //         }
        //         else
        //             std::cout << "  Warning: not MELB";
        //         std::cout << '\n';
        //     }
        // }
    }
    catch (std::exception& err) {
        std::cerr << "ERROR: " << err.what() << '\n';
        exit_code = 2;
    }
    return exit_code;
}

// ----------------------------------------------------------------------
/// Local Variables:
/// eval: (if (fboundp 'eu-rename-buffer) (eu-rename-buffer))
/// End:
<commit_msg>scan serum ids of MELB tables<commit_after>#include <iostream>
#include <string>
#include <vector>
#include <tuple>

#include "acmacs-base/argv.hh"
#include "acmacs-base/filesystem.hh"
#include "acmacs-base/named-type.hh"
#include "acmacs-chart-2/factory-import.hh"
#include "acmacs-chart-2/chart.hh"

// ----------------------------------------------------------------------

// using namespace acmacs::argv;

// struct Options : public argv
// {
//     Options(int a_argc, const char* const a_argv[], on_error on_err = on_error::exit) : argv() { parse(a_argc, a_argv, on_err); }

//     argument<str> period{*this, arg_name{"monthly|yearly|weekly"}, mandatory};
//     argument<Date> start{*this, arg_name{"start-date"}, mandatory};
//     argument<Date> end{*this, arg_name{"end-date"}, mandatory};
//     argument<str> output{*this, arg_name{"output.json"}, dflt{""}};
// };

// ----------------------------------------------------------------------

inline bool is_acmacs_file(const fs::path& path)
{
    if (path.extension() == ".ace")
        return true;
    if (path.extension() == ".bz2") {
        if (path.stem().extension() == ".acd1")
            return true;
    }
    return false;
}

// ----------------------------------------------------------------------

class SerumIds
{
 public:
    using Name = acmacs::chart::Name;
    using Passage = acmacs::chart::Passage;
    using Reassortant = acmacs::chart::Reassortant;
    using Annotations = acmacs::chart::Annotations;
    using SerumId = acmacs::chart::SerumId;
    using SerumIdRoot = acmacs::named_t<std::string, struct SerumIdRootTag>;
    using SerumEntry = std::tuple<Name, Reassortant, Annotations, SerumId, Passage>;
    using TableEntry = std::tuple<acmacs::chart::Lab, acmacs::chart::Assay, acmacs::chart::RbcSpecies, acmacs::chart::TableDate>;
    using Entry = std::tuple<SerumIdRoot, SerumId, Name, Reassortant, Annotations, acmacs::chart::Lab, acmacs::chart::Assay, acmacs::chart::RbcSpecies, acmacs::chart::TableDate, Passage>;

    SerumIds() = default;

    constexpr size_t size() const { return data_.size(); }
    void sort() { std::sort(data_.begin(), data_.end()); /* data_.erase(std::unique(data_.begin(), data_.end()), data_.end()); */ }

    void add(const SerumEntry& serum, const TableEntry& table)
        {
            data_.emplace_back(serum_id_root(serum, table), std::get<SerumId>(serum),
                               std::get<Name>(serum), std::get<Reassortant>(serum), std::get<Annotations>(serum),
                               std::get<acmacs::chart::Lab>(table), std::get<acmacs::chart::Assay>(table), std::get<acmacs::chart::RbcSpecies>(table), std::get<acmacs::chart::TableDate>(table),
                               std::get<Passage>(serum));
        }

    void print() const
        {
            for (const auto& entry : data_)
                std::cout << std::get<SerumIdRoot>(entry) << ' ' << std::get<SerumId>(entry) << ' ' << std::get<Name>(entry) << ' ' << std::get<acmacs::chart::TableDate>(entry) << '\n';
        }

 private:
    std::vector<Entry> data_;

    SerumIdRoot serum_id_root(const SerumEntry& serum, const TableEntry& table) const
        {
            const auto& serum_id = std::get<SerumId>(serum);
            if (std::get<acmacs::chart::Lab>(table) == "MELB") {
                if (serum_id.size() > 6 && (serum_id[0] == 'F' || serum_id[0] == 'R') && serum_id[5] == '-' && serum_id.back() == 'D')
                    return SerumIdRoot(serum_id.substr(0, 5));
                else
                    return SerumIdRoot(serum_id);
            }
            else
                return SerumIdRoot(serum_id);
        }

};

// ----------------------------------------------------------------------

int main(int /*argc*/, const char* /*argv*/[])
{
    int exit_code = 0;
    try {
        // Options opt(argc, argv);
        SerumIds serum_ids;
        size_t charts_processed = 0;
        for (auto& entry : fs::directory_iterator(".")) {
            if (entry.is_regular_file() && is_acmacs_file(entry.path())) {
                // std::cout << entry.path() << '\n';
                auto chart = acmacs::chart::import_from_file(entry.path());
                std::tuple table(chart->info()->lab(), chart->info()->assay(), chart->info()->rbc_species(), chart->info()->date());
                auto sera = chart->sera();
                for (auto serum : *sera)
                    serum_ids.add({serum->name(), serum->reassortant(), serum->annotations(), serum->serum_id(), serum->passage()}, table);
                  // name_id.emplace_back(serum->designation_without_serum_id(), serum->serum_id(), date);
                ++charts_processed;
            }
        }
        std::cout << charts_processed << " charts processed\n";
        std::cout << serum_ids.size() << " entries\n";
        serum_ids.sort();
          // std::cout << serum_ids.size() << " entries\n";
        serum_ids.print();

        // std::vector<std::pair<std::string, std::vector<std::string>>> name_ids;
        // for (const auto& entry : name_id) {
        //     if (name_ids.empty() || name_ids.back().first != entry.first)
        //         name_ids.emplace_back(entry.first, std::vector<std::string>{entry.second});
        //     else
        //         name_ids.back().second.push_back(entry.second);
        // }
        // std::cout << name_ids.size() << " serum names found\n";

        // for (const auto& entry : name_ids) {
        //     std::cout << entry.first << '\n';
        //     for (const auto& serum_id : entry.second) {
        //         std::cout << "  " << serum_id;
        //         if (serum_id[0] == 'F' || serum_id[0] == 'R') {
        //             if (serum_id.back() != 'D')
        //                 std::cout << "  Warning: FIX!";
        //         }
        //         else
        //             std::cout << "  Warning: not MELB";
        //         std::cout << '\n';
        //     }
        // }
    }
    catch (std::exception& err) {
        std::cerr << "ERROR: " << err.what() << '\n';
        exit_code = 2;
    }
    return exit_code;
}

// ----------------------------------------------------------------------
/// Local Variables:
/// eval: (if (fboundp 'eu-rename-buffer) (eu-rename-buffer))
/// End:
<|endoftext|>


<commit_before>/*
 * Copyright 2016 Adam Chyła, adam@chyla.org
 * All rights reserved. Distributed under the terms of the MIT License.
 */

#include "knn_analyzer_object.h"

#include <boost/log/trivial.hpp>
#include <cmath>
#include <limits>
#include <slas/util/run_partially.h>

#include "system.h"

using namespace ::apache::database::detail;
using namespace ::apache::type;
using namespace ::database::type;
using namespace ::type;

namespace apache
{

namespace analyzer
{

namespace detail
{

KnnAnalyzerObjectPtr KnnAnalyzerObject::Create(::database::detail::GeneralDatabaseFunctionsInterfacePtr general_database_functions,
                                               ::apache::database::DatabaseFunctionsPtr apache_database_functions) {
  return KnnAnalyzerObjectPtr(new KnnAnalyzerObject(general_database_functions, apache_database_functions));
}

void KnnAnalyzerObject::Analyze() {
  for (auto agent_name : general_database_functions_->GetAgentNames()) {
    BOOST_LOG_TRIVIAL(debug) << "apache::analyzer::detail::KnnAnalyzerObject::Analyze: Preparing agent: " << agent_name;

    for (auto virtualhost_name : apache_database_functions_->GetVirtualhostNames(agent_name)) {
      BOOST_LOG_TRIVIAL(debug) << "apache::analyzer::detail::KnnAnalyzerObject::Analyze: Preparing virtualhost: " << virtualhost_name;

      auto agent_id = general_database_functions_->GetAgentNameId(agent_name);
      auto virtualhost_id = apache_database_functions_->GetVirtualhostNameId(virtualhost_name);
      BOOST_LOG_TRIVIAL(debug) << "apache::analyzer::detail::KnnAnalyzerObject::Analyze: Agent ID: " << agent_id;
      BOOST_LOG_TRIVIAL(debug) << "apache::analyzer::detail::KnnAnalyzerObject::Analyze: Virtualhost ID " << virtualhost_id;

      AnalyzeVirtualhost(agent_id, virtualhost_id);
    }
  }
}

bool KnnAnalyzerObject::IsAnomalyDetected() const {
  return is_anomaly_detected_;
}

::apache::analyzer::type::KnnAnalyzerSummary KnnAnalyzerObject::GetAnalyzeSummary() const {
  return analyze_summary_;
}

void KnnAnalyzerObject::AnalyzeVirtualhost(const ::database::type::RowId &agent_name_id,
                                           const ::database::type::RowId &virtualhost_name_id) {
  BOOST_LOG_TRIVIAL(debug) << "apache::analyzer::detail::KnnAnalyzerObject::AnalyzeVirtualhost: Function call";
  constexpr RowsCount MAX_ROWS_IN_MEMORY = 100;
  BOOST_LOG_TRIVIAL(debug) << "apache::analyzer::detail::KnnAnalyzerObject::AnalyzeSessions: Max rows in memory: " << MAX_ROWS_IN_MEMORY;

  auto sessions_count = apache_database_functions_->GetNotClassifiedSessionsStatisticsCount(agent_name_id, virtualhost_name_id);
  BOOST_LOG_TRIVIAL(debug) << "apache::analyzer::detail::KnnAnalyzerObject::AnalyzeVirtualhost: Found " << sessions_count << " sessions to analyze";

  analyze_summary_.push_back(type::KnnVirtualhostAnalyzeStatistics());

  util::RunPartially(MAX_ROWS_IN_MEMORY, sessions_count, [&](long long part_count, long long offset) {
    AnalyzeSessions(agent_name_id, virtualhost_name_id, part_count, offset);
  });
}

void KnnAnalyzerObject::AnalyzeSessions(const ::database::type::RowId &agent_name_id,
                                        const ::database::type::RowId &virtualhost_name_id,
                                        unsigned limit,
                                        ::database::type::RowsCount offset) {
  BOOST_LOG_TRIVIAL(debug) << "apache::analyzer::detail::KnnAnalyzerObject::AnalyzeSessions: Function call";
  BOOST_LOG_TRIVIAL(debug) << "apache::analyzer::detail::KnnAnalyzerObject::AnalyzeSessions: Analyzing sessions: agent_name_id=" << agent_name_id << "; virtualhost_name_id=" << virtualhost_name_id << " (limit=" << limit << "; offset=" << offset << ")";
  constexpr RowsCount MAX_ROWS_IN_MEMORY = 100;

  auto sessions_part = apache_database_functions_->GetNotClassifiedSessionStatistics(agent_name_id, virtualhost_name_id, limit, offset);
  BOOST_LOG_TRIVIAL(debug) << "apache::analyzer::detail::KnnAnalyzerObject::AnalyzeSessions: Received " << sessions_part.size() << " sessions statictics";

  type::KnnVirtualhostAnalyzeStatistics &stats = analyze_summary_.at(analyze_summary_.size() - 1);
  stats.agent_id = agent_name_id;
  stats.virtualhost_id = virtualhost_name_id;

  auto learning_set_count = apache_database_functions_->GetLearningSessionsCount(agent_name_id, virtualhost_name_id);
  BOOST_LOG_TRIVIAL(debug) << "apache::analyzer::detail::KnnAnalyzerObject::AnalyzeSessions: Found " << learning_set_count << " sessions in learning set";

  for (auto session : sessions_part) {
    neighbours_table_.SetSession(session);

    util::RunPartially(MAX_ROWS_IN_MEMORY, learning_set_count, [&](long long part_count, long long offset) {
      AnalyzeSessionsWithLearningSet(agent_name_id, virtualhost_name_id, part_count, offset, session);
    });

    auto classification = GetSessionClassification();

    if (classification == ::database::type::Classification::ANOMALY) {
      is_anomaly_detected_ = true;
      stats.anomaly_sessions_ids.push_back(session.id);
    }

    session.classification = classification;
    apache_database_functions_->UpdateSessionStatisticClassification(session.id, session.classification);

    BOOST_LOG_TRIVIAL(debug) << "apache::analyzer::detail::KnnAnalyzerObject::AnalyzeSessions: Is session with id " << session.id << " anomaly? - " << (session.classification == ::database::type::Classification::ANOMALY);
  }
}

void KnnAnalyzerObject::AnalyzeSessionsWithLearningSet(const ::database::type::RowId &agent_name_id,
                                                       const ::database::type::RowId &virtualhost_name_id,
                                                       unsigned limit,
                                                       ::database::type::RowsCount offset,
                                                       const ::apache::type::ApacheSessionEntry &session_to_analyze) {
  BOOST_LOG_TRIVIAL(debug) << "apache::analyzer::detail::KnnAnalyzerObject::AnalyzeSessionsWithLearningSet: Function call";
  auto learning_set_ids = apache_database_functions_->GetLearningSessionsIds(agent_name_id, virtualhost_name_id, limit, offset);

  for (const auto &session_id : learning_set_ids) {
    BOOST_LOG_TRIVIAL(debug) << "apache::analyzer::detail::KnnAnalyzerObject::AnalyzeSessionsWithLearningSet: Comparing session to analyze with id=" << session_to_analyze.id << " to session from learning set " << session_id;

    const auto &learning_set_session = apache_database_functions_->GetOneSessionStatistic(session_id);

    if (learning_set_session.classification != ::database::type::Classification::UNKNOWN)
      neighbours_table_.Add(learning_set_session);
    else
      BOOST_LOG_TRIVIAL(debug) << "apache::analyzer::detail::KnnAnalyzerObject::AnalyzeSessionsWithLearningSet: Unknown session classification (id=" << learning_set_session.id << "(";
  }
}

::database::type::Classification KnnAnalyzerObject::GetSessionClassification() {
  long long i = 0;
  const auto &neighbours = neighbours_table_.Get();

  for (const auto &n : neighbours)
    BOOST_LOG_TRIVIAL(debug) << "apache::analyzer::detail::KnnAnalyzerObject::IsSessionAnomaly: Final nearest neighbour id=" << n.session_id;

  for (const auto &n : neighbours)
    i = i - 1 + 2 * static_cast<int> (n.classification == ::database::type::Classification::ANOMALY);

  ::database::type::Classification c = ::database::type::Classification::UNKNOWN;
  if (neighbours.size() > 0) {
    if (i > 0)
      c = ::database::type::Classification::ANOMALY;
    else
      c = ::database::type::Classification::NORMAL;
  }

  BOOST_LOG_TRIVIAL(debug) << "apache::analyzer::detail::KnnAnalyzerObject::IsSessionAnomaly: Classification: " << static_cast<int> (c);

  return c;
}

KnnAnalyzerObject::KnnAnalyzerObject(::database::detail::GeneralDatabaseFunctionsInterfacePtr general_database_functions,
                                     ::apache::database::DatabaseFunctionsPtr apache_database_functions) :
general_database_functions_(general_database_functions),
apache_database_functions_(apache_database_functions),
is_anomaly_detected_(false) {
}

}

}

}
<commit_msg>Fix Apache log classification<commit_after>/*
 * Copyright 2016 Adam Chyła, adam@chyla.org
 * All rights reserved. Distributed under the terms of the MIT License.
 */

#include "knn_analyzer_object.h"

#include <boost/log/trivial.hpp>
#include <cmath>
#include <limits>
#include <slas/util/run_partially.h>

#include "system.h"

using namespace ::apache::database::detail;
using namespace ::apache::type;
using namespace ::database::type;
using namespace ::type;

namespace apache
{

namespace analyzer
{

namespace detail
{

KnnAnalyzerObjectPtr KnnAnalyzerObject::Create(::database::detail::GeneralDatabaseFunctionsInterfacePtr general_database_functions,
                                               ::apache::database::DatabaseFunctionsPtr apache_database_functions) {
  return KnnAnalyzerObjectPtr(new KnnAnalyzerObject(general_database_functions, apache_database_functions));
}

void KnnAnalyzerObject::Analyze() {
  for (auto agent_name : general_database_functions_->GetAgentNames()) {
    BOOST_LOG_TRIVIAL(debug) << "apache::analyzer::detail::KnnAnalyzerObject::Analyze: Preparing agent: " << agent_name;

    for (auto virtualhost_name : apache_database_functions_->GetVirtualhostNames(agent_name)) {
      BOOST_LOG_TRIVIAL(debug) << "apache::analyzer::detail::KnnAnalyzerObject::Analyze: Preparing virtualhost: " << virtualhost_name;

      auto agent_id = general_database_functions_->GetAgentNameId(agent_name);
      auto virtualhost_id = apache_database_functions_->GetVirtualhostNameId(virtualhost_name);
      BOOST_LOG_TRIVIAL(debug) << "apache::analyzer::detail::KnnAnalyzerObject::Analyze: Agent ID: " << agent_id;
      BOOST_LOG_TRIVIAL(debug) << "apache::analyzer::detail::KnnAnalyzerObject::Analyze: Virtualhost ID " << virtualhost_id;

      AnalyzeVirtualhost(agent_id, virtualhost_id);
    }
  }
}

bool KnnAnalyzerObject::IsAnomalyDetected() const {
  return is_anomaly_detected_;
}

::apache::analyzer::type::KnnAnalyzerSummary KnnAnalyzerObject::GetAnalyzeSummary() const {
  return analyze_summary_;
}

void KnnAnalyzerObject::AnalyzeVirtualhost(const ::database::type::RowId &agent_name_id,
                                           const ::database::type::RowId &virtualhost_name_id) {
  BOOST_LOG_TRIVIAL(debug) << "apache::analyzer::detail::KnnAnalyzerObject::AnalyzeVirtualhost: Function call";
  constexpr RowsCount MAX_ROWS_IN_MEMORY = 100;
  BOOST_LOG_TRIVIAL(debug) << "apache::analyzer::detail::KnnAnalyzerObject::AnalyzeSessions: Max rows in memory: " << MAX_ROWS_IN_MEMORY;

  auto sessions_count = apache_database_functions_->GetNotClassifiedSessionsStatisticsCount(agent_name_id, virtualhost_name_id);
  BOOST_LOG_TRIVIAL(debug) << "apache::analyzer::detail::KnnAnalyzerObject::AnalyzeVirtualhost: Found " << sessions_count << " sessions to analyze";

  analyze_summary_.push_back(type::KnnVirtualhostAnalyzeStatistics());

  util::RunPartially(MAX_ROWS_IN_MEMORY, sessions_count, [&](long long part_count, long long offset) {
    AnalyzeSessions(agent_name_id, virtualhost_name_id, part_count, offset);
  });
}

void KnnAnalyzerObject::AnalyzeSessions(const ::database::type::RowId &agent_name_id,
                                        const ::database::type::RowId &virtualhost_name_id,
                                        unsigned limit,
                                        ::database::type::RowsCount offset) {
  BOOST_LOG_TRIVIAL(debug) << "apache::analyzer::detail::KnnAnalyzerObject::AnalyzeSessions: Function call";
  BOOST_LOG_TRIVIAL(debug) << "apache::analyzer::detail::KnnAnalyzerObject::AnalyzeSessions: Analyzing sessions: agent_name_id=" << agent_name_id << "; virtualhost_name_id=" << virtualhost_name_id << " (limit=" << limit << "; offset=" << offset << ")";
  constexpr RowsCount MAX_ROWS_IN_MEMORY = 100;

  auto sessions_part = apache_database_functions_->GetNotClassifiedSessionStatistics(agent_name_id, virtualhost_name_id, limit, offset);
  BOOST_LOG_TRIVIAL(debug) << "apache::analyzer::detail::KnnAnalyzerObject::AnalyzeSessions: Received " << sessions_part.size() << " sessions statictics";

  type::KnnVirtualhostAnalyzeStatistics &stats = analyze_summary_.at(analyze_summary_.size() - 1);
  stats.agent_id = agent_name_id;
  stats.virtualhost_id = virtualhost_name_id;

  auto learning_set_count = apache_database_functions_->GetLearningSessionsCount(agent_name_id, virtualhost_name_id);
  BOOST_LOG_TRIVIAL(debug) << "apache::analyzer::detail::KnnAnalyzerObject::AnalyzeSessions: Found " << learning_set_count << " sessions in learning set";

  for (auto session : sessions_part) {
    neighbours_table_.SetSession(session);

    util::RunPartially(MAX_ROWS_IN_MEMORY, learning_set_count, [&](long long part_count, long long offset) {
      AnalyzeSessionsWithLearningSet(agent_name_id, virtualhost_name_id, part_count, 0, session);
    });

    auto classification = GetSessionClassification();

    if (classification == ::database::type::Classification::ANOMALY) {
      is_anomaly_detected_ = true;
      stats.anomaly_sessions_ids.push_back(session.id);
    }

    session.classification = classification;
    apache_database_functions_->UpdateSessionStatisticClassification(session.id, session.classification);

    BOOST_LOG_TRIVIAL(debug) << "apache::analyzer::detail::KnnAnalyzerObject::AnalyzeSessions: Is session with id " << session.id << " anomaly? - " << (session.classification == ::database::type::Classification::ANOMALY);
  }
}

void KnnAnalyzerObject::AnalyzeSessionsWithLearningSet(const ::database::type::RowId &agent_name_id,
                                                       const ::database::type::RowId &virtualhost_name_id,
                                                       unsigned limit,
                                                       ::database::type::RowsCount offset,
                                                       const ::apache::type::ApacheSessionEntry &session_to_analyze) {
  BOOST_LOG_TRIVIAL(debug) << "apache::analyzer::detail::KnnAnalyzerObject::AnalyzeSessionsWithLearningSet: Function call";
  auto learning_set_ids = apache_database_functions_->GetLearningSessionsIds(agent_name_id, virtualhost_name_id, limit, offset);

  for (const auto &session_id : learning_set_ids) {
    BOOST_LOG_TRIVIAL(debug) << "apache::analyzer::detail::KnnAnalyzerObject::AnalyzeSessionsWithLearningSet: Comparing session to analyze with id=" << session_to_analyze.id << " to session from learning set " << session_id;

    const auto &learning_set_session = apache_database_functions_->GetOneSessionStatistic(session_id);

    if (learning_set_session.classification != ::database::type::Classification::UNKNOWN)
      neighbours_table_.Add(learning_set_session);
    else
      BOOST_LOG_TRIVIAL(debug) << "apache::analyzer::detail::KnnAnalyzerObject::AnalyzeSessionsWithLearningSet: Unknown session classification (id=" << learning_set_session.id << "(";
  }
}

::database::type::Classification KnnAnalyzerObject::GetSessionClassification() {
  long long i = 0;
  const auto &neighbours = neighbours_table_.Get();

  for (const auto &n : neighbours)
    BOOST_LOG_TRIVIAL(debug) << "apache::analyzer::detail::KnnAnalyzerObject::IsSessionAnomaly: Final nearest neighbour id=" << n.session_id;

  for (const auto &n : neighbours)
    i = i - 1 + 2 * static_cast<int> (n.classification == ::database::type::Classification::ANOMALY);

  ::database::type::Classification c = ::database::type::Classification::UNKNOWN;
  if (neighbours.size() > 0) {
    if (i > 0)
      c = ::database::type::Classification::ANOMALY;
    else
      c = ::database::type::Classification::NORMAL;
  }

  BOOST_LOG_TRIVIAL(debug) << "apache::analyzer::detail::KnnAnalyzerObject::IsSessionAnomaly: Classification: " << static_cast<int> (c);

  return c;
}

KnnAnalyzerObject::KnnAnalyzerObject(::database::detail::GeneralDatabaseFunctionsInterfacePtr general_database_functions,
                                     ::apache::database::DatabaseFunctionsPtr apache_database_functions) :
general_database_functions_(general_database_functions),
apache_database_functions_(apache_database_functions),
is_anomaly_detected_(false) {
}

}

}

}
<|endoftext|>


<commit_before>#include "pivot_path_planner.hpp"

#include <memory>
#include <vector>

#include <rj_constants/constants.hpp>
#include <rj_geometry/pose.hpp>
#include <rj_geometry/util.hpp>

#include "planning/instant.hpp"
#include "planning/planning_params.hpp"
#include "planning/primitives/angle_planning.hpp"
#include "planning/primitives/path_smoothing.hpp"
#include "planning/primitives/trapezoidal_motion.hpp"
#include "planning/primitives/velocity_profiling.hpp"
#include "planning/trajectory.hpp"

namespace planning {
using namespace rj_geometry;

Trajectory PivotPathPlanner::plan(const PlanRequest& request) {
    const RobotInstant& start_instant = request.start;
    const auto& linear_constraints = request.constraints.mot;
    const auto& rotation_constraints = request.constraints.rot;

    rj_geometry::ShapeSet static_obstacles;
    std::vector<DynamicObstacle> dynamic_obstacles;
    fill_obstacles(request, &static_obstacles, &dynamic_obstacles, false);

    const auto& command = std::get<PivotCommand>(request.motion_command);

    double radius = pivot::PARAM_radius_multiplier * kRobotRadius;
    auto pivot_point = command.pivot_point;
    auto pivot_target = command.pivot_target;

    // TODO(Kyle): These need real constants
    if (cached_pivot_target_.has_value() &&
        cached_pivot_target_.value().dist_to(pivot_target) < kRobotMouthWidth / 2 &&
        cached_pivot_point_.has_value() &&
        cached_pivot_point_.value().dist_to(pivot_point) < kRobotMouthWidth / 2) {
        return previous_;
    }

    cached_pivot_target_ = pivot_target;
    cached_pivot_point_ = pivot_point;

    auto final_position = pivot_point + (pivot_point - pivot_target).normalized(radius);
    std::vector<Point> points;

    // max_speed = max_radians * radius
    MotionConstraints new_constraints = request.constraints.mot;
    new_constraints.max_speed =
        std::min(new_constraints.max_speed, rotation_constraints.max_speed * radius) * .5;

    double start_angle = pivot_point.angle_to(start_instant.position());
    double target_angle = pivot_point.angle_to(final_position);
    double angle_change = fix_angle_radians(target_angle - start_angle);

    constexpr double kMaxInterpolationRadians = 10 * M_PI / 180;
    const int interpolations = std::ceil(std::abs(angle_change) / kMaxInterpolationRadians);

    points.push_back(start_instant.position());
    for (int i = 1; i <= interpolations; i++) {
        double percent = (double)i / interpolations;
        double angle = start_angle + angle_change * percent;
        Point point = Point::direction(angle).normalized(radius) + pivot_point;
        points.push_back(point);
    }

    BezierPath path_bezier(points, Point(0, 0), Point(0, 0), linear_constraints);

    Trajectory path = profile_velocity(path_bezier, start_instant.linear_velocity().mag(), 0,
                                       linear_constraints, start_instant.stamp);
    if (!Twist::nearly_equals(path.last().velocity, Twist::zero())) {
        RobotInstant last;
        last.position() = final_position;
        last.velocity = Twist::zero();
        last.stamp = path.end_time() + 100ms;
        path.append_instant(last);
    }
    path.hold_for(RJ::Seconds(3.0));

    AngleFunction function = [pivot_point, pivot_target](const LinearMotionInstant& instant,
                                                         double /*previous_angle*/,
                                                         Eigen::Vector2d* jacobian) -> double {
        Point position = instant.position;
        auto angle_to_pivot_target = position.angle_to(pivot_target);
        auto angle_to_pivot = position.angle_to(pivot_point);

        Point target_point = pivot_point;
        if (abs(angle_to_pivot - angle_to_pivot_target) < degrees_to_radians(10)) {
            target_point = pivot_target;
        }

        auto angle_to_target = position.angle_to(target_point);

        if (jacobian != nullptr) {
            // The angle to the point changes with the dot product of the tangent vector to the circle with the robot's velocity, divided by the radius.
            // Therefore, the rotated vector needs to be divided by the radius twice to get the jacobian.
            *jacobian = (position - target_point).rotate(M_PI / 2);
            *jacobian /= jacobian->squaredNorm();
        }

        return angle_to_target;
    };

    plan_angles(&path, start_instant, AngleFns::face_point(pivot_point), request.constraints.rot);
    path.stamp(RJ::now());

    previous_ = path;
    return path;
}

}  // namespace planning<commit_msg>Use cached pivot point in pivot even when target is constantly changing (#1682)<commit_after>#include "pivot_path_planner.hpp"

#include <memory>
#include <vector>

#include <rj_constants/constants.hpp>
#include <rj_geometry/pose.hpp>
#include <rj_geometry/util.hpp>

#include "planning/instant.hpp"
#include "planning/planning_params.hpp"
#include "planning/primitives/angle_planning.hpp"
#include "planning/primitives/path_smoothing.hpp"
#include "planning/primitives/trapezoidal_motion.hpp"
#include "planning/primitives/velocity_profiling.hpp"
#include "planning/trajectory.hpp"

namespace planning {
using namespace rj_geometry;

Trajectory PivotPathPlanner::plan(const PlanRequest& request) {
    const RobotInstant& start_instant = request.start;
    const auto& linear_constraints = request.constraints.mot;
    const auto& rotation_constraints = request.constraints.rot;

    rj_geometry::ShapeSet static_obstacles;
    std::vector<DynamicObstacle> dynamic_obstacles;
    fill_obstacles(request, &static_obstacles, &dynamic_obstacles, false);

    const auto& command = std::get<PivotCommand>(request.motion_command);

    double radius = pivot::PARAM_radius_multiplier * kRobotRadius;
    auto pivot_point = command.pivot_point;
    auto pivot_target = command.pivot_target;

    const bool pivot_target_unchanged =
        cached_pivot_target_.has_value() &&
        cached_pivot_target_.value().dist_to(pivot_target) < kRobotMouthWidth / 2;
    bool pivot_point_unchanged =
        cached_pivot_point_.has_value() &&
        cached_pivot_point_.value().dist_to(pivot_point) < kRobotMouthWidth / 2;

    // TODO(Kyle): These need real constants
    if (pivot_target_unchanged && pivot_point_unchanged) {
        return previous_;
    }

    if (pivot_point_unchanged) {
        pivot_point = *cached_pivot_point_;
    }
    if (pivot_target_unchanged) {
        pivot_target = *cached_pivot_target_;
    }

    cached_pivot_target_ = pivot_target;
    cached_pivot_point_ = pivot_point;

    auto final_position = pivot_point + (pivot_point - pivot_target).normalized(radius);
    std::vector<Point> points;

    // max_speed = max_radians * radius
    MotionConstraints new_constraints = request.constraints.mot;
    new_constraints.max_speed =
        std::min(new_constraints.max_speed, rotation_constraints.max_speed * radius) * .5;

    double start_angle = pivot_point.angle_to(start_instant.position());
    double target_angle = pivot_point.angle_to(final_position);
    double angle_change = fix_angle_radians(target_angle - start_angle);

    constexpr double kMaxInterpolationRadians = 10 * M_PI / 180;
    const int interpolations = std::ceil(std::abs(angle_change) / kMaxInterpolationRadians);

    points.push_back(start_instant.position());
    for (int i = 1; i <= interpolations; i++) {
        double percent = (double)i / interpolations;
        double angle = start_angle + angle_change * percent;
        Point point = Point::direction(angle).normalized(radius) + pivot_point;
        points.push_back(point);
    }

    BezierPath path_bezier(points, Point(0, 0), Point(0, 0), linear_constraints);

    Trajectory path = profile_velocity(path_bezier, start_instant.linear_velocity().mag(), 0,
                                       linear_constraints, start_instant.stamp);
    if (!Twist::nearly_equals(path.last().velocity, Twist::zero())) {
        RobotInstant last;
        last.position() = final_position;
        last.velocity = Twist::zero();
        last.stamp = path.end_time() + 100ms;
        path.append_instant(last);
    }
    path.hold_for(RJ::Seconds(3.0));

    AngleFunction function = [pivot_point, pivot_target](const LinearMotionInstant& instant,
                                                         double /*previous_angle*/,
                                                         Eigen::Vector2d* jacobian) -> double {
        Point position = instant.position;
        auto angle_to_pivot_target = position.angle_to(pivot_target);
        auto angle_to_pivot = position.angle_to(pivot_point);

        Point target_point = pivot_point;
        if (abs(angle_to_pivot - angle_to_pivot_target) < degrees_to_radians(10)) {
            target_point = pivot_target;
        }

        auto angle_to_target = position.angle_to(target_point);

        if (jacobian != nullptr) {
            // The angle to the point changes with the dot product of the tangent vector to the
            // circle with the robot's velocity, divided by the radius. Therefore, the rotated
            // vector needs to be divided by the radius twice to get the jacobian.
            *jacobian = (position - target_point).rotate(M_PI / 2);
            *jacobian /= jacobian->squaredNorm();
        }

        return angle_to_target;
    };

    plan_angles(&path, start_instant, AngleFns::face_point(pivot_point), request.constraints.rot);
    path.stamp(RJ::now());

    previous_ = path;
    return path;
}

}  // namespace planning<|endoftext|>


<commit_before><commit_msg>Update Pollard Rho.cpp<commit_after><|endoftext|>


<commit_before>#pragma once

#include "fly/types/string/detail/string_classifier.hpp"
#include "fly/types/string/detail/string_traits.hpp"
#include "fly/types/string/string_literal.hpp"

#include <cstdint>
#include <optional>

namespace fly::detail {

/**
 * Helper class to perform lexical analysis of a C-string literal. All methods are constant
 * expressions, allowing for string analysis at compile time.
 *
 * @author Timothy Flynn (trflynn89@pm.me)
 * @version January 3, 2021
 */
template <typename StringType>
class BasicStringLexer
{
    using traits = BasicStringTraits<StringType>;
    using classifier = BasicStringClassifier<StringType>;
    using char_type = typename traits::char_type;
    using view_type = typename traits::view_type;

public:
    /**
     * Constructor. Stores a view into a C-string literal. This class is not interested in the null
     * terminator, thus if provided, it is excluded from the view.
     *
     * @tparam N The size of the C-string literal.
     *
     * @param literals Reference to a C-string literal of size N.
     */
    template <std::size_t N>
    constexpr explicit BasicStringLexer(const char_type (&literals)[N]) noexcept;

    /**
     * @return A string view into the C-string literal.
     */
    constexpr view_type view() const;

    /**
     * @return The lexer's current position into the C-string literal.
     */
    constexpr std::size_t position() const;

    /**
     * If a character is available at the current position (or some offset from the current
     * position) in the C-string literal, return that character.
     *
     * @param offset The offset from the current position to peek.
     *
     * @return If available, the character at the provided offset. Otherwise, an uninitialized
     *         value.
     */
    constexpr std::optional<char_type> peek(std::size_t offset = 0);

    /**
     * If a character is available at the current position in the C-string literal, return that
     * character and advance the current position to the next character.
     *
     * @return If available, the current character. Otherwise, an uninitialized value.
     */
    constexpr std::optional<char_type> consume();

    /**
     * If a character is available at the current position in the C-string literal, and if that
     * character is equivalent to the provided character, advance the current position to the next
     * character.
     *
     * @param ch The character to test with.
     *
     * @return Whether the current character was available and matched the provided character.
     */
    constexpr bool consume_if(char_type ch);

    /**
     * Beginning with the current position, retrieve characters from the current position in the
     * C-string literal and advance the current position to the next character until a character is
     * either not available or not a decimal digit. Convert the retrieved characters to an unsigned
     * number.
     *
     * @return If consumed, the parsed unsigned number. Otherwise, an uninitialized value.
     */
    constexpr std::optional<std::size_t> consume_number();

private:
    /**
     * If a character is available at the current position in the C-string literal, and if that
     * character satisfies the provided condition, advance the current position to the next
     * character.
     *
     * @tparam Condition A callable conditional for testing a character.
     *
     * @param condition The condition to test with.
     *
     * @return Whether the current character was available and satisfied the provided condition.
     */
    template <typename Condition>
    constexpr std::optional<char_type> consume_if(Condition condition);

    static constexpr const auto s_null_terminator = FLY_CHR(char_type, '\0');

    const std::size_t m_size;
    const view_type m_view;

    std::size_t m_index {0};
};

//==================================================================================================
template <typename StringType>
template <std::size_t N>
constexpr BasicStringLexer<StringType>::BasicStringLexer(const char_type (&literals)[N]) noexcept :
    m_size(N - ((literals[N - 1] == s_null_terminator) ? 1 : 0)),
    m_view(literals, m_size)
{
}

//==================================================================================================
template <typename StringType>
constexpr auto BasicStringLexer<StringType>::view() const -> view_type
{
    return m_view;
}

//==================================================================================================
template <typename StringType>
constexpr std::size_t BasicStringLexer<StringType>::position() const
{
    return m_index;
}

//==================================================================================================
template <typename StringType>
constexpr auto BasicStringLexer<StringType>::peek(std::size_t offset) -> std::optional<char_type>
{
    if ((m_index + offset) >= m_size)
    {
        return std::nullopt;
    }

    return *(m_view.data() + m_index + offset);
}

//==================================================================================================
template <typename StringType>
constexpr auto BasicStringLexer<StringType>::consume() -> std::optional<char_type>
{
    if (m_index >= m_size)
    {
        return std::nullopt;
    }

    return *(m_view.data() + m_index++);
}

//==================================================================================================
template <typename StringType>
constexpr bool BasicStringLexer<StringType>::consume_if(char_type ch)
{
    if (auto next = peek(); next && (next.value() == ch))
    {
        ++m_index;
        return true;
    }

    return false;
}

//==================================================================================================
template <typename StringType>
template <typename Condition>
constexpr auto BasicStringLexer<StringType>::consume_if(Condition condition)
    -> std::optional<char_type>
{
    if (auto next = peek(); next && condition(next.value()))
    {
        return consume();
    }

    return std::nullopt;
}

//==================================================================================================
template <typename StringType>
constexpr std::optional<std::size_t> BasicStringLexer<StringType>::consume_number()
{
    bool parsed_number = false;
    std::size_t number = 0;

    while (auto ch = consume_if(classifier::is_digit))
    {
        parsed_number = true;

        number *= 10;
        number += static_cast<std::size_t>(ch.value() - FLY_CHR(char_type, '0'));
    }

    return parsed_number ? std::optional<std::size_t>(number) : std::nullopt;
}

} // namespace fly::detail
<commit_msg>BasicStringLexer may use BasicStringClassifier::size for string size<commit_after>#pragma once

#include "fly/types/string/detail/string_classifier.hpp"
#include "fly/types/string/detail/string_traits.hpp"
#include "fly/types/string/string_literal.hpp"

#include <cstdint>
#include <optional>

namespace fly::detail {

/**
 * Helper class to perform lexical analysis of a C-string literal. All methods are constant
 * expressions, allowing for string analysis at compile time.
 *
 * @author Timothy Flynn (trflynn89@pm.me)
 * @version January 3, 2021
 */
template <typename StringType>
class BasicStringLexer
{
    using traits = BasicStringTraits<StringType>;
    using classifier = BasicStringClassifier<StringType>;
    using char_type = typename traits::char_type;
    using view_type = typename traits::view_type;

public:
    /**
     * Constructor. Stores a view into a C-string literal. This class is not interested in the null
     * terminator, thus if provided, it is excluded from the view.
     *
     * @tparam N The size of the C-string literal.
     *
     * @param literals Reference to a C-string literal of size N.
     */
    template <std::size_t N>
    constexpr explicit BasicStringLexer(const char_type (&literals)[N]) noexcept;

    /**
     * @return A string view into the C-string literal.
     */
    constexpr view_type view() const;

    /**
     * @return The lexer's current position into the C-string literal.
     */
    constexpr std::size_t position() const;

    /**
     * If a character is available at the current position (or some offset from the current
     * position) in the C-string literal, return that character.
     *
     * @param offset The offset from the current position to peek.
     *
     * @return If available, the character at the provided offset. Otherwise, an uninitialized
     *         value.
     */
    constexpr std::optional<char_type> peek(std::size_t offset = 0);

    /**
     * If a character is available at the current position in the C-string literal, return that
     * character and advance the current position to the next character.
     *
     * @return If available, the current character. Otherwise, an uninitialized value.
     */
    constexpr std::optional<char_type> consume();

    /**
     * If a character is available at the current position in the C-string literal, and if that
     * character is equivalent to the provided character, advance the current position to the next
     * character.
     *
     * @param ch The character to test with.
     *
     * @return Whether the current character was available and matched the provided character.
     */
    constexpr bool consume_if(char_type ch);

    /**
     * Beginning with the current position, retrieve characters from the current position in the
     * C-string literal and advance the current position to the next character until a character is
     * either not available or not a decimal digit. Convert the retrieved characters to an unsigned
     * number.
     *
     * @return If consumed, the parsed unsigned number. Otherwise, an uninitialized value.
     */
    constexpr std::optional<std::size_t> consume_number();

private:
    /**
     * If a character is available at the current position in the C-string literal, and if that
     * character satisfies the provided condition, advance the current position to the next
     * character.
     *
     * @tparam Condition A callable conditional for testing a character.
     *
     * @param condition The condition to test with.
     *
     * @return Whether the current character was available and satisfied the provided condition.
     */
    template <typename Condition>
    constexpr std::optional<char_type> consume_if(Condition condition);

    const std::size_t m_size;
    const view_type m_view;

    std::size_t m_index {0};
};

//==================================================================================================
template <typename StringType>
template <std::size_t N>
constexpr BasicStringLexer<StringType>::BasicStringLexer(const char_type (&literals)[N]) noexcept :
    m_size(classifier::size(literals)),
    m_view(literals, m_size)
{
}

//==================================================================================================
template <typename StringType>
constexpr auto BasicStringLexer<StringType>::view() const -> view_type
{
    return m_view;
}

//==================================================================================================
template <typename StringType>
constexpr std::size_t BasicStringLexer<StringType>::position() const
{
    return m_index;
}

//==================================================================================================
template <typename StringType>
constexpr auto BasicStringLexer<StringType>::peek(std::size_t offset) -> std::optional<char_type>
{
    if ((m_index + offset) >= m_size)
    {
        return std::nullopt;
    }

    return *(m_view.data() + m_index + offset);
}

//==================================================================================================
template <typename StringType>
constexpr auto BasicStringLexer<StringType>::consume() -> std::optional<char_type>
{
    if (m_index >= m_size)
    {
        return std::nullopt;
    }

    return *(m_view.data() + m_index++);
}

//==================================================================================================
template <typename StringType>
constexpr bool BasicStringLexer<StringType>::consume_if(char_type ch)
{
    if (auto next = peek(); next && (next.value() == ch))
    {
        ++m_index;
        return true;
    }

    return false;
}

//==================================================================================================
template <typename StringType>
template <typename Condition>
constexpr auto BasicStringLexer<StringType>::consume_if(Condition condition)
    -> std::optional<char_type>
{
    if (auto next = peek(); next && condition(next.value()))
    {
        return consume();
    }

    return std::nullopt;
}

//==================================================================================================
template <typename StringType>
constexpr std::optional<std::size_t> BasicStringLexer<StringType>::consume_number()
{
    bool parsed_number = false;
    std::size_t number = 0;

    while (auto ch = consume_if(classifier::is_digit))
    {
        parsed_number = true;

        number *= 10;
        number += static_cast<std::size_t>(ch.value() - FLY_CHR(char_type, '0'));
    }

    return parsed_number ? std::optional<std::size_t>(number) : std::nullopt;
}

} // namespace fly::detail
<|endoftext|>


<commit_before>/*-------------------------------------------------------------------------
 * drawElements C++ Base Library
 * -----------------------------
 *
 * Copyright 2015 The Android Open Source Project
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *      http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 *
 *//*!
 * \file
 * \brief Cross-thread barrier.
 *//*--------------------------------------------------------------------*/

#include "deSpinBarrier.hpp"
#include "deThread.hpp"
#include "deRandom.hpp"
#include "deInt32.h"

#include <vector>

namespace de
{

SpinBarrier::SpinBarrier (deInt32 numThreads)
	: m_numCores	(deGetNumAvailableLogicalCores())
	, m_numThreads	(numThreads)
	, m_numEntered	(0)
	, m_numLeaving	(0)
	, m_numRemoved	(0)
{
	DE_ASSERT(numThreads > 0);
}

SpinBarrier::~SpinBarrier (void)
{
	DE_ASSERT(m_numEntered == 0 && m_numLeaving == 0);
}

void SpinBarrier::reset (deUint32 numThreads)
{
	// If last threads were removed, m_numEntered > 0 && m_numRemoved > 0
	DE_ASSERT(m_numLeaving == 0);
	DE_ASSERT(numThreads > 0);
	m_numThreads = numThreads;
	m_numEntered = 0;
	m_numLeaving = 0;
	m_numRemoved = 0;
}

inline SpinBarrier::WaitMode getWaitMode (SpinBarrier::WaitMode requested, deUint32 numCores, deInt32 numThreads)
{
	if (requested == SpinBarrier::WAIT_MODE_AUTO)
		return ((deUint32)numThreads <= numCores) ? SpinBarrier::WAIT_MODE_BUSY : SpinBarrier::WAIT_MODE_YIELD;
	else
		return requested;
}

inline void wait (SpinBarrier::WaitMode mode)
{
	DE_ASSERT(mode == SpinBarrier::WAIT_MODE_YIELD || mode == SpinBarrier::WAIT_MODE_BUSY);

	if (mode == SpinBarrier::WAIT_MODE_YIELD)
		deYield();
}

void SpinBarrier::sync (WaitMode requestedMode)
{
	const WaitMode	waitMode	= getWaitMode(requestedMode, m_numCores, m_numThreads);

	deMemoryReadWriteFence();

	// m_numEntered must not be touched until all threads have had
	// a chance to observe it being 0.
	if (m_numLeaving > 0)
	{
		for (;;)
		{
			if (m_numLeaving == 0)
				break;

			wait(waitMode);
		}
	}

	if (deAtomicIncrement32(&m_numEntered) == m_numThreads)
	{
		m_numLeaving  = m_numThreads;
		deMemoryReadWriteFence();

		// m_numLeaving must be set > 0 when manipulating m_numRemoved
		// to prevent other threads from touching m_numRemoved.
		// Since this thread has not been removed, m_numLeaving will be >= 1
		// until m_numLeaving is decremented at the end of this function.
		m_numThreads -= m_numRemoved;
		m_numLeaving -= m_numRemoved;
		m_numRemoved  = 0;

		deMemoryReadWriteFence();
		m_numEntered  = 0;
	}
	else
	{
		for (;;)
		{
			if (m_numEntered == 0)
				break;

			wait(waitMode);
		}
	}

	deAtomicDecrement32(&m_numLeaving);
	deMemoryReadWriteFence();
}

void SpinBarrier::removeThread (WaitMode requestedMode)
{
	const WaitMode	waitMode	= getWaitMode(requestedMode, m_numCores, m_numThreads);

	// Wait for other threads exiting previous barrier
	if (m_numLeaving > 0)
	{
		for (;;)
		{
			if (m_numLeaving == 0)
				break;

			wait(waitMode);
		}
	}

	// Ask for last thread entering barrier to adjust thread count
	deAtomicIncrement32(&m_numRemoved);

	if (deAtomicIncrement32(&m_numEntered) == m_numThreads)
	{
		m_numLeaving  = m_numThreads;
		deMemoryReadWriteFence();

		// See sync() - m_numLeaving > 0 when manipulating m_numRemoved.
		// This thread must not be removed from m_numLeaving until
		// m_numRemoved has been updated.
		m_numThreads -= m_numRemoved;
		m_numLeaving -= (m_numRemoved-1);
		m_numRemoved  = 0;

		deMemoryReadWriteFence();
		m_numEntered  = 0;

		deAtomicDecrement32(&m_numLeaving);
	}
}

namespace
{

void singleThreadTest (SpinBarrier::WaitMode mode)
{
	SpinBarrier barrier(1);

	barrier.sync(mode);
	barrier.sync(mode);
	barrier.sync(mode);
}

class TestThread : public de::Thread
{
public:
	TestThread (SpinBarrier& barrier, volatile deInt32* sharedVar, int numThreads, int threadNdx)
		: m_barrier		(barrier)
		, m_sharedVar	(sharedVar)
		, m_numThreads	(numThreads)
		, m_threadNdx	(threadNdx)
		, m_busyOk		((deUint32)m_numThreads <= deGetNumAvailableLogicalCores())
	{
	}

	void run (void)
	{
		const int	numIters	= 10000;
		de::Random	rnd			(deInt32Hash(m_numThreads) ^ deInt32Hash(m_threadNdx));

		for (int iterNdx = 0; iterNdx < numIters; iterNdx++)
		{
			// Phase 1: count up
			deAtomicIncrement32(m_sharedVar);

			// Verify
			m_barrier.sync(getWaitMode(rnd));

			DE_TEST_ASSERT(*m_sharedVar == m_numThreads);

			m_barrier.sync(getWaitMode(rnd));

			// Phase 2: count down
			deAtomicDecrement32(m_sharedVar);

			// Verify
			m_barrier.sync(getWaitMode(rnd));

			DE_TEST_ASSERT(*m_sharedVar == 0);

			m_barrier.sync(getWaitMode(rnd));
		}
	}

private:
	SpinBarrier&			m_barrier;
	volatile deInt32* const	m_sharedVar;
	const int				m_numThreads;
	const int				m_threadNdx;
	const bool				m_busyOk;

	SpinBarrier::WaitMode getWaitMode (de::Random& rnd)
	{
		static const SpinBarrier::WaitMode	s_allModes[]	=
		{
			SpinBarrier::WAIT_MODE_YIELD,
			SpinBarrier::WAIT_MODE_AUTO,
			SpinBarrier::WAIT_MODE_BUSY,
		};
		const int							numModes		= DE_LENGTH_OF_ARRAY(s_allModes) - (m_busyOk ? 0 : 1);

		return rnd.choose<SpinBarrier::WaitMode>(DE_ARRAY_BEGIN(s_allModes), DE_ARRAY_BEGIN(s_allModes) + numModes);
	}
};

void multiThreadTest (int numThreads)
{
	SpinBarrier					barrier		(numThreads);
	volatile deInt32			sharedVar	= 0;
	std::vector<TestThread*>	threads		(numThreads, static_cast<TestThread*>(DE_NULL));

	for (int ndx = 0; ndx < numThreads; ndx++)
	{
		threads[ndx] = new TestThread(barrier, &sharedVar, numThreads, ndx);
		DE_TEST_ASSERT(threads[ndx]);
		threads[ndx]->start();
	}

	for (int ndx = 0; ndx < numThreads; ndx++)
	{
		threads[ndx]->join();
		delete threads[ndx];
	}

	DE_TEST_ASSERT(sharedVar == 0);
}

void singleThreadRemoveTest (SpinBarrier::WaitMode mode)
{
	SpinBarrier barrier(3);

	barrier.removeThread(mode);
	barrier.removeThread(mode);
	barrier.sync(mode);
	barrier.removeThread(mode);

	barrier.reset(1);
	barrier.sync(mode);

	barrier.reset(2);
	barrier.removeThread(mode);
	barrier.sync(mode);
}

class TestExitThread : public de::Thread
{
public:
	TestExitThread (SpinBarrier& barrier, int numThreads, int threadNdx, SpinBarrier::WaitMode waitMode)
		: m_barrier		(barrier)
		, m_numThreads	(numThreads)
		, m_threadNdx	(threadNdx)
		, m_waitMode	(waitMode)
	{
	}

	void run (void)
	{
		const int	numIters	= 10000;
		de::Random	rnd			(deInt32Hash(m_numThreads) ^ deInt32Hash(m_threadNdx) ^ deInt32Hash((deInt32)m_waitMode));
		const int	invExitProb	= 1000;

		for (int iterNdx = 0; iterNdx < numIters; iterNdx++)
		{
			if (rnd.getInt(0, invExitProb) == 0)
			{
				m_barrier.removeThread(m_waitMode);
				break;
			}
			else
				m_barrier.sync(m_waitMode);
		}
	}

private:
	SpinBarrier&				m_barrier;
	const int					m_numThreads;
	const int					m_threadNdx;
	const SpinBarrier::WaitMode	m_waitMode;
};

void multiThreadRemoveTest (int numThreads, SpinBarrier::WaitMode waitMode)
{
	SpinBarrier						barrier		(numThreads);
	std::vector<TestExitThread*>	threads		(numThreads, static_cast<TestExitThread*>(DE_NULL));

	for (int ndx = 0; ndx < numThreads; ndx++)
	{
		threads[ndx] = new TestExitThread(barrier, numThreads, ndx, waitMode);
		DE_TEST_ASSERT(threads[ndx]);
		threads[ndx]->start();
	}

	for (int ndx = 0; ndx < numThreads; ndx++)
	{
		threads[ndx]->join();
		delete threads[ndx];
	}
}

} // anonymous

void SpinBarrier_selfTest (void)
{
	singleThreadTest(SpinBarrier::WAIT_MODE_YIELD);
	singleThreadTest(SpinBarrier::WAIT_MODE_BUSY);
	singleThreadTest(SpinBarrier::WAIT_MODE_AUTO);
	multiThreadTest(1);
	multiThreadTest(2);
	multiThreadTest(4);
	multiThreadTest(8);
	multiThreadTest(16);

	singleThreadRemoveTest(SpinBarrier::WAIT_MODE_YIELD);
	singleThreadRemoveTest(SpinBarrier::WAIT_MODE_BUSY);
	singleThreadRemoveTest(SpinBarrier::WAIT_MODE_AUTO);
	multiThreadRemoveTest(1, SpinBarrier::WAIT_MODE_BUSY);
	multiThreadRemoveTest(2, SpinBarrier::WAIT_MODE_AUTO);
	multiThreadRemoveTest(4, SpinBarrier::WAIT_MODE_AUTO);
	multiThreadRemoveTest(8, SpinBarrier::WAIT_MODE_AUTO);
	multiThreadRemoveTest(16, SpinBarrier::WAIT_MODE_AUTO);
	multiThreadRemoveTest(1, SpinBarrier::WAIT_MODE_YIELD);
	multiThreadRemoveTest(2, SpinBarrier::WAIT_MODE_YIELD);
	multiThreadRemoveTest(4, SpinBarrier::WAIT_MODE_YIELD);
	multiThreadRemoveTest(8, SpinBarrier::WAIT_MODE_YIELD);
	multiThreadRemoveTest(16, SpinBarrier::WAIT_MODE_YIELD);
}

} // de
<commit_msg>am e36eed77: Fix deSpinBarrier deadlock.<commit_after>/*-------------------------------------------------------------------------
 * drawElements C++ Base Library
 * -----------------------------
 *
 * Copyright 2015 The Android Open Source Project
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *      http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 *
 *//*!
 * \file
 * \brief Cross-thread barrier.
 *//*--------------------------------------------------------------------*/

#include "deSpinBarrier.hpp"
#include "deThread.hpp"
#include "deRandom.hpp"
#include "deInt32.h"

#include <vector>

namespace de
{

SpinBarrier::SpinBarrier (deInt32 numThreads)
	: m_numCores	(deGetNumAvailableLogicalCores())
	, m_numThreads	(numThreads)
	, m_numEntered	(0)
	, m_numLeaving	(0)
	, m_numRemoved	(0)
{
	DE_ASSERT(numThreads > 0);
}

SpinBarrier::~SpinBarrier (void)
{
	DE_ASSERT(m_numEntered == 0 && m_numLeaving == 0);
}

void SpinBarrier::reset (deUint32 numThreads)
{
	// If last threads were removed, m_numEntered > 0 && m_numRemoved > 0
	DE_ASSERT(m_numLeaving == 0);
	DE_ASSERT(numThreads > 0);
	m_numThreads = numThreads;
	m_numEntered = 0;
	m_numLeaving = 0;
	m_numRemoved = 0;
}

inline SpinBarrier::WaitMode getWaitMode (SpinBarrier::WaitMode requested, deUint32 numCores, deInt32 numThreads)
{
	if (requested == SpinBarrier::WAIT_MODE_AUTO)
		return ((deUint32)numThreads <= numCores) ? SpinBarrier::WAIT_MODE_BUSY : SpinBarrier::WAIT_MODE_YIELD;
	else
		return requested;
}

inline void wait (SpinBarrier::WaitMode mode)
{
	DE_ASSERT(mode == SpinBarrier::WAIT_MODE_YIELD || mode == SpinBarrier::WAIT_MODE_BUSY);

	if (mode == SpinBarrier::WAIT_MODE_YIELD)
		deYield();
}

void SpinBarrier::sync (WaitMode requestedMode)
{
	const deInt32	cachedNumThreads	= m_numThreads;
	const WaitMode	waitMode			= getWaitMode(requestedMode, m_numCores, cachedNumThreads);

	deMemoryReadWriteFence();

	// m_numEntered must not be touched until all threads have had
	// a chance to observe it being 0.
	if (m_numLeaving > 0)
	{
		for (;;)
		{
			if (m_numLeaving == 0)
				break;

			wait(waitMode);
		}
	}

	// If m_numRemoved > 0, m_numThreads will decrease. If m_numThreads is decreased
	// just after atomicOp and before comparison, the branch could be taken by multiple
	// threads. Since m_numThreads only changes if all threads are inside the spinbarrier,
	// cached value at snapshotted at the beginning of the function will be equal for
	// all threads.
	if (deAtomicIncrement32(&m_numEntered) == cachedNumThreads)
	{
		// Release all waiting threads. Since this thread has not been removed, m_numLeaving will
		// be >= 1 until m_numLeaving is decremented at the end of this function.
		m_numThreads -= m_numRemoved;
		m_numLeaving  = m_numThreads;
		m_numRemoved  = 0;

		deMemoryReadWriteFence();
		m_numEntered  = 0;
	}
	else
	{
		for (;;)
		{
			if (m_numEntered == 0)
				break;

			wait(waitMode);
		}
	}

	deAtomicDecrement32(&m_numLeaving);
	deMemoryReadWriteFence();
}

void SpinBarrier::removeThread (WaitMode requestedMode)
{
	const deInt32	cachedNumThreads	= m_numThreads;
	const WaitMode	waitMode			= getWaitMode(requestedMode, m_numCores, cachedNumThreads);

	// Wait for other threads exiting previous barrier
	if (m_numLeaving > 0)
	{
		for (;;)
		{
			if (m_numLeaving == 0)
				break;

			wait(waitMode);
		}
	}

	// Ask for last thread entering barrier to adjust thread count
	deAtomicIncrement32(&m_numRemoved);

	// See sync() - use cached value
	if (deAtomicIncrement32(&m_numEntered) == cachedNumThreads)
	{
		// Release all waiting threads.
		m_numThreads -= m_numRemoved;
		m_numLeaving  = m_numThreads;
		m_numRemoved  = 0;

		deMemoryReadWriteFence();
		m_numEntered  = 0;
	}
}

namespace
{

void singleThreadTest (SpinBarrier::WaitMode mode)
{
	SpinBarrier barrier(1);

	barrier.sync(mode);
	barrier.sync(mode);
	barrier.sync(mode);
}

class TestThread : public de::Thread
{
public:
	TestThread (SpinBarrier& barrier, volatile deInt32* sharedVar, int numThreads, int threadNdx)
		: m_barrier		(barrier)
		, m_sharedVar	(sharedVar)
		, m_numThreads	(numThreads)
		, m_threadNdx	(threadNdx)
		, m_busyOk		((deUint32)m_numThreads <= deGetNumAvailableLogicalCores())
	{
	}

	void run (void)
	{
		const int	numIters	= 10000;
		de::Random	rnd			(deInt32Hash(m_numThreads) ^ deInt32Hash(m_threadNdx));

		for (int iterNdx = 0; iterNdx < numIters; iterNdx++)
		{
			// Phase 1: count up
			deAtomicIncrement32(m_sharedVar);

			// Verify
			m_barrier.sync(getWaitMode(rnd));

			DE_TEST_ASSERT(*m_sharedVar == m_numThreads);

			m_barrier.sync(getWaitMode(rnd));

			// Phase 2: count down
			deAtomicDecrement32(m_sharedVar);

			// Verify
			m_barrier.sync(getWaitMode(rnd));

			DE_TEST_ASSERT(*m_sharedVar == 0);

			m_barrier.sync(getWaitMode(rnd));
		}
	}

private:
	SpinBarrier&			m_barrier;
	volatile deInt32* const	m_sharedVar;
	const int				m_numThreads;
	const int				m_threadNdx;
	const bool				m_busyOk;

	SpinBarrier::WaitMode getWaitMode (de::Random& rnd)
	{
		static const SpinBarrier::WaitMode	s_allModes[]	=
		{
			SpinBarrier::WAIT_MODE_YIELD,
			SpinBarrier::WAIT_MODE_AUTO,
			SpinBarrier::WAIT_MODE_BUSY,
		};
		const int							numModes		= DE_LENGTH_OF_ARRAY(s_allModes) - (m_busyOk ? 0 : 1);

		return rnd.choose<SpinBarrier::WaitMode>(DE_ARRAY_BEGIN(s_allModes), DE_ARRAY_BEGIN(s_allModes) + numModes);
	}
};

void multiThreadTest (int numThreads)
{
	SpinBarrier					barrier		(numThreads);
	volatile deInt32			sharedVar	= 0;
	std::vector<TestThread*>	threads		(numThreads, static_cast<TestThread*>(DE_NULL));

	for (int ndx = 0; ndx < numThreads; ndx++)
	{
		threads[ndx] = new TestThread(barrier, &sharedVar, numThreads, ndx);
		DE_TEST_ASSERT(threads[ndx]);
		threads[ndx]->start();
	}

	for (int ndx = 0; ndx < numThreads; ndx++)
	{
		threads[ndx]->join();
		delete threads[ndx];
	}

	DE_TEST_ASSERT(sharedVar == 0);
}

void singleThreadRemoveTest (SpinBarrier::WaitMode mode)
{
	SpinBarrier barrier(3);

	barrier.removeThread(mode);
	barrier.removeThread(mode);
	barrier.sync(mode);
	barrier.removeThread(mode);

	barrier.reset(1);
	barrier.sync(mode);

	barrier.reset(2);
	barrier.removeThread(mode);
	barrier.sync(mode);
}

class TestExitThread : public de::Thread
{
public:
	TestExitThread (SpinBarrier& barrier, int numThreads, int threadNdx, SpinBarrier::WaitMode waitMode)
		: m_barrier		(barrier)
		, m_numThreads	(numThreads)
		, m_threadNdx	(threadNdx)
		, m_waitMode	(waitMode)
	{
	}

	void run (void)
	{
		const int	numIters	= 10000;
		de::Random	rnd			(deInt32Hash(m_numThreads) ^ deInt32Hash(m_threadNdx) ^ deInt32Hash((deInt32)m_waitMode));
		const int	invExitProb	= 1000;

		for (int iterNdx = 0; iterNdx < numIters; iterNdx++)
		{
			if (rnd.getInt(0, invExitProb) == 0)
			{
				m_barrier.removeThread(m_waitMode);
				break;
			}
			else
				m_barrier.sync(m_waitMode);
		}
	}

private:
	SpinBarrier&				m_barrier;
	const int					m_numThreads;
	const int					m_threadNdx;
	const SpinBarrier::WaitMode	m_waitMode;
};

void multiThreadRemoveTest (int numThreads, SpinBarrier::WaitMode waitMode)
{
	SpinBarrier						barrier		(numThreads);
	std::vector<TestExitThread*>	threads		(numThreads, static_cast<TestExitThread*>(DE_NULL));

	for (int ndx = 0; ndx < numThreads; ndx++)
	{
		threads[ndx] = new TestExitThread(barrier, numThreads, ndx, waitMode);
		DE_TEST_ASSERT(threads[ndx]);
		threads[ndx]->start();
	}

	for (int ndx = 0; ndx < numThreads; ndx++)
	{
		threads[ndx]->join();
		delete threads[ndx];
	}
}

} // anonymous

void SpinBarrier_selfTest (void)
{
	singleThreadTest(SpinBarrier::WAIT_MODE_YIELD);
	singleThreadTest(SpinBarrier::WAIT_MODE_BUSY);
	singleThreadTest(SpinBarrier::WAIT_MODE_AUTO);
	multiThreadTest(1);
	multiThreadTest(2);
	multiThreadTest(4);
	multiThreadTest(8);
	multiThreadTest(16);

	singleThreadRemoveTest(SpinBarrier::WAIT_MODE_YIELD);
	singleThreadRemoveTest(SpinBarrier::WAIT_MODE_BUSY);
	singleThreadRemoveTest(SpinBarrier::WAIT_MODE_AUTO);
	multiThreadRemoveTest(1, SpinBarrier::WAIT_MODE_BUSY);
	multiThreadRemoveTest(2, SpinBarrier::WAIT_MODE_AUTO);
	multiThreadRemoveTest(4, SpinBarrier::WAIT_MODE_AUTO);
	multiThreadRemoveTest(8, SpinBarrier::WAIT_MODE_AUTO);
	multiThreadRemoveTest(16, SpinBarrier::WAIT_MODE_AUTO);
	multiThreadRemoveTest(1, SpinBarrier::WAIT_MODE_YIELD);
	multiThreadRemoveTest(2, SpinBarrier::WAIT_MODE_YIELD);
	multiThreadRemoveTest(4, SpinBarrier::WAIT_MODE_YIELD);
	multiThreadRemoveTest(8, SpinBarrier::WAIT_MODE_YIELD);
	multiThreadRemoveTest(16, SpinBarrier::WAIT_MODE_YIELD);
}

} // de
<|endoftext|>