<commit_before>#pragma once
//=========================================================================//
/*!	@file
	@brief	RX24T グループ・DOC 定義
    @author 平松邦仁 (hira@rvf-rc45.net)
	@copyright	Copyright (C) 2017, 2022 Kunihito Hiramatsu @n
				Released under the MIT license @n
				https://github.com/hirakuni45/RX/blob/master/LICENSE
*/
//=========================================================================//
#include "common/device.hpp"

namespace device {

	//+++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++//
	/*!
		@brief  データ演算回路（DOC）
		@param[in]	base	ベース・アドレス
		@param[in]	per		ペリフェラル型
	*/
	//+++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++//
	template <uint32_t base, peripheral per>
	struct doc_t {

		static constexpr auto PERIPHERAL = per;		///< ペリフェラル型

		//-----------------------------------------------------------------//
		/*!
			@brief  DOC コントロールレジスタ（DOCR）
			@param[in]	ofs	オフセット
		*/
		//-----------------------------------------------------------------//
		template <uint32_t ofs>
		struct docr_t : public rw8_t<ofs> {
			typedef rw8_t<ofs> io_;
			using io_::operator =;
			using io_::operator ();
			using io_::operator |=;
			using io_::operator &=;

			bits_rw_t<io_, bitpos::B0, 2>  OMS;
			bit_rw_t <io_, bitpos::B2>     DCSEL;
			bit_rw_t <io_, bitpos::B4>     DOPCIE;
			bit_rw_t <io_, bitpos::B5>     DOPCF;
			bit_rw_t <io_, bitpos::B6>     DOPCFCL;
		};
		typedef docr_t<base + 0x00> DOCR_;
		static  DOCR_ DOCR;

		//-----------------------------------------------------------------//
		/*!
			@brief  DOC データインプットレジスタ（DODIR）
		*/
		//-----------------------------------------------------------------//
		typedef rw16_t<base + 0x02> DODIR_;
		static  DODIR_ DODIR;

		//-----------------------------------------------------------------//
		/*!
			@brief  DOC データセッティングレジスタ（DODSR）
		*/
		//-----------------------------------------------------------------//
		typedef rw16_t<base + 0x04> DODSR_;
		static  DODSR_ DODSR;

	};
	template <uint32_t base, peripheral per>
		typename doc_t<base, per>::DOCR_ doc_t<base, per>::DOCR;
	template <uint32_t base, peripheral per>
		typename doc_t<base, per>::DODIR_ doc_t<base, per>::DODIR;
	template <uint32_t base, peripheral per>
		typename doc_t<base, per>::DODSR_ doc_t<base, per>::DODSR;

#if defined(SIG_RX24T)
	//+++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++//
	/*!
		@brief  データ演算回路（DOC）
		@param[in]	base	ベース・アドレス
		@param[in]	vec		割り込みベクター
	*/
	//+++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++//
	template <uint32_t base, peripheral per, ICU::VECTOR vec>
	struct doc_norm_t : public doc_t<base, per> {

		static constexpr auto I_VEC = vec;	///< 割り込みベクター

	};
	typedef doc_norm_t<0x0008'B080, peripheral::DOC, ICU::VECTOR::DOPCF> DOC;
#endif

#if defined(SIG_RX64M) || defined(SIG_RX71M) || defined(SIG_RX65N) || defined(SIG_RX66T) || defined(SIG_RX72T)
	//+++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++//
	/*!
		@brief  データ演算回路（DOC）
		@param[in]	base	ベース・アドレス
		@param[in]	vec		割り込みベクター
	*/
	//+++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++//
	template <uint32_t base, peripheral per, ICU::VECTOR_BL0 vec>
	struct doc_gbl0_t : public doc_t<base, per> {

		static constexpr auto I_VEC = vec;	///< 割り込みベクター

	};
	typedef doc_gbl0_t<0x0008'B080, peripheral::DOC, ICU::VECTOR_BL0::DOPCI> DOC;
#endif
}
<commit_msg>Update: Add RX63T<commit_after>#pragma once
//=========================================================================//
/*!	@file
	@brief	RX600 グループ・DOC 定義
    @author 平松邦仁 (hira@rvf-rc45.net)
	@copyright	Copyright (C) 2017, 2022 Kunihito Hiramatsu @n
				Released under the MIT license @n
				https://github.com/hirakuni45/RX/blob/master/LICENSE
*/
//=========================================================================//
#include "common/device.hpp"

namespace device {

	//+++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++//
	/*!
		@brief  データ演算回路（DOC）
		@param[in]	base	ベース・アドレス
		@param[in]	per		ペリフェラル型
	*/
	//+++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++//
	template <uint32_t base, peripheral per>
	struct doc_t {

		static constexpr auto PERIPHERAL = per;		///< ペリフェラル型

		//-----------------------------------------------------------------//
		/*!
			@brief  DOC コントロールレジスタ（DOCR）
			@param[in]	ofs	オフセット
		*/
		//-----------------------------------------------------------------//
		template <uint32_t ofs>
		struct docr_t : public rw8_t<ofs> {
			typedef rw8_t<ofs> io_;
			using io_::operator =;
			using io_::operator ();
			using io_::operator |=;
			using io_::operator &=;

			bits_rw_t<io_, bitpos::B0, 2>  OMS;
			bit_rw_t <io_, bitpos::B2>     DCSEL;

			bit_rw_t <io_, bitpos::B4>     DOPCIE;
			bit_rw_t <io_, bitpos::B5>     DOPCF;
			bit_rw_t <io_, bitpos::B6>     DOPCFCL;
		};
		typedef docr_t<base + 0x00> DOCR_;
		static  DOCR_ DOCR;

		//-----------------------------------------------------------------//
		/*!
			@brief  DOC データインプットレジスタ（DODIR）
		*/
		//-----------------------------------------------------------------//
		typedef rw16_t<base + 0x02> DODIR_;
		static  DODIR_ DODIR;

		//-----------------------------------------------------------------//
		/*!
			@brief  DOC データセッティングレジスタ（DODSR）
		*/
		//-----------------------------------------------------------------//
		typedef rw16_t<base + 0x04> DODSR_;
		static  DODSR_ DODSR;

	};
	template <uint32_t base, peripheral per>
		typename doc_t<base, per>::DOCR_ doc_t<base, per>::DOCR;
	template <uint32_t base, peripheral per>
		typename doc_t<base, per>::DODIR_ doc_t<base, per>::DODIR;
	template <uint32_t base, peripheral per>
		typename doc_t<base, per>::DODSR_ doc_t<base, per>::DODSR;

#if defined(SIG_RX63T) || defined(SIG_RX24T)
	//+++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++//
	/*!
		@brief  データ演算回路（DOC）
		@param[in]	base	ベース・アドレス
		@param[in]	vec		割り込みベクター
	*/
	//+++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++//
	template <uint32_t base, peripheral per, ICU::VECTOR vec>
	struct doc_norm_t : public doc_t<base, per> {

		static constexpr auto I_VEC = vec;	///< 割り込みベクター

	};
	typedef doc_norm_t<0x0008'B080, peripheral::DOC, ICU::VECTOR::DOPCF> DOC;
#endif

#if defined(SIG_RX64M) || defined(SIG_RX71M) || defined(SIG_RX65N) || defined(SIG_RX66T) || defined(SIG_RX72T)
	//+++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++//
	/*!
		@brief  データ演算回路（DOC）
		@param[in]	base	ベース・アドレス
		@param[in]	vec		割り込みベクター
	*/
	//+++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++//
	template <uint32_t base, peripheral per, ICU::VECTOR_BL0 vec>
	struct doc_gbl0_t : public doc_t<base, per> {

		static constexpr auto I_VEC = vec;	///< 割り込みベクター

	};
	typedef doc_gbl0_t<0x0008'B080, peripheral::DOC, ICU::VECTOR_BL0::DOPCI> DOC;
#endif
}
<|endoftext|>


<commit_before>/*
 * Copyright (C) 2013 The Android Open Source Project
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *      http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

// #define VERBOSE_DEBUG

#define LOG_TAG "Minikin"
#include <cutils/log.h>

#include "MinikinInternal.h"
#include <minikin/CmapCoverage.h>
#include <minikin/FontCollection.h>

using std::vector;

namespace android {

template <typename T>
static inline T max(T a, T b) {
    return a>b ? a : b;
}

uint32_t FontCollection::sNextId = 0;

FontCollection::FontCollection(const vector<FontFamily*>& typefaces) :
    mMaxChar(0) {
    AutoMutex _l(gMinikinLock);
    mId = sNextId++;
    vector<uint32_t> lastChar;
    size_t nTypefaces = typefaces.size();
#ifdef VERBOSE_DEBUG
    ALOGD("nTypefaces = %d\n", nTypefaces);
#endif
    const FontStyle defaultStyle;
    for (size_t i = 0; i < nTypefaces; i++) {
        FontFamily* family = typefaces[i];
        MinikinFont* typeface = family->getClosestMatch(defaultStyle).font;
        if (typeface == NULL) {
            ALOGE("FontCollection: closest match was null");
            // TODO: we shouldn't hit this, as there should be more robust
            // checks upstream to prevent empty/invalid FontFamily objects
            continue;
        }
        family->RefLocked();
        FontInstance dummy;
        mInstances.push_back(dummy);  // emplace_back would be better
        FontInstance* instance = &mInstances.back();
        instance->mFamily = family;
        instance->mCoverage = new SparseBitSet;
#ifdef VERBOSE_DEBUG
        ALOGD("closest match = %p, family size = %d\n", typeface, family->getNumFonts());
#endif
        const uint32_t cmapTag = MinikinFont::MakeTag('c', 'm', 'a', 'p');
        size_t cmapSize = 0;
        bool ok = typeface->GetTable(cmapTag, NULL, &cmapSize);
        UniquePtr<uint8_t[]> cmapData(new uint8_t[cmapSize]);
        ok = typeface->GetTable(cmapTag, cmapData.get(), &cmapSize);
        CmapCoverage::getCoverage(*instance->mCoverage, cmapData.get(), cmapSize);
#ifdef VERBOSE_DEBUG
        ALOGD("font coverage length=%d, first ch=%x\n", instance->mCoverage->length(),
                instance->mCoverage->nextSetBit(0));
#endif
        mMaxChar = max(mMaxChar, instance->mCoverage->length());
        lastChar.push_back(instance->mCoverage->nextSetBit(0));
    }
    nTypefaces = mInstances.size();
    size_t nPages = (mMaxChar + kPageMask) >> kLogCharsPerPage;
    size_t offset = 0;
    for (size_t i = 0; i < nPages; i++) {
        Range dummy;
        mRanges.push_back(dummy);
        Range* range = &mRanges.back();
#ifdef VERBOSE_DEBUG
        ALOGD("i=%d: range start = %d\n", i, offset);
#endif
        range->start = offset;
        for (size_t j = 0; j < nTypefaces; j++) {
            if (lastChar[j] < (i + 1) << kLogCharsPerPage) {
                const FontInstance* instance = &mInstances[j];
                mInstanceVec.push_back(instance);
                offset++;
                uint32_t nextChar = instance->mCoverage->nextSetBit((i + 1) << kLogCharsPerPage);
#ifdef VERBOSE_DEBUG
                ALOGD("nextChar = %d (j = %d)\n", nextChar, j);
#endif
                lastChar[j] = nextChar;
            }
        }
        range->end = offset;
    }
}

FontCollection::~FontCollection() {
    for (size_t i = 0; i < mInstances.size(); i++) {
        delete mInstances[i].mCoverage;
        mInstances[i].mFamily->UnrefLocked();
    }
}

// Implement heuristic for choosing best-match font. Here are the rules:
// 1. If first font in the collection has the character, it wins.
// 2. If a font matches both language and script, it gets a score of 4.
// 3. If a font matches just language, it gets a score of 2.
// 4. Matching the "compact" or "elegant" variant adds one to the score.
// 5. Highest score wins, with ties resolved to the first font.
const FontCollection::FontInstance* FontCollection::getInstanceForChar(uint32_t ch,
            FontLanguage lang, int variant) const {
    if (ch >= mMaxChar) {
        return NULL;
    }
    const Range& range = mRanges[ch >> kLogCharsPerPage];
#ifdef VERBOSE_DEBUG
    ALOGD("querying range %d:%d\n", range.start, range.end);
#endif
    const FontInstance* bestInstance = NULL;
    int bestScore = -1;
    for (size_t i = range.start; i < range.end; i++) {
        const FontInstance* instance = mInstanceVec[i];
        if (instance->mCoverage->get(ch)) {
            FontFamily* family = instance->mFamily;
            // First font family in collection always matches
            if (mInstances[0].mFamily == family) {
                return instance;
            }
            int score = lang.match(family->lang()) * 2;
            if (variant != 0 && variant == family->variant()) {
                score++;
            }
            if (score > bestScore) {
                bestScore = score;
                bestInstance = instance;
            }
        }
    }
    return bestInstance;
}

const uint32_t NBSP = 0xa0;
const uint32_t ZWJ = 0x200c;
const uint32_t ZWNJ = 0x200d;
// Characters where we want to continue using existing font run instead of
// recomputing the best match in the fallback list.
static const uint32_t stickyWhitelist[] = { '!', ',', '.', ':', ';', '?', NBSP, ZWJ, ZWNJ };

static bool isStickyWhitelisted(uint32_t c) {
    for (size_t i = 0; i < sizeof(stickyWhitelist) / sizeof(stickyWhitelist[0]); i++) {
        if (stickyWhitelist[i] == c) return true;
    }
    return false;
}

void FontCollection::itemize(const uint16_t *string, size_t string_size, FontStyle style,
        vector<Run>* result) const {
    FontLanguage lang = style.getLanguage();
    int variant = style.getVariant();
    const FontInstance* lastInstance = NULL;
    Run* run = NULL;
    int nShorts;
    for (size_t i = 0; i < string_size; i += nShorts) {
        nShorts = 1;
        uint32_t ch = string[i];
        // sigh, decode UTF-16 by hand here
        if ((ch & 0xfc00) == 0xd800) {
            if ((i + 1) < string_size) {
                ch = 0x10000 + ((ch & 0x3ff) << 10) + (string[i + 1] & 0x3ff);
                nShorts = 2;
            }
        }
        // Continue using existing font as long as it has coverage and is whitelisted
        if (lastInstance == NULL
                || !(isStickyWhitelisted(ch) && lastInstance->mCoverage->get(ch))) {
            const FontInstance* instance = getInstanceForChar(ch, lang, variant);
            if (i == 0 || instance != lastInstance) {
                Run dummy;
                result->push_back(dummy);
                run = &result->back();
                if (instance == NULL) {
                    run->fakedFont.font = NULL;
                } else {
                    run->fakedFont = instance->mFamily->getClosestMatch(style);
                }
                lastInstance = instance;
                run->start = i;
            }
        }
        run->end = i + nShorts;
    }
}

MinikinFont* FontCollection::baseFont(FontStyle style) {
    return baseFontFaked(style).font;
}

FakedFont FontCollection::baseFontFaked(FontStyle style) {
    if (mInstances.empty()) {
        return FakedFont();
    }
    return mInstances[0].mFamily->getClosestMatch(style);
}

uint32_t FontCollection::getId() const {
    return mId;
}

}  // namespace android
<commit_msg>Assign non-coverage font runs to base font<commit_after>/*
 * Copyright (C) 2013 The Android Open Source Project
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *      http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

// #define VERBOSE_DEBUG

#define LOG_TAG "Minikin"
#include <cutils/log.h>

#include "MinikinInternal.h"
#include <minikin/CmapCoverage.h>
#include <minikin/FontCollection.h>

using std::vector;

namespace android {

template <typename T>
static inline T max(T a, T b) {
    return a>b ? a : b;
}

uint32_t FontCollection::sNextId = 0;

FontCollection::FontCollection(const vector<FontFamily*>& typefaces) :
    mMaxChar(0) {
    AutoMutex _l(gMinikinLock);
    mId = sNextId++;
    vector<uint32_t> lastChar;
    size_t nTypefaces = typefaces.size();
#ifdef VERBOSE_DEBUG
    ALOGD("nTypefaces = %d\n", nTypefaces);
#endif
    const FontStyle defaultStyle;
    for (size_t i = 0; i < nTypefaces; i++) {
        FontFamily* family = typefaces[i];
        MinikinFont* typeface = family->getClosestMatch(defaultStyle).font;
        if (typeface == NULL) {
            continue;
        }
        family->RefLocked();
        FontInstance dummy;
        mInstances.push_back(dummy);  // emplace_back would be better
        FontInstance* instance = &mInstances.back();
        instance->mFamily = family;
        instance->mCoverage = new SparseBitSet;
#ifdef VERBOSE_DEBUG
        ALOGD("closest match = %p, family size = %d\n", typeface, family->getNumFonts());
#endif
        const uint32_t cmapTag = MinikinFont::MakeTag('c', 'm', 'a', 'p');
        size_t cmapSize = 0;
        bool ok = typeface->GetTable(cmapTag, NULL, &cmapSize);
        UniquePtr<uint8_t[]> cmapData(new uint8_t[cmapSize]);
        ok = typeface->GetTable(cmapTag, cmapData.get(), &cmapSize);
        CmapCoverage::getCoverage(*instance->mCoverage, cmapData.get(), cmapSize);
#ifdef VERBOSE_DEBUG
        ALOGD("font coverage length=%d, first ch=%x\n", instance->mCoverage->length(),
                instance->mCoverage->nextSetBit(0));
#endif
        mMaxChar = max(mMaxChar, instance->mCoverage->length());
        lastChar.push_back(instance->mCoverage->nextSetBit(0));
    }
    nTypefaces = mInstances.size();
    LOG_ALWAYS_FATAL_IF(nTypefaces == 0,
        "Font collection must have at least one valid typeface");
    size_t nPages = (mMaxChar + kPageMask) >> kLogCharsPerPage;
    size_t offset = 0;
    for (size_t i = 0; i < nPages; i++) {
        Range dummy;
        mRanges.push_back(dummy);
        Range* range = &mRanges.back();
#ifdef VERBOSE_DEBUG
        ALOGD("i=%d: range start = %d\n", i, offset);
#endif
        range->start = offset;
        for (size_t j = 0; j < nTypefaces; j++) {
            if (lastChar[j] < (i + 1) << kLogCharsPerPage) {
                const FontInstance* instance = &mInstances[j];
                mInstanceVec.push_back(instance);
                offset++;
                uint32_t nextChar = instance->mCoverage->nextSetBit((i + 1) << kLogCharsPerPage);
#ifdef VERBOSE_DEBUG
                ALOGD("nextChar = %d (j = %d)\n", nextChar, j);
#endif
                lastChar[j] = nextChar;
            }
        }
        range->end = offset;
    }
}

FontCollection::~FontCollection() {
    for (size_t i = 0; i < mInstances.size(); i++) {
        delete mInstances[i].mCoverage;
        mInstances[i].mFamily->UnrefLocked();
    }
}

// Implement heuristic for choosing best-match font. Here are the rules:
// 1. If first font in the collection has the character, it wins.
// 2. If a font matches both language and script, it gets a score of 4.
// 3. If a font matches just language, it gets a score of 2.
// 4. Matching the "compact" or "elegant" variant adds one to the score.
// 5. Highest score wins, with ties resolved to the first font.
const FontCollection::FontInstance* FontCollection::getInstanceForChar(uint32_t ch,
            FontLanguage lang, int variant) const {
    if (ch >= mMaxChar) {
        return NULL;
    }
    const Range& range = mRanges[ch >> kLogCharsPerPage];
#ifdef VERBOSE_DEBUG
    ALOGD("querying range %d:%d\n", range.start, range.end);
#endif
    const FontInstance* bestInstance = NULL;
    int bestScore = -1;
    for (size_t i = range.start; i < range.end; i++) {
        const FontInstance* instance = mInstanceVec[i];
        if (instance->mCoverage->get(ch)) {
            FontFamily* family = instance->mFamily;
            // First font family in collection always matches
            if (mInstances[0].mFamily == family) {
                return instance;
            }
            int score = lang.match(family->lang()) * 2;
            if (variant != 0 && variant == family->variant()) {
                score++;
            }
            if (score > bestScore) {
                bestScore = score;
                bestInstance = instance;
            }
        }
    }
    if (bestInstance == NULL) {
        bestInstance = &mInstances[0];
    }
    return bestInstance;
}

const uint32_t NBSP = 0xa0;
const uint32_t ZWJ = 0x200c;
const uint32_t ZWNJ = 0x200d;
// Characters where we want to continue using existing font run instead of
// recomputing the best match in the fallback list.
static const uint32_t stickyWhitelist[] = { '!', ',', '.', ':', ';', '?', NBSP, ZWJ, ZWNJ };

static bool isStickyWhitelisted(uint32_t c) {
    for (size_t i = 0; i < sizeof(stickyWhitelist) / sizeof(stickyWhitelist[0]); i++) {
        if (stickyWhitelist[i] == c) return true;
    }
    return false;
}

void FontCollection::itemize(const uint16_t *string, size_t string_size, FontStyle style,
        vector<Run>* result) const {
    FontLanguage lang = style.getLanguage();
    int variant = style.getVariant();
    const FontInstance* lastInstance = NULL;
    Run* run = NULL;
    int nShorts;
    for (size_t i = 0; i < string_size; i += nShorts) {
        nShorts = 1;
        uint32_t ch = string[i];
        // sigh, decode UTF-16 by hand here
        if ((ch & 0xfc00) == 0xd800) {
            if ((i + 1) < string_size) {
                ch = 0x10000 + ((ch & 0x3ff) << 10) + (string[i + 1] & 0x3ff);
                nShorts = 2;
            }
        }
        // Continue using existing font as long as it has coverage and is whitelisted
        if (lastInstance == NULL
                || !(isStickyWhitelisted(ch) && lastInstance->mCoverage->get(ch))) {
            const FontInstance* instance = getInstanceForChar(ch, lang, variant);
            if (i == 0 || instance != lastInstance) {
                Run dummy;
                result->push_back(dummy);
                run = &result->back();
                if (instance == NULL) {
                    run->fakedFont.font = NULL;
                } else {
                    run->fakedFont = instance->mFamily->getClosestMatch(style);
                }
                lastInstance = instance;
                run->start = i;
            }
        }
        run->end = i + nShorts;
    }
}

MinikinFont* FontCollection::baseFont(FontStyle style) {
    return baseFontFaked(style).font;
}

FakedFont FontCollection::baseFontFaked(FontStyle style) {
    if (mInstances.empty()) {
        return FakedFont();
    }
    return mInstances[0].mFamily->getClosestMatch(style);
}

uint32_t FontCollection::getId() const {
    return mId;
}

}  // namespace android
<|endoftext|>


<commit_before>/******************************************************************************
 *                    _   _____   __________                                  *
 *                   | | / / _ | / __/_  __/     Visibility                   *
 *                   | |/ / __ |_\ \  / /          Across                     *
 *                   |___/_/ |_/___/ /_/       Space and Time                 *
 *                                                                            *
 * This file is part of VAST. It is subject to the license terms in the       *
 * LICENSE file found in the top-level directory of this distribution and at  *
 * http://vast.io/license. No part of VAST, including this file, may be       *
 * copied, modified, propagated, or distributed except according to the terms *
 * contained in the LICENSE file.                                             *
 ******************************************************************************/

#include <caf/all.hpp>

#include "vast/concept/printable/std/chrono.hpp"
#include "vast/concept/printable/to_string.hpp"
#include "vast/concept/printable/vast/bitmap.hpp"
#include "vast/concept/printable/vast/event.hpp"
#include "vast/concept/printable/vast/expression.hpp"
#include "vast/concept/printable/vast/uuid.hpp"
#include "vast/detail/assert.hpp"
#include "vast/detail/fill_status_map.hpp"
#include "vast/event.hpp"
#include "vast/expression_visitors.hpp"
#include "vast/logger.hpp"
#include "vast/table_slice.hpp"
#include "vast/to_events.hpp"

#include "vast/system/archive.hpp"
#include "vast/system/atoms.hpp"
#include "vast/system/exporter.hpp"

#include "vast/detail/assert.hpp"

using namespace std::chrono;
using namespace std::string_literals;
using namespace caf;

namespace vast {
namespace system {

namespace {

void ship_results(stateful_actor<exporter_state>* self) {
  VAST_TRACE("");
  if (self->state.results.empty() || self->state.query.requested == 0) {
    return;
  }
  VAST_DEBUG(self, "relays", self->state.results.size(), "events");
  message msg;
  if (self->state.results.size() <= self->state.query.requested) {
    self->state.query.requested -= self->state.results.size();
    self->state.query.shipped += self->state.results.size();
    msg = make_message(std::move(self->state.results));
    self->state.results = {};
  } else {
    std::vector<event> remainder;
    remainder.reserve(self->state.results.size() - self->state.query.requested);
    auto begin = self->state.results.begin() + self->state.query.requested;
    auto end = self->state.results.end();
    std::move(begin, end, std::back_inserter(remainder));
    self->state.results.resize(self->state.query.requested);
    msg = make_message(std::move(self->state.results));
    self->state.results = std::move(remainder);
    self->state.query.shipped += self->state.query.requested;
    self->state.query.requested = 0;
  }
  self->send(self->state.sink, msg);
}

void report_statistics(stateful_actor<exporter_state>* self) {
  auto& st = self->state;
  timespan runtime = steady_clock::now() - st.start;
  st.query.runtime = runtime;
  VAST_INFO(self, "shipped", st.query.shipped, "results out of",
            st.query.processed, "candidates in", vast::to_string(runtime));
  self->send(st.sink, st.id, st.query);
  if (st.accountant) {
    auto hits = rank(st.hits);
    auto processed = st.query.processed;
    auto shipped = st.query.shipped;
    auto results = shipped + st.results.size();
    auto selectivity = double(results) / processed;
    auto msg = report{{"exporter.hits", hits},
                      {"exporter.processed", processed},
                      {"exporter.results", results},
                      {"exporter.shipped", shipped},
                      {"exporter.selectivity", selectivity},
                      {"exporter.runtime", runtime}};
    self->send(st.accountant, msg);
  }
}

void shutdown(stateful_actor<exporter_state>* self, caf::error err) {
  VAST_DEBUG(self, "initiates shutdown with error", self->system().render(err));
  self->send_exit(self, std::move(err));
}

void shutdown(stateful_actor<exporter_state>* self) {
  if (has_continuous_option(self->state.options))
    return;
  VAST_DEBUG(self, "initiates shutdown");
  self->send_exit(self, exit_reason::normal);
}

void request_more_hits(stateful_actor<exporter_state>* self) {
  const auto& st = self->state;
  if (!has_historical_option(st.options))
    return;
  auto waiting_for_hits =
    st.query.received == st.query.scheduled;
  auto need_more_results = st.query.requested > 0;
  auto have_no_inflight_requests =
    st.query.lookups_issued == st.query.lookups_complete;
  // If we're (1) no longer waiting for index hits, (2) still need more
  // results, and (3) have no inflight requests to the archive, we ask
  // the index for more hits.
  if (!waiting_for_hits && need_more_results && have_no_inflight_requests) {
    auto remaining = st.query.expected - st.query.received;
    VAST_ASSERT(remaining > 0);
    // TODO: Figure out right number of partitions to ask for. For now, we
    // bound the number by an arbitrary constant.
    auto n = std::min(remaining, size_t{2});
    VAST_DEBUG(self, "asks index to process", n, "more partitions");
    self->send(st.index, st.id, n);
  }
}

} // namespace <anonymous>

caf::settings exporter_state::status() {
  caf::settings result;
  put(result, "hits", rank(hits));
  put(result, "start", caf::deep_to_string(start));
  put(result, "id", to_string(id));
  put(result, "expression", to_string(expr));
  return result;
}

behavior exporter(stateful_actor<exporter_state>* self, expression expr,
                  query_options options) {
  auto eu = self->system().dummy_execution_unit();
  self->state.sink = actor_pool::make(eu, actor_pool::broadcast());
  if (auto a = self->system().registry().get(accountant_atom::value)) {
    self->state.accountant = actor_cast<accountant_type>(a);
    self->send(self->state.accountant, announce_atom::value, self->name());
  }
  self->state.options = options;
  self->state.expr = std::move(expr);
  if (has_continuous_option(options))
    VAST_DEBUG(self, "has continuous query option");
  self->set_exit_handler(
    [=](const exit_msg& msg) {
      VAST_DEBUG(self, "received exit from", msg.source, "with reason:", msg.reason);
      self->send<message_priority::high>(self->state.index, self->state.id, 0);
      self->send(self->state.sink, sys_atom::value, delete_atom::value);
      self->send_exit(self->state.sink, msg.reason);
      self->quit(msg.reason);
      if (msg.reason != exit_reason::kill)
        report_statistics(self);
    }
  );
  self->set_down_handler(
    [=](const down_msg& msg) {
      VAST_DEBUG(self, "received DOWN from", msg.source);
      if (has_continuous_option(self->state.options)
          && (msg.source == self->state.archive
              || msg.source == self->state.index))
        report_statistics(self);
    }
  );
  auto finished = [&](const query_status& qs) -> bool {
    return qs.received == qs.expected
      && qs.lookups_issued == qs.lookups_complete;
  };
  auto handle_batch = [=](std::vector<event> candidates) {
    auto& st = self->state;
    VAST_DEBUG(self, "got batch of", candidates.size(), "events");
    auto sender = self->current_sender();
    for (auto& candidate : candidates) {
      auto& checker = st.checkers[candidate.type()];
      // Construct a candidate checker if we don't have one for this type.
      if (caf::holds_alternative<caf::none_t>(checker)) {
        auto x = tailor(st.expr, candidate.type());
        if (!x) {
          VAST_ERROR(self, "failed to tailor expression:",
                     self->system().render(x.error()));
          ship_results(self);
          self->send_exit(self, exit_reason::normal);
          return;
        }
        checker = std::move(*x);
        VAST_DEBUG(self, "tailored AST to", candidate.type() << ':', checker);
      }
      // Perform candidate check and keep event as result on success.
      if (caf::visit(event_evaluator{candidate}, checker))
        st.results.push_back(std::move(candidate));
      else
        VAST_DEBUG(self, "ignores false positive:", candidate);
    }
    st.query.processed += candidates.size();
    ship_results(self);
  };
  return {
    // The INDEX (or the EVALUATOR, to be more precise) sends us a series of
    // `ids` in response to an expression (query), terminated by 'done'.
    [=](ids& hits) -> caf::result<void> {
      auto& st = self->state;
      // Skip results that arrive before we got our lookup handle from the
      // INDEX actor.
      if (st.query.expected == 0)
        return caf::skip;
      // Add `hits` to the total result set and update all stats.
      timespan runtime = steady_clock::now() - st.start;
      st.query.runtime = runtime;
      auto count = rank(hits);
      if (st.accountant) {
        auto r = report{};
        if (st.hits.empty())
          r.push_back({"exporter.hits.first", runtime});
        r.push_back({"exporter.hits.arrived", runtime});
        r.push_back({"exporter.hits.count", count});
        self->send(st.accountant, r);
      }
      if (count == 0) {
        VAST_WARNING(self, "got empty hits");
      } else {
        VAST_DEBUG(self, "got", count, "index hits in [", (select(hits, 1)),
                   ',', (select(hits, -1) + 1), ')');
        st.hits |= hits;
        VAST_DEBUG(self, "forwards hits to archive");
        // FIXME: restrict according to configured limit.
        ++st.query.lookups_issued;
        self->send(st.archive, std::move(hits));
      }
      return caf::unit;
    },
    [=](table_slice_ptr slice) {
      handle_batch(to_events(*slice, self->state.hits));
    },
    [=](done_atom) {
      // Figure out if we're done by bumping the counter for `received` and
      // check whether it reaches `expected`.
      auto& st = self->state;
      timespan runtime = steady_clock::now() - st.start;
      st.query.runtime = runtime;
      st.query.received += st.query.scheduled;
      if (st.query.received < st.query.expected) {
        VAST_DEBUG(self, "received", st.query.received, '/',
                   st.query.expected, "ID sets");
        request_more_hits(self);
      } else {
        VAST_DEBUG(self, "received all", st.query.expected,
                   "ID set(s) in", vast::to_string(runtime));
        if (st.accountant)
          self->send(st.accountant, "exporter.hits.runtime", runtime);
        if (finished(st.query))
          shutdown(self);
      }
    },
    [=](done_atom, const caf::error& err) {
      auto& st = self->state;
      auto sender = self->current_sender();
      if (sender == st.archive) {
        if (err)
          VAST_DEBUG(self, "received error from archive:",
              self->system().render(err));
        ++st.query.lookups_complete;
      }
      if (finished(st.query))
        shutdown(self);
    },
    [=](extract_atom) {
      VAST_DEBUG(self, "got request to extract all events");
      if (self->state.query.requested == max_events) {
        VAST_WARNING(self, "ignores extract request, already getting all");
        return;
      }
      self->state.query.requested = max_events;
      ship_results(self);
      request_more_hits(self);
    },
    [=](extract_atom, uint64_t requested) {
      if (self->state.query.requested == max_events) {
        VAST_WARNING(self, "ignores extract request, already getting all");
        return;
      }
      auto n = std::min(max_events - requested, requested);
      self->state.query.requested += n;
      VAST_DEBUG(self, "got request to extract", n, "new events in addition to",
                 self->state.query.requested, "pending results");
      ship_results(self);
      request_more_hits(self);
    },
    [=](status_atom) {
      auto result = self->state.status();
      detail::fill_status_map(result, self);
      return result;
    },
    [=](const archive_type& archive) {
      VAST_DEBUG(self, "registers archive", archive);
      self->state.archive = archive;
      if (has_continuous_option(self->state.options))
        self->monitor(archive);
      // Register self at the archive
      if (has_historical_option(self->state.options))
        self->send(archive, exporter_atom::value, self);
    },
    [=](index_atom, const actor& index) {
      VAST_DEBUG(self, "registers index", index);
      self->state.index = index;
      if (has_continuous_option(self->state.options))
        self->monitor(index);
    },
    [=](sink_atom, const actor& sink) {
      VAST_DEBUG(self, "registers sink", sink);
      self->send(self->state.sink, sys_atom::value, put_atom::value, sink);
      self->monitor(self->state.sink);
    },
    [=](importer_atom, const std::vector<actor>& importers) {
      // Register for events at running IMPORTERs.
      if (has_continuous_option(self->state.options))
        for (auto& x : importers)
          self->send(x, exporter_atom::value, self);
    },
    [=](run_atom) {
      VAST_INFO(self, "executes query", self->state.expr);
      self->state.start = steady_clock::now();
      if (!has_historical_option(self->state.options))
        return;
      self->request(self->state.index, infinite, self->state.expr).then(
        [=](const uuid& lookup, uint32_t partitions, uint32_t scheduled) {
          VAST_DEBUG(self, "got lookup handle", lookup << ", scheduled",
                     scheduled << '/' << partitions, "partitions");
          self->state.id = lookup;
          if (partitions > 0) {
            self->state.query.expected = partitions;
            self->state.query.scheduled = scheduled;
          } else {
            shutdown(self);
          }
        },
        [=](const error& e) {
          shutdown(self, e);
        }
      );
    },
    [=](caf::stream<table_slice_ptr> in) {
      return self->make_sink(
        in,
        [](caf::unit_t&) {
          // nop
        },
        [=](caf::unit_t&, const table_slice_ptr& slice) {
          // TODO: port to new table slice API
          auto candidates = to_events(*slice);
          handle_batch(std::move(candidates));
        },
        [=](caf::unit_t&, const error& err) {
          VAST_IGNORE_UNUSED(err);
          VAST_ERROR(self, "got error during streaming: ", err);
        }
      );
    },
  };
}

} // namespace system
} // namespace vast
<commit_msg>Polish EXPORTER logging output<commit_after>/******************************************************************************
 *                    _   _____   __________                                  *
 *                   | | / / _ | / __/_  __/     Visibility                   *
 *                   | |/ / __ |_\ \  / /          Across                     *
 *                   |___/_/ |_/___/ /_/       Space and Time                 *
 *                                                                            *
 * This file is part of VAST. It is subject to the license terms in the       *
 * LICENSE file found in the top-level directory of this distribution and at  *
 * http://vast.io/license. No part of VAST, including this file, may be       *
 * copied, modified, propagated, or distributed except according to the terms *
 * contained in the LICENSE file.                                             *
 ******************************************************************************/

#include <caf/all.hpp>

#include "vast/concept/printable/std/chrono.hpp"
#include "vast/concept/printable/to_string.hpp"
#include "vast/concept/printable/vast/bitmap.hpp"
#include "vast/concept/printable/vast/event.hpp"
#include "vast/concept/printable/vast/expression.hpp"
#include "vast/concept/printable/vast/uuid.hpp"
#include "vast/detail/assert.hpp"
#include "vast/detail/fill_status_map.hpp"
#include "vast/event.hpp"
#include "vast/expression_visitors.hpp"
#include "vast/logger.hpp"
#include "vast/table_slice.hpp"
#include "vast/to_events.hpp"

#include "vast/system/archive.hpp"
#include "vast/system/atoms.hpp"
#include "vast/system/exporter.hpp"

#include "vast/detail/assert.hpp"

using namespace std::chrono;
using namespace std::string_literals;
using namespace caf;

namespace vast {
namespace system {

namespace {

void ship_results(stateful_actor<exporter_state>* self) {
  VAST_TRACE("");
  if (self->state.results.empty() || self->state.query.requested == 0) {
    return;
  }
  VAST_DEBUG(self, "relays", self->state.results.size(), "events");
  message msg;
  if (self->state.results.size() <= self->state.query.requested) {
    self->state.query.requested -= self->state.results.size();
    self->state.query.shipped += self->state.results.size();
    msg = make_message(std::move(self->state.results));
    self->state.results = {};
  } else {
    std::vector<event> remainder;
    remainder.reserve(self->state.results.size() - self->state.query.requested);
    auto begin = self->state.results.begin() + self->state.query.requested;
    auto end = self->state.results.end();
    std::move(begin, end, std::back_inserter(remainder));
    self->state.results.resize(self->state.query.requested);
    msg = make_message(std::move(self->state.results));
    self->state.results = std::move(remainder);
    self->state.query.shipped += self->state.query.requested;
    self->state.query.requested = 0;
  }
  self->send(self->state.sink, msg);
}

void report_statistics(stateful_actor<exporter_state>* self) {
  auto& st = self->state;
  timespan runtime = steady_clock::now() - st.start;
  st.query.runtime = runtime;
  VAST_INFO(self, "processed", st.query.processed, "candidates in",
            vast::to_string(runtime), "and shipped", st.query.shipped,
            "results");
  self->send(st.sink, st.id, st.query);
  if (st.accountant) {
    auto hits = rank(st.hits);
    auto processed = st.query.processed;
    auto shipped = st.query.shipped;
    auto results = shipped + st.results.size();
    auto selectivity = double(results) / processed;
    auto msg = report{{"exporter.hits", hits},
                      {"exporter.processed", processed},
                      {"exporter.results", results},
                      {"exporter.shipped", shipped},
                      {"exporter.selectivity", selectivity},
                      {"exporter.runtime", runtime}};
    self->send(st.accountant, msg);
  }
}

void shutdown(stateful_actor<exporter_state>* self, caf::error err) {
  VAST_DEBUG(self, "initiates shutdown with error", self->system().render(err));
  self->send_exit(self, std::move(err));
}

void shutdown(stateful_actor<exporter_state>* self) {
  if (has_continuous_option(self->state.options))
    return;
  VAST_DEBUG(self, "initiates shutdown");
  self->send_exit(self, exit_reason::normal);
}

void request_more_hits(stateful_actor<exporter_state>* self) {
  const auto& st = self->state;
  if (!has_historical_option(st.options))
    return;
  auto waiting_for_hits =
    st.query.received == st.query.scheduled;
  auto need_more_results = st.query.requested > 0;
  auto have_no_inflight_requests =
    st.query.lookups_issued == st.query.lookups_complete;
  // If we're (1) no longer waiting for index hits, (2) still need more
  // results, and (3) have no inflight requests to the archive, we ask
  // the index for more hits.
  if (!waiting_for_hits && need_more_results && have_no_inflight_requests) {
    auto remaining = st.query.expected - st.query.received;
    VAST_ASSERT(remaining > 0);
    // TODO: Figure out right number of partitions to ask for. For now, we
    // bound the number by an arbitrary constant.
    auto n = std::min(remaining, size_t{2});
    VAST_DEBUG(self, "asks index to process", n, "more partitions");
    self->send(st.index, st.id, n);
  }
}

} // namespace <anonymous>

caf::settings exporter_state::status() {
  caf::settings result;
  put(result, "hits", rank(hits));
  put(result, "start", caf::deep_to_string(start));
  put(result, "id", to_string(id));
  put(result, "expression", to_string(expr));
  return result;
}

behavior exporter(stateful_actor<exporter_state>* self, expression expr,
                  query_options options) {
  auto eu = self->system().dummy_execution_unit();
  self->state.sink = actor_pool::make(eu, actor_pool::broadcast());
  if (auto a = self->system().registry().get(accountant_atom::value)) {
    self->state.accountant = actor_cast<accountant_type>(a);
    self->send(self->state.accountant, announce_atom::value, self->name());
  }
  self->state.options = options;
  self->state.expr = std::move(expr);
  if (has_continuous_option(options))
    VAST_DEBUG(self, "has continuous query option");
  self->set_exit_handler(
    [=](const exit_msg& msg) {
      VAST_DEBUG(self, "received exit from", msg.source, "with reason:", msg.reason);
      self->send<message_priority::high>(self->state.index, self->state.id, 0);
      self->send(self->state.sink, sys_atom::value, delete_atom::value);
      self->send_exit(self->state.sink, msg.reason);
      self->quit(msg.reason);
      if (msg.reason != exit_reason::kill)
        report_statistics(self);
    }
  );
  self->set_down_handler(
    [=](const down_msg& msg) {
      VAST_DEBUG(self, "received DOWN from", msg.source);
      if (has_continuous_option(self->state.options)
          && (msg.source == self->state.archive
              || msg.source == self->state.index))
        report_statistics(self);
    }
  );
  auto finished = [&](const query_status& qs) -> bool {
    return qs.received == qs.expected
      && qs.lookups_issued == qs.lookups_complete;
  };
  auto handle_batch = [=](std::vector<event> candidates) {
    auto& st = self->state;
    VAST_DEBUG(self, "got batch of", candidates.size(), "events");
    auto sender = self->current_sender();
    for (auto& candidate : candidates) {
      auto& checker = st.checkers[candidate.type()];
      // Construct a candidate checker if we don't have one for this type.
      if (caf::holds_alternative<caf::none_t>(checker)) {
        auto x = tailor(st.expr, candidate.type());
        if (!x) {
          VAST_ERROR(self, "failed to tailor expression:",
                     self->system().render(x.error()));
          ship_results(self);
          self->send_exit(self, exit_reason::normal);
          return;
        }
        checker = std::move(*x);
        VAST_DEBUG(self, "tailored AST to", candidate.type() << ':', checker);
      }
      // Perform candidate check and keep event as result on success.
      if (caf::visit(event_evaluator{candidate}, checker))
        st.results.push_back(std::move(candidate));
      else
        VAST_DEBUG(self, "ignores false positive:", candidate);
    }
    st.query.processed += candidates.size();
    ship_results(self);
  };
  return {
    // The INDEX (or the EVALUATOR, to be more precise) sends us a series of
    // `ids` in response to an expression (query), terminated by 'done'.
    [=](ids& hits) -> caf::result<void> {
      auto& st = self->state;
      // Skip results that arrive before we got our lookup handle from the
      // INDEX actor.
      if (st.query.expected == 0)
        return caf::skip;
      // Add `hits` to the total result set and update all stats.
      timespan runtime = steady_clock::now() - st.start;
      st.query.runtime = runtime;
      auto count = rank(hits);
      if (st.accountant) {
        auto r = report{};
        if (st.hits.empty())
          r.push_back({"exporter.hits.first", runtime});
        r.push_back({"exporter.hits.arrived", runtime});
        r.push_back({"exporter.hits.count", count});
        self->send(st.accountant, r);
      }
      if (count == 0) {
        VAST_WARNING(self, "got empty hits");
      } else {
        VAST_DEBUG(self, "got", count, "index hits in [", (select(hits, 1)),
                   ',', (select(hits, -1) + 1), ')');
        st.hits |= hits;
        VAST_DEBUG(self, "forwards hits to archive");
        // FIXME: restrict according to configured limit.
        ++st.query.lookups_issued;
        self->send(st.archive, std::move(hits));
      }
      return caf::unit;
    },
    [=](table_slice_ptr slice) {
      handle_batch(to_events(*slice, self->state.hits));
    },
    [=](done_atom) {
      // Figure out if we're done by bumping the counter for `received` and
      // check whether it reaches `expected`.
      auto& st = self->state;
      timespan runtime = steady_clock::now() - st.start;
      st.query.runtime = runtime;
      st.query.received += st.query.scheduled;
      if (st.query.received < st.query.expected) {
        VAST_DEBUG(self, "received", st.query.received, '/',
                   st.query.expected, "ID sets");
        request_more_hits(self);
      } else {
        VAST_DEBUG(self, "received all", st.query.expected,
                   "ID set(s) in", vast::to_string(runtime));
        if (st.accountant)
          self->send(st.accountant, "exporter.hits.runtime", runtime);
        if (finished(st.query))
          shutdown(self);
      }
    },
    [=](done_atom, const caf::error& err) {
      auto& st = self->state;
      auto sender = self->current_sender();
      if (sender == st.archive) {
        if (err)
          VAST_DEBUG(self, "received error from archive:",
              self->system().render(err));
        ++st.query.lookups_complete;
      }
      if (finished(st.query))
        shutdown(self);
    },
    [=](extract_atom) {
      VAST_DEBUG(self, "got request to extract all events");
      if (self->state.query.requested == max_events) {
        VAST_WARNING(self, "ignores extract request, already getting all");
        return;
      }
      self->state.query.requested = max_events;
      ship_results(self);
      request_more_hits(self);
    },
    [=](extract_atom, uint64_t requested) {
      if (self->state.query.requested == max_events) {
        VAST_WARNING(self, "ignores extract request, already getting all");
        return;
      }
      auto n = std::min(max_events - requested, requested);
      self->state.query.requested += n;
      VAST_DEBUG(self, "got request to extract", n, "new events in addition to",
                 self->state.query.requested, "pending results");
      ship_results(self);
      request_more_hits(self);
    },
    [=](status_atom) {
      auto result = self->state.status();
      detail::fill_status_map(result, self);
      return result;
    },
    [=](const archive_type& archive) {
      VAST_DEBUG(self, "registers archive", archive);
      self->state.archive = archive;
      if (has_continuous_option(self->state.options))
        self->monitor(archive);
      // Register self at the archive
      if (has_historical_option(self->state.options))
        self->send(archive, exporter_atom::value, self);
    },
    [=](index_atom, const actor& index) {
      VAST_DEBUG(self, "registers index", index);
      self->state.index = index;
      if (has_continuous_option(self->state.options))
        self->monitor(index);
    },
    [=](sink_atom, const actor& sink) {
      VAST_DEBUG(self, "registers sink", sink);
      self->send(self->state.sink, sys_atom::value, put_atom::value, sink);
      self->monitor(self->state.sink);
    },
    [=](importer_atom, const std::vector<actor>& importers) {
      // Register for events at running IMPORTERs.
      if (has_continuous_option(self->state.options))
        for (auto& x : importers)
          self->send(x, exporter_atom::value, self);
    },
    [=](run_atom) {
      VAST_INFO(self, "executes query:", self->state.expr);
      self->state.start = steady_clock::now();
      if (!has_historical_option(self->state.options))
        return;
      self->request(self->state.index, infinite, self->state.expr).then(
        [=](const uuid& lookup, uint32_t partitions, uint32_t scheduled) {
          VAST_DEBUG(self, "got lookup handle", lookup << ", scheduled",
                     scheduled << '/' << partitions, "partitions");
          self->state.id = lookup;
          if (partitions > 0) {
            self->state.query.expected = partitions;
            self->state.query.scheduled = scheduled;
          } else {
            shutdown(self);
          }
        },
        [=](const error& e) {
          shutdown(self, e);
        }
      );
    },
    [=](caf::stream<table_slice_ptr> in) {
      return self->make_sink(
        in,
        [](caf::unit_t&) {
          // nop
        },
        [=](caf::unit_t&, const table_slice_ptr& slice) {
          // TODO: port to new table slice API
          auto candidates = to_events(*slice);
          handle_batch(std::move(candidates));
        },
        [=](caf::unit_t&, const error& err) {
          VAST_IGNORE_UNUSED(err);
          VAST_ERROR(self, "got error during streaming: ", err);
        }
      );
    },
  };
}

} // namespace system
} // namespace vast
<|endoftext|>


<commit_before>#include "Scheduler.h"
#include <cstdio>

using namespace std;
using namespace boost;

void Scheduler::addConnection(Connection *c)
{
    printf("addConnection: source [ %p idx %d ] -> sink [ %p idx %d]\n",
        c->m_sourceBlock, c->m_sourceIndex, c->m_sinkBlock, c->m_sinkIndex);

    vertex_t vert_source, vert_sink;
    bool found_source = false, found_sink = false;

    /* see if the source or sink vertex (or both) is already in the graph */
    auto v_it = vertices(m_graph);
    for (auto it = v_it.first; it != v_it.second; ++it)
    {
        vertex_t v = *it;

        if (m_graph[v] == c->m_sourceBlock)
        {
            printf("- source block %p is already in the graph.\n",
                c->m_sourceBlock);

            vert_source = v;
            found_source = true;
        }
        if (m_graph[v] == c->m_sinkBlock)
        {
            printf("- sink block %p is already in the graph.\n",
                c->m_sinkBlock);

            vert_sink = v;
            found_sink = true;
        }

        if (found_source && found_sink) break;
    }

    /* if source and/or sink were not found, add new vertex(es) to the graph */
    if (!found_source)
    {
        printf("- source block %p is not in the graph; creating.\n",
            c->m_sourceBlock);

        vert_source = add_vertex(m_graph);
        m_graph[vert_source] = c->m_sourceBlock;
    }
    if (!found_sink)
    {
        printf("- sink block %p is not in the graph; creating.\n",
            c->m_sinkBlock);

        vert_sink = add_vertex(m_graph);
        m_graph[vert_sink] = c->m_sinkBlock;
    }

    edge_t edge;
    bool found_edge = false;

    /* see if the edge is already in the graph */
    auto e_it = edges(m_graph);
    for (auto it = e_it.first; it != e_it.second; ++it)
    {
        edge_t e = *it;
        pair<int, int> indices = m_graph[e];

        vertex_t v1 = source(e, m_graph);
        vertex_t v2 = target(e, m_graph);

        if (m_graph[v1] == c->m_sourceBlock && m_graph[v2] == c->m_sinkBlock &&
            indices.first == c->m_sourceIndex &&
            indices.second == c->m_sinkIndex)
        {
            printf("- edge is already in the graph!\n");

            edge = e;
            found_edge = true;

            break;
        }
    }

    /* if edge was not found, add new edge to the graph */
    if (!found_edge)
    {
        printf("- edge does not exist in the graph; creating.\n");

        bool b;
        tie(edge, b) = add_edge(vert_source, vert_sink, m_graph);
        m_graph[edge] = pair<int, int>(c->m_sourceIndex, c->m_sinkIndex);

        /* bool b is supposed to be set false if the graph is configured to
         * disallow parallel edges (see documentation for add_edge);
         * currently this is not how our graph is set up.
         * if this can be set, then we don't need to hunt for duplicate edges */
    }
    else
    {
        /* probably want to throw an exception here instead of this nonsense */
        printf("- refusing to add a duplicate edge\n");
    }

    dumpGraph();

    m_needCheck = true;
}

void Scheduler::checkGraph(void)
{
    printf("checkGraph: checking graph validity\n");

    int errors = 0;

    /* ensure that no input is connected to more than one output
     * (no two edges should have the same sink block AND sink index) */
    auto e_it = edges(m_graph);
    for (auto it1 = e_it.first; it1 != e_it.second; ++it1)
    {
        edge_t e1 = *it1;

        Block *sinkBlock1 = m_graph[target(e1, m_graph)];
        int sinkIndex1 = m_graph[e1].second;

        for (auto it2 = e_it.first; it2 != e_it.second; ++it2)
        {
            if (it1 == it2) continue;

            edge_t e2 = *it2;

            Block *sinkBlock2 = m_graph[target(e2, m_graph)];
            int sinkIndex2 = m_graph[e2].second;

            if (sinkBlock1 == sinkBlock2 && sinkIndex1 == sinkIndex2)
            {
                printf("- sink [ %p idx %d ] has more than one source "
                    "connected!\n", sinkBlock1, sinkIndex1);
                ++errors;
            }
        }
    }

    printf("- %d errors\n", errors);
    if (errors != 0)
    {
        throw new runtime_error("Graph validation failed");
    }

    m_needCheck = false;
}

void Scheduler::dumpGraph(void)
{
    printf("\n====================== m_graph summary ======================\n");

    printf("%lu vertices\n", num_vertices(m_graph));
    auto v_it = vertices(m_graph);
    for (auto it = v_it.first; it != v_it.second; ++it)
    {
        vertex_t v = *it;
        printf(" + vert: %p\n", m_graph[v]);
    }

    printf("\n%lu edges\n", num_edges(m_graph));
    auto e_it = edges(m_graph);
    for (auto it = e_it.first; it != e_it.second; ++it)
    {
        edge_t e = *it;
        std::pair<int, int> indices = m_graph[e];

        vertex_t v1 = source(e, m_graph);
        vertex_t v2 = target(e, m_graph);

        printf(" + edge: [ %p idx %d ] -> [ %p idx %d ]\n",
            m_graph[v1], indices.first, m_graph[v2], indices.second);
    }

    printf("=============================================================\n\n");
}

void Scheduler::run(void)
{
    if (m_needCheck)
    {
        checkGraph();
    }

#if 0
    /* Call work on all blocks */
    for (boost::tie(i, end) = vertices(m_graph); i != end; ++i)
    {
        printf("run: calling work on %p\n", *i);
        (*i)->work();
    }

    /* Move samples from source blocks to sink blocks */
    graph_traits < adjacency_list <> >::vertex_iterator i, end;
    for (boost::tie(i, end) = vertices(m_graph); i != end; ++i)
    {
        while (!(*i)->outputEmpty())
        {
            int sample = (*i)->popOutput();
            /* Iterate over all sinks attached to this block */
            graph_traits < adjacency_list <> >::adjacency_iterator ai, a_end;
            for (boost::tie(ai, a_end) = adjacent_vertices(*i, m_graph);
                ai != a_end; ++ai)
            {
                printf("");
                (*ai)->pushInput(sample);
            }
        }
    }
#endif
}
<commit_msg>checkGraph: remove erroneous 'new' in throw statement<commit_after>#include "Scheduler.h"
#include <cstdio>

using namespace std;
using namespace boost;

void Scheduler::addConnection(Connection *c)
{
    printf("addConnection: source [ %p idx %d ] -> sink [ %p idx %d]\n",
        c->m_sourceBlock, c->m_sourceIndex, c->m_sinkBlock, c->m_sinkIndex);

    vertex_t vert_source, vert_sink;
    bool found_source = false, found_sink = false;

    /* see if the source or sink vertex (or both) is already in the graph */
    auto v_it = vertices(m_graph);
    for (auto it = v_it.first; it != v_it.second; ++it)
    {
        vertex_t v = *it;

        if (m_graph[v] == c->m_sourceBlock)
        {
            printf("- source block %p is already in the graph.\n",
                c->m_sourceBlock);

            vert_source = v;
            found_source = true;
        }
        if (m_graph[v] == c->m_sinkBlock)
        {
            printf("- sink block %p is already in the graph.\n",
                c->m_sinkBlock);

            vert_sink = v;
            found_sink = true;
        }

        if (found_source && found_sink) break;
    }

    /* if source and/or sink were not found, add new vertex(es) to the graph */
    if (!found_source)
    {
        printf("- source block %p is not in the graph; creating.\n",
            c->m_sourceBlock);

        vert_source = add_vertex(m_graph);
        m_graph[vert_source] = c->m_sourceBlock;
    }
    if (!found_sink)
    {
        printf("- sink block %p is not in the graph; creating.\n",
            c->m_sinkBlock);

        vert_sink = add_vertex(m_graph);
        m_graph[vert_sink] = c->m_sinkBlock;
    }

    edge_t edge;
    bool found_edge = false;

    /* see if the edge is already in the graph */
    auto e_it = edges(m_graph);
    for (auto it = e_it.first; it != e_it.second; ++it)
    {
        edge_t e = *it;
        pair<int, int> indices = m_graph[e];

        vertex_t v1 = source(e, m_graph);
        vertex_t v2 = target(e, m_graph);

        if (m_graph[v1] == c->m_sourceBlock && m_graph[v2] == c->m_sinkBlock &&
            indices.first == c->m_sourceIndex &&
            indices.second == c->m_sinkIndex)
        {
            printf("- edge is already in the graph!\n");

            edge = e;
            found_edge = true;

            break;
        }
    }

    /* if edge was not found, add new edge to the graph */
    if (!found_edge)
    {
        printf("- edge does not exist in the graph; creating.\n");

        bool b;
        tie(edge, b) = add_edge(vert_source, vert_sink, m_graph);
        m_graph[edge] = pair<int, int>(c->m_sourceIndex, c->m_sinkIndex);

        /* bool b is supposed to be set false if the graph is configured to
         * disallow parallel edges (see documentation for add_edge);
         * currently this is not how our graph is set up.
         * if this can be set, then we don't need to hunt for duplicate edges */
    }
    else
    {
        /* probably want to throw an exception here instead of this nonsense */
        printf("- refusing to add a duplicate edge\n");
    }

    dumpGraph();

    m_needCheck = true;
}

void Scheduler::checkGraph(void)
{
    printf("checkGraph: checking graph validity\n");

    int errors = 0;

    /* ensure that no input is connected to more than one output
     * (no two edges should have the same sink block AND sink index) */
    auto e_it = edges(m_graph);
    for (auto it1 = e_it.first; it1 != e_it.second; ++it1)
    {
        edge_t e1 = *it1;

        Block *sinkBlock1 = m_graph[target(e1, m_graph)];
        int sinkIndex1 = m_graph[e1].second;

        for (auto it2 = e_it.first; it2 != e_it.second; ++it2)
        {
            if (it1 == it2) continue;

            edge_t e2 = *it2;

            Block *sinkBlock2 = m_graph[target(e2, m_graph)];
            int sinkIndex2 = m_graph[e2].second;

            if (sinkBlock1 == sinkBlock2 && sinkIndex1 == sinkIndex2)
            {
                printf("- sink [ %p idx %d ] has more than one source "
                    "connected!\n", sinkBlock1, sinkIndex1);
                ++errors;
            }
        }
    }

    printf("- %d errors\n", errors);
    if (errors != 0)
    {
        throw runtime_error("Graph validation failed");
    }

    m_needCheck = false;
}

void Scheduler::dumpGraph(void)
{
    printf("\n====================== m_graph summary ======================\n");

    printf("%lu vertices\n", num_vertices(m_graph));
    auto v_it = vertices(m_graph);
    for (auto it = v_it.first; it != v_it.second; ++it)
    {
        vertex_t v = *it;
        printf(" + vert: %p\n", m_graph[v]);
    }

    printf("\n%lu edges\n", num_edges(m_graph));
    auto e_it = edges(m_graph);
    for (auto it = e_it.first; it != e_it.second; ++it)
    {
        edge_t e = *it;
        std::pair<int, int> indices = m_graph[e];

        vertex_t v1 = source(e, m_graph);
        vertex_t v2 = target(e, m_graph);

        printf(" + edge: [ %p idx %d ] -> [ %p idx %d ]\n",
            m_graph[v1], indices.first, m_graph[v2], indices.second);
    }

    printf("=============================================================\n\n");
}

void Scheduler::run(void)
{
    if (m_needCheck)
    {
        checkGraph();
    }

#if 0
    /* Call work on all blocks */
    for (boost::tie(i, end) = vertices(m_graph); i != end; ++i)
    {
        printf("run: calling work on %p\n", *i);
        (*i)->work();
    }

    /* Move samples from source blocks to sink blocks */
    graph_traits < adjacency_list <> >::vertex_iterator i, end;
    for (boost::tie(i, end) = vertices(m_graph); i != end; ++i)
    {
        while (!(*i)->outputEmpty())
        {
            int sample = (*i)->popOutput();
            /* Iterate over all sinks attached to this block */
            graph_traits < adjacency_list <> >::adjacency_iterator ai, a_end;
            for (boost::tie(ai, a_end) = adjacent_vertices(*i, m_graph);
                ai != a_end; ++ai)
            {
                printf("");
                (*ai)->pushInput(sample);
            }
        }
    }
#endif
}
<|endoftext|>


<commit_before>/*
 * Copyright (C) 2015, Nils Moehrle
 * TU Darmstadt - Graphics, Capture and Massively Parallel Computing
 * All rights reserved.
 *
 * This software may be modified and distributed under the terms
 * of the BSD 3-Clause license. See the LICENSE.txt file for details.
 */

#include <set>
#include <map>

#include <util/file_system.h>
#include <mve/image_tools.h>
#include <mve/image_io.h>

#include "texture_atlas.h"

TextureAtlas::TextureAtlas(unsigned int size) :
    size(size), padding(size >> 7), finalized(false) {

    bin = RectangularBin::create(size, size);
    image = mve::ByteImage::create(size, size, 3);
    validity_mask = mve::ByteImage::create(size, size, 1);
}

/**
  * Copies the src image into the dest image at the given position,
  * optionally adding a border.
  * @warning asserts that the given src image fits into the given dest image.
  */
void copy_into(mve::ByteImage::ConstPtr src, int x, int y,
    mve::ByteImage::Ptr dest, int border = 0) {

    assert(x >= 0 && x + src->width() + 2 * border <= dest->width());
    assert(y >= 0 && y + src->height() + 2 * border <= dest->height());

    for (int i = 0; i < src->width() + 2 * border; ++i) {
        for(int j = 0; j < src->height() + 2 * border; j++) {
            int sx = i - border;
            int sy = j - border;

            if (sx < 0 || sx >= src->width() || sy < 0 || sy >= src->height())
                continue;

            for (int c = 0; c < src->channels(); ++c) {
                dest->at(x + i, y + j, c) = src->at(sx, sy, c);
            }
        }
    }
}

typedef std::vector<std::pair<int, int> > PixelVector;
typedef std::set<std::pair<int, int> > PixelSet;

bool
TextureAtlas::insert(TexturePatch::ConstPtr texture_patch) {
    if (finalized) {
        throw util::Exception("No insertion possible, TextureAtlas already finalized");
    }

    assert(bin != NULL);
    assert(validity_mask != NULL);

    int const width = texture_patch->get_width() + 2 * padding;
    int const height = texture_patch->get_height() + 2 * padding;
    Rect<int> rect(0, 0, width, height);
    if (!bin->insert(&rect)) return false;

    /* Update texture atlas and its validity mask. */
    mve::ByteImage::Ptr patch_image = mve::image::float_to_byte_image(
        texture_patch->get_image(), 0.0f, 1.0f);
    mve::image::gamma_correct(patch_image, 1.0f / 2.2f);

    copy_into(patch_image, rect.min_x, rect.min_y, image, padding);
    mve::ByteImage::ConstPtr patch_validity_mask = texture_patch->get_validity_mask();
    copy_into(patch_validity_mask, rect.min_x, rect.min_y, validity_mask, padding);

    TexturePatch::Faces const & patch_faces = texture_patch->get_faces();
    TexturePatch::Texcoords const & patch_texcoords = texture_patch->get_texcoords();

    /* Calculate the offset of the texture patches' relative texture coordinates */
    math::Vec2f offset = math::Vec2f(rect.min_x + padding, rect.min_y + padding);

    faces.insert(faces.end(), patch_faces.begin(), patch_faces.end());

    /* Calculate the final textcoords of the faces. */
    for (std::size_t i = 0; i < patch_faces.size(); ++i) {
        for (int j = 0; j < 3; ++j) {
            math::Vec2f rel_texcoord(patch_texcoords[i * 3 + j]);
            math::Vec2f texcoord = rel_texcoord + offset;

            texcoord[0] = texcoord[0] / this->size;
            texcoord[1] = texcoord[1] / this->size;
            texcoords.push_back(texcoord);
        }
    }
    return true;
}

void
TextureAtlas::apply_edge_padding(void) {
    assert(image != NULL);
    assert(validity_mask != NULL);

    const int width = image->width();
    const int height = image->height();

    math::Matrix<float, 3, 3> gauss;
    gauss[0] = 1.0f; gauss[1] = 2.0f; gauss[2] = 1.0f;
    gauss[3] = 2.0f; gauss[4] = 4.0f; gauss[5] = 2.0f;
    gauss[6] = 1.0f; gauss[7] = 2.0f; gauss[8] = 1.0f;
    gauss /= 16.0f;

    /* Calculate the set of invalid pixels at the border of texture patches. */
    PixelSet invalid_border_pixels;
    for (int y = 0; y < height; ++y) {
        for (int x = 0; x < width; ++x) {
            if (validity_mask->at(x, y, 0) == 255) continue;

            /* Check the direct neighbourhood of all invalid pixels. */
            for (int j = -1; j <= 1; ++j) {
                for (int i = -1; i <= 1; ++i) {
                    int nx = x + i;
                    int ny = y + j;
                    /* If the invalid pixel has a valid neighbour: */
                    if (0 <= nx && nx < width &&
                        0 <= ny && ny < height &&
                        validity_mask->at(nx, ny, 0) == 255) {

                        /* Add the pixel to the set of invalid border pixels. */
                        invalid_border_pixels.insert(std::pair<int, int>(x, y));
                    }
                }
            }
        }
    }

    mve::ByteImage::Ptr new_validity_mask = validity_mask->duplicate();

    /* Iteratively dilate border pixels until padding constants are reached. */
    for (unsigned int n = 0; n <= padding; ++n) {
        PixelVector new_valid_pixels;

        PixelSet::iterator it = invalid_border_pixels.begin();
        for (;it != invalid_border_pixels.end(); it++) {
            int x = it->first;
            int y = it->second;

            bool now_valid = false;
            /* Calculate new pixel value. */
            for (int c = 0; c < 3; ++c) {
                float norm = 0.0f;
                float value = 0.0f;
                for (int j = -1; j <= 1; ++j) {
                    for (int i = -1; i <= 1; ++i) {
                        int nx = x + i;
                        int ny = y + j;
                        if (0 <= nx && nx < width &&
                            0 <= ny && ny < height &&
                            new_validity_mask->at(nx, ny, 0) == 255) {

                            float w = gauss[(j + 1) * 3 + (i + 1)];
                            norm += w;
                            value += (image->at(nx, ny, c) / 255.0f) * w;
                        }
                    }
                }

                if (norm == 0.0f)
                    continue;

                now_valid = true;
                image->at(x, y, c) = (value / norm) * 255.0f;
            }

            if (now_valid) {
                new_valid_pixels.push_back(*it);
            }
        }

        invalid_border_pixels.clear();

        /* Mark the new valid pixels valid in the validity mask. */
        for (std::size_t i = 0; i < new_valid_pixels.size(); ++i) {
             int x = new_valid_pixels[i].first;
             int y = new_valid_pixels[i].second;

             new_validity_mask->at(x, y, 0) = 255;
        }

        /* Calculate the set of invalid pixels at the border of the valid area. */
        for (std::size_t i = 0; i < new_valid_pixels.size(); ++i) {
            int x = new_valid_pixels[i].first;
            int y = new_valid_pixels[i].second;

            for (int j = -1; j <= 1; ++j) {
                 for (int i = -1; i <= 1; ++i) {
                     int nx = x + i;
                     int ny = y + j;
                     if (0 <= nx && nx < width &&
                         0 <= ny && ny < height &&
                         new_validity_mask->at(nx, ny, 0) == 0) {

                         invalid_border_pixels.insert(std::pair<int, int>(nx, ny));
                    }
                }
            }
        }
    }
}

struct VectorCompare {
    bool operator()(math::Vec2f const & lhs, math::Vec2f const & rhs) {
        return lhs[0] < rhs[0] || (lhs[0] == rhs[0] && lhs[1] < rhs[1]);
    }
};

typedef std::map<math::Vec2f, std::size_t, VectorCompare> TexcoordMap;

void
TextureAtlas::merge_texcoords() {
    Texcoords tmp; tmp.swap(this->texcoords);

    TexcoordMap texcoord_map;
    for (math::Vec2f const & texcoord : tmp) {
        TexcoordMap::iterator iter = texcoord_map.find(texcoord);
        if (iter == texcoord_map.end()) {
            std::size_t texcoord_id = this->texcoords.size();
            texcoord_map[texcoord] = texcoord_id;
            this->texcoords.push_back(texcoord);
            this->texcoord_ids.push_back(texcoord_id);
        } else {
            this->texcoord_ids.push_back(iter->second);
        }
    }

}

void
TextureAtlas::finalize() {
    if (finalized) {
        throw util::Exception("TextureAtlas already finalized");
    }

    this->bin.reset();
    this->apply_edge_padding();
    this->validity_mask.reset();
    this->merge_texcoords();

    this->finalized = true;
}
<commit_msg>Qualify VectorCompare::operator() as const<commit_after>/*
 * Copyright (C) 2015, Nils Moehrle
 * TU Darmstadt - Graphics, Capture and Massively Parallel Computing
 * All rights reserved.
 *
 * This software may be modified and distributed under the terms
 * of the BSD 3-Clause license. See the LICENSE.txt file for details.
 */

#include <set>
#include <map>

#include <util/file_system.h>
#include <mve/image_tools.h>
#include <mve/image_io.h>

#include "texture_atlas.h"

TextureAtlas::TextureAtlas(unsigned int size) :
    size(size), padding(size >> 7), finalized(false) {

    bin = RectangularBin::create(size, size);
    image = mve::ByteImage::create(size, size, 3);
    validity_mask = mve::ByteImage::create(size, size, 1);
}

/**
  * Copies the src image into the dest image at the given position,
  * optionally adding a border.
  * @warning asserts that the given src image fits into the given dest image.
  */
void copy_into(mve::ByteImage::ConstPtr src, int x, int y,
    mve::ByteImage::Ptr dest, int border = 0) {

    assert(x >= 0 && x + src->width() + 2 * border <= dest->width());
    assert(y >= 0 && y + src->height() + 2 * border <= dest->height());

    for (int i = 0; i < src->width() + 2 * border; ++i) {
        for(int j = 0; j < src->height() + 2 * border; j++) {
            int sx = i - border;
            int sy = j - border;

            if (sx < 0 || sx >= src->width() || sy < 0 || sy >= src->height())
                continue;

            for (int c = 0; c < src->channels(); ++c) {
                dest->at(x + i, y + j, c) = src->at(sx, sy, c);
            }
        }
    }
}

typedef std::vector<std::pair<int, int> > PixelVector;
typedef std::set<std::pair<int, int> > PixelSet;

bool
TextureAtlas::insert(TexturePatch::ConstPtr texture_patch) {
    if (finalized) {
        throw util::Exception("No insertion possible, TextureAtlas already finalized");
    }

    assert(bin != NULL);
    assert(validity_mask != NULL);

    int const width = texture_patch->get_width() + 2 * padding;
    int const height = texture_patch->get_height() + 2 * padding;
    Rect<int> rect(0, 0, width, height);
    if (!bin->insert(&rect)) return false;

    /* Update texture atlas and its validity mask. */
    mve::ByteImage::Ptr patch_image = mve::image::float_to_byte_image(
        texture_patch->get_image(), 0.0f, 1.0f);
    mve::image::gamma_correct(patch_image, 1.0f / 2.2f);

    copy_into(patch_image, rect.min_x, rect.min_y, image, padding);
    mve::ByteImage::ConstPtr patch_validity_mask = texture_patch->get_validity_mask();
    copy_into(patch_validity_mask, rect.min_x, rect.min_y, validity_mask, padding);

    TexturePatch::Faces const & patch_faces = texture_patch->get_faces();
    TexturePatch::Texcoords const & patch_texcoords = texture_patch->get_texcoords();

    /* Calculate the offset of the texture patches' relative texture coordinates */
    math::Vec2f offset = math::Vec2f(rect.min_x + padding, rect.min_y + padding);

    faces.insert(faces.end(), patch_faces.begin(), patch_faces.end());

    /* Calculate the final textcoords of the faces. */
    for (std::size_t i = 0; i < patch_faces.size(); ++i) {
        for (int j = 0; j < 3; ++j) {
            math::Vec2f rel_texcoord(patch_texcoords[i * 3 + j]);
            math::Vec2f texcoord = rel_texcoord + offset;

            texcoord[0] = texcoord[0] / this->size;
            texcoord[1] = texcoord[1] / this->size;
            texcoords.push_back(texcoord);
        }
    }
    return true;
}

void
TextureAtlas::apply_edge_padding(void) {
    assert(image != NULL);
    assert(validity_mask != NULL);

    const int width = image->width();
    const int height = image->height();

    math::Matrix<float, 3, 3> gauss;
    gauss[0] = 1.0f; gauss[1] = 2.0f; gauss[2] = 1.0f;
    gauss[3] = 2.0f; gauss[4] = 4.0f; gauss[5] = 2.0f;
    gauss[6] = 1.0f; gauss[7] = 2.0f; gauss[8] = 1.0f;
    gauss /= 16.0f;

    /* Calculate the set of invalid pixels at the border of texture patches. */
    PixelSet invalid_border_pixels;
    for (int y = 0; y < height; ++y) {
        for (int x = 0; x < width; ++x) {
            if (validity_mask->at(x, y, 0) == 255) continue;

            /* Check the direct neighbourhood of all invalid pixels. */
            for (int j = -1; j <= 1; ++j) {
                for (int i = -1; i <= 1; ++i) {
                    int nx = x + i;
                    int ny = y + j;
                    /* If the invalid pixel has a valid neighbour: */
                    if (0 <= nx && nx < width &&
                        0 <= ny && ny < height &&
                        validity_mask->at(nx, ny, 0) == 255) {

                        /* Add the pixel to the set of invalid border pixels. */
                        invalid_border_pixels.insert(std::pair<int, int>(x, y));
                    }
                }
            }
        }
    }

    mve::ByteImage::Ptr new_validity_mask = validity_mask->duplicate();

    /* Iteratively dilate border pixels until padding constants are reached. */
    for (unsigned int n = 0; n <= padding; ++n) {
        PixelVector new_valid_pixels;

        PixelSet::iterator it = invalid_border_pixels.begin();
        for (;it != invalid_border_pixels.end(); it++) {
            int x = it->first;
            int y = it->second;

            bool now_valid = false;
            /* Calculate new pixel value. */
            for (int c = 0; c < 3; ++c) {
                float norm = 0.0f;
                float value = 0.0f;
                for (int j = -1; j <= 1; ++j) {
                    for (int i = -1; i <= 1; ++i) {
                        int nx = x + i;
                        int ny = y + j;
                        if (0 <= nx && nx < width &&
                            0 <= ny && ny < height &&
                            new_validity_mask->at(nx, ny, 0) == 255) {

                            float w = gauss[(j + 1) * 3 + (i + 1)];
                            norm += w;
                            value += (image->at(nx, ny, c) / 255.0f) * w;
                        }
                    }
                }

                if (norm == 0.0f)
                    continue;

                now_valid = true;
                image->at(x, y, c) = (value / norm) * 255.0f;
            }

            if (now_valid) {
                new_valid_pixels.push_back(*it);
            }
        }

        invalid_border_pixels.clear();

        /* Mark the new valid pixels valid in the validity mask. */
        for (std::size_t i = 0; i < new_valid_pixels.size(); ++i) {
             int x = new_valid_pixels[i].first;
             int y = new_valid_pixels[i].second;

             new_validity_mask->at(x, y, 0) = 255;
        }

        /* Calculate the set of invalid pixels at the border of the valid area. */
        for (std::size_t i = 0; i < new_valid_pixels.size(); ++i) {
            int x = new_valid_pixels[i].first;
            int y = new_valid_pixels[i].second;

            for (int j = -1; j <= 1; ++j) {
                 for (int i = -1; i <= 1; ++i) {
                     int nx = x + i;
                     int ny = y + j;
                     if (0 <= nx && nx < width &&
                         0 <= ny && ny < height &&
                         new_validity_mask->at(nx, ny, 0) == 0) {

                         invalid_border_pixels.insert(std::pair<int, int>(nx, ny));
                    }
                }
            }
        }
    }
}

struct VectorCompare {
    bool operator()(math::Vec2f const & lhs, math::Vec2f const & rhs) const {
        return lhs[0] < rhs[0] || (lhs[0] == rhs[0] && lhs[1] < rhs[1]);
    }
};

typedef std::map<math::Vec2f, std::size_t, VectorCompare> TexcoordMap;

void
TextureAtlas::merge_texcoords() {
    Texcoords tmp; tmp.swap(this->texcoords);

    TexcoordMap texcoord_map;
    for (math::Vec2f const & texcoord : tmp) {
        TexcoordMap::iterator iter = texcoord_map.find(texcoord);
        if (iter == texcoord_map.end()) {
            std::size_t texcoord_id = this->texcoords.size();
            texcoord_map[texcoord] = texcoord_id;
            this->texcoords.push_back(texcoord);
            this->texcoord_ids.push_back(texcoord_id);
        } else {
            this->texcoord_ids.push_back(iter->second);
        }
    }

}

void
TextureAtlas::finalize() {
    if (finalized) {
        throw util::Exception("TextureAtlas already finalized");
    }

    this->bin.reset();
    this->apply_edge_padding();
    this->validity_mask.reset();
    this->merge_texcoords();

    this->finalized = true;
}
<|endoftext|>


<commit_before>#include <XdmfArray.h>
#include <XdmfHDF.h>

#include <mpi.h>

/// Simple memory buffer implementation that keeps track of it's stream pointer.
class Buffer {
private:
  std::size_t m_size;
  char* m_data;
  char* m_put;
  char* m_tell;

public:
  Buffer( std::size_t size ) :
    m_size( size ),
    m_data( new char[size] ),
    m_put( m_data ),
    m_tell( m_data )
  {}

  ~Buffer() {
    delete [] m_data;
  }

  /// put a single value into the buffer
  template< typename T >
  void put( const T& t ) {
    std::size_t size = sizeof( T );
    memcpy( m_put, &t, size );
    m_put += size;
  }

  /// copy a contiguous block into the buffer
  void put( void* data, std::size_t size ) {
    memcpy( m_put, data, size );
    m_put += size;
  }

  /// Copy a single value into the buffer.
  template< typename T >
  T tell() {
    std::size_t size = sizeof( T );
    T tmp;
    memcpy( &tmp, m_tell, size );
    m_tell += size;
    return tmp;
  }

  /// copy a contiguous block of data from the buffer to an already allocated
  /// location
  void tell( void* out, std::size_t size ) {
    memcpy( out, m_tell, size );
    m_tell += size;
  }

  std::size_t size() const {
    return m_size;
  }

  char* pointer() {
    return m_data;
  }

  void reset() {
    m_put = m_data;
    m_tell = m_data;
  }
};

/// Callback implements parallel IO by communicating to rank 0 in MPI_COMM_WORLD.
class CommunicationCallback :
  public XdmfOpenCallback,
  public XdmfWriteCallback,
  public XdmfCloseCallback,
  public XdmfReadCallback
{
private:
  int mCommRank;
  int mCommSize;
public:

  CommunicationCallback() {
    MPI_Comm_size( MPI_COMM_WORLD, &mCommSize );
    MPI_Comm_rank( MPI_COMM_WORLD, &mCommRank );
  }

  XdmfInt32 DoOpen(
    XdmfHeavyData* ds,
    XdmfConstString name,
    XdmfConstString access )
  {
    if ( mCommRank == 0 ) {
      return ds->DoOpen( name, access );
    } else {
      return XDMF_SUCCESS;
    }
  }

  XdmfInt32 DoClose( XdmfHeavyData* ds )
  {
    if ( mCommRank == 0 ) {
      return ds->DoClose();
    } else {
      return XDMF_SUCCESS;
    }
  }

  XdmfInt32 DoWrite( XdmfHeavyData* ds, XdmfArray* array )
  {
    // this is a really bad implementation that assumes rank 0 has the same data
    // size as everyone else, but we're really just going for a simple
    // example here.  The real coalescing implementation will require a few more
    // classes to handle buffering the data cleanly and robustly.

    XdmfInt64 start[1], stride[1], count[1];
    XdmfInt32 slab_rank = ds->GetHyperSlab( start, stride, count );
    std::size_t slab_info_size =
      sizeof( XdmfInt32 ) // slab rank
      + slab_rank * sizeof( XdmfInt64 ) * 3; // start, stride, and count
    Buffer buf( slab_info_size + array->GetCoreLength() );

    if ( mCommRank != 0 ) {
      // copy local data to the buffer for sending
      buf.put( slab_rank );
      for ( int i = 0; i < slab_rank; ++i ) {
        buf.put( start[i] );
        buf.put( stride[i] );
        buf.put( count[i] );
      }
      buf.put( array->GetDataPointer(), array->GetCoreLength() );
      MPI_Send(
        buf.pointer(),
        buf.size(),
        MPI_BYTE,
        0,
        0,
        MPI_COMM_WORLD );
    } else {
      // first, it's easy to write my own data
      ds->DoWrite( array );

      int processes_received = 1; // I've written local data
      while ( processes_received < mCommSize ) {
        MPI_Recv(
          buf.pointer(),
          buf.size(),
          MPI_BYTE,
          MPI_ANY_SOURCE,
          0,
          MPI_COMM_WORLD,
          0 );
        processes_received++;

        // pull the information from the buffer
        buf.reset();
        slab_rank = buf.tell< XdmfInt32 >();
        for( int i = 0; i < slab_rank; ++i ) {
          start[i] = buf.tell< XdmfInt64 >();
          stride[i] = buf.tell< XdmfInt64 >();
          count[i] = buf.tell< XdmfInt64 >();
        }
        ds->SelectHyperSlab( start, stride, count );
        XdmfArray* recv = new XdmfArray;
        recv->CopyShape( array );
        buf.tell( recv->GetDataPointer(), recv->GetCoreLength() );
        ds->DoWrite( recv );
        delete recv;
      }
    }

    return XDMF_SUCCESS;
  }

  XdmfArray* DoRead( XdmfHeavyData* ds, XdmfArray* array )
  {
    if ( mCommRank == 0 ) {
      return ds->DoRead( array );
    } else {
      return NULL;
    }
  }
};

char const * const kDatasetName = "FILE:TestFile.h5:/XdmfHDFMPI";

int main( int argc, char* argv[] ) {
  MPI_Init( &argc, &argv );

  int rank;
  MPI_Comm_rank( MPI_COMM_WORLD, &rank );

  XdmfHDF* H5 = new XdmfHDF();
  CommunicationCallback* cb = new CommunicationCallback;
  H5->setOpenCallback( cb );
  H5->setWriteCallback( cb );
  H5->setCloseCallback(cb );
  XdmfArray* MyData = new XdmfArray();

  MyData->SetNumberType( XDMF_FLOAT32_TYPE );
  MyData->SetNumberOfElements( 25 );
  MyData->Generate( rank * 25, rank*25 + 24 );

  H5->CopyType( MyData );
  XdmfInt64 dims[1], start[1], stride[1], count[1];
  dims[0] = 100;
  H5->SetShape( 1, dims );
  start[0] = rank * 25;
  stride[0] = 1;
  count[0] = 25;
  H5->SelectHyperSlab( start, stride, count );
  H5->Open( kDatasetName, "w" );
  H5->Write( MyData );
  H5->Close();

  bool failure = false;

  XdmfHDF* H5In = new XdmfHDF();
  H5In->setReadCallback( cb );
  H5In->setOpenCallback( cb );
  H5In->Open( kDatasetName, "r" );
  XdmfArray* result = H5In->Read();

  if ( result ) {
    for ( size_t i = 0; i < 100; ++i ) {
      float value = result->GetValueAsFloat32( i );
      std::cout << i << " " << value << std::endl;
      failure = ( value != i );
    }
  }

  MPI_Finalize();

  delete H5;
  delete cb;
  delete MyData;
  delete H5In;
  delete result;

  if ( failure ) {
    return -1;
  } else {
    return 0;
  }
};

<commit_msg>fix compilation error for mpich<commit_after>#include <mpi.h>
#include <XdmfArray.h>
#include <XdmfHDF.h>

/// Simple memory buffer implementation that keeps track of it's stream pointer.
class Buffer {
private:
  std::size_t m_size;
  char* m_data;
  char* m_put;
  char* m_tell;

public:
  Buffer( std::size_t size ) :
    m_size( size ),
    m_data( new char[size] ),
    m_put( m_data ),
    m_tell( m_data )
  {}

  ~Buffer() {
    delete [] m_data;
  }

  /// put a single value into the buffer
  template< typename T >
  void put( const T& t ) {
    std::size_t size = sizeof( T );
    memcpy( m_put, &t, size );
    m_put += size;
  }

  /// copy a contiguous block into the buffer
  void put( void* data, std::size_t size ) {
    memcpy( m_put, data, size );
    m_put += size;
  }

  /// Copy a single value into the buffer.
  template< typename T >
  T tell() {
    std::size_t tsize = sizeof( T );
    T tmp;
    memcpy( &tmp, m_tell, tsize );
    m_tell += tsize;
    return tmp;
  }

  /// copy a contiguous block of data from the buffer to an already allocated
  /// location
  void tell( void* out, std::size_t size ) {
    memcpy( out, m_tell, size );
    m_tell += size;
  }

  std::size_t size() const {
    return m_size;
  }

  char* pointer() {
    return m_data;
  }

  void reset() {
    m_put = m_data;
    m_tell = m_data;
  }
};

/// Callback implements parallel IO by communicating to rank 0 in MPI_COMM_WORLD.
class CommunicationCallback :
  public XdmfOpenCallback,
  public XdmfWriteCallback,
  public XdmfCloseCallback,
  public XdmfReadCallback
{
private:
  int mCommRank;
  int mCommSize;
public:

  CommunicationCallback() {
    MPI_Comm_size( MPI_COMM_WORLD, &mCommSize );
    MPI_Comm_rank( MPI_COMM_WORLD, &mCommRank );
  }

  XdmfInt32 DoOpen(
    XdmfHeavyData* ds,
    XdmfConstString name,
    XdmfConstString access )
  {
    if ( mCommRank == 0 ) {
      return ds->DoOpen( name, access );
    } else {
      return XDMF_SUCCESS;
    }
  }

  XdmfInt32 DoClose( XdmfHeavyData* ds )
  {
    if ( mCommRank == 0 ) {
      return ds->DoClose();
    } else {
      return XDMF_SUCCESS;
    }
  }

  XdmfInt32 DoWrite( XdmfHeavyData* ds, XdmfArray* array )
  {
    // this is a really bad implementation that assumes rank 0 has the same data
    // size as everyone else, but we're really just going for a simple
    // example here.  The real coalescing implementation will require a few more
    // classes to handle buffering the data cleanly and robustly.

    XdmfInt64 start[1], stride[1], count[1];
    XdmfInt32 slab_rank = ds->GetHyperSlab( start, stride, count );
    std::size_t slab_info_size =
      sizeof( XdmfInt32 ) // slab rank
      + slab_rank * sizeof( XdmfInt64 ) * 3; // start, stride, and count
    Buffer buf( slab_info_size + array->GetCoreLength() );

    if ( mCommRank != 0 ) {
      // copy local data to the buffer for sending
      buf.put( slab_rank );
      for ( int i = 0; i < slab_rank; ++i ) {
        buf.put( start[i] );
        buf.put( stride[i] );
        buf.put( count[i] );
      }
      buf.put( array->GetDataPointer(), array->GetCoreLength() );
      MPI_Send(
        buf.pointer(),
        buf.size(),
        MPI_BYTE,
        0,
        0,
        MPI_COMM_WORLD );
    } else {
      // first, it's easy to write my own data
      ds->DoWrite( array );

      int processes_received = 1; // I've written local data
      while ( processes_received < mCommSize ) {
        MPI_Recv(
          buf.pointer(),
          buf.size(),
          MPI_BYTE,
          MPI_ANY_SOURCE,
          0,
          MPI_COMM_WORLD,
          0 );
        processes_received++;

        // pull the information from the buffer
        buf.reset();
        slab_rank = buf.tell< XdmfInt32 >();
        for( int i = 0; i < slab_rank; ++i ) {
          start[i] = buf.tell< XdmfInt64 >();
          stride[i] = buf.tell< XdmfInt64 >();
          count[i] = buf.tell< XdmfInt64 >();
        }
        ds->SelectHyperSlab( start, stride, count );
        XdmfArray* recv = new XdmfArray;
        recv->CopyShape( array );
        buf.tell( recv->GetDataPointer(), recv->GetCoreLength() );
        ds->DoWrite( recv );
        delete recv;
      }
    }

    return XDMF_SUCCESS;
  }

  XdmfArray* DoRead( XdmfHeavyData* ds, XdmfArray* array )
  {
    if ( mCommRank == 0 ) {
      return ds->DoRead( array );
    } else {
      return NULL;
    }
  }
};

char const * const kDatasetName = "FILE:TestFile.h5:/XdmfHDFMPI";

int main( int argc, char* argv[] ) {
  MPI_Init( &argc, &argv );

  int rank;
  MPI_Comm_rank( MPI_COMM_WORLD, &rank );

  XdmfHDF* H5 = new XdmfHDF();
  CommunicationCallback* cb = new CommunicationCallback;
  H5->setOpenCallback( cb );
  H5->setWriteCallback( cb );
  H5->setCloseCallback(cb );
  XdmfArray* MyData = new XdmfArray();

  MyData->SetNumberType( XDMF_FLOAT32_TYPE );
  MyData->SetNumberOfElements( 25 );
  MyData->Generate( rank * 25, rank*25 + 24 );

  H5->CopyType( MyData );
  XdmfInt64 dims[1], start[1], stride[1], count[1];
  dims[0] = 100;
  H5->SetShape( 1, dims );
  start[0] = rank * 25;
  stride[0] = 1;
  count[0] = 25;
  H5->SelectHyperSlab( start, stride, count );
  H5->Open( kDatasetName, "w" );
  H5->Write( MyData );
  H5->Close();

  bool failure = false;

  XdmfHDF* H5In = new XdmfHDF();
  H5In->setReadCallback( cb );
  H5In->setOpenCallback( cb );
  H5In->Open( kDatasetName, "r" );
  XdmfArray* result = H5In->Read();

  if ( result ) {
    for ( size_t i = 0; i < 100; ++i ) {
      float value = result->GetValueAsFloat32( i );
      std::cout << i << " " << value << std::endl;
      failure = ( value != i );
    }
  }

  MPI_Finalize();

  delete H5;
  delete cb;
  delete MyData;
  delete H5In;
  delete result;

  if ( failure ) {
    return -1;
  } else {
    return 0;
  }
};

<|endoftext|>


<commit_before>#include "bf/generator.h"

#include <fstream>

int main(int argc, char **argv) {
    bf::generator bfg;

    // Helper function to read ascii int from the input
    auto read_int = [&bfg](bf::var &var) {
        auto input = bfg.new_var("input");
        auto ascii_zero = bfg.new_var("ascii_zero", '0');
        input->read_input();
        bfg.while_begin(*input);
        {
            auto cmp_res = bfg.new_var("cmp_res");
            cmp_res->copy(*input);
            cmp_res->greater_equal(*ascii_zero);
            bfg.if_begin(*cmp_res);
            {
                // Add input to variable
                var.multiply(10);
                input->subtract('0'); // ascii to int
                var.add(*input);
                // Read next char
                input->read_input();
            }
            bfg.if_end(*cmp_res);

            auto not_res = bfg.new_var("not_res");
            not_res->bool_not(*cmp_res);
            bfg.if_begin(*not_res);
            {
                // End input cols
                input->set(0);
            }
            bfg.if_end(*not_res);
        }
        bfg.while_end(*input);
    };

    // Read input
    auto cols = bfg.new_var("cols");
    auto rows = bfg.new_var("rows");

    bfg.print("Cols? ");
    read_int(*cols);
    bfg.print("Rows? ");
    read_int(*rows);

    bfg.print("\n");

    // ------------------------------------------------------------------------

    // Adjacent underscores in the current row
    auto underscores = bfg.new_var("underscores", 1);

    // For each row // for (i = row; i > 0; --i)
    auto i = bfg.new_var("i");
    i->copy(*rows);
    bfg.while_begin(*i);
    {
        // Just print an X for the first column
        bfg.print("X");

        // Column position, count < underscores -> print "_"
        auto count = bfg.new_var("count");

        // For each col but the first // for (j = cols - 1; j > 0; --j)
        auto j = bfg.new_var("j");
        j->copy(*cols);
        j->decrement();
        bfg.while_begin(*j);
        {
            // If count < underscores
            auto cmp_res = bfg.new_var("cmp_res");
            cmp_res->copy(*count);
            cmp_res->lower_than(*underscores);
            bfg.if_begin(*cmp_res);
            {
                // Print "_" and ++count
                bfg.print("_");
                count->increment();
            }
            bfg.if_end(*cmp_res);

            // Else (count >= underscores)
            auto not_res = bfg.new_var("not_res");
            not_res->bool_not(*cmp_res);
            bfg.if_begin(*not_res);
            {
                // Print "X" and reset count
                bfg.print("X");
                count->set(0);
            }
            bfg.if_end(*not_res);

            j->decrement();
        }
        bfg.while_end(*j);
        bfg.print("\n");

        // More underscores in the next row
        underscores->increment();
        i->decrement();
    }
    bfg.while_end(*i);

    // Print to file
    std::ofstream out("Ueb3Aufg2.bf");
    out << bfg;
    //auto minimal_code = bfg.minimal_code();
    //out << minimal_code;

    return 0;
}
<commit_msg>Renamed some variables.<commit_after>#include "bf/generator.h"

#include <fstream>

int main(int argc, char **argv) {
    bf::generator bfg;

    // Helper function to read ascii int from the input
    auto read_int = [&bfg](bf::var &var) {
        auto input = bfg.new_var("input");
        auto ascii_zero = bfg.new_var("ascii_zero", '0');
        input->read_input();
        bfg.while_begin(*input);
        {
            auto is_number = bfg.new_var("is_number");
            is_number->copy(*input);
            is_number->greater_equal(*ascii_zero);
            bfg.if_begin(*is_number);
            {
                // Add input to variable
                var.multiply(10);
                input->subtract('0'); // ascii to int
                var.add(*input);
                // Read next char
                input->read_input();
            }
            bfg.if_end(*is_number);

            auto no_number = bfg.new_var("no_number");
            no_number->bool_not(*is_number);
            bfg.if_begin(*no_number);
            {
                // End input cols
                input->set(0);
            }
            bfg.if_end(*no_number);
        }
        bfg.while_end(*input);
    };

    // Read input
    auto cols = bfg.new_var("cols");
    auto rows = bfg.new_var("rows");

    bfg.print("Cols? ");
    read_int(*cols);
    bfg.print("Rows? ");
    read_int(*rows);

    bfg.print("\n");

    // ------------------------------------------------------------------------

    // Adjacent underscores in the current row
    auto underscores = bfg.new_var("underscores", 1);

    // For each row // for (i = row; i > 0; --i)
    auto i = bfg.new_var("i");
    i->copy(*rows);
    bfg.while_begin(*i);
    {
        // Just print an X for the first column
        bfg.print("X");

        // Column position, count < underscores -> print "_"
        auto count = bfg.new_var("count");

        // For each col but the first // for (j = cols - 1; j > 0; --j)
        auto j = bfg.new_var("j");
        j->copy(*cols);
        j->decrement();
        bfg.while_begin(*j);
        {
            // If count < underscores
            auto cmp_res = bfg.new_var("cmp_res");
            cmp_res->copy(*count);
            cmp_res->lower_than(*underscores);
            bfg.if_begin(*cmp_res);
            {
                // Print "_" and ++count
                bfg.print("_");
                count->increment();
            }
            bfg.if_end(*cmp_res);

            // Else (count >= underscores)
            auto not_res = bfg.new_var("not_res");
            not_res->bool_not(*cmp_res);
            bfg.if_begin(*not_res);
            {
                // Print "X" and reset count
                bfg.print("X");
                count->set(0);
            }
            bfg.if_end(*not_res);

            j->decrement();
        }
        bfg.while_end(*j);
        bfg.print("\n");

        // More underscores in the next row
        underscores->increment();
        i->decrement();
    }
    bfg.while_end(*i);

    // Print to file
    std::ofstream out("Ueb3Aufg2.bf");
    out << bfg;
    //auto minimal_code = bfg.minimal_code();
    //out << minimal_code;

    return 0;
}
<|endoftext|>