<commit_before>/** \file    augment_time_aspects.cc
 *  \brief   A tool for adding normalised time references to MARC-21 datasets.
 *  \author  Dr. Johannes Ruscheinski
 */

/*
    Copyright (C) 2019-2021, Library of the University of Tübingen

    This program is free software: you can redistribute it and/or modify
    it under the terms of the GNU Affero General Public License as
    published by the Free Software Foundation, either version 3 of the
    License, or (at your option) any later version.

    This program is distributed in the hope that it will be useful,
    but WITHOUT ANY WARRANTY; without even the implied warranty of
    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
    GNU Affero General Public License for more details.

    You should have received a copy of the GNU Affero General Public License
    along with this program.  If not, see <http://www.gnu.org/licenses/>.
*/

#include <unordered_map>
#include <cstdlib>
#include "Compiler.h"
#include "MARC.h"
#include "RangeUtil.h"
#include "RegexMatcher.h"
#include "StringUtil.h"
#include "util.h"

namespace {

static const std::vector<std::string> TIME_ASPECT_GND_LINKING_TAGS{ "689" };
static const std::vector<std::string> KEYWORD_PREFIXES{ "Geschichte ", "Geistesgeschichte ", "Ideengeschichte ", "Kirchengeschichte ",
                                                         "Sozialgeschichte ", "Vor- und Frühgeschichte ", "Weltgeschichte ", "Prognose " };

inline std::vector<std::string>::const_iterator FindFirstPrefixMatch(const std::string &s, const std::vector<std::string> &prefixes) {
    for (auto prefix(prefixes.cbegin()); prefix != prefixes.cend(); ++prefix) {
        if (StringUtil::StartsWith(s, *prefix))
            return prefix;
    }

    return prefixes.cend();
}

void LoadAuthorityData(MARC::Reader * const reader,
                       std::unordered_map<std::string, std::string> * const authority_ppns_to_time_codes_map,
                       std::unordered_map<std::string, std::string> * const authority_ppns_to_time_categories_map)
{
    unsigned total_count(0);
    while (auto record = reader->read()) {
        ++total_count;

        bool found_548(false);

        auto _548_field(record.findTag("548"));
        while (_548_field != record.end()) {
            if (_548_field->hasSubfieldWithValue('i', "Zeitraum")) {
                const std::string free_form_range_candidate(_548_field->getFirstSubfieldWithCode('a'));
                std::string range;
                if (RangeUtil::ConvertTextToTimeRange(free_form_range_candidate, &range, /* special_case_centuries = */ true)) {
                    (*authority_ppns_to_time_codes_map)[record.getControlNumber()] = range;
                    found_548 = true;
                    const auto _150_field(record.findTag("150"));
                    if (_150_field != record.end() and _150_field->hasSubfield('a')) {
                        std::string _150a_subfield = _150_field->getFirstSubfieldWithCode('a');
                        _150a_subfield = _150a_subfield + " " + free_form_range_candidate;
                        (*authority_ppns_to_time_categories_map)[record.getControlNumber()] = _150a_subfield;
                    }
                    break;
                } else
                    LOG_WARNING("can't convert \"" + free_form_range_candidate + "\" to a time range!");
            }
            _548_field++;
        }
        if (found_548 == false) {
            auto _450_field(record.findTag("450"));
            while (_450_field != record.end()) {
                if (_450_field->hasSubfield('a')) {
                    std::string _450a_subfield = _450_field->getFirstSubfieldWithCode('a');
                    const auto matched_prefix(FindFirstPrefixMatch(_450a_subfield, KEYWORD_PREFIXES));
                    if (matched_prefix != KEYWORD_PREFIXES.cend()) {
                        std::string range;
                        if (RangeUtil::ConvertTextToTimeRange(_450a_subfield.substr(matched_prefix->length()), &range)) {
                            (*authority_ppns_to_time_codes_map)[record.getControlNumber()] = range;
                            (*authority_ppns_to_time_categories_map)[record.getControlNumber()] = _450a_subfield;
                            break;
                        }
                    }
                }
                _450_field++;
            }
        }
    }

    LOG_INFO("found " + std::to_string(authority_ppns_to_time_codes_map->size()) + " time aspect records among "
             + std::to_string(total_count) + " authority records.");
}

void CollectAuthorityPPNs(const MARC::Record &record, const MARC::Tag &linking_field,
                          std::vector<std::string> * const authority_ppns)
{
    for (const auto &field : record.getTagRange(linking_field)) {
        const MARC::Subfields subfields(field.getSubfields());
        for (const auto &subfield : subfields) {
            if (subfield.code_ == '0' and StringUtil::StartsWith(subfield.value_, "(DE-627)"))
                authority_ppns->emplace_back(subfield.value_.substr(__builtin_strlen("(DE-627)")));
        }
    }
}

void ProcessRecords(MARC::Reader * const reader, MARC::Writer * const writer,
                    const std::unordered_map<std::string, std::string> &authority_ppns_to_time_codes_map,
                    const std::unordered_map<std::string, std::string> &authority_ppns_to_time_categories_map)
{
    unsigned total_count(0), augmented_count(0);
    while (auto record = reader->read()) {
        ++total_count;

        std::string range;
        std::string category;
        for (const std::string &tag : TIME_ASPECT_GND_LINKING_TAGS) {
            for (const auto &time_aspect_field : record.getTagRange(tag)) {
                auto a_subfield(time_aspect_field.getFirstSubfieldWithCode('a'));
                const auto matched_prefix(FindFirstPrefixMatch(a_subfield, KEYWORD_PREFIXES));
                if (matched_prefix == KEYWORD_PREFIXES.cend())
                    continue;

                // Special handling of ranges like "Kirchengeschichte Anfänge-1600":
                if (StringUtil::StartsWith(a_subfield, "Kirchengeschichte Anfänge-"))
                    a_subfield = "Kirchengeschichte 30" + a_subfield.substr(__builtin_strlen("Kirchengeschichte Anfänge"));

                if (RangeUtil::ConvertTextToTimeRange(a_subfield.substr(matched_prefix->length()), &range)) {
                    category = a_subfield;
                    goto augment_record;
                }
            }

            std::vector<std::string> authority_ppns;
            CollectAuthorityPPNs(record, tag, &authority_ppns);
            for (const auto &authority_ppn : authority_ppns) {
                const auto authority_ppn_and_time_code(authority_ppns_to_time_codes_map.find(authority_ppn));
                if (authority_ppn_and_time_code != authority_ppns_to_time_codes_map.cend()) {
                    range = authority_ppn_and_time_code->second;
                    const auto authority_ppn_and_category_code(authority_ppns_to_time_categories_map.find(authority_ppn));
                    if (authority_ppn_and_category_code != authority_ppns_to_time_categories_map.cend()) {
                        category = authority_ppn_and_category_code->second;
                    }
                    goto augment_record;
                }
            }
        }

augment_record:
        if (not range.empty()) {
            record.insertField("TIM", { { 'a', range }, {'b', category.length() == 0 ? RangeUtil::ConvertTimeRangeToText(range) : category} });
            ++augmented_count;
        }

        writer->write(record);
    }

    LOG_INFO("augmented " + std::to_string(augmented_count) + " of " + std::to_string(total_count) + " records.");
}

} // unnamed namespace

int Main(int argc, char **argv) {
    if (argc != 4)
        ::Usage("ixtheo_titles authority_records augmented_ixtheo_titles");

    const std::string title_input_filename(argv[1]);
    const std::string authority_filename(argv[2]);
    const std::string title_output_filename(argv[3]);
    if (unlikely(title_input_filename == title_output_filename))
        LOG_ERROR("Title input file name equals title output file name!");
    if (unlikely(title_input_filename == authority_filename))
        LOG_ERROR("Title input file name equals authority file name!");
    if (unlikely(title_output_filename == authority_filename))
        LOG_ERROR("Title output file name equals authority file name!");

    auto authority_reader(MARC::Reader::Factory(authority_filename));
    std::unordered_map<std::string, std::string> authority_ppns_to_time_codes_map;
    std::unordered_map<std::string, std::string> authority_ppns_to_time_categories_map;
    LoadAuthorityData(authority_reader.get(), &authority_ppns_to_time_codes_map, &authority_ppns_to_time_categories_map);

    auto title_reader(MARC::Reader::Factory(title_input_filename));
    auto title_writer(MARC::Writer::Factory(title_output_filename));
    ProcessRecords(title_reader.get(), title_writer.get(), authority_ppns_to_time_codes_map, authority_ppns_to_time_categories_map);

    return EXIT_SUCCESS;
}
<commit_msg>Improved logging.<commit_after>/** \file    augment_time_aspects.cc
 *  \brief   A tool for adding normalised time references to MARC-21 datasets.
 *  \author  Dr. Johannes Ruscheinski
 */

/*
    Copyright (C) 2019-2021, Library of the University of Tübingen

    This program is free software: you can redistribute it and/or modify
    it under the terms of the GNU Affero General Public License as
    published by the Free Software Foundation, either version 3 of the
    License, or (at your option) any later version.

    This program is distributed in the hope that it will be useful,
    but WITHOUT ANY WARRANTY; without even the implied warranty of
    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
    GNU Affero General Public License for more details.

    You should have received a copy of the GNU Affero General Public License
    along with this program.  If not, see <http://www.gnu.org/licenses/>.
*/

#include <unordered_map>
#include <cstdlib>
#include "Compiler.h"
#include "MARC.h"
#include "RangeUtil.h"
#include "RegexMatcher.h"
#include "StringUtil.h"
#include "util.h"

namespace {

static const std::vector<std::string> TIME_ASPECT_GND_LINKING_TAGS{ "689" };
static const std::vector<std::string> KEYWORD_PREFIXES{ "Geschichte ", "Geistesgeschichte ", "Ideengeschichte ", "Kirchengeschichte ",
                                                         "Sozialgeschichte ", "Vor- und Frühgeschichte ", "Weltgeschichte ", "Prognose " };

inline std::vector<std::string>::const_iterator FindFirstPrefixMatch(const std::string &s, const std::vector<std::string> &prefixes) {
    for (auto prefix(prefixes.cbegin()); prefix != prefixes.cend(); ++prefix) {
        if (StringUtil::StartsWith(s, *prefix))
            return prefix;
    }

    return prefixes.cend();
}

void LoadAuthorityData(MARC::Reader * const reader,
                       std::unordered_map<std::string, std::string> * const authority_ppns_to_time_codes_map,
                       std::unordered_map<std::string, std::string> * const authority_ppns_to_time_categories_map)
{
    unsigned total_count(0);
    while (auto record = reader->read()) {
        ++total_count;

        bool found_548(false);

        auto _548_field(record.findTag("548"));
        while (_548_field != record.end()) {
            if (_548_field->hasSubfieldWithValue('i', "Zeitraum")) {
                const std::string free_form_range_candidate(_548_field->getFirstSubfieldWithCode('a'));
                std::string range;
                if (RangeUtil::ConvertTextToTimeRange(free_form_range_candidate, &range, /* special_case_centuries = */ true)) {
                    (*authority_ppns_to_time_codes_map)[record.getControlNumber()] = range;
                    found_548 = true;
                    const auto _150_field(record.findTag("150"));
                    if (_150_field != record.end() and _150_field->hasSubfield('a')) {
                        std::string _150a_subfield = _150_field->getFirstSubfieldWithCode('a');
                        _150a_subfield = _150a_subfield + " " + free_form_range_candidate;
                        (*authority_ppns_to_time_categories_map)[record.getControlNumber()] = _150a_subfield;
                    }
                    break;
                } else
                    LOG_WARNING("can't convert \"" + free_form_range_candidate + "\" to a time range! (PPN: "
                                + record.getControlNumber() + ")");
            }
            _548_field++;
        }
        if (found_548 == false) {
            auto _450_field(record.findTag("450"));
            while (_450_field != record.end()) {
                if (_450_field->hasSubfield('a')) {
                    std::string _450a_subfield = _450_field->getFirstSubfieldWithCode('a');
                    const auto matched_prefix(FindFirstPrefixMatch(_450a_subfield, KEYWORD_PREFIXES));
                    if (matched_prefix != KEYWORD_PREFIXES.cend()) {
                        std::string range;
                        if (RangeUtil::ConvertTextToTimeRange(_450a_subfield.substr(matched_prefix->length()), &range)) {
                            (*authority_ppns_to_time_codes_map)[record.getControlNumber()] = range;
                            (*authority_ppns_to_time_categories_map)[record.getControlNumber()] = _450a_subfield;
                            break;
                        }
                    }
                }
                _450_field++;
            }
        }
    }

    LOG_INFO("found " + std::to_string(authority_ppns_to_time_codes_map->size()) + " time aspect records among "
             + std::to_string(total_count) + " authority records.");
}

void CollectAuthorityPPNs(const MARC::Record &record, const MARC::Tag &linking_field,
                          std::vector<std::string> * const authority_ppns)
{
    for (const auto &field : record.getTagRange(linking_field)) {
        const MARC::Subfields subfields(field.getSubfields());
        for (const auto &subfield : subfields) {
            if (subfield.code_ == '0' and StringUtil::StartsWith(subfield.value_, "(DE-627)"))
                authority_ppns->emplace_back(subfield.value_.substr(__builtin_strlen("(DE-627)")));
        }
    }
}

void ProcessRecords(MARC::Reader * const reader, MARC::Writer * const writer,
                    const std::unordered_map<std::string, std::string> &authority_ppns_to_time_codes_map,
                    const std::unordered_map<std::string, std::string> &authority_ppns_to_time_categories_map)
{
    unsigned total_count(0), augmented_count(0);
    while (auto record = reader->read()) {
        ++total_count;

        std::string range;
        std::string category;
        for (const std::string &tag : TIME_ASPECT_GND_LINKING_TAGS) {
            for (const auto &time_aspect_field : record.getTagRange(tag)) {
                auto a_subfield(time_aspect_field.getFirstSubfieldWithCode('a'));
                const auto matched_prefix(FindFirstPrefixMatch(a_subfield, KEYWORD_PREFIXES));
                if (matched_prefix == KEYWORD_PREFIXES.cend())
                    continue;

                // Special handling of ranges like "Kirchengeschichte Anfänge-1600":
                if (StringUtil::StartsWith(a_subfield, "Kirchengeschichte Anfänge-"))
                    a_subfield = "Kirchengeschichte 30" + a_subfield.substr(__builtin_strlen("Kirchengeschichte Anfänge"));

                if (RangeUtil::ConvertTextToTimeRange(a_subfield.substr(matched_prefix->length()), &range)) {
                    category = a_subfield;
                    goto augment_record;
                }
            }

            std::vector<std::string> authority_ppns;
            CollectAuthorityPPNs(record, tag, &authority_ppns);
            for (const auto &authority_ppn : authority_ppns) {
                const auto authority_ppn_and_time_code(authority_ppns_to_time_codes_map.find(authority_ppn));
                if (authority_ppn_and_time_code != authority_ppns_to_time_codes_map.cend()) {
                    range = authority_ppn_and_time_code->second;
                    const auto authority_ppn_and_category_code(authority_ppns_to_time_categories_map.find(authority_ppn));
                    if (authority_ppn_and_category_code != authority_ppns_to_time_categories_map.cend()) {
                        category = authority_ppn_and_category_code->second;
                    }
                    goto augment_record;
                }
            }
        }

augment_record:
        if (not range.empty()) {
            record.insertField("TIM", { { 'a', range }, {'b', category.length() == 0 ? RangeUtil::ConvertTimeRangeToText(range) : category} });
            ++augmented_count;
        }

        writer->write(record);
    }

    LOG_INFO("augmented " + std::to_string(augmented_count) + " of " + std::to_string(total_count) + " records.");
}

} // unnamed namespace

int Main(int argc, char **argv) {
    if (argc != 4)
        ::Usage("ixtheo_titles authority_records augmented_ixtheo_titles");

    const std::string title_input_filename(argv[1]);
    const std::string authority_filename(argv[2]);
    const std::string title_output_filename(argv[3]);
    if (unlikely(title_input_filename == title_output_filename))
        LOG_ERROR("Title input file name equals title output file name!");
    if (unlikely(title_input_filename == authority_filename))
        LOG_ERROR("Title input file name equals authority file name!");
    if (unlikely(title_output_filename == authority_filename))
        LOG_ERROR("Title output file name equals authority file name!");

    auto authority_reader(MARC::Reader::Factory(authority_filename));
    std::unordered_map<std::string, std::string> authority_ppns_to_time_codes_map;
    std::unordered_map<std::string, std::string> authority_ppns_to_time_categories_map;
    LoadAuthorityData(authority_reader.get(), &authority_ppns_to_time_codes_map, &authority_ppns_to_time_categories_map);

    auto title_reader(MARC::Reader::Factory(title_input_filename));
    auto title_writer(MARC::Writer::Factory(title_output_filename));
    ProcessRecords(title_reader.get(), title_writer.get(), authority_ppns_to_time_codes_map, authority_ppns_to_time_categories_map);

    return EXIT_SUCCESS;
}
<|endoftext|>


<commit_before>/* Copyright 2020 The TensorFlow Authors. All Rights Reserved.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
==============================================================================*/
#include "tensorflow/lite/delegates/flex/allowlisted_flex_ops.h"

#include <set>

#include <gtest/gtest.h>
#include "tensorflow/core/framework/op_kernel.h"
#include "tensorflow/lite/delegates/flex/allowlisted_flex_ops_internal.h"

namespace tflite {
namespace flex {

// Get all cpu kernels registered in Tensorflow.
std::set<string> GetAllCpuKernels() {
  auto is_cpu_kernel = [](const tensorflow::KernelDef& def) {
    return (def.device_type() == "CPU" || def.device_type() == "DEFAULT");
  };

  tensorflow::KernelList kernel_list =
      tensorflow::GetFilteredRegisteredKernels(is_cpu_kernel);
  std::set<string> result;

  for (int i = 0; i < kernel_list.kernel_size(); ++i) {
    tensorflow::KernelDef kernel_def = kernel_list.kernel(i);
    result.insert(kernel_def.op());
  }
  return result;
}

// Test if every flex op has their kernel included in the flex delegate library.
// This test must be run on both Linux and Android.
TEST(AllowlistedFlexOpsTest, EveryOpHasKernel) {
  const std::set<std::string>& allowlist = GetFlexAllowlist();
  std::set<string> all_kernels = GetAllCpuKernels();

  for (const std::string& op_name : allowlist) {
    EXPECT_EQ(all_kernels.count(op_name), 1)
        << op_name << " op is added to flex allowlist "
        << "but its kernel is not found.";
  }
}
}  // namespace flex
}  // namespace tflite

int main(int argc, char** argv) {
  ::testing::InitGoogleTest(&argc, argv);
  return RUN_ALL_TESTS();
}
<commit_msg>Use `std::string` instead of `string`<commit_after>/* Copyright 2020 The TensorFlow Authors. All Rights Reserved.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
==============================================================================*/
#include "tensorflow/lite/delegates/flex/allowlisted_flex_ops.h"

#include <set>

#include <gtest/gtest.h>
#include "tensorflow/core/framework/op_kernel.h"
#include "tensorflow/lite/delegates/flex/allowlisted_flex_ops_internal.h"

namespace tflite {
namespace flex {

// Get all cpu kernels registered in Tensorflow.
std::set<std::string> GetAllCpuKernels() {
  auto is_cpu_kernel = [](const tensorflow::KernelDef& def) {
    return (def.device_type() == "CPU" || def.device_type() == "DEFAULT");
  };

  tensorflow::KernelList kernel_list =
      tensorflow::GetFilteredRegisteredKernels(is_cpu_kernel);
  std::set<std::string> result;

  for (int i = 0; i < kernel_list.kernel_size(); ++i) {
    tensorflow::KernelDef kernel_def = kernel_list.kernel(i);
    result.insert(kernel_def.op());
  }
  return result;
}

// Test if every flex op has their kernel included in the flex delegate library.
// This test must be run on both Linux and Android.
TEST(AllowlistedFlexOpsTest, EveryOpHasKernel) {
  const std::set<std::string>& allowlist = GetFlexAllowlist();
  std::set<std::string> all_kernels = GetAllCpuKernels();

  for (const std::string& op_name : allowlist) {
    EXPECT_EQ(all_kernels.count(op_name), 1)
        << op_name << " op is added to flex allowlist "
        << "but its kernel is not found.";
  }
}
}  // namespace flex
}  // namespace tflite

int main(int argc, char** argv) {
  ::testing::InitGoogleTest(&argc, argv);
  return RUN_ALL_TESTS();
}
<|endoftext|>


<commit_before>/**
 * \file
 * \brief FpuSignalTestCase class implementation for ARMv7-M
 *
 * \author Copyright (C) 2015-2019 Kamil Szczygiel https://distortec.com https://freddiechopin.info
 *
 * \par License
 * This Source Code Form is subject to the terms of the Mozilla Public License, v. 2.0. If a copy of the MPL was not
 * distributed with this file, You can obtain one at https://mozilla.org/MPL/2.0/.
 */

#include "ARMv7-M-FpuSignalTestCase.hpp"

#include "distortos/chip/CMSIS-proxy.h"

/// configuration required by FpuSignalTestCase
#define ARMV7_M_FPU_SIGNAL_TEST_CASE_ENABLED __FPU_PRESENT == 1 && __FPU_USED == 1 && DISTORTOS_SIGNALS_ENABLE == 1 && \
		DISTORTOS_MAIN_THREAD_SIGNAL_ACTIONS > 0 && DISTORTOS_MAIN_THREAD_QUEUED_SIGNALS > 0

#if ARMV7_M_FPU_SIGNAL_TEST_CASE_ENABLED == 1

#include "ARMv7-M-checkFpuRegisters.hpp"
#include "ARMv7-M-setFpuRegisters.hpp"

#include "distortos/DynamicThread.hpp"
#include "distortos/statistics.hpp"
#include "distortos/ThisThread.hpp"
#include "distortos/ThisThread-Signals.hpp"

#include "estd/ScopeGuard.hpp"

#include <cerrno>

#endif	// ARMV7_M_FPU_SIGNAL_TEST_CASE_ENABLED == 1

namespace distortos
{

namespace test
{

#if ARMV7_M_FPU_SIGNAL_TEST_CASE_ENABLED == 1

namespace
{

/*---------------------------------------------------------------------------------------------------------------------+
| local types
+---------------------------------------------------------------------------------------------------------------------*/

/// values used in single test stage
struct Stage
{
	/// value written to all FPU registers in main thread, don't use FPU in main thread if 0
	uint32_t threadValue;

	/// value written to "lower" FPU registers in "sender" before queuing of signal, skip this step if 0
	uint32_t senderValueBefore;

	/// value written to "lower" FPU registers in "sender" after queuing of signal, skip this step if 0
	uint32_t senderValueAfter;

	/// signal value, written to "lower" FPU registers in handler, don't use FPU in handler if 0
	int signalValue;
};

/// description of single test phase
struct Phase
{
	/// type of "sender" function
	using Sender = int(Thread&, const Stage&);

	/// reference to "sender" function
	Sender& sender;

	/// expected number of context switches for single stage executed in this phase
	decltype(statistics::getContextSwitchCount()) contextSwitchCount;
};

/// context shared with interrupt
struct Context
{
	/// shared return value
	int sharedRet;

	/// pointer to test stage
	const Stage* stage;

	/// pointer to thread to which the signal will be queued
	Thread* thread;
};

/*---------------------------------------------------------------------------------------------------------------------+
| local objects
+---------------------------------------------------------------------------------------------------------------------*/

/// tested signal number
constexpr uint8_t testSignalNumber {16};

/// size of stack for test thread, bytes
constexpr size_t testThreadStackSize {512};

/// pointer to context shared with interrupt
Context* volatile context;

/*---------------------------------------------------------------------------------------------------------------------+
| local functions
+---------------------------------------------------------------------------------------------------------------------*/

/**
 * \brief Disables FPU context for current thread by clearing FPCA bit in CONTROL register.
 */

void disableFpuContext()
{
	const auto control = __get_CONTROL();
	__set_CONTROL(control & ~CONTROL_FPCA_Msk);
}

/**
 * \brief Signal handler
 *
 * Sets "lower" FPU registers (s0-s15 and fpscr) using the value queued with signal, unless this value is 0.
 *
 * \param [in] signalInformation is a reference to received SignalInformation object
 */

void handler(const SignalInformation& signalInformation)
{
	const auto value = signalInformation.getValue().sival_int;
	if (value == 0)
		return;

	setFpuRegisters(value, false);
}

/**
 * \brief Tests whether FPU context is active for current thread.
 *
 * \return true if FPU context is active for current thread (FPCA bit in CONTROL register is set), false otherwise
 */

bool isFpuContextActive()
{
	const auto control = __get_CONTROL();
	return (control & CONTROL_FPCA_Msk) != 0;
}

/**
 * \brief Test wrapper for Thread::queueSignal() that also modifies FPU registers.
 *
 * \param [in] thread is a reference to thread to which the signal will be queued
 * \param [in] stage is a reference to test stage
 * \param [in] full is the \a full argument passed to setFpuRegisters()
 * \param [in] sharedRet is a reference to int variable which will be written with 0 on success, error code otherwise
 */

void queueSignalWrapper(Thread& thread, const Stage& stage, const bool full, int& sharedRet)
{
	if (stage.senderValueBefore != 0)	// should FPU be used at the beginning of "sender"?
		setFpuRegisters(stage.senderValueBefore, full);

	sharedRet = thread.queueSignal(testSignalNumber, sigval{stage.signalValue});

	if (stage.senderValueAfter != 0)	// should FPU be used at th"sender""sender"?
		setFpuRegisters(stage.senderValueAfter, full);
}

/**
 * \brief Queues signal from interrupt (via manually triggered UsageFault) to current thread.
 *
 * \param [in] thread is a reference to thread to which the signal will be queued
 * \param [in] stage is a reference to test stage
 *
 * \return 0 on success, error code otherwise:
 * - error codes returned by Thread::queueSignal();
 */

int queueSignalFromInterrupt(Thread& thread, const Stage& stage)
{
	Context localContext {};
	context = &localContext;
	const auto contextScopeGuard = estd::makeScopeGuard(
			[]()
			{
				context = {};
			});

	context->thread = &thread;
	context->stage = &stage;

	SCB->CCR |= SCB_CCR_DIV_0_TRP_Msk;			// enable trapping of "divide by 0" by UsageFault
	SCB->SHCSR |= SCB_SHCSR_USGFAULTENA_Msk;	// enable UsageFault

	volatile int a {};
	volatile int b {};
	a /= b;	// trigger UsageFault via "divide by 0"

	return context->sharedRet;
}

/**
 * \brief Queues signal from thread to non-current thread.
 *
 * \param [in] thread is a reference to thread to which the signal will be queued
 * \param [in] stage is a reference to test stage
 *
 * \return 0 on success, error code otherwise:
 * - error codes returned by Thread::queueSignal();
 */

int queueSignalFromThread(Thread& thread, const Stage& stage)
{
	constexpr decltype(FpuSignalTestCase::getTestCasePriority()) highPriority
	{
			FpuSignalTestCase::getTestCasePriority() + 1
	};

	int sharedRet {EINVAL};
	auto testThread = makeAndStartDynamicThread({testThreadStackSize, highPriority}, queueSignalWrapper,
			std::ref(thread), std::ref(stage), true, std::ref(sharedRet));
	testThread.join();

	return sharedRet;
}

/*---------------------------------------------------------------------------------------------------------------------+
| local constants
+---------------------------------------------------------------------------------------------------------------------*/

/// test phases
const Phase phases[]
{
		{queueSignalFromInterrupt, 0},
		{queueSignalFromThread, 2},
};

/// test stages
const Stage stages[]
{
		{0, 0, 0, 0},								// don't use FPU in all contexts
		{0, 0, 0, 0x0b39fa96},						// in handler
		{0, 0, 0x8606151d, 0},						// in "sender" (at the end)
		{0, 0, 0x8bdd3b5e, 0x7d21d1c6},				// in "sender" (at the end) and in handler
		{0, 0x2f884196, 0, 0},						// in "sender" (at the beginning)
		{0, 0x0b0bbc86, 0, 0x0e43811b},				// in "sender" (at the beginning) and in handler
		{0, 0xb72b2917, 0x8c27baa7, 0},				// in "sender"
		{0, 0xa83b80c2, 0xd2b7dd4d, 0x626ca399},	// in "sender" and in handler
		{0xb4e40525, 0, 0, 0},						// in main thread
		{0x772bdf91, 0, 0, 0x0bb325b7},				// in main thread and in handler
		{0x8a19625e, 0, 0x32378f7b, 0},				// in main thread and in "sender" (at the end)
		{0xd17a21db, 0, 0xaa807a91, 0x03caf264},	// in main thread, in "sender" (at the end) and in handler
		{0xe4b44073, 0xa88c0cf5, 0, 0},				// in main thread and in "sender" (at the beginning)
		{0xb94c722b, 0x5f8ca773, 0, 0x288301cf},	// in main thread, in "sender" (at the beginning) and in handler
		{0x347ecfc5, 0xcb4a3584, 0x5a8bf219, 0},	// in main thread and in "sender"
		{0x788ed92e, 0x4b0ddff9, 0x73776a21, 0x48fb1969},	// in all contexts
};

}	// namespace

#endif	// ARMV7_M_FPU_SIGNAL_TEST_CASE_ENABLED == 1

/*---------------------------------------------------------------------------------------------------------------------+
| protected functions
+---------------------------------------------------------------------------------------------------------------------*/

bool FpuSignalTestCase::finalize() const
{
#if ARMV7_M_FPU_SIGNAL_TEST_CASE_ENABLED == 1

	disableFpuContext();

#endif	// ARMV7_M_FPU_SIGNAL_TEST_CASE_ENABLED == 1

	return SignalsTestCaseCommon::finalize();
}

/*---------------------------------------------------------------------------------------------------------------------+
| private functions
+---------------------------------------------------------------------------------------------------------------------*/

bool FpuSignalTestCase::run_() const
{
#if ARMV7_M_FPU_SIGNAL_TEST_CASE_ENABLED == 1

	const auto contextSwitchCount = statistics::getContextSwitchCount();
	auto expectedContextSwitchCount = decltype(contextSwitchCount){};

	{
		int ret;
		std::tie(ret, std::ignore) = ThisThread::Signals::setSignalAction(testSignalNumber,
				SignalAction{handler, SignalSet{SignalSet::empty}});
		if (ret != 0)
			return false;
	}

	auto& currentThread = ThisThread::get();

	for (auto& phase : phases)
		for (auto& stage : stages)
		{
			disableFpuContext();

			decltype(setFpuRegisters({}, {})) fpscr {};
			if (stage.threadValue != 0)	// should FPU be used in main thread?
				fpscr = setFpuRegisters(stage.threadValue, true);

			if (phase.sender(currentThread, stage) != 0)
				return false;

			// FPU context may be active only if FPU was used in main thread
			if (isFpuContextActive() != (stage.threadValue != 0))
				return false;

			if (stage.threadValue != 0)	// was FPU used in main thread?
				if (checkFpuRegisters(stage.threadValue, fpscr) == false)
					return false;

			expectedContextSwitchCount += phase.contextSwitchCount;
			if (statistics::getContextSwitchCount() - contextSwitchCount != expectedContextSwitchCount)
				return false;
		}

	if (statistics::getContextSwitchCount() - contextSwitchCount != expectedContextSwitchCount)
		return false;

#endif	// ARMV7_M_FPU_SIGNAL_TEST_CASE_ENABLED == 1

	return true;
}

}	// namespace test

}	// namespace distortos

/*---------------------------------------------------------------------------------------------------------------------+
| global functions
+---------------------------------------------------------------------------------------------------------------------*/

/**
 * \brief UsageFault_Handler() for ARMv7-M
 */

extern "C" void UsageFault_Handler()
{
	SCB->CFSR = SCB_CFSR_DIVBYZERO_Msk;			// clear "divide by 0" flag
	SCB->CCR &= ~SCB_CCR_DIV_0_TRP_Msk;			// disable trapping of "divide by 0" by UsageFault
	SCB->SHCSR &= ~SCB_SHCSR_USGFAULTENA_Msk;	// disable UsageFault

	using namespace distortos::test;
	queueSignalWrapper(*context->thread, *context->stage, false, context->sharedRet);
}
<commit_msg>Fix build test of ARMv7-M-FpuSignalTestCase<commit_after>/**
 * \file
 * \brief FpuSignalTestCase class implementation for ARMv7-M
 *
 * \author Copyright (C) 2015-2021 Kamil Szczygiel https://distortec.com https://freddiechopin.info
 *
 * \par License
 * This Source Code Form is subject to the terms of the Mozilla Public License, v. 2.0. If a copy of the MPL was not
 * distributed with this file, You can obtain one at https://mozilla.org/MPL/2.0/.
 */

#include "ARMv7-M-FpuSignalTestCase.hpp"

#include "distortos/chip/CMSIS-proxy.h"

/// configuration required by FpuSignalTestCase
#define ARMV7_M_FPU_SIGNAL_TEST_CASE_ENABLED __FPU_PRESENT == 1 && __FPU_USED == 1 && DISTORTOS_SIGNALS_ENABLE == 1 && \
		DISTORTOS_MAIN_THREAD_SIGNAL_ACTIONS > 0 && DISTORTOS_MAIN_THREAD_QUEUED_SIGNALS > 0

#if ARMV7_M_FPU_SIGNAL_TEST_CASE_ENABLED == 1

#include "ARMv7-M-checkFpuRegisters.hpp"
#include "ARMv7-M-setFpuRegisters.hpp"

#include "distortos/DynamicThread.hpp"
#include "distortos/statistics.hpp"
#include "distortos/ThisThread.hpp"
#include "distortos/ThisThread-Signals.hpp"

#include "estd/ScopeGuard.hpp"

#include <cerrno>

#endif	// ARMV7_M_FPU_SIGNAL_TEST_CASE_ENABLED == 1

namespace distortos
{

namespace test
{

#if ARMV7_M_FPU_SIGNAL_TEST_CASE_ENABLED == 1

namespace
{

/*---------------------------------------------------------------------------------------------------------------------+
| local types
+---------------------------------------------------------------------------------------------------------------------*/

/// values used in single test stage
struct Stage
{
	/// value written to all FPU registers in main thread, don't use FPU in main thread if 0
	uint32_t threadValue;

	/// value written to "lower" FPU registers in "sender" before queuing of signal, skip this step if 0
	uint32_t senderValueBefore;

	/// value written to "lower" FPU registers in "sender" after queuing of signal, skip this step if 0
	uint32_t senderValueAfter;

	/// signal value, written to "lower" FPU registers in handler, don't use FPU in handler if 0
	int signalValue;
};

/// description of single test phase
struct Phase
{
	/// type of "sender" function
	using Sender = int(Thread&, const Stage&);

	/// reference to "sender" function
	Sender& sender;

	/// expected number of context switches for single stage executed in this phase
	decltype(statistics::getContextSwitchCount()) contextSwitchCount;
};

/// context shared with interrupt
struct Context
{
	/// shared return value
	int sharedRet;

	/// pointer to test stage
	const Stage* stage;

	/// pointer to thread to which the signal will be queued
	Thread* thread;
};

/*---------------------------------------------------------------------------------------------------------------------+
| local objects
+---------------------------------------------------------------------------------------------------------------------*/

/// tested signal number
constexpr uint8_t testSignalNumber {16};

/// size of stack for test thread, bytes
constexpr size_t testThreadStackSize {512};

/// pointer to context shared with interrupt
Context* volatile context;

/*---------------------------------------------------------------------------------------------------------------------+
| local functions
+---------------------------------------------------------------------------------------------------------------------*/

/**
 * \brief Disables FPU context for current thread by clearing FPCA bit in CONTROL register.
 */

void disableFpuContext()
{
	const auto control = __get_CONTROL();
	__set_CONTROL(control & ~CONTROL_FPCA_Msk);
}

/**
 * \brief Signal handler
 *
 * Sets "lower" FPU registers (s0-s15 and fpscr) using the value queued with signal, unless this value is 0.
 *
 * \param [in] signalInformation is a reference to received SignalInformation object
 */

void handler(const SignalInformation& signalInformation)
{
	const auto value = signalInformation.getValue().sival_int;
	if (value == 0)
		return;

	setFpuRegisters(value, false);
}

/**
 * \brief Tests whether FPU context is active for current thread.
 *
 * \return true if FPU context is active for current thread (FPCA bit in CONTROL register is set), false otherwise
 */

bool isFpuContextActive()
{
	const auto control = __get_CONTROL();
	return (control & CONTROL_FPCA_Msk) != 0;
}

/**
 * \brief Test wrapper for Thread::queueSignal() that also modifies FPU registers.
 *
 * \param [in] thread is a reference to thread to which the signal will be queued
 * \param [in] stage is a reference to test stage
 * \param [in] full is the \a full argument passed to setFpuRegisters()
 * \param [in] sharedRet is a reference to int variable which will be written with 0 on success, error code otherwise
 */

void queueSignalWrapper(Thread& thread, const Stage& stage, const bool full, int& sharedRet)
{
	if (stage.senderValueBefore != 0)	// should FPU be used at the beginning of "sender"?
		setFpuRegisters(stage.senderValueBefore, full);

	sharedRet = thread.queueSignal(testSignalNumber, sigval{stage.signalValue});

	if (stage.senderValueAfter != 0)	// should FPU be used at th"sender""sender"?
		setFpuRegisters(stage.senderValueAfter, full);
}

/**
 * \brief Queues signal from interrupt (via manually triggered UsageFault) to current thread.
 *
 * \param [in] thread is a reference to thread to which the signal will be queued
 * \param [in] stage is a reference to test stage
 *
 * \return 0 on success, error code otherwise:
 * - error codes returned by Thread::queueSignal();
 */

int queueSignalFromInterrupt(Thread& thread, const Stage& stage)
{
	Context localContext {};
	context = &localContext;
	const auto contextScopeGuard = estd::makeScopeGuard(
			[]()
			{
				context = {};
			});

	context->thread = &thread;
	context->stage = &stage;

	SCB->CCR |= SCB_CCR_DIV_0_TRP_Msk;			// enable trapping of "divide by 0" by UsageFault
	SCB->SHCSR |= SCB_SHCSR_USGFAULTENA_Msk;	// enable UsageFault

	volatile int a {};
	volatile int b {};
	a /= b;	// trigger UsageFault via "divide by 0"

	return context->sharedRet;
}

/**
 * \brief Queues signal from thread to non-current thread.
 *
 * \param [in] thread is a reference to thread to which the signal will be queued
 * \param [in] stage is a reference to test stage
 *
 * \return 0 on success, error code otherwise:
 * - error codes returned by Thread::queueSignal();
 */

int queueSignalFromThread(Thread& thread, const Stage& stage)
{
	constexpr decltype(FpuSignalTestCase::getTestCasePriority()) highPriority
	{
			FpuSignalTestCase::getTestCasePriority() + 1
	};

	int sharedRet {EINVAL};
	auto testThread = makeAndStartDynamicThread({testThreadStackSize, highPriority}, queueSignalWrapper,
			std::ref(thread), std::ref(stage), true, std::ref(sharedRet));
	testThread.join();

	return sharedRet;
}

/*---------------------------------------------------------------------------------------------------------------------+
| local constants
+---------------------------------------------------------------------------------------------------------------------*/

/// test phases
const Phase phases[]
{
		{queueSignalFromInterrupt, 0},
		{queueSignalFromThread, 2},
};

/// test stages
const Stage stages[]
{
		{0, 0, 0, 0},								// don't use FPU in all contexts
		{0, 0, 0, 0x0b39fa96},						// in handler
		{0, 0, 0x8606151d, 0},						// in "sender" (at the end)
		{0, 0, 0x8bdd3b5e, 0x7d21d1c6},				// in "sender" (at the end) and in handler
		{0, 0x2f884196, 0, 0},						// in "sender" (at the beginning)
		{0, 0x0b0bbc86, 0, 0x0e43811b},				// in "sender" (at the beginning) and in handler
		{0, 0xb72b2917, 0x8c27baa7, 0},				// in "sender"
		{0, 0xa83b80c2, 0xd2b7dd4d, 0x626ca399},	// in "sender" and in handler
		{0xb4e40525, 0, 0, 0},						// in main thread
		{0x772bdf91, 0, 0, 0x0bb325b7},				// in main thread and in handler
		{0x8a19625e, 0, 0x32378f7b, 0},				// in main thread and in "sender" (at the end)
		{0xd17a21db, 0, 0xaa807a91, 0x03caf264},	// in main thread, in "sender" (at the end) and in handler
		{0xe4b44073, 0xa88c0cf5, 0, 0},				// in main thread and in "sender" (at the beginning)
		{0xb94c722b, 0x5f8ca773, 0, 0x288301cf},	// in main thread, in "sender" (at the beginning) and in handler
		{0x347ecfc5, 0xcb4a3584, 0x5a8bf219, 0},	// in main thread and in "sender"
		{0x788ed92e, 0x4b0ddff9, 0x73776a21, 0x48fb1969},	// in all contexts
};

}	// namespace

#endif	// ARMV7_M_FPU_SIGNAL_TEST_CASE_ENABLED == 1

/*---------------------------------------------------------------------------------------------------------------------+
| protected functions
+---------------------------------------------------------------------------------------------------------------------*/

bool FpuSignalTestCase::finalize() const
{
#if ARMV7_M_FPU_SIGNAL_TEST_CASE_ENABLED == 1

	disableFpuContext();

#endif	// ARMV7_M_FPU_SIGNAL_TEST_CASE_ENABLED == 1

	return SignalsTestCaseCommon::finalize();
}

/*---------------------------------------------------------------------------------------------------------------------+
| private functions
+---------------------------------------------------------------------------------------------------------------------*/

bool FpuSignalTestCase::run_() const
{
#if ARMV7_M_FPU_SIGNAL_TEST_CASE_ENABLED == 1

	const auto contextSwitchCount = statistics::getContextSwitchCount();
	auto expectedContextSwitchCount = decltype(contextSwitchCount){};

	{
		int ret;
		std::tie(ret, std::ignore) = ThisThread::Signals::setSignalAction(testSignalNumber,
				SignalAction{handler, SignalSet{SignalSet::empty}});
		if (ret != 0)
			return false;
	}

	auto& currentThread = ThisThread::get();

	for (auto& phase : phases)
		for (auto& stage : stages)
		{
			disableFpuContext();

			decltype(setFpuRegisters({}, {})) fpscr {};
			if (stage.threadValue != 0)	// should FPU be used in main thread?
				fpscr = setFpuRegisters(stage.threadValue, true);

			if (phase.sender(currentThread, stage) != 0)
				return false;

			// FPU context may be active only if FPU was used in main thread
			if (isFpuContextActive() != (stage.threadValue != 0))
				return false;

			if (stage.threadValue != 0)	// was FPU used in main thread?
				if (checkFpuRegisters(stage.threadValue, fpscr) == false)
					return false;

			expectedContextSwitchCount += phase.contextSwitchCount;
			if (statistics::getContextSwitchCount() - contextSwitchCount != expectedContextSwitchCount)
				return false;
		}

	if (statistics::getContextSwitchCount() - contextSwitchCount != expectedContextSwitchCount)
		return false;

#endif	// ARMV7_M_FPU_SIGNAL_TEST_CASE_ENABLED == 1

	return true;
}

}	// namespace test

}	// namespace distortos

#if ARMV7_M_FPU_SIGNAL_TEST_CASE_ENABLED == 1

/*---------------------------------------------------------------------------------------------------------------------+
| global functions
+---------------------------------------------------------------------------------------------------------------------*/

/**
 * \brief UsageFault_Handler() for ARMv7-M
 */

extern "C" void UsageFault_Handler()
{
	SCB->CFSR = SCB_CFSR_DIVBYZERO_Msk;			// clear "divide by 0" flag
	SCB->CCR &= ~SCB_CCR_DIV_0_TRP_Msk;			// disable trapping of "divide by 0" by UsageFault
	SCB->SHCSR &= ~SCB_SHCSR_USGFAULTENA_Msk;	// disable UsageFault

	using namespace distortos::test;
	queueSignalWrapper(*context->thread, *context->stage, false, context->sharedRet);
}

#endif	// ARMV7_M_FPU_SIGNAL_TEST_CASE_ENABLED == 1
<|endoftext|>


<commit_before>/*
 * Copyright(c) Sophist Solutions, Inc. 1990-2020.  All rights reserved
 */
#ifndef _Stroika_Foundation_Configuration_Version_inl_
#define _Stroika_Foundation_Configuration_Version_inl_ 1

/*
 ********************************************************************************
 ***************************** Implementation Details ***************************
 ********************************************************************************
 */
#if defined(__cplusplus)
namespace Stroika::Foundation::Configuration {
#endif

    /*
     ********************************************************************************
     *********************************** Version ************************************
     ********************************************************************************
     */
    constexpr Version::Version ()
        : fMajorVer{0}
        , fMinorVer{0}
        , fVerStage{VersionStage::eSTART}
        , fVerSubStage{0}
        , fFinalBuild{0}
    {
    }
    constexpr Version::Version (Binary32BitFullVersionType fullVersionNumber)
        : fMajorVer{static_cast<uint8_t> ((fullVersionNumber >> 24) & 0x8f)}
        , fMinorVer{static_cast<uint8_t> ((fullVersionNumber >> 16) & 0xff)}
        , fVerStage{static_cast<VersionStage> ((fullVersionNumber >> 13) & 0x07)}
        , fVerSubStage{static_cast<uint16_t> ((fullVersionNumber >> 1) & 0xfff)}
        , fFinalBuild{static_cast<bool> (fullVersionNumber & 1)}
    {
        Assert (fVerSubStage < kMaxVersionSubStage); // we got shift/mask right above
    }
    constexpr Version::Version (uint8_t majorVer, uint8_t minorVer, VersionStage verStage, uint16_t verSubStage, bool finalBuild)
        : fMajorVer{majorVer}
        , fMinorVer{minorVer}
        , fVerStage{verStage}
        , fVerSubStage{verSubStage}
        , fFinalBuild{finalBuild}
    {
        // @todo validate arg verSubStage < kMaxVersionSubStage
    }
    constexpr Binary32BitFullVersionType Version::AsFullVersionNum () const
    {
        // @todo validate arg verSubStage < kMaxVersionSubStage
        return Stroika_Make_FULL_VERSION (fMajorVer, fMinorVer, ((uint8_t)fVerStage), fVerSubStage, static_cast<int> (fFinalBuild));
    }
    inline Characters::String Version::ToString () const
    {
        return AsPrettyVersionString ();
    }
#if __cpp_impl_three_way_comparison >= 201907
    constexpr strong_ordering Version::operator<=> (const Version& rhs) const
    {
        return make_signed_t<Binary32BitFullVersionType> (AsFullVersionNum ()) <=> make_signed_t<Binary32BitFullVersionType> (rhs.AsFullVersionNum ());
    }
    constexpr bool Version::operator== (const Version& rhs) const
    {
        return make_signed_t<Binary32BitFullVersionType> (AsFullVersionNum ()) == make_signed_t<Binary32BitFullVersionType> (rhs.AsFullVersionNum ());
    }
#endif

    /*
     ********************************************************************************
     ***************************** Version operators ********************************
     ********************************************************************************
     */
#if __cpp_impl_three_way_comparison < 201907
    constexpr bool operator< (const Version& lhs, const Version& rhs)
    {
        return make_signed_t<Binary32BitFullVersionType> (lhs.AsFullVersionNum ()) < make_signed_t<Binary32BitFullVersionType> (rhs.AsFullVersionNum ()));
    }
    constexpr bool operator<= (const Version& lhs, const Version& rhs)
    {
        return make_signed_t<Binary32BitFullVersionType> (lhs.AsFullVersionNum ()) <= make_signed_t<Binary32BitFullVersionType> (rhs.AsFullVersionNum ()));
    }
    constexpr bool operator== (const Version& lhs, const Version& rhs)
    {
        return make_signed_t<Binary32BitFullVersionType> (lhs.AsFullVersionNum ()) == make_signed_t<Binary32BitFullVersionType> (rhs.AsFullVersionNum ()));
    }
    constexpr bool operator!= (const Version& lhs, const Version& rhs)
    {
        return make_signed_t<Binary32BitFullVersionType> (lhs.AsFullVersionNum ()) != make_signed_t<Binary32BitFullVersionType> (rhs.AsFullVersionNum ()));
    }
    constexpr bool operator>= (const Version& lhs, const Version& rhs)
    {
        return make_signed_t<Binary32BitFullVersionType> (lhs.AsFullVersionNum ()) >= make_signed_t<Binary32BitFullVersionType> (rhs.AsFullVersionNum ()));
    }
    constexpr bool operator> (const Version& lhs, const Version& rhs)
    {
        return make_signed_t<Binary32BitFullVersionType> (lhs.AsFullVersionNum ()) > make_signed_t<Binary32BitFullVersionType> (rhs.AsFullVersionNum ()));
    }
#endif

#if defined(__cplusplus)
}
#endif

#endif /*_Stroika_Foundation_Configuration_Version_inl_*/
<commit_msg>fixed typo<commit_after>/*
 * Copyright(c) Sophist Solutions, Inc. 1990-2020.  All rights reserved
 */
#ifndef _Stroika_Foundation_Configuration_Version_inl_
#define _Stroika_Foundation_Configuration_Version_inl_ 1

/*
 ********************************************************************************
 ***************************** Implementation Details ***************************
 ********************************************************************************
 */
#if defined(__cplusplus)
namespace Stroika::Foundation::Configuration {
#endif

    /*
     ********************************************************************************
     *********************************** Version ************************************
     ********************************************************************************
     */
    constexpr Version::Version ()
        : fMajorVer{0}
        , fMinorVer{0}
        , fVerStage{VersionStage::eSTART}
        , fVerSubStage{0}
        , fFinalBuild{0}
    {
    }
    constexpr Version::Version (Binary32BitFullVersionType fullVersionNumber)
        : fMajorVer{static_cast<uint8_t> ((fullVersionNumber >> 24) & 0x8f)}
        , fMinorVer{static_cast<uint8_t> ((fullVersionNumber >> 16) & 0xff)}
        , fVerStage{static_cast<VersionStage> ((fullVersionNumber >> 13) & 0x07)}
        , fVerSubStage{static_cast<uint16_t> ((fullVersionNumber >> 1) & 0xfff)}
        , fFinalBuild{static_cast<bool> (fullVersionNumber & 1)}
    {
        Assert (fVerSubStage < kMaxVersionSubStage); // we got shift/mask right above
    }
    constexpr Version::Version (uint8_t majorVer, uint8_t minorVer, VersionStage verStage, uint16_t verSubStage, bool finalBuild)
        : fMajorVer{majorVer}
        , fMinorVer{minorVer}
        , fVerStage{verStage}
        , fVerSubStage{verSubStage}
        , fFinalBuild{finalBuild}
    {
        // @todo validate arg verSubStage < kMaxVersionSubStage
    }
    constexpr Binary32BitFullVersionType Version::AsFullVersionNum () const
    {
        // @todo validate arg verSubStage < kMaxVersionSubStage
        return Stroika_Make_FULL_VERSION (fMajorVer, fMinorVer, ((uint8_t)fVerStage), fVerSubStage, static_cast<int> (fFinalBuild));
    }
    inline Characters::String Version::ToString () const
    {
        return AsPrettyVersionString ();
    }
#if __cpp_impl_three_way_comparison >= 201907
    constexpr strong_ordering Version::operator<=> (const Version& rhs) const
    {
        return make_signed_t<Binary32BitFullVersionType> (AsFullVersionNum ()) <=> make_signed_t<Binary32BitFullVersionType> (rhs.AsFullVersionNum ());
    }
    constexpr bool Version::operator== (const Version& rhs) const
    {
        return make_signed_t<Binary32BitFullVersionType> (AsFullVersionNum ()) == make_signed_t<Binary32BitFullVersionType> (rhs.AsFullVersionNum ());
    }
#endif

    /*
     ********************************************************************************
     ***************************** Version operators ********************************
     ********************************************************************************
     */
#if __cpp_impl_three_way_comparison < 201907
    constexpr bool operator< (const Version& lhs, const Version& rhs)
    {
        return make_signed_t<Binary32BitFullVersionType> (lhs.AsFullVersionNum ()) < make_signed_t<Binary32BitFullVersionType> (rhs.AsFullVersionNum ());
    }
    constexpr bool operator<= (const Version& lhs, const Version& rhs)
    {
        return make_signed_t<Binary32BitFullVersionType> (lhs.AsFullVersionNum ()) <= make_signed_t<Binary32BitFullVersionType> (rhs.AsFullVersionNum ());
    }
    constexpr bool operator== (const Version& lhs, const Version& rhs)
    {
        return make_signed_t<Binary32BitFullVersionType> (lhs.AsFullVersionNum ()) == make_signed_t<Binary32BitFullVersionType> (rhs.AsFullVersionNum ());
    }
    constexpr bool operator!= (const Version& lhs, const Version& rhs)
    {
        return make_signed_t<Binary32BitFullVersionType> (lhs.AsFullVersionNum ()) != make_signed_t<Binary32BitFullVersionType> (rhs.AsFullVersionNum ());
    }
    constexpr bool operator>= (const Version& lhs, const Version& rhs)
    {
        return make_signed_t<Binary32BitFullVersionType> (lhs.AsFullVersionNum ()) >= make_signed_t<Binary32BitFullVersionType> (rhs.AsFullVersionNum ());
    }
    constexpr bool operator> (const Version& lhs, const Version& rhs)
    {
        return make_signed_t<Binary32BitFullVersionType> (lhs.AsFullVersionNum ()) > make_signed_t<Binary32BitFullVersionType> (rhs.AsFullVersionNum ());
    }
#endif

#if defined(__cplusplus)
}
#endif

#endif /*_Stroika_Foundation_Configuration_Version_inl_*/
<|endoftext|>


<commit_before>// $Id$
// vim:tabstop=2
/***********************************************************************
 Moses - factored phrase-based language decoder
 Copyright (C) 2010 Hieu Hoang

 This library is free software; you can redistribute it and/or
 modify it under the terms of the GNU Lesser General Public
 License as published by the Free Software Foundation; either
 version 2.1 of the License, or (at your option) any later version.

 This library is distributed in the hope that it will be useful,
 but WITHOUT ANY WARRANTY; without even the implied warranty of
 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
 Lesser General Public License for more details.

 You should have received a copy of the GNU Lesser General Public
 License along with this library; if not, write to the Free Software
 Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301  USA
 ***********************************************************************/

#include "ChartTrellisNode.h"
#include "ChartHypothesis.h"
#include "ScoreComponentCollection.h"

using namespace std;

namespace Moses
{

ChartTrellisNode::ChartTrellisNode(const ChartHypothesis *hypo)
  :m_hypo(hypo)
{
  const std::vector<const ChartHypothesis*> &prevHypos = hypo->GetPrevHypos();

  m_edge.reserve(prevHypos.size());
  for (size_t ind = 0; ind < prevHypos.size(); ++ind) {
    const ChartHypothesis *prevHypo = prevHypos[ind];
    ChartTrellisNode *child = new ChartTrellisNode(prevHypo);
    m_edge.push_back(child);
  }

  assert(m_hypo);
}

ChartTrellisNode::ChartTrellisNode(const ChartTrellisNode &origNode
                         , const ChartTrellisNode &soughtNode
                         , const ChartHypothesis &replacementHypo
                         , ScoreComponentCollection	&scoreChange
                         , const ChartTrellisNode *&nodeChanged)
{
  if (&origNode.GetHypothesis() == &soughtNode.GetHypothesis()) {
    // this node should be replaced
    m_hypo = &replacementHypo;
    nodeChanged = this;

    // scores
    assert(scoreChange.GetWeightedScore() == 0); // should only be changing 1 node

    scoreChange = replacementHypo.GetScoreBreakdown();
    scoreChange.MinusEquals(origNode.GetHypothesis().GetScoreBreakdown());

    float deltaScore = scoreChange.GetWeightedScore();
    assert(deltaScore <= 0.0005);

    // follow prev hypos back to beginning
    const std::vector<const ChartHypothesis*> &prevHypos = replacementHypo.GetPrevHypos();
    vector<const ChartHypothesis*>::const_iterator iter;
    assert(m_edge.empty());
    m_edge.reserve(prevHypos.size());
    for (iter = prevHypos.begin(); iter != prevHypos.end(); ++iter) {
      const ChartHypothesis *prevHypo = *iter;
      ChartTrellisNode *prevNode = new ChartTrellisNode(prevHypo);
      m_edge.push_back(prevNode);
    }

  } else {
    // not the node we're looking for. Copy as-is and continue finding node
    m_hypo = &origNode.GetHypothesis();
    NodeChildren::const_iterator iter;
    assert(m_edge.empty());
    m_edge.reserve(origNode.m_edge.size());
    for (iter = origNode.m_edge.begin(); iter != origNode.m_edge.end(); ++iter) {
      const ChartTrellisNode &prevNode = **iter;
      ChartTrellisNode *newPrevNode = new ChartTrellisNode(prevNode, soughtNode, replacementHypo, scoreChange, nodeChanged);
      m_edge.push_back(newPrevNode);
    }
  }

  assert(m_hypo);
}

ChartTrellisNode::~ChartTrellisNode()
{
  RemoveAllInColl(m_edge);
}

Phrase ChartTrellisNode::GetOutputPhrase() const
{
  // exactly like same fn in hypothesis, but use trellis nodes instead of prevHypos pointer
  Phrase ret(Output, ARRAY_SIZE_INCR);

  const Phrase &currTargetPhrase = m_hypo->GetCurrTargetPhrase();
  for (size_t pos = 0; pos < currTargetPhrase.GetSize(); ++pos) {
    const Word &word = currTargetPhrase.GetWord(pos);
    if (word.IsNonTerminal()) {
      // non-term. fill out with prev hypo
      size_t nonTermInd = m_hypo->GetCoveredChartSpanTargetOrder(pos);
      const ChartTrellisNode &childNode = GetChild(nonTermInd);
      Phrase childPhrase = childNode.GetOutputPhrase();
      ret.Append(childPhrase);
    } else {
      ret.AddWord(word);
    }
  }

  return ret;
}

std::ostream& operator<<(std::ostream &out, const ChartTrellisNode &node)
{
  out << "*   " << node.GetHypothesis() << endl;

  ChartTrellisNode::NodeChildren::const_iterator iter;
  for (iter = node.GetChildren().begin(); iter != node.GetChildren().end(); ++iter) {
    out << **iter;
  }

  return out;
}

}

<commit_msg>derivation of n-best list output<commit_after>// $Id$
// vim:tabstop=2
/***********************************************************************
 Moses - factored phrase-based language decoder
 Copyright (C) 2010 Hieu Hoang

 This library is free software; you can redistribute it and/or
 modify it under the terms of the GNU Lesser General Public
 License as published by the Free Software Foundation; either
 version 2.1 of the License, or (at your option) any later version.

 This library is distributed in the hope that it will be useful,
 but WITHOUT ANY WARRANTY; without even the implied warranty of
 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
 Lesser General Public License for more details.

 You should have received a copy of the GNU Lesser General Public
 License along with this library; if not, write to the Free Software
 Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301  USA
 ***********************************************************************/

#include "ChartTrellisNode.h"
#include "ChartHypothesis.h"
#include "ScoreComponentCollection.h"
#include "StaticData.h"

using namespace std;

namespace Moses
{

ChartTrellisNode::ChartTrellisNode(const ChartHypothesis *hypo)
  :m_hypo(hypo)
{
  const std::vector<const ChartHypothesis*> &prevHypos = hypo->GetPrevHypos();

  m_edge.reserve(prevHypos.size());
  for (size_t ind = 0; ind < prevHypos.size(); ++ind) {
    const ChartHypothesis *prevHypo = prevHypos[ind];
    ChartTrellisNode *child = new ChartTrellisNode(prevHypo);
    m_edge.push_back(child);
  }

  assert(m_hypo);
}

ChartTrellisNode::ChartTrellisNode(const ChartTrellisNode &origNode
                         , const ChartTrellisNode &soughtNode
                         , const ChartHypothesis &replacementHypo
                         , ScoreComponentCollection	&scoreChange
                         , const ChartTrellisNode *&nodeChanged)
{
  if (&origNode.GetHypothesis() == &soughtNode.GetHypothesis()) {
    // this node should be replaced
    m_hypo = &replacementHypo;
    nodeChanged = this;

    // scores
    assert(scoreChange.GetWeightedScore() == 0); // should only be changing 1 node

    scoreChange = replacementHypo.GetScoreBreakdown();
    scoreChange.MinusEquals(origNode.GetHypothesis().GetScoreBreakdown());

    float deltaScore = scoreChange.GetWeightedScore();
    assert(deltaScore <= 0.0005);

    // follow prev hypos back to beginning
    const std::vector<const ChartHypothesis*> &prevHypos = replacementHypo.GetPrevHypos();
    vector<const ChartHypothesis*>::const_iterator iter;
    assert(m_edge.empty());
    m_edge.reserve(prevHypos.size());
    for (iter = prevHypos.begin(); iter != prevHypos.end(); ++iter) {
      const ChartHypothesis *prevHypo = *iter;
      ChartTrellisNode *prevNode = new ChartTrellisNode(prevHypo);
      m_edge.push_back(prevNode);
    }

  } else {
    // not the node we're looking for. Copy as-is and continue finding node
    m_hypo = &origNode.GetHypothesis();
    NodeChildren::const_iterator iter;
    assert(m_edge.empty());
    m_edge.reserve(origNode.m_edge.size());
    for (iter = origNode.m_edge.begin(); iter != origNode.m_edge.end(); ++iter) {
      const ChartTrellisNode &prevNode = **iter;
      ChartTrellisNode *newPrevNode = new ChartTrellisNode(prevNode, soughtNode, replacementHypo, scoreChange, nodeChanged);
      m_edge.push_back(newPrevNode);
    }
  }

  assert(m_hypo);
}

ChartTrellisNode::~ChartTrellisNode()
{
  RemoveAllInColl(m_edge);
}

Phrase ChartTrellisNode::GetOutputPhrase() const
{
  // exactly like same fn in hypothesis, but use trellis nodes instead of prevHypos pointer
  Phrase ret(Output, ARRAY_SIZE_INCR);

  const ChartTranslationOption &transOpt = m_hypo->GetTranslationOption();

  VERBOSE(3, "Trans Opt:" << transOpt << std::endl);

  const Phrase &currTargetPhrase = m_hypo->GetCurrTargetPhrase();
  for (size_t pos = 0; pos < currTargetPhrase.GetSize(); ++pos) {
    const Word &word = currTargetPhrase.GetWord(pos);
    if (word.IsNonTerminal()) {
      // non-term. fill out with prev hypo
      size_t nonTermInd = m_hypo->GetCoveredChartSpanTargetOrder(pos);
      const ChartTrellisNode &childNode = GetChild(nonTermInd);
      Phrase childPhrase = childNode.GetOutputPhrase();
      ret.Append(childPhrase);
    } else {
      ret.AddWord(word);
    }
  }

  return ret;
}

std::ostream& operator<<(std::ostream &out, const ChartTrellisNode &node)
{
  out << "*   " << node.GetHypothesis() << endl;

  ChartTrellisNode::NodeChildren::const_iterator iter;
  for (iter = node.GetChildren().begin(); iter != node.GetChildren().end(); ++iter) {
    out << **iter;
  }

  return out;
}

}

<|endoftext|>


<commit_before>// $Id$

/***********************************************************************
Moses - factored phrase-based language decoder
Copyright (C) 2006 University of Edinburgh

This library is free software; you can redistribute it and/or
modify it under the terms of the GNU Lesser General Public
License as published by the Free Software Foundation; either
version 2.1 of the License, or (at your option) any later version.

This library is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
Lesser General Public License for more details.

You should have received a copy of the GNU Lesser General Public
License along with this library; if not, write to the Free Software
Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301  USA
***********************************************************************/

#include <cassert>
#include <cstring>
#include <iostream>
#include <stdlib.h>
#include "lm/binary_format.hh"
#include "lm/enumerate_vocab.hh"
#include "lm/model.hh"

#include "LanguageModelKen.h"
#include "FFState.h"
#include "TypeDef.h"
#include "Util.h"
#include "FactorCollection.h"
#include "Phrase.h"
#include "InputFileStream.h"
#include "StaticData.h"
#include "ChartHypothesis.h"

using namespace std;

namespace Moses
{

LanguageModelKenBase::~LanguageModelKenBase() {}

namespace
{

  class LanguageModelChartStateKenLM : public FFState
  {
  private:
    lm::ngram::ChartState m_state;
    const ChartHypothesis *m_hypo;

  public:
    explicit LanguageModelChartStateKenLM(const ChartHypothesis &hypo)
        :m_hypo(&hypo)
    {}

    const ChartHypothesis* GetHypothesis() const { return m_hypo; }

    const lm::ngram::ChartState &GetChartState() const { return m_state; }
    lm::ngram::ChartState &GetChartState() { return m_state; }

    int Compare(const FFState& o) const
    {
      const LanguageModelChartStateKenLM &other = static_cast<const LanguageModelChartStateKenLM&>(o);
      int ret = m_state.Compare(other.m_state);
      return ret;
    }
  };

class MappingBuilder : public lm::ngram::EnumerateVocab
{
public:
  MappingBuilder(FactorCollection &factorCollection, std::vector<lm::WordIndex> &mapping)
    : m_factorCollection(factorCollection), m_mapping(mapping) {}

  void Add(lm::WordIndex index, const StringPiece &str) {
    std::size_t factorId = m_factorCollection.AddFactor(str)->GetId();
    if (m_mapping.size() <= factorId) {
      // 0 is <unk> :-)
      m_mapping.resize(factorId + 1);
    }
    m_mapping[factorId] = index;
  }

private:
  FactorCollection &m_factorCollection;
  std::vector<lm::WordIndex> &m_mapping;
};

struct KenLMState : public FFState {
  lm::ngram::State state;
  int Compare(const FFState &o) const {
    const KenLMState &other = static_cast<const KenLMState &>(o);
    if (state.length < other.state.length) return -1;
    if (state.length > other.state.length) return 1;
    return std::memcmp(state.words, other.state.words, sizeof(lm::WordIndex) * state.length);
  }
};

/*
 * An implementation of single factor LM using Ken's code.
 */
template <class Model> class LanguageModelKen : public LanguageModelKenBase
{
private:
  Model *m_ngram;
  std::vector<lm::WordIndex> m_lmIdLookup;
  bool m_lazy;
  KenLMState m_nullContextState;
  KenLMState m_beginSentenceState;

  void TranslateIDs(const std::vector<const Word*> &contextFactor, lm::WordIndex *indices) const;

public:
  LanguageModelKen(bool lazy);
  ~LanguageModelKen();

  bool Load(const std::string &filePath
            , FactorType factorType
            , size_t nGramOrder);

  LMResult GetValueGivenState(const std::vector<const Word*> &contextFactor, FFState &state) const {
    return GetKenFullScoreGivenState(contextFactor, state);
  }
  LMKenResult GetKenFullScoreGivenState(const std::vector<const Word*> &contextFactor, FFState &state) const;

  LMResult GetValueForgotState(const std::vector<const Word*> &contextFactor, FFState &outState) const {
    return GetKenFullScoreForgotState(contextFactor, outState);
  }
  LMKenResult GetKenFullScoreForgotState(const std::vector<const Word*> &contextFactor, FFState &outState) const;

  void GetState(const std::vector<const Word*> &contextFactor, FFState &outState) const;

  const FFState *GetNullContextState() const;
  const FFState *GetBeginSentenceState() const;
  FFState *NewState(const FFState *from = NULL) const;

  lm::WordIndex GetLmID(const std::string &str) const;

  void CleanUpAfterSentenceProcessing() {}
  void InitializeBeforeSentenceProcessing() {}

  FFState *EvaluateChart(const ChartHypothesis& cur_hypo,
                         int featureID,
                         ScoreComponentCollection *accumulator,
                         const LanguageModel *feature) const;
};

template <class Model>
FFState *LanguageModelKen<Model>::EvaluateChart(
    const ChartHypothesis& hypo,
    int featureID,
    ScoreComponentCollection *accumulator,
    const LanguageModel *feature) const
{
  LanguageModelChartStateKenLM *newState = new LanguageModelChartStateKenLM(hypo);
  lm::ngram::RuleScore<Model> ruleScore(*m_ngram, newState->GetChartState());
  const AlignmentInfo::NonTermIndexMap &nonTermIndexMap = hypo.GetCurrTargetPhrase().GetAlignmentInfo().GetNonTermIndexMap();

  const size_t size = hypo.GetCurrTargetPhrase().GetSize();
  size_t phrasePos = 0;
  // Special cases for first word.
  if (size) {
    const Word &word = hypo.GetCurrTargetPhrase().GetWord(0);
    if (word == GetSentenceStartArray()) {
      // Begin of sentence
      ruleScore.BeginSentence();
      phrasePos++;
    } else if (word.IsNonTerminal()) {
      // Non-terminal is first so we can copy instead of rescoring.
      const ChartHypothesis *prevHypo = hypo.GetPrevHypo(nonTermIndexMap[phrasePos]);
      const lm::ngram::ChartState &prevState = static_cast<const LanguageModelChartStateKenLM*>(prevHypo->GetFFState(featureID))->GetChartState();
      ruleScore.BeginNonTerminal(prevState, prevHypo->GetScoreBreakdown().GetScoresForProducer(feature)[0]);
      phrasePos++;
    }
  }

  for (; phrasePos < size; phrasePos++) {
    const Word &word = hypo.GetCurrTargetPhrase().GetWord(phrasePos);
    if (word.IsNonTerminal()) {
      const ChartHypothesis *prevHypo = hypo.GetPrevHypo(nonTermIndexMap[phrasePos]);
      const lm::ngram::ChartState &prevState = static_cast<const LanguageModelChartStateKenLM*>(prevHypo->GetFFState(featureID))->GetChartState();
      ruleScore.NonTerminal(prevState, prevHypo->GetScoreBreakdown().GetScoresForProducer(feature)[0]);
    } else {
      std::size_t factor = word.GetFactor(GetFactorType())->GetId();
      lm::WordIndex new_word = (factor >= m_lmIdLookup.size() ? 0 : m_lmIdLookup[factor]);
      ruleScore.Terminal(new_word);
    }
  }

  accumulator->Assign(feature, ruleScore.Finish());
  return newState;
}

template <class Model> void LanguageModelKen<Model>::TranslateIDs(const std::vector<const Word*> &contextFactor, lm::WordIndex *indices) const
{
  FactorType factorType = GetFactorType();
  // set up context
  for (size_t i = 0 ; i < contextFactor.size(); i++) {
    std::size_t factor = contextFactor[i]->GetFactor(factorType)->GetId();
    lm::WordIndex new_word = (factor >= m_lmIdLookup.size() ? 0 : m_lmIdLookup[factor]);
    indices[contextFactor.size() - 1 - i] = new_word;
  }
}

template <class Model> LanguageModelKen<Model>::LanguageModelKen(bool lazy)
  :m_ngram(NULL), m_lazy(lazy)
{
}

template <class Model> LanguageModelKen<Model>::~LanguageModelKen()
{
  delete m_ngram;
}

template <class Model> bool LanguageModelKen<Model>::Load(const std::string &filePath,
    FactorType factorType,
    size_t /*nGramOrder*/)
{
  m_factorType = factorType;
  m_filePath   = filePath;

  FactorCollection &factorCollection = FactorCollection::Instance();
  m_sentenceStart = factorCollection.AddFactor(BOS_);
  m_sentenceStartArray[m_factorType] = m_sentenceStart;
  m_sentenceEnd = factorCollection.AddFactor(EOS_);
  m_sentenceEndArray[m_factorType] = m_sentenceEnd;

  MappingBuilder builder(factorCollection, m_lmIdLookup);
  lm::ngram::Config config;

  IFVERBOSE(1) {
    config.messages = &std::cerr;
  }
  else {
    config.messages = NULL;
  }

  config.enumerate_vocab = &builder;
  config.load_method = m_lazy ? util::LAZY : util::POPULATE_OR_READ;

  try {
    m_ngram = new Model(filePath.c_str(), config);
  } catch (std::exception &e) {
    std::cerr << e.what() << std::endl;
    abort();
  }
  m_nGramOrder  = m_ngram->Order();

  m_nullContextState.state = m_ngram->NullContextState();
  m_beginSentenceState.state = m_ngram->BeginSentenceState();
  return true;
}

template <class Model> LMKenResult LanguageModelKen<Model>::GetKenFullScoreGivenState(const std::vector<const Word*> &contextFactor, FFState &state) const
{
  LMKenResult result;
  if (contextFactor.empty()) {
    result.score = 0.0;
    result.unknown = false;
    result.ngram_length = 0;
    return result;
  }
  lm::ngram::State &realState = static_cast<KenLMState&>(state).state;
  std::size_t factor = contextFactor.back()->GetFactor(GetFactorType())->GetId();
  lm::WordIndex new_word = (factor >= m_lmIdLookup.size() ? 0 : m_lmIdLookup[factor]);
  lm::ngram::State copied(realState);
  lm::FullScoreReturn ret(m_ngram->FullScore(copied, new_word, realState));

  result.score = TransformLMScore(ret.prob);
  result.unknown = (new_word == 0);
  result.ngram_length = ret.ngram_length;
  return result;
}

template <class Model> LMKenResult LanguageModelKen<Model>::GetKenFullScoreForgotState(const vector<const Word*> &contextFactor, FFState &outState) const
{
  LMKenResult result;
  if (contextFactor.empty()) {
    static_cast<KenLMState&>(outState).state = m_ngram->NullContextState();
    result.score = 0.0;
    result.unknown = false;
    result.ngram_length = 0;
    return result;
  }

  lm::WordIndex indices[contextFactor.size()];
  TranslateIDs(contextFactor, indices);

  lm::FullScoreReturn ret(m_ngram->FullScoreForgotState(indices + 1, indices + contextFactor.size(), indices[0], static_cast<KenLMState&>(outState).state));

  result.score = TransformLMScore(ret.prob);
  result.unknown = (indices[0] == 0);
  result.ngram_length = ret.ngram_length;
  return result;
}

template <class Model> void LanguageModelKen<Model>::GetState(const std::vector<const Word*> &contextFactor, FFState &outState) const
{
  if (contextFactor.empty()) {
    static_cast<KenLMState&>(outState).state = m_ngram->NullContextState();
    return;
  }
  lm::WordIndex indices[contextFactor.size()];
  TranslateIDs(contextFactor, indices);
  m_ngram->GetState(indices, indices + contextFactor.size(), static_cast<KenLMState&>(outState).state);
}

template <class Model> const FFState *LanguageModelKen<Model>::GetNullContextState() const
{
  return &m_nullContextState;
}

template <class Model> const FFState *LanguageModelKen<Model>::GetBeginSentenceState() const
{
  return &m_beginSentenceState;
}

template <class Model> FFState *LanguageModelKen<Model>::NewState(const FFState *from) const
{
  KenLMState *ret = new KenLMState;
  if (from) {
    ret->state = static_cast<const KenLMState&>(*from).state;
  }
  return ret;
}

template <class Model> lm::WordIndex LanguageModelKen<Model>::GetLmID(const std::string &str) const
{
  return m_ngram->GetVocabulary().Index(str);
}

} // namespace

LanguageModelSingleFactor *ConstructKenLM(const std::string &file, bool lazy)
{
  lm::ngram::ModelType model_type;
  if (lm::ngram::RecognizeBinary(file.c_str(), model_type)) {
    switch(model_type) {
    case lm::ngram::HASH_PROBING:
      return new LanguageModelKen<lm::ngram::ProbingModel>(lazy);
    case lm::ngram::TRIE_SORTED:
      return new LanguageModelKen<lm::ngram::TrieModel>(lazy);
    case lm::ngram::QUANT_TRIE_SORTED:
      return new LanguageModelKen<lm::ngram::QuantTrieModel>(lazy);
    case lm::ngram::ARRAY_TRIE_SORTED:
      return new LanguageModelKen<lm::ngram::ArrayTrieModel>(lazy);
    case lm::ngram::QUANT_ARRAY_TRIE_SORTED:
      return new LanguageModelKen<lm::ngram::QuantArrayTrieModel>(lazy);
    default:
      std::cerr << "Unrecognized kenlm model type " << model_type << std::endl;
      abort();
    }
  } else {
    return new LanguageModelKen<lm::ngram::ProbingModel>(lazy);
  }
}

}

<commit_msg>compile error due to last commit<commit_after>// $Id$

/***********************************************************************
Moses - factored phrase-based language decoder
Copyright (C) 2006 University of Edinburgh

This library is free software; you can redistribute it and/or
modify it under the terms of the GNU Lesser General Public
License as published by the Free Software Foundation; either
version 2.1 of the License, or (at your option) any later version.

This library is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
Lesser General Public License for more details.

You should have received a copy of the GNU Lesser General Public
License along with this library; if not, write to the Free Software
Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301  USA
***********************************************************************/

#include <cassert>
#include <cstring>
#include <iostream>
#include <stdlib.h>
#include "lm/binary_format.hh"
#include "lm/enumerate_vocab.hh"
#include "lm/model.hh"

#include "LanguageModelKen.h"
#include "FFState.h"
#include "TypeDef.h"
#include "Util.h"
#include "FactorCollection.h"
#include "Phrase.h"
#include "InputFileStream.h"
#include "StaticData.h"
#include "ChartHypothesis.h"

using namespace std;

namespace Moses
{

LanguageModelKenBase::~LanguageModelKenBase() {}

namespace
{

  class LanguageModelChartStateKenLM : public FFState
  {
  private:
    lm::ngram::ChartState m_state;
    const ChartHypothesis *m_hypo;

  public:
    explicit LanguageModelChartStateKenLM(const ChartHypothesis &hypo)
        :m_hypo(&hypo)
    {}

    const ChartHypothesis* GetHypothesis() const { return m_hypo; }

    const lm::ngram::ChartState &GetChartState() const { return m_state; }
    lm::ngram::ChartState &GetChartState() { return m_state; }

    int Compare(const FFState& o) const
    {
      const LanguageModelChartStateKenLM &other = static_cast<const LanguageModelChartStateKenLM&>(o);
      int ret = m_state.Compare(other.m_state);
      return ret;
    }
  };

class MappingBuilder : public lm::ngram::EnumerateVocab
{
public:
  MappingBuilder(FactorCollection &factorCollection, std::vector<lm::WordIndex> &mapping)
    : m_factorCollection(factorCollection), m_mapping(mapping) {}

  void Add(lm::WordIndex index, const StringPiece &str) {
    std::size_t factorId = m_factorCollection.AddFactor(str.data())->GetId();
    if (m_mapping.size() <= factorId) {
      // 0 is <unk> :-)
      m_mapping.resize(factorId + 1);
    }
    m_mapping[factorId] = index;
  }

private:
  FactorCollection &m_factorCollection;
  std::vector<lm::WordIndex> &m_mapping;
};

struct KenLMState : public FFState {
  lm::ngram::State state;
  int Compare(const FFState &o) const {
    const KenLMState &other = static_cast<const KenLMState &>(o);
    if (state.length < other.state.length) return -1;
    if (state.length > other.state.length) return 1;
    return std::memcmp(state.words, other.state.words, sizeof(lm::WordIndex) * state.length);
  }
};

/*
 * An implementation of single factor LM using Ken's code.
 */
template <class Model> class LanguageModelKen : public LanguageModelKenBase
{
private:
  Model *m_ngram;
  std::vector<lm::WordIndex> m_lmIdLookup;
  bool m_lazy;
  KenLMState m_nullContextState;
  KenLMState m_beginSentenceState;

  void TranslateIDs(const std::vector<const Word*> &contextFactor, lm::WordIndex *indices) const;

public:
  LanguageModelKen(bool lazy);
  ~LanguageModelKen();

  bool Load(const std::string &filePath
            , FactorType factorType
            , size_t nGramOrder);

  LMResult GetValueGivenState(const std::vector<const Word*> &contextFactor, FFState &state) const {
    return GetKenFullScoreGivenState(contextFactor, state);
  }
  LMKenResult GetKenFullScoreGivenState(const std::vector<const Word*> &contextFactor, FFState &state) const;

  LMResult GetValueForgotState(const std::vector<const Word*> &contextFactor, FFState &outState) const {
    return GetKenFullScoreForgotState(contextFactor, outState);
  }
  LMKenResult GetKenFullScoreForgotState(const std::vector<const Word*> &contextFactor, FFState &outState) const;

  void GetState(const std::vector<const Word*> &contextFactor, FFState &outState) const;

  const FFState *GetNullContextState() const;
  const FFState *GetBeginSentenceState() const;
  FFState *NewState(const FFState *from = NULL) const;

  lm::WordIndex GetLmID(const std::string &str) const;

  void CleanUpAfterSentenceProcessing() {}
  void InitializeBeforeSentenceProcessing() {}

  FFState *EvaluateChart(const ChartHypothesis& cur_hypo,
                         int featureID,
                         ScoreComponentCollection *accumulator,
                         const LanguageModel *feature) const;
};

template <class Model>
FFState *LanguageModelKen<Model>::EvaluateChart(
    const ChartHypothesis& hypo,
    int featureID,
    ScoreComponentCollection *accumulator,
    const LanguageModel *feature) const
{
  LanguageModelChartStateKenLM *newState = new LanguageModelChartStateKenLM(hypo);
  lm::ngram::RuleScore<Model> ruleScore(*m_ngram, newState->GetChartState());
  const AlignmentInfo::NonTermIndexMap &nonTermIndexMap = hypo.GetCurrTargetPhrase().GetAlignmentInfo().GetNonTermIndexMap();

  const size_t size = hypo.GetCurrTargetPhrase().GetSize();
  size_t phrasePos = 0;
  // Special cases for first word.
  if (size) {
    const Word &word = hypo.GetCurrTargetPhrase().GetWord(0);
    if (word == GetSentenceStartArray()) {
      // Begin of sentence
      ruleScore.BeginSentence();
      phrasePos++;
    } else if (word.IsNonTerminal()) {
      // Non-terminal is first so we can copy instead of rescoring.
      const ChartHypothesis *prevHypo = hypo.GetPrevHypo(nonTermIndexMap[phrasePos]);
      const lm::ngram::ChartState &prevState = static_cast<const LanguageModelChartStateKenLM*>(prevHypo->GetFFState(featureID))->GetChartState();
      ruleScore.BeginNonTerminal(prevState, prevHypo->GetScoreBreakdown().GetScoresForProducer(feature)[0]);
      phrasePos++;
    }
  }

  for (; phrasePos < size; phrasePos++) {
    const Word &word = hypo.GetCurrTargetPhrase().GetWord(phrasePos);
    if (word.IsNonTerminal()) {
      const ChartHypothesis *prevHypo = hypo.GetPrevHypo(nonTermIndexMap[phrasePos]);
      const lm::ngram::ChartState &prevState = static_cast<const LanguageModelChartStateKenLM*>(prevHypo->GetFFState(featureID))->GetChartState();
      ruleScore.NonTerminal(prevState, prevHypo->GetScoreBreakdown().GetScoresForProducer(feature)[0]);
    } else {
      std::size_t factor = word.GetFactor(GetFactorType())->GetId();
      lm::WordIndex new_word = (factor >= m_lmIdLookup.size() ? 0 : m_lmIdLookup[factor]);
      ruleScore.Terminal(new_word);
    }
  }

  accumulator->Assign(feature, ruleScore.Finish());
  return newState;
}

template <class Model> void LanguageModelKen<Model>::TranslateIDs(const std::vector<const Word*> &contextFactor, lm::WordIndex *indices) const
{
  FactorType factorType = GetFactorType();
  // set up context
  for (size_t i = 0 ; i < contextFactor.size(); i++) {
    std::size_t factor = contextFactor[i]->GetFactor(factorType)->GetId();
    lm::WordIndex new_word = (factor >= m_lmIdLookup.size() ? 0 : m_lmIdLookup[factor]);
    indices[contextFactor.size() - 1 - i] = new_word;
  }
}

template <class Model> LanguageModelKen<Model>::LanguageModelKen(bool lazy)
  :m_ngram(NULL), m_lazy(lazy)
{
}

template <class Model> LanguageModelKen<Model>::~LanguageModelKen()
{
  delete m_ngram;
}

template <class Model> bool LanguageModelKen<Model>::Load(const std::string &filePath,
    FactorType factorType,
    size_t /*nGramOrder*/)
{
  m_factorType = factorType;
  m_filePath   = filePath;

  FactorCollection &factorCollection = FactorCollection::Instance();
  m_sentenceStart = factorCollection.AddFactor(BOS_);
  m_sentenceStartArray[m_factorType] = m_sentenceStart;
  m_sentenceEnd = factorCollection.AddFactor(EOS_);
  m_sentenceEndArray[m_factorType] = m_sentenceEnd;

  MappingBuilder builder(factorCollection, m_lmIdLookup);
  lm::ngram::Config config;

  IFVERBOSE(1) {
    config.messages = &std::cerr;
  }
  else {
    config.messages = NULL;
  }

  config.enumerate_vocab = &builder;
  config.load_method = m_lazy ? util::LAZY : util::POPULATE_OR_READ;

  try {
    m_ngram = new Model(filePath.c_str(), config);
  } catch (std::exception &e) {
    std::cerr << e.what() << std::endl;
    abort();
  }
  m_nGramOrder  = m_ngram->Order();

  m_nullContextState.state = m_ngram->NullContextState();
  m_beginSentenceState.state = m_ngram->BeginSentenceState();
  return true;
}

template <class Model> LMKenResult LanguageModelKen<Model>::GetKenFullScoreGivenState(const std::vector<const Word*> &contextFactor, FFState &state) const
{
  LMKenResult result;
  if (contextFactor.empty()) {
    result.score = 0.0;
    result.unknown = false;
    result.ngram_length = 0;
    return result;
  }
  lm::ngram::State &realState = static_cast<KenLMState&>(state).state;
  std::size_t factor = contextFactor.back()->GetFactor(GetFactorType())->GetId();
  lm::WordIndex new_word = (factor >= m_lmIdLookup.size() ? 0 : m_lmIdLookup[factor]);
  lm::ngram::State copied(realState);
  lm::FullScoreReturn ret(m_ngram->FullScore(copied, new_word, realState));

  result.score = TransformLMScore(ret.prob);
  result.unknown = (new_word == 0);
  result.ngram_length = ret.ngram_length;
  return result;
}

template <class Model> LMKenResult LanguageModelKen<Model>::GetKenFullScoreForgotState(const vector<const Word*> &contextFactor, FFState &outState) const
{
  LMKenResult result;
  if (contextFactor.empty()) {
    static_cast<KenLMState&>(outState).state = m_ngram->NullContextState();
    result.score = 0.0;
    result.unknown = false;
    result.ngram_length = 0;
    return result;
  }

  lm::WordIndex indices[contextFactor.size()];
  TranslateIDs(contextFactor, indices);

  lm::FullScoreReturn ret(m_ngram->FullScoreForgotState(indices + 1, indices + contextFactor.size(), indices[0], static_cast<KenLMState&>(outState).state));

  result.score = TransformLMScore(ret.prob);
  result.unknown = (indices[0] == 0);
  result.ngram_length = ret.ngram_length;
  return result;
}

template <class Model> void LanguageModelKen<Model>::GetState(const std::vector<const Word*> &contextFactor, FFState &outState) const
{
  if (contextFactor.empty()) {
    static_cast<KenLMState&>(outState).state = m_ngram->NullContextState();
    return;
  }
  lm::WordIndex indices[contextFactor.size()];
  TranslateIDs(contextFactor, indices);
  m_ngram->GetState(indices, indices + contextFactor.size(), static_cast<KenLMState&>(outState).state);
}

template <class Model> const FFState *LanguageModelKen<Model>::GetNullContextState() const
{
  return &m_nullContextState;
}

template <class Model> const FFState *LanguageModelKen<Model>::GetBeginSentenceState() const
{
  return &m_beginSentenceState;
}

template <class Model> FFState *LanguageModelKen<Model>::NewState(const FFState *from) const
{
  KenLMState *ret = new KenLMState;
  if (from) {
    ret->state = static_cast<const KenLMState&>(*from).state;
  }
  return ret;
}

template <class Model> lm::WordIndex LanguageModelKen<Model>::GetLmID(const std::string &str) const
{
  return m_ngram->GetVocabulary().Index(str);
}

} // namespace

LanguageModelSingleFactor *ConstructKenLM(const std::string &file, bool lazy)
{
  lm::ngram::ModelType model_type;
  if (lm::ngram::RecognizeBinary(file.c_str(), model_type)) {
    switch(model_type) {
    case lm::ngram::HASH_PROBING:
      return new LanguageModelKen<lm::ngram::ProbingModel>(lazy);
    case lm::ngram::TRIE_SORTED:
      return new LanguageModelKen<lm::ngram::TrieModel>(lazy);
    case lm::ngram::QUANT_TRIE_SORTED:
      return new LanguageModelKen<lm::ngram::QuantTrieModel>(lazy);
    case lm::ngram::ARRAY_TRIE_SORTED:
      return new LanguageModelKen<lm::ngram::ArrayTrieModel>(lazy);
    case lm::ngram::QUANT_ARRAY_TRIE_SORTED:
      return new LanguageModelKen<lm::ngram::QuantArrayTrieModel>(lazy);
    default:
      std::cerr << "Unrecognized kenlm model type " << model_type << std::endl;
      abort();
    }
  } else {
    return new LanguageModelKen<lm::ngram::ProbingModel>(lazy);
  }
}

}

<|endoftext|>