<commit_before>// ============================= BIT ALGORITHM ============================== //
// Project:         The C++ Bit Library
// Name:            bit_algorithm.hpp
// Description:     Optimized versions of algorithms for bit manipulation
// Creator:         Vincent Reverdy
// Contributor(s):  Vincent Reverdy [2015-2017]
//                  Maghav Kumar [2016-2017]
// License:         BSD 3-Clause License
// ========================================================================== //
#ifndef _BIT_ALGORITHM_HPP_INCLUDED
#define _BIT_ALGORITHM_HPP_INCLUDED
// ========================================================================== //

// ================================ PREAMBLE ================================ //
// C++ standard library
// Project sources
#include "bit_details.hpp"
#include "bit_value.hpp"
#include "bit_reference.hpp"
#include "bit_pointer.hpp"
#include "bit_iterator.hpp"
// Third-party libraries
// Miscellaneous
namespace bit {
// ========================================================================== //

/* ***************************** BIT CONSTANTS ****************************** */
// Bit constants
static constexpr bit_value bit0(0U);
static constexpr bit_value bit1(1U);
/* ************************************************************************** */

/* ************************** STANDARD ALGORITHMS *************************** */
// Non-modifying sequence operations
template <class InputIt>
typename bit_iterator<InputIt>::difference_type
count(
    bit_iterator<InputIt> first,
    bit_iterator<InputIt> last,
    bit_value value
);

// Modifying sequence operations
template <class BidirIt>
void reverse(
    bit_iterator<BidirIt> first,
    bit_iterator<BidirIt> last
);
/* ************************************************************************** */

// ------------------- NON-MODIFYING SEQUENCE OPERATIONS -------------------- //
// Counts the number of bits equal to the provided bit value
template <class InputIt>
typename bit_iterator<InputIt>::difference_type
count(
    bit_iterator<InputIt> first,
    bit_iterator<InputIt> last,
    bit_value value
)
{
    // Assertions
    _assert_range_viability(first, last);

    // Types and constants
    using underlying_type = typename bit_iterator<InputIt>::underlying_type;
    using difference_type = typename bit_iterator<InputIt>::difference_type;
    constexpr difference_type digits = binary_digits<underlying_type>::value;

    // Initialization
    difference_type result = 0;
    auto it = first.base();

    // Computation when bits belong to several underlying values
    if (first.base() != last.base()) {
        if (first.position() != 0) {
            result = _popcnt(*first.base() >> first.position());
            ++it;
        }
        for (; it != last.base(); ++it) {
            result += _popcnt(*it);
        }
        if (last.position() != 0) {
            result += _popcnt(*last.base() << (digits - last.position()));
        }
    // Computation when bits belong to the same underlying value
    } else {
        result = _popcnt(_bextr<underlying_type>(
            *first.base(),
            first.position(),
            last.position() - first.position()
        ));
    }

    // Negates when the number of zero bits is requested
    if (!static_cast<bool>(value)) {
        result = std::distance(first, last) - result;
    }

    // Finalization
    return result;
}
// -------------------------------------------------------------------------- //

// --------------------- MODIFYING SEQUENCE OPERATIONS ---------------------- //
// Reverses the order of the bits in the provided range
template <class BidirIt>
void reverse(
    bit_iterator<BidirIt> first,
    bit_iterator<BidirIt> last
)
{
    // Assertions
    _assert_range_viability(first, last);

    // Types and constants
    using underlying_type = typename bit_iterator<BidirIt>::underlying_type;
    using size_type = typename bit_iterator<BidirIt>::size_type;
    constexpr size_type digits = binary_digits<underlying_type>::value;

    // Initialization
    const bool is_first_aligned = first.position() == 0;
    const bool is_last_aligned = last.position() == 0;
    size_type gap = (digits - last.position()) * !is_last_aligned;
    auto it = first.base();
    underlying_type first_value = {};
    underlying_type last_value = {};

    // Reverse when bit iterators are aligned
    if (is_first_aligned && is_last_aligned) {
        std::reverse(first.base(), last.base());
        for (; it !=  last.base(); ++it) {
            *it = _bitswap(*it);
        }
    // Reverse when bit iterators do not belong to the same underlying value
    } else if (first.base() != last.base()) {
        // Save first and last element
        first_value = *first.base();
        last_value = *std::prev(last.base(), is_last_aligned);
        // Reverse the underlying sequence
        std::reverse(first.base(), std::next(last.base(), !is_last_aligned));
        // Shift the underlying sequence to the left
        if (first.position() < gap) {
            it = first.base();
            gap = gap - first.position();
            for (; it != last.base(); ++it) {
                *it = _shld<underlying_type>(*it, *std::next(it), gap);
            }
            *it <<= gap;
            it = first.base();
        // Shift the underlying sequence to the right
        } else if (first.position() > gap) {
            it = std::prev(last.base(), is_last_aligned);
            gap = first.position() - gap;
            for (; it != first.base(); --it) {
                *it = _shrd<underlying_type>(*it, *std::prev(it), gap);
            }
            *it >>= gap;
            it = first.base();
        }
        // Bitswap every element of the underlying sequence
        for (; it != std::next(last.base(), !is_last_aligned); ++it) {
            *it = _bitswap(*it);
        }
        // Blend bits of the first element
        if (!is_first_aligned) {
            *first.base() = _bitblend<underlying_type>(
                first_value,
                *first.base(),
                first.position(),
                digits - first.position()
            );
        }
        // Blend bits of the last element
        if (!is_last_aligned) {
            *last.base() = _bitblend<underlying_type>(
                *last.base(),
                last_value,
                last.position(),
                digits - last.position()
            );
        }
    // Reverse when bit iterators belong to the same underlying value
    } else {
        *it = _bitblend<underlying_type>(
            *it,
            _bitswap(*it >> first.position()) >> gap,
            first.position(),
            last.position() - first.position()
        );
    }
}
// -------------------------------------------------------------------------- //

// ========================================================================== //
} // namespace bit
#endif // _BIT_ALGORITHM_HPP_INCLUDED
// ========================================================================== //
<commit_msg>Few modifications to deal with the issue of operator <<, >> and implicit conversions to int.<commit_after>// ============================= BIT ALGORITHM ============================== //
// Project:         The C++ Bit Library
// Name:            bit_algorithm.hpp
// Description:     Optimized versions of algorithms for bit manipulation
// Creator:         Vincent Reverdy
// Contributor(s):  Vincent Reverdy [2015-2017]
//                  Maghav Kumar [2016-2017]
// License:         BSD 3-Clause License
// ========================================================================== //
#ifndef _BIT_ALGORITHM_HPP_INCLUDED
#define _BIT_ALGORITHM_HPP_INCLUDED
// ========================================================================== //

// ================================ PREAMBLE ================================ //
// C++ standard library
// Project sources
#include "bit_details.hpp"
#include "bit_value.hpp"
#include "bit_reference.hpp"
#include "bit_pointer.hpp"
#include "bit_iterator.hpp"
// Third-party libraries
// Miscellaneous
namespace bit {
// ========================================================================== //

/* ***************************** BIT CONSTANTS ****************************** */
// Bit constants
static constexpr bit_value bit0(0U);
static constexpr bit_value bit1(1U);
/* ************************************************************************** */

/* ************************** STANDARD ALGORITHMS *************************** */
// Non-modifying sequence operations
template <class InputIt>
typename bit_iterator<InputIt>::difference_type
count(
    bit_iterator<InputIt> first,
    bit_iterator<InputIt> last,
    bit_value value
);

// Modifying sequence operations
template <class BidirIt>
void reverse(
    bit_iterator<BidirIt> first,
    bit_iterator<BidirIt> last
);
/* ************************************************************************** */

// ------------------- NON-MODIFYING SEQUENCE OPERATIONS -------------------- //
// Counts the number of bits equal to the provided bit value
template <class InputIt>
typename bit_iterator<InputIt>::difference_type
count(
    bit_iterator<InputIt> first,
    bit_iterator<InputIt> last,
    bit_value value
)
{
    // Assertions
    _assert_range_viability(first, last);

    // Types and constants
    using underlying_type = typename bit_iterator<InputIt>::underlying_type;
    using difference_type = typename bit_iterator<InputIt>::difference_type;
    constexpr difference_type digits = binary_digits<underlying_type>::value;

    // Initialization
    difference_type result = 0;
    auto it = first.base();
    underlying_type first_value = {};
    underlying_type last_value = {};

    // Computation when bits belong to several underlying values
    if (first.base() != last.base()) {
        if (first.position() != 0) {
            first_value = *first.base() >> first.position();
            result = _popcnt(first_value);
            ++it;
        }
        for (; it != last.base(); ++it) {
            result += _popcnt(*it);
        }
        if (last.position() != 0) {
            last_value = *last.base() << (digits - last.position());
            result += _popcnt(last_value);
        }
    // Computation when bits belong to the same underlying value
    } else {
        result = _popcnt(_bextr<underlying_type>(
            *first.base(),
            first.position(),
            last.position() - first.position()
        ));
    }

    // Negates when the number of zero bits is requested
    if (!static_cast<bool>(value)) {
        result = std::distance(first, last) - result;
    }

    // Finalization
    return result;
}
// -------------------------------------------------------------------------- //

// --------------------- MODIFYING SEQUENCE OPERATIONS ---------------------- //
// Reverses the order of the bits in the provided range
template <class BidirIt>
void reverse(
    bit_iterator<BidirIt> first,
    bit_iterator<BidirIt> last
)
{
    // Assertions
    _assert_range_viability(first, last);

    // Types and constants
    using underlying_type = typename bit_iterator<BidirIt>::underlying_type;
    using size_type = typename bit_iterator<BidirIt>::size_type;
    constexpr size_type digits = binary_digits<underlying_type>::value;

    // Initialization
    const bool is_first_aligned = first.position() == 0;
    const bool is_last_aligned = last.position() == 0;
    size_type gap = (digits - last.position()) * !is_last_aligned;
    auto it = first.base();
    underlying_type first_value = {};
    underlying_type last_value = {};

    // Reverse when bit iterators are aligned
    if (is_first_aligned && is_last_aligned) {
        std::reverse(first.base(), last.base());
        for (; it !=  last.base(); ++it) {
            *it = _bitswap(*it);
        }
    // Reverse when bit iterators do not belong to the same underlying value
    } else if (first.base() != last.base()) {
        // Save first and last element
        first_value = *first.base();
        last_value = *std::prev(last.base(), is_last_aligned);
        // Reverse the underlying sequence
        std::reverse(first.base(), std::next(last.base(), !is_last_aligned));
        // Shift the underlying sequence to the left
        if (first.position() < gap) {
            it = first.base();
            gap = gap - first.position();
            for (; it != last.base(); ++it) {
                *it = _shld<underlying_type>(*it, *std::next(it), gap);
            }
            *it <<= gap;
            it = first.base();
        // Shift the underlying sequence to the right
        } else if (first.position() > gap) {
            it = std::prev(last.base(), is_last_aligned);
            gap = first.position() - gap;
            for (; it != first.base(); --it) {
                *it = _shrd<underlying_type>(*it, *std::prev(it), gap);
            }
            *it >>= gap;
            it = first.base();
        }
        // Bitswap every element of the underlying sequence
        for (; it != std::next(last.base(), !is_last_aligned); ++it) {
            *it = _bitswap(*it);
        }
        // Blend bits of the first element
        if (!is_first_aligned) {
            *first.base() = _bitblend<underlying_type>(
                first_value,
                *first.base(),
                first.position(),
                digits - first.position()
            );
        }
        // Blend bits of the last element
        if (!is_last_aligned) {
            *last.base() = _bitblend<underlying_type>(
                *last.base(),
                last_value,
                last.position(),
                digits - last.position()
            );
        }
    // Reverse when bit iterators belong to the same underlying value
    } else {
        *it = _bitblend<underlying_type>(
            *it,
            _bitswap<underlying_type>(*it >> first.position()) >> gap,
            first.position(),
            last.position() - first.position()
        );
    }
}
// -------------------------------------------------------------------------- //

// ========================================================================== //
} // namespace bit
#endif // _BIT_ALGORITHM_HPP_INCLUDED
// ========================================================================== //
<|endoftext|>


<commit_before>/** \file    augment_authority_data_with_translations.cc
 *  \brief   Extract translations from database amd augment the normdata file
 *  \author  Johannes Riedl
 */

/*
    Copyright (C) 2016, 2017 Library of the University of Tübingen

    This program is free software: you can redistribute it and/or modify
    it under the terms of the GNU Affero General Public License as
    published by the Free Software Foundation, either version 3 of the
    License, or (at your option) any later version.

    This program is distributed in the hope that it will be useful,
    but WITHOUT ANY WARRANTY; without even the implied warranty of
    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
    GNU Affero General Public License for more details.

    You should have received a copy of the GNU Affero General Public License
    along with this program.  If not, see <http://www.gnu.org/licenses/>.
*/

#include <iostream>
#include <string>
#include <vector>
#include <tuple>
#include <unordered_set>
#include <utility>
#include <cstdlib>
#include "Compiler.h"
#include "DbConnection.h"
#include "DbResultSet.h"
#include "DbRow.h"
#include "IniFile.h"
#include "MarcReader.h"
#include "MarcRecord.h"
#include "MarcWriter.h"
#include "StringUtil.h"
#include "Subfields.h"
#include "util.h"

// Save language code, translation, origin, status
typedef std::tuple<std::string, std::string, std::string, std::string> OneTranslation;

static unsigned record_count(0);
static unsigned modified_count(0);

void Usage() {
    std::cerr << "Usage: " << ::progname << " authority_data_input authority_data_output\n";
    std::exit(EXIT_FAILURE);
}

inline bool IsReliableSynonym(const std::string &status) {
    return status == "replaced_synonym" or status == "new_synonym" or status == "derived_synonym";
}

std::string ReplaceAngleBracketsWithParentheses(const std::string &value) {
    return StringUtil::Map(value, "<>", "()");
}

void ExtractTranslations(DbConnection * const db_connection, std::map<std::string,
                         std::vector<OneTranslation> > * const all_translations)
{
    db_connection->queryOrDie("SELECT DISTINCT ppn FROM keyword_translations");
    DbResultSet ppn_result_set(db_connection->getLastResultSet());
    while (const DbRow ppn_row = ppn_result_set.getNextRow()) {
        std::string ppn = ppn_row["ppn"];
        db_connection->queryOrDie("SELECT ppn, language_code, translation, origin, status FROM keyword_translations "
                                  "WHERE ppn='" + ppn + "'");
        DbResultSet result_set(db_connection->getLastResultSet());
        std::vector<OneTranslation> translations;
        while (const DbRow row = result_set.getNextRow()) {
            // We are not interested in synonym fields as we will directly derive synonyms from the translation field
            // Furthermore we insert keywords where the german translation is the reference and needs no further
            // inserting
            if (not IsReliableSynonym(row["status"]) and row["language_code"] != "ger") {
                std::string translation(row["translation"]);
                // Handle '#'-separated synonyms appropriately
                if (translation.find("#") == std::string::npos and not translation.empty())
                    translations.emplace_back(ReplaceAngleBracketsWithParentheses(translation),
                                              row["language_code"], row["origin"], row["status"]);
                else {
                    std::vector<std::string> primary_and_synonyms;
                    StringUtil::SplitThenTrim(translation, "#", " \t\n", &primary_and_synonyms);
		    // Use the first translation as non-synonmym
                    if (primary_and_synonyms.size() > 0) {
                        translations.emplace_back(primary_and_synonyms[0], row["language_code"], row["origin"],
                                              row["status"]);
                        // Add further synonyms as derived synonyms
                        for (auto synonyms(std::next(primary_and_synonyms.cbegin()));
                            synonyms != primary_and_synonyms.cend(); ++synonyms)
                            translations.emplace_back(ReplaceAngleBracketsWithParentheses(*synonyms),
                                                  row["language_code"], row["origin"], "derived_synonym");
                    }
                }
            }
        }
        all_translations->insert(std::make_pair(ppn, translations));
    }
}

std::string MapLanguageCode(const std::string lang_code) {
    if (lang_code == "ger")
        return "de";
    if (lang_code == "eng")
        return "en";
    if (lang_code == "fre")
        return "fr";
    if (lang_code == "dut")
        return "nl";
    if (lang_code == "ita")
        return "it";
    if (lang_code == "spa")
        return "es";
    if (lang_code == "hant")
        return "zh-Hant";
    if (lang_code == "hans")
        return "zh-Hans";
    if (lang_code == "por")
        return "pt";
    if (lang_code == "rus")
        return "ru";
    if (lang_code == "gre")
        return "el";
    logger->error("Unknown language code " + lang_code);
}

void InsertTranslation(MarcRecord * const record, const char indicator1, const char indicator2,
                       const std::string &term, const std::string &language_code, const std::string &status)
{
    Subfields subfields(indicator1, indicator2);
    subfields.addSubfield('a', term);
    subfields.addSubfield('9', "L:" + language_code);
    subfields.addSubfield('9', "Z:" + std::string(IsReliableSynonym(status) ? "VW" : "AF"));
    subfields.addSubfield('2', "IxTheo");
    record->insertField("750", subfields);
}

size_t GetFieldIndexForExistingTranslation(const MarcRecord *record, const std::vector<size_t> &field_indices,
                                           const std::string &language_code, const std::string &status) {
    // We can have several either previously existing or already inserted synonyms, so don't replace synonyms
    if (IsReliableSynonym(status))
        return MarcRecord::FIELD_NOT_FOUND;

    for (auto field_index : field_indices) {
        Subfields subfields_present(record->getSubfields(field_index));
        if (subfields_present.hasSubfieldWithValue('2', "IxTheo") and
            subfields_present.hasSubfieldWithValue('9', "L:" + language_code) and
            subfields_present.hasSubfieldWithValue('9', "Z:AF")) {
                return field_index;
}
    }
    return MarcRecord::FIELD_NOT_FOUND;
}

void ProcessRecord(MarcRecord * const record,
                   const std::map<std::string, std::vector<OneTranslation> > &all_translations)
{
    const std::string ppn(record->getControlNumber());
    auto one_translation(all_translations.find(ppn));

    if (one_translation != all_translations.cend()) {
        // We only insert/replace IxTheo-Translations
        for (const auto &one_lang_translation : one_translation->second) {
            std::string term(std::get<0>(one_lang_translation));
            std::string language_code(std::get<1>(one_lang_translation));
            std::string origin(std::get<2>(one_lang_translation));
            std::string status(std::get<3>(one_lang_translation));

            // Skip non-derived synonyms, german terms and unreliable translations
            if ((status != "derived_synonym" and StringUtil::EndsWith(status, "synonym")) or status == "unreliable"
                or language_code == "ger")
                continue;

            // Don't touch MACS translations and leave alone alone authoritative IxTheo Translations from BSZ
            std::vector<size_t> field_indices;
            if (record->getFieldIndices("750", &field_indices) > 0) {
                const size_t field_index(GetFieldIndexForExistingTranslation(record, field_indices, language_code,
                                                                             status));
                if (field_index == MarcRecord::FIELD_NOT_FOUND)
                    InsertTranslation(record, ' ', '6', term, language_code, status);
            } else
                InsertTranslation(record, ' ', '6', term, language_code, status);
        }
        ++modified_count;
    }
}

void AugmentNormdata(MarcReader * const marc_reader, MarcWriter *marc_writer, const std::map<std::string,
                     std::vector<OneTranslation> > &all_translations)
{
    // Read in all PPNs from authority data
    while (MarcRecord record = marc_reader->read()) {
        ProcessRecord(&record, all_translations);
        marc_writer->write(record);
        ++record_count;
    }

    std::cerr << "Modified " << modified_count << " of " << record_count << " entries.\n";
}

const std::string CONF_FILE_PATH("/usr/local/var/lib/tuelib/translations.conf");

int main(int argc, char **argv) {
    ::progname = argv[0];

    if (argc != 3)
        Usage();

    const std::string marc_input_filename(argv[1]);
    const std::string marc_output_filename(argv[2]);

    if (unlikely(marc_input_filename == marc_output_filename))
        logger->error("Input file equals output file");

    try {
        std::unique_ptr<MarcReader> marc_reader(MarcReader::Factory(marc_input_filename));
        std::unique_ptr<MarcWriter> marc_writer(MarcWriter::Factory(marc_output_filename));

        const IniFile ini_file(CONF_FILE_PATH);
        const std::string sql_database(ini_file.getString("Database", "sql_database"));
        const std::string sql_username(ini_file.getString("Database", "sql_username"));
        const std::string sql_password(ini_file.getString("Database", "sql_password"));
        DbConnection db_connection(sql_database, sql_username, sql_password);

        std::map<std::string, std::vector<OneTranslation> > all_translations;
        ExtractTranslations(&db_connection, &all_translations);

        AugmentNormdata(marc_reader.get(), marc_writer.get(), all_translations);
    } catch (const std::exception &x) {
        logger->error("caught exception: " + std::string(x.what()));
    }
}
<commit_msg>Minor corrections<commit_after>/** \file    augment_authority_data_with_translations.cc
 *  \brief   Extract translations from database amd augment the normdata file
 *  \author  Johannes Riedl
 */

/*
    Copyright (C) 2016, 2017 Library of the University of Tübingen

    This program is free software: you can redistribute it and/or modify
    it under the terms of the GNU Affero General Public License as
    published by the Free Software Foundation, either version 3 of the
    License, or (at your option) any later version.

    This program is distributed in the hope that it will be useful,
    but WITHOUT ANY WARRANTY; without even the implied warranty of
    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
    GNU Affero General Public License for more details.

    You should have received a copy of the GNU Affero General Public License
    along with this program.  If not, see <http://www.gnu.org/licenses/>.
*/

#include <iostream>
#include <string>
#include <vector>
#include <tuple>
#include <unordered_set>
#include <utility>
#include <cstdlib>
#include "Compiler.h"
#include "DbConnection.h"
#include "DbResultSet.h"
#include "DbRow.h"
#include "IniFile.h"
#include "MarcReader.h"
#include "MarcRecord.h"
#include "MarcWriter.h"
#include "StringUtil.h"
#include "Subfields.h"
#include "util.h"

namespace {
// Save language code, translation, origin, status
typedef std::tuple<std::string, std::string, std::string, std::string> OneTranslation;

static unsigned record_count(0);
static unsigned modified_count(0);

void Usage() {
    std::cerr << "Usage: " << ::progname << " authority_data_input authority_data_output\n";
    std::exit(EXIT_FAILURE);
}

inline bool IsReliableSynonym(const std::string &status) {
    return status == "replaced_synonym" or status == "new_synonym" or status == "derived_synonym";
}

std::string ReplaceAngleBracketsWithParentheses(const std::string &value) {
    return StringUtil::Map(value, "<>", "()");
}

void ExtractTranslations(DbConnection * const db_connection, std::map<std::string,
                         std::vector<OneTranslation> > * const all_translations)
{
    db_connection->queryOrDie("SELECT DISTINCT ppn FROM keyword_translations");
    DbResultSet ppn_result_set(db_connection->getLastResultSet());
    while (const DbRow ppn_row = ppn_result_set.getNextRow()) {
        std::string ppn = ppn_row["ppn"];
        db_connection->queryOrDie("SELECT ppn, language_code, translation, origin, status FROM keyword_translations "
                                  "WHERE ppn='" + ppn + "'");
        DbResultSet result_set(db_connection->getLastResultSet());
        std::vector<OneTranslation> translations;
        while (const DbRow row = result_set.getNextRow()) {
            // We are not interested in synonym fields as we will directly derive synonyms from the translation field
            // Furthermore we insert keywords where the german translation is the reference and needs no further
            // inserting
            if (not IsReliableSynonym(row["status"]) and row["language_code"] != "ger") {
                std::string translation(row["translation"]);
                // Handle '#'-separated synonyms appropriately
                if (translation.find("#") == std::string::npos and not translation.empty())
                    translations.emplace_back(ReplaceAngleBracketsWithParentheses(translation),
                                              row["language_code"], row["origin"], row["status"]);
                else {
                    std::vector<std::string> primary_and_synonyms;
                    StringUtil::SplitThenTrim(translation, "#", " \t\n", &primary_and_synonyms);
		    // Use the first translation as non-synonmym
                    if (primary_and_synonyms.size() > 0) {
                        translations.emplace_back(primary_and_synonyms[0], row["language_code"], row["origin"],
                                              row["status"]);
                        // Add further synonyms as derived synonyms
                        for (auto synonyms(std::next(primary_and_synonyms.cbegin()));
                            synonyms != primary_and_synonyms.cend(); ++synonyms)
                            translations.emplace_back(ReplaceAngleBracketsWithParentheses(*synonyms),
                                                  row["language_code"], row["origin"], "derived_synonym");
                    }
                }
            }
        }
        all_translations->insert(std::make_pair(ppn, translations));
    }
}

void InsertTranslation(MarcRecord * const record, const char indicator1, const char indicator2,
                       const std::string &term, const std::string &language_code, const std::string &status)
{
    Subfields subfields(indicator1, indicator2);
    subfields.addSubfield('a', term);
    subfields.addSubfield('9', "L:" + language_code);
    subfields.addSubfield('9', "Z:" + std::string(IsReliableSynonym(status) ? "VW" : "AF"));
    subfields.addSubfield('2', "IxTheo");
    record->insertField("750", subfields);
}

size_t GetFieldIndexForExistingTranslation(const MarcRecord *record, const std::vector<size_t> &field_indices,
                                           const std::string &language_code, const std::string &status) {
    // We can have several either previously existing or already inserted synonyms, so don't replace synonyms
    if (IsReliableSynonym(status))
        return MarcRecord::FIELD_NOT_FOUND;

    for (auto field_index : field_indices) {
        Subfields subfields_present(record->getSubfields(field_index));
        if (subfields_present.hasSubfieldWithValue('2', "IxTheo") and
            subfields_present.hasSubfieldWithValue('9', "L:" + language_code) and
            subfields_present.hasSubfieldWithValue('9', "Z:AF"))
                return field_index;
    }
    return MarcRecord::FIELD_NOT_FOUND;
}

void ProcessRecord(MarcRecord * const record,
                   const std::map<std::string, std::vector<OneTranslation> > &all_translations)
{
    const std::string ppn(record->getControlNumber());
    auto one_translation(all_translations.find(ppn));

    if (one_translation != all_translations.cend()) {
        // We only insert/replace IxTheo-Translations
        for (const auto &one_lang_translation : one_translation->second) {
            std::string term(std::get<0>(one_lang_translation));
            std::string language_code(std::get<1>(one_lang_translation));
            std::string origin(std::get<2>(one_lang_translation));
            std::string status(std::get<3>(one_lang_translation));

            // Skip non-derived synonyms, german terms and unreliable translations
            if ((status != "derived_synonym" and StringUtil::EndsWith(status, "synonym")) or status == "unreliable"
                or language_code == "ger")
                continue;

            // Don't touch MACS translations and leave alone authoritative IxTheo Translations from BSZ
            std::vector<size_t> field_indices;
            if (record->getFieldIndices("750", &field_indices) > 0) {
                const size_t field_index(GetFieldIndexForExistingTranslation(record, field_indices, language_code,
                                                                             status));
                if (field_index == MarcRecord::FIELD_NOT_FOUND)
                    InsertTranslation(record, ' ', '6', term, language_code, status);
            } else
                InsertTranslation(record, ' ', '6', term, language_code, status);
        }
        ++modified_count;
    }
}

void AugmentNormdata(MarcReader * const marc_reader, MarcWriter *marc_writer, const std::map<std::string,
                     std::vector<OneTranslation> > &all_translations)
{
    // Read in all PPNs from authority data
    while (MarcRecord record = marc_reader->read()) {
        ProcessRecord(&record, all_translations);
        marc_writer->write(record);
        ++record_count;
    }

    std::cerr << "Modified " << modified_count << " of " << record_count << " entries.\n";
}

} //unnamed namespace

const std::string CONF_FILE_PATH("/usr/local/var/lib/tuelib/translations.conf");

int main(int argc, char **argv) {
    ::progname = argv[0];

    if (argc != 3)
        Usage();

    const std::string marc_input_filename(argv[1]);
    const std::string marc_output_filename(argv[2]);

    if (unlikely(marc_input_filename == marc_output_filename))
        logger->error("Input file equals output file");

    try {
        std::unique_ptr<MarcReader> marc_reader(MarcReader::Factory(marc_input_filename));
        std::unique_ptr<MarcWriter> marc_writer(MarcWriter::Factory(marc_output_filename));

        const IniFile ini_file(CONF_FILE_PATH);
        const std::string sql_database(ini_file.getString("Database", "sql_database"));
        const std::string sql_username(ini_file.getString("Database", "sql_username"));
        const std::string sql_password(ini_file.getString("Database", "sql_password"));
        DbConnection db_connection(sql_database, sql_username, sql_password);

        std::map<std::string, std::vector<OneTranslation> > all_translations;
        ExtractTranslations(&db_connection, &all_translations);

        AugmentNormdata(marc_reader.get(), marc_writer.get(), all_translations);
    } catch (const std::exception &x) {
        logger->error("caught exception: " + std::string(x.what()));
    }
}
<|endoftext|>


<commit_before>/*
 *  GF256.cpp
 *  zxing
 *
 *  Created by Christian Brunschen on 05/05/2008.
 *  Copyright 2008 Google UK. All rights reserved.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *      http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

#include <vector>
#include <iostream>
#include <zxing/common/reedsolomon/GF256.h>
#include <zxing/common/reedsolomon/GF256Poly.h>
#include <zxing/common/IllegalArgumentException.h>
#include <zxing/common/Array.h>
#include <zxing/common/Counted.h>

namespace zxing {
using namespace std;

static inline ArrayRef<int> makeArray(int value) {
  ArrayRef<int> valuesRef(new Array<int> (value, 1));
  return valuesRef;
}

static inline Ref<GF256Poly> refPoly(GF256 &field, int value) {
  ArrayRef<int> values(makeArray(value));
  Ref<GF256Poly> result(new GF256Poly(field, values));
  return result;
}

GF256::GF256(int primitive) :
    exp_(256, (const int)0), log_(256, (const int)0), zero_(refPoly(*this, 0)), one_(refPoly(*this, 1)) {
  int x = 1;
  for (int i = 0; i < 256; i++) {
    exp_[i] = x;
    x <<= 1;
    if (x >= 0x100) {
      x ^= primitive;
    }
  }

  // log(0) == 0, but should never be used
  log_[0] = 0;
  for (int i = 0; i < 255; i++) {
    log_[exp_[i]] = i;
  }
}

Ref<GF256Poly> GF256::getZero() {
  return zero_;
}

Ref<GF256Poly> GF256::getOne() {
  return one_;
}

Ref<GF256Poly> GF256::buildMonomial(int degree, int coefficient) {
#ifdef DEBUG
  cout << __FUNCTION__ << "\n";
#endif
  if (degree < 0) {
    throw IllegalArgumentException("Degree must be non-negative");
  }
  if (coefficient == 0) {
    return zero_;
  }
  int nCoefficients = degree + 1;
  ArrayRef<int> coefficients(new Array<int> (nCoefficients));
  coefficients[0] = coefficient;
  Ref<GF256Poly> result(new GF256Poly(*this, coefficients));
  return result;
}

int GF256::addOrSubtract(int a, int b) {
  return a ^ b;
}

int GF256::exp(int a) {
  return exp_[a];
}

int GF256::log(int a) {
  if (a == 0) {
    throw IllegalArgumentException("Cannot take the logarithm of 0");
  }
  return log_[a];
}

int GF256::inverse(int a) {
  if (a == 0) {
    throw IllegalArgumentException("Cannot calculate the inverse of 0");
  }
  return exp_[255 - log_[a]];
}

int GF256::multiply(int a, int b) {
  if (a == 0 || b == 0) {
    return 0;
  }
  if (a == 1) {
    return b;
  }
  if (b == 1) {
    return a;
  }
  return exp_[(log_[a] + log_[b]) % 255];
}

GF256 GF256::QR_CODE_FIELD(0x011D); // x^8 + x^4 + x^3 + x^2 + 1
GF256 GF256::DATA_MATRIX_FIELD(0x012D); // x^8 + x^5 + x^3 + x^2 + 1

ostream& operator<<(ostream& out, const GF256& field) {
  out << "Field[\nexp=(";
  out << field.exp_[0];
  for (int i = 1; i < 256; i++) {
    out << "," << field.exp_[i];
  }
  out << "),\nlog=(";
  out << field.log_[0];
  for (int i = 1; i < 256; i++) {
    out << "," << field.log_[i];
  }
  out << ")\n]";
  return out;
}

}
<commit_msg>Same change as Revision r1395 for C++ port: Small speedup, per issue 422<commit_after>/*
 *  GF256.cpp
 *  zxing
 *
 *  Created by Christian Brunschen on 05/05/2008.
 *  Copyright 2008 Google UK. All rights reserved.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *      http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

#include <vector>
#include <iostream>
#include <zxing/common/reedsolomon/GF256.h>
#include <zxing/common/reedsolomon/GF256Poly.h>
#include <zxing/common/IllegalArgumentException.h>
#include <zxing/common/Array.h>
#include <zxing/common/Counted.h>

namespace zxing {
using namespace std;

static inline ArrayRef<int> makeArray(int value) {
  ArrayRef<int> valuesRef(new Array<int> (value, 1));
  return valuesRef;
}

static inline Ref<GF256Poly> refPoly(GF256 &field, int value) {
  ArrayRef<int> values(makeArray(value));
  Ref<GF256Poly> result(new GF256Poly(field, values));
  return result;
}

GF256::GF256(int primitive) :
    exp_(256, (const int)0), log_(256, (const int)0), zero_(refPoly(*this, 0)), one_(refPoly(*this, 1)) {
  int x = 1;
  for (int i = 0; i < 256; i++) {
    exp_[i] = x;
    x <<= 1;
    if (x >= 0x100) {
      x ^= primitive;
    }
  }

  // log(0) == 0, but should never be used
  log_[0] = 0;
  for (int i = 0; i < 255; i++) {
    log_[exp_[i]] = i;
  }
}

Ref<GF256Poly> GF256::getZero() {
  return zero_;
}

Ref<GF256Poly> GF256::getOne() {
  return one_;
}

Ref<GF256Poly> GF256::buildMonomial(int degree, int coefficient) {
#ifdef DEBUG
  cout << __FUNCTION__ << "\n";
#endif
  if (degree < 0) {
    throw IllegalArgumentException("Degree must be non-negative");
  }
  if (coefficient == 0) {
    return zero_;
  }
  int nCoefficients = degree + 1;
  ArrayRef<int> coefficients(new Array<int> (nCoefficients));
  coefficients[0] = coefficient;
  Ref<GF256Poly> result(new GF256Poly(*this, coefficients));
  return result;
}

int GF256::addOrSubtract(int a, int b) {
  return a ^ b;
}

int GF256::exp(int a) {
  return exp_[a];
}

int GF256::log(int a) {
  if (a == 0) {
    throw IllegalArgumentException("Cannot take the logarithm of 0");
  }
  return log_[a];
}

int GF256::inverse(int a) {
  if (a == 0) {
    throw IllegalArgumentException("Cannot calculate the inverse of 0");
  }
  return exp_[255 - log_[a]];
}

int GF256::multiply(int a, int b) {
  if (a == 0 || b == 0) {
    return 0;
  }
  int logSum = log_[a] + log_[b];
  // index is a sped-up alternative to logSum % 255 since sum
  // is in [0,510]. Thanks to jmsachs for the idea
  return exp_[(logSum & 0xFF) + (logSum >> 8)];
}

GF256 GF256::QR_CODE_FIELD(0x011D); // x^8 + x^4 + x^3 + x^2 + 1
GF256 GF256::DATA_MATRIX_FIELD(0x012D); // x^8 + x^5 + x^3 + x^2 + 1

ostream& operator<<(ostream& out, const GF256& field) {
  out << "Field[\nexp=(";
  out << field.exp_[0];
  for (int i = 1; i < 256; i++) {
    out << "," << field.exp_[i];
  }
  out << "),\nlog=(";
  out << field.log_[0];
  for (int i = 1; i < 256; i++) {
    out << "," << field.log_[i];
  }
  out << ")\n]";
  return out;
}

}
<|endoftext|>


<commit_before>/**
 * This file is part of the CernVM File System.
 */

#include "upload_local.h"

#include <errno.h>

#include "logging.h"
#include "compression.h"
#include "util.h"

#include "file_processing/char_buffer.h"

using namespace upload;

LocalUploader::LocalUploader(const SpoolerDefinition &spooler_definition) :
  AbstractUploader(spooler_definition),
  upstream_path_(spooler_definition.spooler_configuration),
  temporary_path_(spooler_definition.temporary_path)
{
  assert (spooler_definition.IsValid() &&
          spooler_definition.driver_type == SpoolerDefinition::Local);

  atomic_init32(&copy_errors_);
}

bool LocalUploader::WillHandle(const SpoolerDefinition &spooler_definition) {
  return spooler_definition.driver_type == SpoolerDefinition::Local;
}

unsigned int LocalUploader::GetNumberOfErrors() const {
  return atomic_read32(&copy_errors_);
}

void LocalUploader::WorkerThread() {
  bool running = true;
  while (running) {
    UploadJob job = AcquireNewJob();
    switch (job.type) {
      case UploadJob::Upload:
        Upload(job.stream_handle,
               job.buffer,
               job.callback);
        break;
      case UploadJob::Commit:
        FinalizeStreamedUpload(job.stream_handle,
                               job.content_hash,
                               job.hash_suffix);
        break;
      case UploadJob::Terminate:
        running = false;
        break;
      default:
        const bool unknown_job_type = false;
        assert (unknown_job_type);
        break;
    }
  }
}

void LocalUploader::FileUpload(const std::string &local_path,
                               const std::string &remote_path,
                               const callback_t   *callback) {
  // create destination in backend storage temporary directory
  std::string tmp_path = CreateTempPath(temporary_path_ + "/upload", 0666);
  if (tmp_path.empty()) {
    LogCvmfs(kLogSpooler, kLogVerboseMsg, "failed to create temp path for "
                                          "upload of file '%s' (errno: %d)",
             local_path.c_str(), errno);
    atomic_inc32(&copy_errors_);
    Respond(callback, UploaderResults(1, local_path));
    return;
  }

  // copy file into controlled temporary directory location
  int retval  = CopyPath2Path(local_path, tmp_path);
  int retcode = retval ? 0 : 100;
  if (retcode != 0) {
    LogCvmfs(kLogSpooler, kLogVerboseMsg, "failed to copy file '%s' to staging "
                                          "area: '%s'",
             local_path.c_str(), tmp_path.c_str());
    atomic_inc32(&copy_errors_);
    Respond(callback, UploaderResults(retcode, local_path));
    return;
  }

  // move the file in place (atomic operation)
  retcode = Move(tmp_path, remote_path);
  if (retcode != 0) {
    LogCvmfs(kLogSpooler, kLogVerboseMsg, "failed to move file '%s' from the "
                                          "staging area to the final location: "
                                          "'%s'",
             tmp_path.c_str(), remote_path.c_str());
    atomic_inc32(&copy_errors_);
  }
  Respond(callback, UploaderResults(retcode, local_path));
}

int LocalUploader::CreateAndOpenTemporaryChunkFile(std::string *path) const {
  const std::string tmp_path = CreateTempPath(temporary_path_ + "/" + "chunk",
                                              0644);
  if (tmp_path.empty()) {
    LogCvmfs(kLogSpooler, kLogStderr, "Failed to create temp file for upload of "
                                      "file chunk.");
    atomic_inc32(&copy_errors_);
    return -1;
  }

  const int tmp_fd = open(tmp_path.c_str(), O_WRONLY);
  if (tmp_fd < 0) {
    LogCvmfs(kLogSpooler, kLogStderr, "Failed to open temp file '%s' for upload "
                                      "of file chunk (errno: %d)",
             tmp_path.c_str(), errno);
    unlink(tmp_path.c_str());
    atomic_inc32(&copy_errors_);
    return tmp_fd;
  }

  *path = tmp_path;
  return tmp_fd;
}

UploadStreamHandle* LocalUploader::InitStreamedUpload(
                                                   const callback_t *callback) {
  std::string tmp_path;
  const int tmp_fd = CreateAndOpenTemporaryChunkFile(&tmp_path);
  if (tmp_fd < 0) {
    return NULL;
  }

  return new LocalStreamHandle(callback, tmp_fd, tmp_path);
}

void LocalUploader::Upload(UploadStreamHandle  *handle,
                           CharBuffer          *buffer,
                           const callback_t    *callback) {
  assert (buffer->IsInitialized());
  LocalStreamHandle *local_handle = static_cast<LocalStreamHandle*>(handle);

  const size_t bytes_written = write(local_handle->file_descriptor,
                                     buffer->ptr(),
                                     buffer->used_bytes());
  if (bytes_written != buffer->used_bytes()) {
    const int cpy_errno = errno;
    LogCvmfs(kLogSpooler, kLogVerboseMsg, "failed to write %d bytes to '%s' "
                                          "(errno: %d)",
             buffer->used_bytes(),
             local_handle->temporary_path.c_str(),
             cpy_errno);
    atomic_inc32(&copy_errors_);
    Respond(callback, UploaderResults(cpy_errno, buffer));
    return;
  }

  Respond(callback, UploaderResults(0, buffer));
}

void LocalUploader::FinalizeStreamedUpload(UploadStreamHandle *handle,
                                           const shash::Any    content_hash,
                                           const std::string   hash_suffix) {
  int retval = 0;
  LocalStreamHandle *local_handle = static_cast<LocalStreamHandle*>(handle);

  retval = close(local_handle->file_descriptor);
  if (retval != 0) {
    const int cpy_errno = errno;
    LogCvmfs(kLogSpooler, kLogVerboseMsg, "failed to close temp file '%s' "
                                          "(errno: %d)",
             local_handle->temporary_path.c_str(), cpy_errno);
    atomic_inc32(&copy_errors_);
    Respond(handle->commit_callback, UploaderResults(cpy_errno));
    return;
  }

  const std::string final_path = upstream_path_ + "/data" +
                                 content_hash.MakePath(1, 2) +
                                 hash_suffix;

  retval = rename(local_handle->temporary_path.c_str(), final_path.c_str());
  if (retval != 0) {
    const int cpy_errno = errno;
    LogCvmfs(kLogSpooler, kLogVerboseMsg, "failed to move temp file '%s' to "
                                          "final location '%s' (errno: %d)",
             local_handle->temporary_path.c_str(),
             final_path.c_str(),
             cpy_errno);
    atomic_inc32(&copy_errors_);
    Respond(handle->commit_callback, UploaderResults(cpy_errno));
    return;
  }

  const callback_t *callback = handle->commit_callback;
  delete local_handle;

  Respond(callback, UploaderResults(0));
}

bool LocalUploader::Remove(const std::string& file_to_delete) {
  if (! Peek(file_to_delete)) {
    return false;
  }

  const int retval = unlink((upstream_path_ + "/" + file_to_delete).c_str());
  return retval == 0;
}

bool LocalUploader::Peek(const std::string& path) const {
  return FileExists(upstream_path_ + "/" + path);
}

int LocalUploader::Move(const std::string &local_path,
                        const std::string &remote_path) const {
  const std::string destination_path = upstream_path_ + "/" + remote_path;

  // make sure the file has the right permissions
  int retval  = chmod(local_path.c_str(), 0666);
  int retcode = (retval == 0) ? 0 : 101;
  if (retcode != 0) {
    LogCvmfs(kLogSpooler, kLogVerboseMsg, "failed to set file permission '%s' "
                                          "errno: %d",
             local_path.c_str(), errno);
    return retcode;
  }

  // move the file in place
  retval  = rename(local_path.c_str(), destination_path.c_str());
  retcode = (retval == 0) ? 0 : errno;
  if (retcode != 0) {
    LogCvmfs(kLogSpooler, kLogVerboseMsg, "failed to move file '%s' to '%s' "
                                          "errno: %d",
             local_path.c_str(), remote_path.c_str(), errno);
  }

  return retcode;
}
<commit_msg>FIX: print errno in error message<commit_after>/**
 * This file is part of the CernVM File System.
 */

#include "upload_local.h"

#include <errno.h>

#include "logging.h"
#include "compression.h"
#include "util.h"

#include "file_processing/char_buffer.h"

using namespace upload;

LocalUploader::LocalUploader(const SpoolerDefinition &spooler_definition) :
  AbstractUploader(spooler_definition),
  upstream_path_(spooler_definition.spooler_configuration),
  temporary_path_(spooler_definition.temporary_path)
{
  assert (spooler_definition.IsValid() &&
          spooler_definition.driver_type == SpoolerDefinition::Local);

  atomic_init32(&copy_errors_);
}

bool LocalUploader::WillHandle(const SpoolerDefinition &spooler_definition) {
  return spooler_definition.driver_type == SpoolerDefinition::Local;
}

unsigned int LocalUploader::GetNumberOfErrors() const {
  return atomic_read32(&copy_errors_);
}

void LocalUploader::WorkerThread() {
  bool running = true;
  while (running) {
    UploadJob job = AcquireNewJob();
    switch (job.type) {
      case UploadJob::Upload:
        Upload(job.stream_handle,
               job.buffer,
               job.callback);
        break;
      case UploadJob::Commit:
        FinalizeStreamedUpload(job.stream_handle,
                               job.content_hash,
                               job.hash_suffix);
        break;
      case UploadJob::Terminate:
        running = false;
        break;
      default:
        const bool unknown_job_type = false;
        assert (unknown_job_type);
        break;
    }
  }
}

void LocalUploader::FileUpload(const std::string &local_path,
                               const std::string &remote_path,
                               const callback_t   *callback) {
  // create destination in backend storage temporary directory
  std::string tmp_path = CreateTempPath(temporary_path_ + "/upload", 0666);
  if (tmp_path.empty()) {
    LogCvmfs(kLogSpooler, kLogVerboseMsg, "failed to create temp path for "
                                          "upload of file '%s' (errno: %d)",
             local_path.c_str(), errno);
    atomic_inc32(&copy_errors_);
    Respond(callback, UploaderResults(1, local_path));
    return;
  }

  // copy file into controlled temporary directory location
  int retval  = CopyPath2Path(local_path, tmp_path);
  int retcode = retval ? 0 : 100;
  if (retcode != 0) {
    LogCvmfs(kLogSpooler, kLogVerboseMsg, "failed to copy file '%s' to staging "
                                          "area: '%s'",
             local_path.c_str(), tmp_path.c_str());
    atomic_inc32(&copy_errors_);
    Respond(callback, UploaderResults(retcode, local_path));
    return;
  }

  // move the file in place (atomic operation)
  retcode = Move(tmp_path, remote_path);
  if (retcode != 0) {
    LogCvmfs(kLogSpooler, kLogVerboseMsg, "failed to move file '%s' from the "
                                          "staging area to the final location: "
                                          "'%s'",
             tmp_path.c_str(), remote_path.c_str());
    atomic_inc32(&copy_errors_);
  }
  Respond(callback, UploaderResults(retcode, local_path));
}

int LocalUploader::CreateAndOpenTemporaryChunkFile(std::string *path) const {
  const std::string tmp_path = CreateTempPath(temporary_path_ + "/" + "chunk",
                                              0644);
  if (tmp_path.empty()) {
    LogCvmfs(kLogSpooler, kLogStderr, "Failed to create temp file for upload of "
                                      "file chunk (errno: %d).", errno);
    atomic_inc32(&copy_errors_);
    return -1;
  }

  const int tmp_fd = open(tmp_path.c_str(), O_WRONLY);
  if (tmp_fd < 0) {
    LogCvmfs(kLogSpooler, kLogStderr, "Failed to open temp file '%s' for upload "
                                      "of file chunk (errno: %d)",
             tmp_path.c_str(), errno);
    unlink(tmp_path.c_str());
    atomic_inc32(&copy_errors_);
    return tmp_fd;
  }

  *path = tmp_path;
  return tmp_fd;
}

UploadStreamHandle* LocalUploader::InitStreamedUpload(
                                                   const callback_t *callback) {
  std::string tmp_path;
  const int tmp_fd = CreateAndOpenTemporaryChunkFile(&tmp_path);
  if (tmp_fd < 0) {
    return NULL;
  }

  return new LocalStreamHandle(callback, tmp_fd, tmp_path);
}

void LocalUploader::Upload(UploadStreamHandle  *handle,
                           CharBuffer          *buffer,
                           const callback_t    *callback) {
  assert (buffer->IsInitialized());
  LocalStreamHandle *local_handle = static_cast<LocalStreamHandle*>(handle);

  const size_t bytes_written = write(local_handle->file_descriptor,
                                     buffer->ptr(),
                                     buffer->used_bytes());
  if (bytes_written != buffer->used_bytes()) {
    const int cpy_errno = errno;
    LogCvmfs(kLogSpooler, kLogVerboseMsg, "failed to write %d bytes to '%s' "
                                          "(errno: %d)",
             buffer->used_bytes(),
             local_handle->temporary_path.c_str(),
             cpy_errno);
    atomic_inc32(&copy_errors_);
    Respond(callback, UploaderResults(cpy_errno, buffer));
    return;
  }

  Respond(callback, UploaderResults(0, buffer));
}

void LocalUploader::FinalizeStreamedUpload(UploadStreamHandle *handle,
                                           const shash::Any    content_hash,
                                           const std::string   hash_suffix) {
  int retval = 0;
  LocalStreamHandle *local_handle = static_cast<LocalStreamHandle*>(handle);

  retval = close(local_handle->file_descriptor);
  if (retval != 0) {
    const int cpy_errno = errno;
    LogCvmfs(kLogSpooler, kLogVerboseMsg, "failed to close temp file '%s' "
                                          "(errno: %d)",
             local_handle->temporary_path.c_str(), cpy_errno);
    atomic_inc32(&copy_errors_);
    Respond(handle->commit_callback, UploaderResults(cpy_errno));
    return;
  }

  const std::string final_path = upstream_path_ + "/data" +
                                 content_hash.MakePath(1, 2) +
                                 hash_suffix;

  retval = rename(local_handle->temporary_path.c_str(), final_path.c_str());
  if (retval != 0) {
    const int cpy_errno = errno;
    LogCvmfs(kLogSpooler, kLogVerboseMsg, "failed to move temp file '%s' to "
                                          "final location '%s' (errno: %d)",
             local_handle->temporary_path.c_str(),
             final_path.c_str(),
             cpy_errno);
    atomic_inc32(&copy_errors_);
    Respond(handle->commit_callback, UploaderResults(cpy_errno));
    return;
  }

  const callback_t *callback = handle->commit_callback;
  delete local_handle;

  Respond(callback, UploaderResults(0));
}

bool LocalUploader::Remove(const std::string& file_to_delete) {
  if (! Peek(file_to_delete)) {
    return false;
  }

  const int retval = unlink((upstream_path_ + "/" + file_to_delete).c_str());
  return retval == 0;
}

bool LocalUploader::Peek(const std::string& path) const {
  return FileExists(upstream_path_ + "/" + path);
}

int LocalUploader::Move(const std::string &local_path,
                        const std::string &remote_path) const {
  const std::string destination_path = upstream_path_ + "/" + remote_path;

  // make sure the file has the right permissions
  int retval  = chmod(local_path.c_str(), 0666);
  int retcode = (retval == 0) ? 0 : 101;
  if (retcode != 0) {
    LogCvmfs(kLogSpooler, kLogVerboseMsg, "failed to set file permission '%s' "
                                          "errno: %d",
             local_path.c_str(), errno);
    return retcode;
  }

  // move the file in place
  retval  = rename(local_path.c_str(), destination_path.c_str());
  retcode = (retval == 0) ? 0 : errno;
  if (retcode != 0) {
    LogCvmfs(kLogSpooler, kLogVerboseMsg, "failed to move file '%s' to '%s' "
                                          "errno: %d",
             local_path.c_str(), remote_path.c_str(), errno);
  }

  return retcode;
}
<|endoftext|>


<commit_before>///
/// @file  P2.cpp
/// @brief 2nd partial sieve function.
///        P2(x, y) counts the numbers <= x that have exactly 2 prime
///        factors each exceeding the a-th prime.
///
/// Copyright (C) 2016 Kim Walisch, <kim.walisch@gmail.com>
///
/// This file is distributed under the BSD License. See the COPYING
/// file in the top level directory.
///

#include <primecount-internal.hpp>
#include <primesieve.hpp>
#include <aligned_vector.hpp>
#include <BitSieve.hpp>
#include <generate.hpp>
#include <int128.hpp>
#include <min_max.hpp>
#include <pmath.hpp>
#include <Wheel.hpp>

#include <stdint.h>
#include <algorithm>
#include <iostream>
#include <iomanip>
#include <vector>

#ifdef _OPENMP
  #include <omp.h>
#endif

using namespace std;
using namespace primecount;

namespace {

/// Calculate the segments per thread.
/// The idea is to gradually increase the segments per thread (based
/// on elapsed time) in order to keep all CPU cores busy.
///
int64_t balanceLoad(int64_t segments_per_thread, double start_time)
{
  double seconds = get_wtime() - start_time;

  if (seconds < 30)
    segments_per_thread *= 2;
  else if (segments_per_thread >= 4)
    segments_per_thread -= segments_per_thread / 4;

  return segments_per_thread;
}

/// Cross-off the multiples inside [low, high[
/// of the primes <= sqrt(high - 1).
///
void cross_off(BitSieve& sieve,
               vector<int32_t> primes,
               Wheel& wheel,
               int64_t c,
               int64_t low,
               int64_t high)
{
  int64_t pi_sqrt_high = pi_bsearch(primes, isqrt(high - 1));

  for (int64_t i = c + 1; i <= pi_sqrt_high; i++)
  {
    int64_t prime = primes[i];
    int64_t m = wheel[i].next_multiple;
    int64_t wheel_index = wheel[i].wheel_index;

    // cross-off the multiples of prime inside [low, high[
    for (; m < high; m += prime * Wheel::next_multiple_factor(&wheel_index))
      sieve.unset(m - low);

    wheel[i].set(m, wheel_index);
  }
}

template <typename T>
T P2_OpenMP_thread(T x,
                   int64_t y,
                   int64_t segment_size,
                   int64_t segments_per_thread,
                   int64_t thread_num,
                   int64_t low,
                   int64_t limit,
                   int64_t& pix,
                   int64_t& pix_count,
                   vector<int32_t>& primes)
{
  pix = 0;
  pix_count = 0;
  low += thread_num * segments_per_thread * segment_size;
  limit = min(low + segments_per_thread * segment_size, limit);
  int64_t size = pi_bsearch(primes, isqrt(limit)) + 1;
  int64_t start = (int64_t) max(x / limit + 1, y);
  int64_t stop  = (int64_t) min(x / low, isqrt(x));
  T P2_thread = 0;

  // P2_thread = \sum_{i = pi[start]}^{pi[stop]} pi(x / primes[i]) - pi(low - 1)
  // We use a reverse prime iterator to calculate P2_thread
  primesieve::iterator pi(stop + 1, start);
  int64_t prime = pi.previous_prime();

  bool sieve_primes = true;
  Wheel wheel(primes, size, low, sieve_primes);
  BitSieve sieve(segment_size);

  // segmented sieve of Eratosthenes
  for (; low < limit; low += segment_size)
  {
    // current segment = interval [low, high[
    int64_t high = min(low + segment_size, limit);
    int64_t x_div_prime = 0;
    int64_t j = 0;
    int64_t c = 6;

    // pre-sieve the multiples of the first c primes
    sieve.pre_sieve(c, low, sieve_primes);

    // cross-off the multiples of the primes <= sqrt(high - 1)
    cross_off(sieve, primes, wheel, c, low, high);

    while (prime >= start &&
           (x_div_prime = (int64_t) (x / prime)) < high)
    {
      int64_t next_count = x_div_prime - low;
      pix += sieve.count(j, next_count);
      j = next_count + 1;
      pix_count++;
      P2_thread += pix;
      prime = pi.previous_prime();
    }

    pix += sieve.count(j, (high - 1) - low);
  }

  return P2_thread;
}

/// P2(x, y) counts the numbers <= x that have exactly 2 prime
/// factors each exceeding the a-th prime, a = pi(y).
/// Space complexity: O((x / y)^(1/2)).
///
template <typename T>
T P2_OpenMP_master(T x, int64_t y, int threads)
{
#if __cplusplus >= 201103L
  static_assert(prt::is_signed<T>::value,
                "P2(T x, ...): T must be signed integer type");
#endif

  if (x < 4)
    return 0;

  T a = pi_legendre(y, threads);
  T b = pi_legendre((int64_t) isqrt(x), threads);

  if (a >= b)
    return 0;

  int64_t low = 2;
  int64_t limit = (int64_t)(x / max(y, 1));
  int64_t sqrt_limit = isqrt(limit);
  int64_t segment_size = max(sqrt_limit, 1 << 12);
  int64_t segments_per_thread = 64;
  threads = validate_threads(threads, limit);

  vector<int32_t> primes = generate_primes(sqrt_limit);
  aligned_vector<int64_t> pix(threads);
  aligned_vector<int64_t> pix_counts(threads);

  // \sum_{i=a+1}^{b} pi(x / primes[i]) - (i - 1)
  T p2 = 0;
  T pix_total = 0;

  // \sum_{i=a+1}^{b} -(i - 1)
  p2 = (a - 2) * (a + 1) / 2 - (b - 2) * (b + 1) / 2;

  // \sum_{i=a+1}^{b} pi(x / primes[i])
  while (low < limit)
  {
    int64_t segments = ceil_div(limit - low, segment_size);
    threads = in_between(1, threads, segments);
    segments_per_thread = in_between(1, segments_per_thread, ceil_div(segments, threads));
    double time = get_wtime();

    #pragma omp parallel for \
        num_threads(threads) reduction(+: p2)
    for (int i = 0; i < threads; i++)
      p2 += P2_OpenMP_thread(x, y, segment_size, segments_per_thread,
         i, low, limit, pix[i], pix_counts[i], primes);

    low += segments_per_thread * threads * segment_size;
    segments_per_thread = balanceLoad(segments_per_thread, time);

    // Add missing sum contributions in order
    for (int i = 0; i < threads; i++)
    {
      p2 += pix_total * pix_counts[i];
      pix_total += pix[i];
    }

    if (print_status())
    {
      double percent = get_percent((double) low, (double) limit);
      cout << "\rStatus: " << fixed << setprecision(get_status_precision(x))
           << percent << '%' << flush;
    }
  }

  return p2;
}

} // namespace

namespace primecount {

int64_t P2(int64_t x, int64_t y, int threads)
{
#ifdef HAVE_MPI
  if (mpi_num_procs() > 1)
    return P2_mpi(x, y, threads);
#endif

  print("");
  print("=== P2(x, y) ===");
  print("Computation of the 2nd partial sieve function");
  print(x, y, threads);

  double time = get_wtime();
  int64_t p2 = P2_OpenMP_master(x, y, threads);

  print("P2", p2, time);
  return p2;
}

#ifdef HAVE_INT128_T

int128_t P2(int128_t x, int64_t y, int threads)
{
#ifdef HAVE_MPI
  if (mpi_num_procs() > 1)
    return P2_mpi(x, y, threads);
#endif

  print("");
  print("=== P2(x, y) ===");
  print("Computation of the 2nd partial sieve function");
  print(x, y, threads);

  double time = get_wtime();
  int128_t p2 = P2_OpenMP_master(x, y, threads);

  print("P2", p2, time);
  return p2;
}

#endif

} // namespace primecount
<commit_msg>30% speed up<commit_after>///
/// @file  P2.cpp
/// @brief 2nd partial sieve function.
///        P2(x, y) counts the numbers <= x that have exactly 2 prime
///        factors each exceeding the a-th prime.
///
/// Copyright (C) 2016 Kim Walisch, <kim.walisch@gmail.com>
///
/// This file is distributed under the BSD License. See the COPYING
/// file in the top level directory.
///

#include <primecount-internal.hpp>
#include <primesieve.hpp>
#include <aligned_vector.hpp>
#include <int128.hpp>
#include <min_max.hpp>
#include <pmath.hpp>

#include <stdint.h>
#include <algorithm>
#include <iostream>
#include <iomanip>

#ifdef _OPENMP
  #include <omp.h>
#endif

using namespace std;
using namespace primecount;

namespace {

/// Calculate the segment_size per thread.
/// The idea is to gradually increase the segment_size per
/// thread in order to keep all CPU cores busy.
///
int64_t balanceLoad(int64_t segment_size, double start_time)
{
  double seconds = get_wtime() - start_time;

  if (seconds < 30)
    segment_size *= 2;
  else if (segment_size >= 4)
    segment_size -= segment_size / 4;

  return segment_size;
}

template <typename T>
int64_t count_primes(primesieve::iterator& it,
                     int64_t& prime,
                     T stop)
{
  int64_t primes = 0;

  for (; prime <= stop; primes++)
    prime = it.next_prime();

  return primes;
}

template <typename T>
T P2_OpenMP_thread(T x,
                   int64_t y,
                   int64_t segment_size,
                   int64_t thread_num,
                   int64_t low,
                   int64_t limit,
                   int64_t& pix,
                   int64_t& pix_count)
{
  pix = 0;
  pix_count = 0;
  low += thread_num * segment_size;
  limit = min(low + segment_size, limit);
  int64_t start = (int64_t) max(x / limit + 1, y);
  int64_t stop  = (int64_t) min(x / low, isqrt(x));

  primesieve::iterator rit(stop + 1, start);
  primesieve::iterator it(low - 1, limit);

  int64_t next = it.next_prime();
  int64_t prime = rit.previous_prime();
  T P2_thread = 0;

  // P2_thread = \sum_{i = pi[start]}^{pi[stop]} pi(x / primes[i])
  while (prime >= start &&
         x / prime < limit)
  {
    pix += count_primes(it, next, x / prime);
    P2_thread += pix;
    pix_count++;
    prime = rit.previous_prime();
  }

  pix += count_primes(it, next, limit);

  return P2_thread;
}

/// P2(x, y) counts the numbers <= x that have exactly 2 prime
/// factors each exceeding the a-th prime, a = pi(y).
/// Space complexity: O((x / y)^(1/2)).
///
template <typename T>
T P2_OpenMP_master(T x, int64_t y, int threads)
{
#if __cplusplus >= 201103L
  static_assert(prt::is_signed<T>::value,
                "P2(T x, ...): T must be signed integer type");
#endif

  if (x < 4)
    return 0;

  T a = pi_legendre(y, threads);
  T b = pi_legendre((int64_t) isqrt(x), threads);

  if (a >= b)
    return 0;

  int64_t low = 2;
  int64_t limit = (int64_t)(x / max(y, 1));
  int64_t segment_size = 1 << 18;
  threads = validate_threads(threads, limit);

  aligned_vector<int64_t> pix(threads);
  aligned_vector<int64_t> pix_counts(threads);

  // \sum_{i=a+1}^{b} pi(x / primes[i]) - (i - 1)
  T p2 = 0;
  T pix_total = 0;

  // \sum_{i=a+1}^{b} -(i - 1)
  p2 = (a - 2) * (a + 1) / 2 - (b - 2) * (b + 1) / 2;

  // \sum_{i=a+1}^{b} pi(x / primes[i])
  while (low < limit)
  {
    int64_t segments = ceil_div(limit - low, segment_size);
    threads = in_between(1, threads, segments);
    double time = get_wtime();

    #pragma omp parallel for num_threads(threads) reduction(+: p2)
    for (int i = 0; i < threads; i++)
      p2 += P2_OpenMP_thread(x, y, segment_size, i,
         low, limit, pix[i], pix_counts[i]);

    low += threads * segment_size;
    segment_size = balanceLoad(segment_size, time);

    // Add missing sum contributions in order
    for (int i = 0; i < threads; i++)
    {
      p2 += pix_total * pix_counts[i];
      pix_total += pix[i];
    }

    if (print_status())
    {
      double percent = get_percent((double) low, (double) limit);
      cout << "\rStatus: " << fixed << setprecision(get_status_precision(x))
           << percent << '%' << flush;
    }
  }

  return p2;
}

} // namespace

namespace primecount {

int64_t P2(int64_t x, int64_t y, int threads)
{
#ifdef HAVE_MPI
  if (mpi_num_procs() > 1)
    return P2_mpi(x, y, threads);
#endif

  print("");
  print("=== P2(x, y) ===");
  print("Computation of the 2nd partial sieve function");
  print(x, y, threads);

  double time = get_wtime();
  int64_t p2 = P2_OpenMP_master(x, y, threads);

  print("P2", p2, time);
  return p2;
}

#ifdef HAVE_INT128_T

int128_t P2(int128_t x, int64_t y, int threads)
{
#ifdef HAVE_MPI
  if (mpi_num_procs() > 1)
    return P2_mpi(x, y, threads);
#endif

  print("");
  print("=== P2(x, y) ===");
  print("Computation of the 2nd partial sieve function");
  print(x, y, threads);

  double time = get_wtime();
  int128_t p2 = P2_OpenMP_master(x, y, threads);

  print("P2", p2, time);
  return p2;
}

#endif

} // namespace primecount
<|endoftext|>


<commit_before>#include	<cstdlib>
#include	<cmath>
#include	<fstream>
#include	<sstream>
#include	<iostream>
#include	<iomanip>
#include	<string>
#include	<vector>
#include	<algorithm>
#include	<exception>
#include	<sys/time.h>
#include	"modules/htmTree.h"
#include	"modules/kdTree.h"
#include        "misc.h"
#include        "feat.h"
#include        "structs.h"
#include        "collision.h"
#include        "global.h"
//reduce redistributes, updates  07/02/15 rnc
int main(int argc, char **argv) {
	//// Initializations ---------------------------------------------
	srand48(1234); // Make sure we have reproducability
	check_args(argc);
	Time t, time; // t for global, time for local
	init_time(t);
	Feat F;
    MTL M;

	// Read parameters file //
	F.readInputFile(argv[1]);
	printFile(argv[1]);
	// Read Secretfile
    // Secret contains the identity of each target: QSO-Ly-a, QSO-tracers, LRG, ELG, fake QSO, fake LRG, SS, SF
    Gals Secret;
    printf("# Read %s galaxies from %s \n",f(F.Ngal).c_str(),F.Secretfile.c_str());
    Secret=read_Secretfile(F.Secretfile,F);
    F.Ngal= Secret.size();

    std::vector<int> count;
    count=count_galaxies(Secret);
    printf(" Number of galaxies by type, QSO-Ly-a, QSO-tracers, LRG, ELG, fake QSO, fake LRG, SS, SF\n");
    for(int i=0;i<8;i++){printf (" type %d number  %d  \n",i, count[i]);}
    //read the three input files
    MTL Targ=read_MTLfile(F.Targfile,F,0,0);
    MTL SStars=read_MTLfile(F.SStarsfile,F,1,0);
    MTL SkyF=read_MTLfile(F.SkyFfile,F,0,1);
    //combine the three input files

    M=Targ;

    printf(" M size %d \n",M.size());
    M.insert(M.end(),SStars.begin(),SStars.end());
    printf(" M size %d \n",M.size());
    M.insert(M.end(),SkyF.begin(),SkyF.end());
    printf(" M size %d \n",M.size());

    assign_priority_class(Targ);

    //establish priority classes
    std::vector <int> count_class(Targ.priority_list.size(),0);
    printf("Number in each priority class.  The last two are SF and SS.\n");
    for(int i;i<Targ.size();++i){
        count_class[Targ[i].priority_class]+=1;
    }
    for(int i;i<Targ.priority_list.size();++i){
        printf("  class  %d  number  %d\n",i,count_class[i]);
    }

    PP pp;
	pp.read_fiber_positions(F);
	F.Nfiber = pp.fp.size()/2;
	F.Npetal = max(pp.spectrom)+1;
    F.Nfbp = (int) (F.Nfiber/F.Npetal);// fibers per petal = 500
	pp.get_neighbors(F); pp.compute_fibsofsp(F);

    //P is original list of plates
	Plates P = read_plate_centers(F);

    printf(" future plates %d\n",P.size());
    F.Nplate=P.size();
	printf("# Read %s plate centers from %s and %d fibers from %s\n",f(F.Nplate).c_str(),F.tileFile.c_str(),F.Nfiber,F.fibFile.c_str());

	// Computes geometries of cb and fh: pieces of positioner - used to determine possible collisions
	F.cb = create_cb(); // cb=central body
	F.fh = create_fh(); // fh=fiber holder

	//// Collect available galaxies <-> tilefibers --------------------
	// HTM Tree of galaxies
	const double MinTreeSize = 0.01;
	init_time_at(time,"# Start building HTM tree",t);
	htmTree<struct target> T(M,MinTreeSize);
	print_time(time,"# ... took :");//T.stats();

	// For plates/fibers, collect available galaxies; done in parallel
    collect_galaxies_for_all(M,T,P,pp,F);

	// For each galaxy, computes available tilefibers  G[i].av_tfs = [(j1,k1),(j2,k2),..]
	collect_available_tilefibers(M,P,F);

	//results_on_inputs("doc/figs/",G,P,F,true);

	//// Assignment |||||||||||||||||||||||||||||||||||||||||||||||||||
    printf(" Nplate %d  Ngal %d   Nfiber %d \n", F.Nplate, F.Ngal, F.Nfiber);
    Assignment A(M,F);

	print_time(t,"# Start assignment at : ");

	// Make a plan ----------------------------------------------------
    // Plans whole survey without sky fibers, standard stars
    // assumes maximum number of observations needed for QSOs, LRGs

    simple_assign(M,P,pp,F,A);

    //check to see if there are tiles with no galaxies
    //need to keep mapping of old tile list to new tile list
    for (int j=0;j<F.Nplate ;++j){
        bool not_done=true;
        for(int k=0;k<F.Nfiber && not_done;++k){
            if(A.TF[j][k]!=-1){
                A.suborder.push_back(j);
                not_done=false;

            }
        }
    }
    F.NUsedplate=A.suborder.size();
    printf(" Plates after screening %d \n",F.NUsedplate);

    //if(F.diagnose)diagnostic(M,G,F,A);

    print_hist("Unused fibers",5,histogram(A.unused_fbp(pp,F),5),false); // Hist of unused fibs

	// Smooth out distribution of free fibers, and increase the number of assignments

	for (int i=0; i<1; i++) redistribute_tf(M,P,pp,F,A);// more iterations will improve performance slightly
	for (int i=0; i<1; i++) {
        improve(M,P,pp,F,A);
		redistribute_tf(M,P,pp,F,A);
	}

	print_hist("Unused fibers",5,histogram(A.unused_fbp(pp,F),5),false);
    //try assigning SF and SS before real time assignment
    for (int j=0;j<F.Nplate;++j){
        int js=A.suborder[j];
        A.next_plate=js;
        assign_sf_ss(js,M,P,pp,F,A); // Assign SS and SF for each tile
        assign_unused(js,M,P,pp,F,A);
    }

    init_time_at(time,"# Begin real time assignment",t);

	//Execute plan, updating targets at intervals

    for(int i=0;i<F.pass_intervals.size()&&F.pass_intervals[i]<F.Nplate;++i){
        printf(" before pass = %d  at %d  tiles\n",i,F.pass_intervals[i]);
        //display_results("doc/figs/",G,P,pp,F,A,true);
        //execute this phase (i) of survey
        //A.next_plate=F.pass_intervals[i];
        for (int jj=F.pass_intervals[i]; jj<F.Nplate; jj++) {
            int j = A.suborder[A.next_plate];
            printf("  next plate is %d \n",j);
            assign_sf_ss(j,M,P,pp,F,A); // Assign SS and SF
            assign_unused(j,M,P,pp,F,A);
            //A.next_plate++;
        }
        //update target information for this interval
        //A.next_plate=F.pass_intervals[i];
        for (int jj=F.pass_intervals[i]; jj<F.pass_intervals[i+1]&&jj<F.Nplate; jj++) {
            //int j = A.suborder[A.next_plate];
            int js=A.suborder[jj];
            // Update corrects all future occurrences of wrong QSOs etc and tries to observe something else
            if (0<=js-F.Analysis) update_plan_from_one_obs(Secret,M,P,pp,F,A,F.Nplate-1); else printf("\n");
            //A.next_plate++;
        }
        /*
        if(A.next_plate<F.Nplate){
        redistribute_tf(M,P,pp,F,A);
        redistribute_tf(M,P,pp,F,A);
        improve(M,P,pp,F,A);
        redistribute_tf(M,P,pp,F,A);
        }
         */
        if(F.diagnose)diagnostic(M,Secret,F,A);
    }

	// Results -------------------------------------------------------
    if (F.PrintAscii) for (int j=0; j<F.NUsedplate; j++){
        write_FAtile_ascii(A.suborder[j],F.outDir,M,P,pp,F,A);
    }

    if (F.PrintFits) for (int j=0; j<F.NUsedplate; j++){
        fa_write(A.suborder[j],F.outDir,M,P,pp,F,A); // Write output
    }

	display_results("doc/figs/",Secret,M,P,pp,F,A,true);
	if (F.Verif) A.verif(P,M,pp,F); // Verification that the assignment is sane

	print_time(t,"# Finished !... in");

	return(0);

}
<commit_msg>new diagnostic<commit_after>#include	<cstdlib>
#include	<cmath>
#include	<fstream>
#include	<sstream>
#include	<iostream>
#include	<iomanip>
#include	<string>
#include	<vector>
#include	<algorithm>
#include	<exception>
#include	<sys/time.h>
#include	"modules/htmTree.h"
#include	"modules/kdTree.h"
#include        "misc.h"
#include        "feat.h"
#include        "structs.h"
#include        "collision.h"
#include        "global.h"
//reduce redistributes, updates  07/02/15 rnc
int main(int argc, char **argv) {
	//// Initializations ---------------------------------------------
	srand48(1234); // Make sure we have reproducability
	check_args(argc);
	Time t, time; // t for global, time for local
	init_time(t);
	Feat F;
    MTL M;

	// Read parameters file //
	F.readInputFile(argv[1]);
	printFile(argv[1]);
	// Read Secretfile
    // Secret contains the identity of each target: QSO-Ly-a, QSO-tracers, LRG, ELG, fake QSO, fake LRG, SS, SF
    Gals Secret;
    printf("# Read %s galaxies from %s \n",f(F.Ngal).c_str(),F.Secretfile.c_str());
    Secret=read_Secretfile(F.Secretfile,F);
    F.Ngal= Secret.size();

    std::vector<int> count;
    count=count_galaxies(Secret);
    printf(" Number of galaxies by type, QSO-Ly-a, QSO-tracers, LRG, ELG, fake QSO, fake LRG, SS, SF\n");
    for(int i=0;i<8;i++){printf (" type %d number  %d  \n",i, count[i]);}
    //read the three input files
    MTL Targ=read_MTLfile(F.Targfile,F,0,0);
    MTL SStars=read_MTLfile(F.SStarsfile,F,1,0);
    MTL SkyF=read_MTLfile(F.SkyFfile,F,0,1);
    //combine the three input files

    M=Targ;

    printf(" M size %d \n",M.size());
    M.insert(M.end(),SStars.begin(),SStars.end());
    printf(" M size %d \n",M.size());
    M.insert(M.end(),SkyF.begin(),SkyF.end());
    printf(" M size %d \n",M.size());

    assign_priority_class(Targ);

    //establish priority classes
    std::vector <int> count_class(Targ.priority_list.size(),0);
    printf("Number in each priority class.  The last two are SF and SS.\n");
    for(int i;i<Targ.size();++i){
        count_class[Targ[i].priority_class]+=1;
    }
    for(int i;i<Targ.priority_list.size();++i){
        printf("  class  %d  number  %d\n",i,count_class[i]);
    }

    PP pp;
	pp.read_fiber_positions(F);
	F.Nfiber = pp.fp.size()/2;
	F.Npetal = max(pp.spectrom)+1;
    F.Nfbp = (int) (F.Nfiber/F.Npetal);// fibers per petal = 500
	pp.get_neighbors(F); pp.compute_fibsofsp(F);

    //P is original list of plates
	Plates P = read_plate_centers(F);

    printf(" future plates %d\n",P.size());
    F.Nplate=P.size();
	printf("# Read %s plate centers from %s and %d fibers from %s\n",f(F.Nplate).c_str(),F.tileFile.c_str(),F.Nfiber,F.fibFile.c_str());

	// Computes geometries of cb and fh: pieces of positioner - used to determine possible collisions
	F.cb = create_cb(); // cb=central body
	F.fh = create_fh(); // fh=fiber holder

	//// Collect available galaxies <-> tilefibers --------------------
	// HTM Tree of galaxies
	const double MinTreeSize = 0.01;
	init_time_at(time,"# Start building HTM tree",t);
	htmTree<struct target> T(M,MinTreeSize);
	print_time(time,"# ... took :");//T.stats();

	// For plates/fibers, collect available galaxies; done in parallel
    collect_galaxies_for_all(M,T,P,pp,F);

	// For each galaxy, computes available tilefibers  G[i].av_tfs = [(j1,k1),(j2,k2),..]
	collect_available_tilefibers(M,P,F);

	//results_on_inputs("doc/figs/",G,P,F,true);

	//// Assignment |||||||||||||||||||||||||||||||||||||||||||||||||||
    printf(" Nplate %d  Ngal %d   Nfiber %d \n", F.Nplate, F.Ngal, F.Nfiber);
    Assignment A(M,F);

	print_time(t,"# Start assignment at : ");

	// Make a plan ----------------------------------------------------
    // Plans whole survey without sky fibers, standard stars
    // assumes maximum number of observations needed for QSOs, LRGs

    simple_assign(M,P,pp,F,A);

    //check to see if there are tiles with no galaxies
    //need to keep mapping of old tile list to new tile list
    for (int j=0;j<F.Nplate ;++j){
        bool not_done=true;
        for(int k=0;k<F.Nfiber && not_done;++k){
            if(A.TF[j][k]!=-1){
                A.suborder.push_back(j);
                not_done=false;

            }
        }
    }
    F.NUsedplate=A.suborder.size();
    printf(" Plates after screening %d \n",F.NUsedplate);

    //if(F.diagnose)diagnostic(M,G,F,A);

    print_hist("Unused fibers",5,histogram(A.unused_fbp(pp,F),5),false); // Hist of unused fibs

	// Smooth out distribution of free fibers, and increase the number of assignments

	for (int i=0; i<1; i++) redistribute_tf(M,P,pp,F,A);// more iterations will improve performance slightly
	for (int i=0; i<1; i++) {
        improve(M,P,pp,F,A);
		redistribute_tf(M,P,pp,F,A);
	}

	print_hist("Unused fibers",5,histogram(A.unused_fbp(pp,F),5),false);
    //try assigning SF and SS before real time assignment
    for (int j=0;j<F.NUsedplate;++j){
        printf(" before assign_sf_ss j= %d\n",j);
        int js=A.suborder[j];
        A.next_plate=js;
        assign_sf_ss(js,M,P,pp,F,A); // Assign SS and SF for each tile
        printf("before assign_unused ]n");
        assign_unused(js,M,P,pp,F,A);
    }

    init_time_at(time,"# Begin real time assignment",t);

	//Execute plan, updating targets at intervals

    for(int i=0;i<F.pass_intervals.size()&&F.pass_intervals[i]<F.Nplate;++i){
        printf(" before pass = %d  at %d  tiles\n",i,F.pass_intervals[i]);
        //display_results("doc/figs/",G,P,pp,F,A,true);
        //execute this phase (i) of survey
        //A.next_plate=F.pass_intervals[i];
        for (int jj=F.pass_intervals[i]; jj<F.Nplate; jj++) {
            int j = A.suborder[A.next_plate];
            printf("  next plate is %d \n",j);
            assign_sf_ss(j,M,P,pp,F,A); // Assign SS and SF
            assign_unused(j,M,P,pp,F,A);
            //A.next_plate++;
        }
        //update target information for this interval
        //A.next_plate=F.pass_intervals[i];
        for (int jj=F.pass_intervals[i]; jj<F.pass_intervals[i+1]&&jj<F.Nplate; jj++) {
            //int j = A.suborder[A.next_plate];
            int js=A.suborder[jj];
            // Update corrects all future occurrences of wrong QSOs etc and tries to observe something else
            if (0<=js-F.Analysis) update_plan_from_one_obs(Secret,M,P,pp,F,A,F.Nplate-1); else printf("\n");
            //A.next_plate++;
        }
        /*
        if(A.next_plate<F.Nplate){
        redistribute_tf(M,P,pp,F,A);
        redistribute_tf(M,P,pp,F,A);
        improve(M,P,pp,F,A);
        redistribute_tf(M,P,pp,F,A);
        }
         */
        if(F.diagnose)diagnostic(M,Secret,F,A);
    }

	// Results -------------------------------------------------------
    if (F.PrintAscii) for (int j=0; j<F.NUsedplate; j++){
        write_FAtile_ascii(A.suborder[j],F.outDir,M,P,pp,F,A);
    }

    if (F.PrintFits) for (int j=0; j<F.NUsedplate; j++){
        fa_write(A.suborder[j],F.outDir,M,P,pp,F,A); // Write output
    }

	display_results("doc/figs/",Secret,M,P,pp,F,A,true);
	if (F.Verif) A.verif(P,M,pp,F); // Verification that the assignment is sane

	print_time(t,"# Finished !... in");

	return(0);

}
<|endoftext|>


<commit_before>#include <mcl/op.hpp>
#include <mcl/util.hpp>
#include <mcl/conversion.hpp>
#ifdef USE_MONT_FP
#include <mcl/fp_generator.hpp>
#endif
#include <mcl/fp_proto.hpp>

namespace mcl { namespace fp {

#ifdef USE_MONT_FP
FpGenerator *Op::createFpGenerator()
{
	return new FpGenerator();
}
void Op::destroyFpGenerator(FpGenerator *fg)
{
	delete fg;
}
#else
FpGenerator *Op::createFpGenerator()
{
	return 0;
}
void Op::destroyFpGenerator(FpGenerator *)
{
}
#endif

inline const char *verifyStr(bool *isMinus, int *base, const std::string& str)
{
	const char *p = str.c_str();
	if (*p == '-') {
		*isMinus = true;
		p++;
	} else {
		*isMinus = false;
	}
	if (p[0] == '0') {
		if (p[1] == 'x') {
			if (*base != 0 && *base != 16) {
				throw cybozu::Exception("fp:verifyStr:bad base") << *base << str;
			}
			*base = 16;
			p += 2;
		} else if (p[1] == 'b') {
			if (*base != 0 && *base != 2) {
				throw cybozu::Exception("fp:verifyStr:bad base") << *base << str;
			}
			*base = 2;
			p += 2;
		}
	}
	if (*base == 0) *base = 10;
	if (*p == '\0') throw cybozu::Exception("fp:verifyStr:str is empty");
	return p;
}

bool strToMpzArray(size_t *pBitSize, Unit *y, size_t maxBitSize, mpz_class& x, const std::string& str, int base)
{
	bool isMinus;
	const char *p = verifyStr(&isMinus, &base, str);
	if (!Gmp::setStr(x, p, base)) {
		throw cybozu::Exception("fp:strToMpzArray:bad format") << str;
	}
	const size_t bitSize = Gmp::getBitSize(x);
	if (bitSize > maxBitSize) throw cybozu::Exception("fp:strToMpzArray:too large str") << str << bitSize << maxBitSize;
	if (pBitSize) *pBitSize = bitSize;
	Gmp::getArray(y, (maxBitSize + UnitBitSize - 1) / UnitBitSize, x);
	return isMinus;
}

template<size_t bitSize>
struct OpeFunc {
	static const size_t N = (bitSize + UnitBitSize - 1) / UnitBitSize;
	static inline void set_mpz_t(mpz_t& z, const Unit* p, int n = (int)N)
	{
		z->_mp_alloc = n;
		z->_mp_size = getNonZeroArraySize(p, n);
		z->_mp_d = (mp_limb_t*)const_cast<Unit*>(p);
	}
	static inline void set_zero(mpz_t& z, Unit *p, size_t n)
	{
		z->_mp_alloc = (int)n;
		z->_mp_size = 0;
		z->_mp_d = (mp_limb_t*)p;
	}
	static inline void clearC(Unit *x)
	{
		clearArray(x, 0, N);
	}
	static inline void copyC(Unit *y, const Unit *x)
	{
		copyArray(y, x, N);
	}
	static inline void addC(Unit *z, const Unit *x, const Unit *y, const Unit *p)
	{
		Unit ret[N + 2]; // not N + 1
		mpz_t mz, mx, my, mp;
		set_zero(mz, ret, N + 2);
		set_mpz_t(mx, x);
		set_mpz_t(my, y);
		set_mpz_t(mp, p);
		mpz_add(mz, mx, my);
		if (mpz_cmp(mz, mp) >= 0) {
			mpz_sub(mz, mz, mp);
		}
		Gmp::getArray(z, N, mz);
	}
	static inline void subC(Unit *z, const Unit *x, const Unit *y, const Unit *p)
	{
		Unit ret[N + 1];
		mpz_t mz, mx, my;
		set_zero(mz, ret, N + 1);
		set_mpz_t(mx, x);
		set_mpz_t(my, y);
		mpz_sub(mz, mx, my);
		if (mpz_sgn(mz) < 0) {
			mpz_t mp;
			set_mpz_t(mp, p);
			mpz_add(mz, mz, mp);
		}
		Gmp::getArray(z, N, mz);
	}
	static inline void mulPreC(Unit *z, const Unit *x, const Unit *y)
	{
		mpz_t mx, my, mz;
		set_zero(mz, z, N * 2);
		set_mpz_t(mx, x);
		set_mpz_t(my, y);
		mpz_mul(mz, mx, my);
		Gmp::getArray(z, N * 2, mz);
	}
	// x[N * 2] -> y[N]
	static inline void modC(Unit *y, const Unit *x, const Unit *p)
	{
		mpz_t mx, my, mp;
		set_mpz_t(mx, x, N * 2);
		set_mpz_t(my, y);
		set_mpz_t(mp, p);
		mpz_mod(my, mx, mp);
		clearArray(y, my->_mp_size, N);
	}
	static inline void invOp(Unit *y, const Unit *x, const Op& op)
	{
		mpz_class my;
		mpz_t mx, mp;
		set_mpz_t(mx, x);
		set_mpz_t(mp, op.p);
		mpz_invert(my.get_mpz_t(), mx, mp);
		Gmp::getArray(y, N, my);
	}
	/*
		inv(x/R) = (1/x)R -toMont-> 1/x -toMont-> (1/x)R^-1
	*/
	static void invMontOp(Unit *y, const Unit *x, const Op& op)
	{
		invOp(y, x, op);
		op.toMont(y, y);
		op.toMont(y, y);
	}
	static inline bool isZeroC(const Unit *x)
	{
		return isZeroArray(x, N);
	}
	static inline void negC(Unit *y, const Unit *x, const Unit *p)
	{
		if (isZeroC(x)) {
			if (x != y) clearC(y);
			return;
		}
		subC(y, p, x, p);
	}
};

#ifdef MCL_USE_LLVM
	#define SET_OP_LLVM(n) \
		addP = mcl_fp_add ## n ##S; \
		subP = mcl_fp_sub ## n ##S; \
		mulPreP = mcl_fp_mulPre ## n; \
		mont = mcl_fp_mont ## n;
#else
	#define SET_OP_LLVM(n)
#endif

#define SET_OP(n) \
		N = n / UnitBitSize; \
		isZero = OpeFunc<n>::isZeroC; \
		clear = OpeFunc<n>::clearC; \
		copy = OpeFunc<n>::copyC; \
		negP = OpeFunc<n>::negC; \
		if (useMont) { \
			invOp = OpeFunc<n>::invMontOp; \
		} else { \
			invOp = OpeFunc<n>::invOp; \
		} \
		addP = OpeFunc<n>::addC; \
		subP = OpeFunc<n>::subC; \
		mulPreP = OpeFunc<n>::mulPreC; \
		modP = OpeFunc<n>::modC; \
		SET_OP_LLVM(n)

#ifdef USE_MONT_FP
inline void invOpForMont(Unit *y, const Unit *x, const Op& op)
{
	Unit r[maxOpUnitSize];
	int k = op.preInv(r, x);
	/*
		xr = 2^k
		R = 2^(N * 64)
		get r2^(-k)R^2 = r 2^(N * 64 * 2 - k)
	*/
	op.mul(y, r, op.invTbl.data() + k * op.N);
}

static void initInvTbl(Op& op)
{
	const size_t N = op.N;
	const size_t invTblN = N * sizeof(Unit) * 8 * 2;
	op.invTbl.resize(invTblN * N);
	Unit *tbl = op.invTbl.data() + (invTblN - 1) * N;
	Unit t[maxOpUnitSize] = {};
	t[0] = 2;
	op.toMont(tbl, t);
	for (size_t i = 0; i < invTblN - 1; i++) {
		op.add(tbl - N, tbl, tbl);
		tbl -= N;
	}
}
#endif

static void initForMont(Op& op, const Unit *p)
{
	const size_t N = op.N;
	assert(N >= 2);
	mpz_class t = 1;
	Gmp::getArray(op.one, N, t);
	t = (t << (N * 64)) % op.mp;
	t = (t * t) % op.mp;
	Gmp::getArray(op.RR, N, t);
	op.rp = getMontgomeryCoeff(p[0]);
#ifdef USE_MONT_FP
	FpGenerator *fg = op.fg;
	if (fg == 0) return;
	fg->init(p, (int)N);

	op.neg = Xbyak::CastTo<void2u>(fg->neg_);
	op.add = Xbyak::CastTo<void3u>(fg->add_);
	op.sub = Xbyak::CastTo<void3u>(fg->sub_);
	op.mul = Xbyak::CastTo<void3u>(fg->mul_);
	op.preInv = Xbyak::CastTo<int2u>(op.fg->preInv_);
	op.invOp = &invOpForMont;

	initInvTbl(op);
#endif
}

void Op::init(const std::string& mstr, int base, size_t maxBitSize, Mode mode)
{
	cybozu::disable_warning_unused_variable(mode);
	bool isMinus = fp::strToMpzArray(&bitSize, p, maxBitSize, mp, mstr, base);
	if (isMinus) throw cybozu::Exception("Op:init:mstr is minus") << mstr;
	if (mp == 0) throw cybozu::Exception("Op:init:mstr is zero") << mstr;

	const size_t roundBit = (bitSize + UnitBitSize - 1) & ~(UnitBitSize - 1);
	switch (roundBit) {
	case 32:
	case 64:
	case 96:
	case 128: SET_OP(128); break;
	case 192: SET_OP(192); break;
	case 256: SET_OP(256); break;
	case 320: SET_OP(320); break;
	case 384: SET_OP(384); break;
	case 448: SET_OP(448); break;
	case 512: SET_OP(512); break;
#if CYBOZU_OS_BIT == 64
	case 576: SET_OP(576); break;
#else
	case 160: SET_OP(160); break;
	case 224: SET_OP(224); break;
	case 288: SET_OP(288); break;
	case 352: SET_OP(352); break;
	case 416: SET_OP(416); break;
	case 480: SET_OP(480); break;
	case 544: SET_OP(544); break;
#endif
	default:
		throw cybozu::Exception("Op::init:not:support") << mstr;
	}
#ifdef MCL_USE_LLVM
	if (mp == mpz_class("0xfffffffffffffffffffffffffffffffeffffffffffffffff")) {
		mul = &mcl_fp_mul_NIST_P192; // slower than MontFp192
	}
#endif
	if (useMont) {
		fp::initForMont(*this, p);
	}
	sq.set(mp);
}

void arrayToStr(std::string& str, const Unit *x, size_t n, int base, bool withPrefix)
{
	switch (base) {
	case 10:
		{
			mpz_class t;
			Gmp::setArray(t, x, n);
			Gmp::getStr(str, t, 10);
		}
		return;
	case 16:
		mcl::fp::toStr16(str, x, n, withPrefix);
		return;
	case 2:
		mcl::fp::toStr2(str, x, n, withPrefix);
		return;
	default:
		throw cybozu::Exception("fp:arrayToStr:bad base") << base;
	}
}

void copyAndMask(Unit *y, const void *x, size_t xByteSize, const Op& op, bool doMask)
{
	const size_t fpByteSize = sizeof(Unit) * op.N;
	if (xByteSize > fpByteSize) {
		if (!doMask) throw cybozu::Exception("fp:copyAndMask:bad size") << xByteSize << fpByteSize;
		xByteSize = fpByteSize;
	}
	memcpy(y, x, xByteSize);
	memset((char *)y + xByteSize, 0, fpByteSize - xByteSize);
	if (!doMask) {
		if (compareArray(y, op.p, op.N) >= 0) throw cybozu::Exception("fp:copyAndMask:large x");
		return;
	}
	maskArray(y, op.N, op.bitSize - 1);
	assert(compareArray(y, op.p, op.N) < 0);
}

} } // mcl::fp

<commit_msg>disable useMont for NIST_P192<commit_after>#include <mcl/op.hpp>
#include <mcl/util.hpp>
#include <mcl/conversion.hpp>
#ifdef USE_MONT_FP
#include <mcl/fp_generator.hpp>
#endif
#include <mcl/fp_proto.hpp>

namespace mcl { namespace fp {

#ifdef USE_MONT_FP
FpGenerator *Op::createFpGenerator()
{
	return new FpGenerator();
}
void Op::destroyFpGenerator(FpGenerator *fg)
{
	delete fg;
}
#else
FpGenerator *Op::createFpGenerator()
{
	return 0;
}
void Op::destroyFpGenerator(FpGenerator *)
{
}
#endif

inline const char *verifyStr(bool *isMinus, int *base, const std::string& str)
{
	const char *p = str.c_str();
	if (*p == '-') {
		*isMinus = true;
		p++;
	} else {
		*isMinus = false;
	}
	if (p[0] == '0') {
		if (p[1] == 'x') {
			if (*base != 0 && *base != 16) {
				throw cybozu::Exception("fp:verifyStr:bad base") << *base << str;
			}
			*base = 16;
			p += 2;
		} else if (p[1] == 'b') {
			if (*base != 0 && *base != 2) {
				throw cybozu::Exception("fp:verifyStr:bad base") << *base << str;
			}
			*base = 2;
			p += 2;
		}
	}
	if (*base == 0) *base = 10;
	if (*p == '\0') throw cybozu::Exception("fp:verifyStr:str is empty");
	return p;
}

bool strToMpzArray(size_t *pBitSize, Unit *y, size_t maxBitSize, mpz_class& x, const std::string& str, int base)
{
	bool isMinus;
	const char *p = verifyStr(&isMinus, &base, str);
	if (!Gmp::setStr(x, p, base)) {
		throw cybozu::Exception("fp:strToMpzArray:bad format") << str;
	}
	const size_t bitSize = Gmp::getBitSize(x);
	if (bitSize > maxBitSize) throw cybozu::Exception("fp:strToMpzArray:too large str") << str << bitSize << maxBitSize;
	if (pBitSize) *pBitSize = bitSize;
	Gmp::getArray(y, (maxBitSize + UnitBitSize - 1) / UnitBitSize, x);
	return isMinus;
}

template<size_t bitSize>
struct OpeFunc {
	static const size_t N = (bitSize + UnitBitSize - 1) / UnitBitSize;
	static inline void set_mpz_t(mpz_t& z, const Unit* p, int n = (int)N)
	{
		z->_mp_alloc = n;
		z->_mp_size = getNonZeroArraySize(p, n);
		z->_mp_d = (mp_limb_t*)const_cast<Unit*>(p);
	}
	static inline void set_zero(mpz_t& z, Unit *p, size_t n)
	{
		z->_mp_alloc = (int)n;
		z->_mp_size = 0;
		z->_mp_d = (mp_limb_t*)p;
	}
	static inline void clearC(Unit *x)
	{
		clearArray(x, 0, N);
	}
	static inline void copyC(Unit *y, const Unit *x)
	{
		copyArray(y, x, N);
	}
	static inline void addC(Unit *z, const Unit *x, const Unit *y, const Unit *p)
	{
		Unit ret[N + 2]; // not N + 1
		mpz_t mz, mx, my, mp;
		set_zero(mz, ret, N + 2);
		set_mpz_t(mx, x);
		set_mpz_t(my, y);
		set_mpz_t(mp, p);
		mpz_add(mz, mx, my);
		if (mpz_cmp(mz, mp) >= 0) {
			mpz_sub(mz, mz, mp);
		}
		Gmp::getArray(z, N, mz);
	}
	static inline void subC(Unit *z, const Unit *x, const Unit *y, const Unit *p)
	{
		Unit ret[N + 1];
		mpz_t mz, mx, my;
		set_zero(mz, ret, N + 1);
		set_mpz_t(mx, x);
		set_mpz_t(my, y);
		mpz_sub(mz, mx, my);
		if (mpz_sgn(mz) < 0) {
			mpz_t mp;
			set_mpz_t(mp, p);
			mpz_add(mz, mz, mp);
		}
		Gmp::getArray(z, N, mz);
	}
	static inline void mulPreC(Unit *z, const Unit *x, const Unit *y)
	{
		mpz_t mx, my, mz;
		set_zero(mz, z, N * 2);
		set_mpz_t(mx, x);
		set_mpz_t(my, y);
		mpz_mul(mz, mx, my);
		Gmp::getArray(z, N * 2, mz);
	}
	// x[N * 2] -> y[N]
	static inline void modC(Unit *y, const Unit *x, const Unit *p)
	{
		mpz_t mx, my, mp;
		set_mpz_t(mx, x, N * 2);
		set_mpz_t(my, y);
		set_mpz_t(mp, p);
		mpz_mod(my, mx, mp);
		clearArray(y, my->_mp_size, N);
	}
	static inline void invOp(Unit *y, const Unit *x, const Op& op)
	{
		mpz_class my;
		mpz_t mx, mp;
		set_mpz_t(mx, x);
		set_mpz_t(mp, op.p);
		mpz_invert(my.get_mpz_t(), mx, mp);
		Gmp::getArray(y, N, my);
	}
	/*
		inv(x/R) = (1/x)R -toMont-> 1/x -toMont-> (1/x)R^-1
	*/
	static void invMontOp(Unit *y, const Unit *x, const Op& op)
	{
		invOp(y, x, op);
		op.toMont(y, y);
		op.toMont(y, y);
	}
	static inline bool isZeroC(const Unit *x)
	{
		return isZeroArray(x, N);
	}
	static inline void negC(Unit *y, const Unit *x, const Unit *p)
	{
		if (isZeroC(x)) {
			if (x != y) clearC(y);
			return;
		}
		subC(y, p, x, p);
	}
};

#ifdef MCL_USE_LLVM
	#define SET_OP_LLVM(n) \
		addP = mcl_fp_add ## n ##S; \
		subP = mcl_fp_sub ## n ##S; \
		mulPreP = mcl_fp_mulPre ## n; \
		mont = mcl_fp_mont ## n;
#else
	#define SET_OP_LLVM(n)
#endif

#define SET_OP(n) \
		N = n / UnitBitSize; \
		isZero = OpeFunc<n>::isZeroC; \
		clear = OpeFunc<n>::clearC; \
		copy = OpeFunc<n>::copyC; \
		negP = OpeFunc<n>::negC; \
		if (useMont) { \
			invOp = OpeFunc<n>::invMontOp; \
		} else { \
			invOp = OpeFunc<n>::invOp; \
		} \
		addP = OpeFunc<n>::addC; \
		subP = OpeFunc<n>::subC; \
		mulPreP = OpeFunc<n>::mulPreC; \
		modP = OpeFunc<n>::modC; \
		SET_OP_LLVM(n)

#ifdef USE_MONT_FP
inline void invOpForMont(Unit *y, const Unit *x, const Op& op)
{
	Unit r[maxOpUnitSize];
	int k = op.preInv(r, x);
	/*
		xr = 2^k
		R = 2^(N * 64)
		get r2^(-k)R^2 = r 2^(N * 64 * 2 - k)
	*/
	op.mul(y, r, op.invTbl.data() + k * op.N);
}

static void initInvTbl(Op& op)
{
	const size_t N = op.N;
	const size_t invTblN = N * sizeof(Unit) * 8 * 2;
	op.invTbl.resize(invTblN * N);
	Unit *tbl = op.invTbl.data() + (invTblN - 1) * N;
	Unit t[maxOpUnitSize] = {};
	t[0] = 2;
	op.toMont(tbl, t);
	for (size_t i = 0; i < invTblN - 1; i++) {
		op.add(tbl - N, tbl, tbl);
		tbl -= N;
	}
}
#endif

static void initForMont(Op& op, const Unit *p)
{
	const size_t N = op.N;
	assert(N >= 2);
	mpz_class t = 1;
	Gmp::getArray(op.one, N, t);
	t = (t << (N * 64)) % op.mp;
	t = (t * t) % op.mp;
	Gmp::getArray(op.RR, N, t);
	op.rp = getMontgomeryCoeff(p[0]);
#ifdef USE_MONT_FP
	FpGenerator *fg = op.fg;
	if (fg == 0) return;
	fg->init(p, (int)N);

	op.neg = Xbyak::CastTo<void2u>(fg->neg_);
	op.add = Xbyak::CastTo<void3u>(fg->add_);
	op.sub = Xbyak::CastTo<void3u>(fg->sub_);
	op.mul = Xbyak::CastTo<void3u>(fg->mul_);
	op.preInv = Xbyak::CastTo<int2u>(op.fg->preInv_);
	op.invOp = &invOpForMont;

	initInvTbl(op);
#endif
}

void Op::init(const std::string& mstr, int base, size_t maxBitSize, Mode mode)
{
	cybozu::disable_warning_unused_variable(mode);
	bool isMinus = fp::strToMpzArray(&bitSize, p, maxBitSize, mp, mstr, base);
	if (isMinus) throw cybozu::Exception("Op:init:mstr is minus") << mstr;
	if (mp == 0) throw cybozu::Exception("Op:init:mstr is zero") << mstr;

	const size_t roundBit = (bitSize + UnitBitSize - 1) & ~(UnitBitSize - 1);
	switch (roundBit) {
	case 32:
	case 64:
	case 96:
	case 128: SET_OP(128); break;
	case 192: SET_OP(192); break;
	case 256: SET_OP(256); break;
	case 320: SET_OP(320); break;
	case 384: SET_OP(384); break;
	case 448: SET_OP(448); break;
	case 512: SET_OP(512); break;
#if CYBOZU_OS_BIT == 64
	case 576: SET_OP(576); break;
#else
	case 160: SET_OP(160); break;
	case 224: SET_OP(224); break;
	case 288: SET_OP(288); break;
	case 352: SET_OP(352); break;
	case 416: SET_OP(416); break;
	case 480: SET_OP(480); break;
	case 544: SET_OP(544); break;
#endif
	default:
		throw cybozu::Exception("Op::init:not:support") << mstr;
	}
#ifdef MCL_USE_LLVM
	if (mode == FP_AUTO && mp == mpz_class("0xfffffffffffffffffffffffffffffffeffffffffffffffff")) {
		mul = &mcl_fp_mul_NIST_P192;
		useMont = false;
	}
#endif
	if (useMont) {
		fp::initForMont(*this, p);
	}
	sq.set(mp);
}

void arrayToStr(std::string& str, const Unit *x, size_t n, int base, bool withPrefix)
{
	switch (base) {
	case 10:
		{
			mpz_class t;
			Gmp::setArray(t, x, n);
			Gmp::getStr(str, t, 10);
		}
		return;
	case 16:
		mcl::fp::toStr16(str, x, n, withPrefix);
		return;
	case 2:
		mcl::fp::toStr2(str, x, n, withPrefix);
		return;
	default:
		throw cybozu::Exception("fp:arrayToStr:bad base") << base;
	}
}

void copyAndMask(Unit *y, const void *x, size_t xByteSize, const Op& op, bool doMask)
{
	const size_t fpByteSize = sizeof(Unit) * op.N;
	if (xByteSize > fpByteSize) {
		if (!doMask) throw cybozu::Exception("fp:copyAndMask:bad size") << xByteSize << fpByteSize;
		xByteSize = fpByteSize;
	}
	memcpy(y, x, xByteSize);
	memset((char *)y + xByteSize, 0, fpByteSize - xByteSize);
	if (!doMask) {
		if (compareArray(y, op.p, op.N) >= 0) throw cybozu::Exception("fp:copyAndMask:large x");
		return;
	}
	maskArray(y, op.N, op.bitSize - 1);
	assert(compareArray(y, op.p, op.N) < 0);
}

} } // mcl::fp

<|endoftext|>