<commit_before>/*
 * Distributed under the OSI-approved Apache License, Version 2.0.  See
 * accompanying file Copyright.txt for details.
 *
 * InSituMPISchedules.cpp
 * common functions for in situ MPI Writer and Reader
 * It requires MPI
 *
 *  Created on: Dec 25, 2017
 *      Author: Norbert Podhorszki pnorbert@ornl.gov
 */

#include "InSituMPISchedules.h"

#include "adios2/helper/adiosMath.h"
#include "adios2/helper/adiosMemory.h"

#include <iostream>

namespace adios2
{

namespace insitumpi
{

int GetNumberOfRequests(
    const std::map<std::string, SubFileInfoMap> &variablesSubFileInfo) noexcept
{
    int n = 0;
    for (const auto &variableNamePair : variablesSubFileInfo)
    {
        // <writer, <steps, <SubFileInfo>>>
        for (const auto &subFileIndexPair : variableNamePair.second)
        {
            // <steps, <SubFileInfo>>  but there is only one step
            for (const auto &stepPair : subFileIndexPair.second)
            {
                // <SubFileInfo>
                for (const auto &sfi : stepPair.second)
                {
                    n++;
                }
            }
        }
    }
    return n;
}

int FixSeeksToZeroOffset(
    std::map<std::string, SubFileInfoMap> &variablesSubFileInfo) noexcept
{
    int n = 0;
    for (auto &variableNamePair : variablesSubFileInfo)
    {
        // <writer, <steps, <SubFileInfo>>>
        for (auto &subFileIndexPair : variableNamePair.second)
        {
            // <steps, <SubFileInfo>>  but there is only one step
            for (auto &stepPair : subFileIndexPair.second)
            {
                // <SubFileInfo>
                for (auto &sfi : stepPair.second)
                {
                    FixSeeksToZeroOffset(sfi);
                    n++;
                }
            }
        }
    }
    return n;
}

void FixSeeksToZeroOffset(SubFileInfo &record) noexcept
{
    size_t nElements = record.Seeks.second - record.Seeks.first + 1;
    size_t pos =
        LinearIndex(record.BlockBox, record.IntersectionBox.first, true);
    record.Seeks.first = pos;
    record.Seeks.second = pos + nElements - 1;
}

std::vector<std::vector<char>> SerializeLocalReadSchedule(
    const int nWriters,
    const std::map<std::string, SubFileInfoMap> &variablesSubFileInfo) noexcept
{
    std::vector<std::vector<char>> buffers(nWriters);

    // Create a buffer for each writer
    std::vector<int> nVarPerWriter(nWriters);
    for (size_t i = 0; i < nWriters; i++)
    {
        nVarPerWriter[i] = 0;
        InsertToBuffer(buffers[i], &nVarPerWriter[i],
                       1); // allocate first 4 bytes
    }

    for (const auto &variableNamePair : variablesSubFileInfo)
    {
        const std::string variableName(variableNamePair.first);
        // <writer, <steps, <SubFileInfo>>>
        for (const auto &subFileIndexPair : variableNamePair.second)
        {
            const size_t subFileIndex = subFileIndexPair.first; // writer
            auto &lrs = buffers[subFileIndex];
            // <steps, <SubFileInfo>>  but there is only one step
            for (const auto &stepPair : subFileIndexPair.second)
            {
                // LocalReadSchedule sfi = subFileIndexPair.second[0];
                const std::vector<SubFileInfo> &sfi = stepPair.second;
                SerializeLocalReadSchedule(buffers[subFileIndex], variableName,
                                           sfi);
                ++nVarPerWriter[subFileIndex];
                break; // there is only one step here
            }
        }
    }

    // Record the number of actually requested variables for each buffer
    for (int i = 0; i < nWriters; i++)
    {
        size_t pos = 0;
        CopyToBuffer(buffers[i], pos, &nVarPerWriter[i]);
    }
    return buffers;
}

void SerializeLocalReadSchedule(std::vector<char> &buffer,
                                const std::string varName,
                                const LocalReadSchedule lrs) noexcept
{
    const int nameLen = varName.size();
    InsertToBuffer(buffer, &nameLen, 1);
    InsertToBuffer(buffer, varName.data(), nameLen);
    const int nSubFileInfos = lrs.size();
    InsertToBuffer(buffer, &nSubFileInfos, 1);
    for (const auto &blockInfo : lrs)
    {
        SerializeSubFileInfo(buffer, blockInfo);
    }
}

void SerializeSubFileInfo(std::vector<char> &buffer,
                          const SubFileInfo record) noexcept
{
    SerializeBox(buffer, record.BlockBox);
    SerializeBox(buffer, record.IntersectionBox);
    SerializeBox(buffer, record.Seeks);
}

void SerializeBox(std::vector<char> &buffer, const Box<Dims> box) noexcept
{
    const int nDims = box.first.size();
    InsertToBuffer(buffer, &nDims, 1);
    InsertToBuffer(buffer, box.first.data(), nDims);
    InsertToBuffer(buffer, box.second.data(), nDims);
}

void SerializeBox(std::vector<char> &buffer, const Box<size_t> box) noexcept
{
    InsertToBuffer(buffer, &box.first, 1);
    InsertToBuffer(buffer, &box.second, 1);
}

int GetNumberOfRequestsInWriteScheduleMap(WriteScheduleMap &map) noexcept
{
    int n;
    //<variable <reader, <SubFileInfo>>>
    for (auto &variableNamePair : map)
    {
        // <reader, <SubFileInfo>>
        for (auto &readerPair : variableNamePair.second)
        {
            // <SubFileInfo>
            for (auto &sfi : readerPair.second)
            {
                n++;
            }
        }
    }
    return n;
}

WriteScheduleMap
DeserializeReadSchedule(const std::vector<std::vector<char>> &buffers) noexcept
{
    WriteScheduleMap map;

    for (int i = 0; i < buffers.size(); i++)
    {
        const auto &buffer = buffers[i];
        LocalReadScheduleMap lrsm = DeserializeReadSchedule(buffer);
        for (const auto &varSchedule : lrsm)
        {
            map[varSchedule.first][i] = varSchedule.second;
        }
    }
    return map;
}

LocalReadScheduleMap
DeserializeReadSchedule(const std::vector<char> &buffer) noexcept
{
    LocalReadScheduleMap map;
    size_t pos = 0;
    int nVars = ReadValue<int>(buffer, pos);
    for (int i = 0; i < nVars; i++)
    {
        int nameLen = ReadValue<int>(buffer, pos);
        char name[nameLen + 1];
        CopyFromBuffer(buffer, pos, name, nameLen);
        name[nameLen] = '\0';
        int nSubFileInfos = ReadValue<int>(buffer, pos);
        std::vector<SubFileInfo> sfis;
        sfis.reserve(nSubFileInfos);
        for (int j = 0; j < nSubFileInfos; j++)
        {
            sfis.push_back(DeserializeSubFileInfo(buffer, pos));
        }
        map[name] = sfis;
    }
    return map;
}

SubFileInfo DeserializeSubFileInfo(const std::vector<char> &buffer,
                                   size_t &position) noexcept
{
    SubFileInfo record;
    record.BlockBox = DeserializeBoxDims(buffer, position);
    record.IntersectionBox = DeserializeBoxDims(buffer, position);
    record.Seeks = DeserializeBoxSizet(buffer, position);
    return record;
}

Box<Dims> DeserializeBoxDims(const std::vector<char> &buffer,
                             size_t &position) noexcept
{
    Box<Dims> box;
    int nDims = ReadValue<int>(buffer, position);
    std::vector<size_t> start(nDims);
    std::vector<size_t> count(nDims);
    CopyFromBuffer(buffer, position, start.data(), nDims);
    CopyFromBuffer(buffer, position, count.data(), nDims);
    return Box<Dims>(start, count);
}

Box<size_t> DeserializeBoxSizet(const std::vector<char> &buffer,
                                size_t &position) noexcept
{
    size_t first, second;
    CopyFromBuffer(buffer, position, &first, 1);
    CopyFromBuffer(buffer, position, &second, 1);
    return Box<size_t>(first, second);
}

void PrintReadScheduleMap(const WriteScheduleMap &map) noexcept
{
    // <variableName, <reader, <SubFileInfo>>>
    for (const auto &variableNamePair : map)
    {
        std::cout << "{ var = " << variableNamePair.first << " ";
        // <reader, <SubFileInfo>>
        for (const auto &readerPair : variableNamePair.second)
        {
            const size_t readerRank = readerPair.first;
            std::cout << "{ reader = " << readerPair.first << " ";
            const std::vector<SubFileInfo> &sfis = readerPair.second;
            for (const auto &sfi : sfis)
            {
                std::cout << "<";
                PrintSubFileInfo(sfi);
                std::cout << "> ";
            }
            std::cout << "} ";
        }
        std::cout << "} ";
    }
}

void PrintSubFileInfo(const SubFileInfo &sfi) noexcept
{
    std::cout << "(block=";
    PrintBox(sfi.BlockBox);
    std::cout << ", intersection=";
    PrintBox(sfi.IntersectionBox);
    std::cout << ", seeks=";
    PrintBox(sfi.Seeks);
    std::cout << ")";
}

void PrintBox(const Box<Dims> &box) noexcept
{
    std::cout << "[";
    PrintDims(box.first);
    std::cout << ",";
    PrintDims(box.second);
    std::cout << "]";
}

void PrintBox(const Box<size_t> &box) noexcept
{
    std::cout << "{" << box.first << "," << box.second << "}";
}

void PrintDims(const Dims &dims) noexcept
{
    std::cout << "{";
    for (int i = 0; i < dims.size(); i++)
    {
        std::cout << dims[i];
        if (i < dims.size() - 1)
            std::cout << ",";
    }
    std::cout << "}";
}

} // end namespace insitumpi

} // end namespace adios2
<commit_msg>Fix uninitialized variable<commit_after>/*
 * Distributed under the OSI-approved Apache License, Version 2.0.  See
 * accompanying file Copyright.txt for details.
 *
 * InSituMPISchedules.cpp
 * common functions for in situ MPI Writer and Reader
 * It requires MPI
 *
 *  Created on: Dec 25, 2017
 *      Author: Norbert Podhorszki pnorbert@ornl.gov
 */

#include "InSituMPISchedules.h"

#include "adios2/helper/adiosMath.h"
#include "adios2/helper/adiosMemory.h"

#include <iostream>

namespace adios2
{

namespace insitumpi
{

int GetNumberOfRequests(
    const std::map<std::string, SubFileInfoMap> &variablesSubFileInfo) noexcept
{
    int n = 0;
    for (const auto &variableNamePair : variablesSubFileInfo)
    {
        // <writer, <steps, <SubFileInfo>>>
        for (const auto &subFileIndexPair : variableNamePair.second)
        {
            // <steps, <SubFileInfo>>  but there is only one step
            for (const auto &stepPair : subFileIndexPair.second)
            {
                // <SubFileInfo>
                for (const auto &sfi : stepPair.second)
                {
                    n++;
                }
            }
        }
    }
    return n;
}

int FixSeeksToZeroOffset(
    std::map<std::string, SubFileInfoMap> &variablesSubFileInfo) noexcept
{
    int n = 0;
    for (auto &variableNamePair : variablesSubFileInfo)
    {
        // <writer, <steps, <SubFileInfo>>>
        for (auto &subFileIndexPair : variableNamePair.second)
        {
            // <steps, <SubFileInfo>>  but there is only one step
            for (auto &stepPair : subFileIndexPair.second)
            {
                // <SubFileInfo>
                for (auto &sfi : stepPair.second)
                {
                    FixSeeksToZeroOffset(sfi);
                    n++;
                }
            }
        }
    }
    return n;
}

void FixSeeksToZeroOffset(SubFileInfo &record) noexcept
{
    size_t nElements = record.Seeks.second - record.Seeks.first + 1;
    size_t pos =
        LinearIndex(record.BlockBox, record.IntersectionBox.first, true);
    record.Seeks.first = pos;
    record.Seeks.second = pos + nElements - 1;
}

std::vector<std::vector<char>> SerializeLocalReadSchedule(
    const int nWriters,
    const std::map<std::string, SubFileInfoMap> &variablesSubFileInfo) noexcept
{
    std::vector<std::vector<char>> buffers(nWriters);

    // Create a buffer for each writer
    std::vector<int> nVarPerWriter(nWriters);
    for (size_t i = 0; i < nWriters; i++)
    {
        nVarPerWriter[i] = 0;
        InsertToBuffer(buffers[i], &nVarPerWriter[i],
                       1); // allocate first 4 bytes
    }

    for (const auto &variableNamePair : variablesSubFileInfo)
    {
        const std::string variableName(variableNamePair.first);
        // <writer, <steps, <SubFileInfo>>>
        for (const auto &subFileIndexPair : variableNamePair.second)
        {
            const size_t subFileIndex = subFileIndexPair.first; // writer
            auto &lrs = buffers[subFileIndex];
            // <steps, <SubFileInfo>>  but there is only one step
            for (const auto &stepPair : subFileIndexPair.second)
            {
                // LocalReadSchedule sfi = subFileIndexPair.second[0];
                const std::vector<SubFileInfo> &sfi = stepPair.second;
                SerializeLocalReadSchedule(buffers[subFileIndex], variableName,
                                           sfi);
                ++nVarPerWriter[subFileIndex];
                break; // there is only one step here
            }
        }
    }

    // Record the number of actually requested variables for each buffer
    for (int i = 0; i < nWriters; i++)
    {
        size_t pos = 0;
        CopyToBuffer(buffers[i], pos, &nVarPerWriter[i]);
    }
    return buffers;
}

void SerializeLocalReadSchedule(std::vector<char> &buffer,
                                const std::string varName,
                                const LocalReadSchedule lrs) noexcept
{
    const int nameLen = varName.size();
    InsertToBuffer(buffer, &nameLen, 1);
    InsertToBuffer(buffer, varName.data(), nameLen);
    const int nSubFileInfos = lrs.size();
    InsertToBuffer(buffer, &nSubFileInfos, 1);
    for (const auto &blockInfo : lrs)
    {
        SerializeSubFileInfo(buffer, blockInfo);
    }
}

void SerializeSubFileInfo(std::vector<char> &buffer,
                          const SubFileInfo record) noexcept
{
    SerializeBox(buffer, record.BlockBox);
    SerializeBox(buffer, record.IntersectionBox);
    SerializeBox(buffer, record.Seeks);
}

void SerializeBox(std::vector<char> &buffer, const Box<Dims> box) noexcept
{
    const int nDims = box.first.size();
    InsertToBuffer(buffer, &nDims, 1);
    InsertToBuffer(buffer, box.first.data(), nDims);
    InsertToBuffer(buffer, box.second.data(), nDims);
}

void SerializeBox(std::vector<char> &buffer, const Box<size_t> box) noexcept
{
    InsertToBuffer(buffer, &box.first, 1);
    InsertToBuffer(buffer, &box.second, 1);
}

int GetNumberOfRequestsInWriteScheduleMap(WriteScheduleMap &map) noexcept
{
    int n = 0;
    //<variable <reader, <SubFileInfo>>>
    for (auto &variableNamePair : map)
    {
        // <reader, <SubFileInfo>>
        for (auto &readerPair : variableNamePair.second)
        {
            // <SubFileInfo>
            for (auto &sfi : readerPair.second)
            {
                n++;
            }
        }
    }
    return n;
}

WriteScheduleMap
DeserializeReadSchedule(const std::vector<std::vector<char>> &buffers) noexcept
{
    WriteScheduleMap map;

    for (int i = 0; i < buffers.size(); i++)
    {
        const auto &buffer = buffers[i];
        LocalReadScheduleMap lrsm = DeserializeReadSchedule(buffer);
        for (const auto &varSchedule : lrsm)
        {
            map[varSchedule.first][i] = varSchedule.second;
        }
    }
    return map;
}

LocalReadScheduleMap
DeserializeReadSchedule(const std::vector<char> &buffer) noexcept
{
    LocalReadScheduleMap map;
    size_t pos = 0;
    int nVars = ReadValue<int>(buffer, pos);
    for (int i = 0; i < nVars; i++)
    {
        int nameLen = ReadValue<int>(buffer, pos);
        char name[nameLen + 1];
        CopyFromBuffer(buffer, pos, name, nameLen);
        name[nameLen] = '\0';
        int nSubFileInfos = ReadValue<int>(buffer, pos);
        std::vector<SubFileInfo> sfis;
        sfis.reserve(nSubFileInfos);
        for (int j = 0; j < nSubFileInfos; j++)
        {
            sfis.push_back(DeserializeSubFileInfo(buffer, pos));
        }
        map[name] = sfis;
    }
    return map;
}

SubFileInfo DeserializeSubFileInfo(const std::vector<char> &buffer,
                                   size_t &position) noexcept
{
    SubFileInfo record;
    record.BlockBox = DeserializeBoxDims(buffer, position);
    record.IntersectionBox = DeserializeBoxDims(buffer, position);
    record.Seeks = DeserializeBoxSizet(buffer, position);
    return record;
}

Box<Dims> DeserializeBoxDims(const std::vector<char> &buffer,
                             size_t &position) noexcept
{
    Box<Dims> box;
    int nDims = ReadValue<int>(buffer, position);
    std::vector<size_t> start(nDims);
    std::vector<size_t> count(nDims);
    CopyFromBuffer(buffer, position, start.data(), nDims);
    CopyFromBuffer(buffer, position, count.data(), nDims);
    return Box<Dims>(start, count);
}

Box<size_t> DeserializeBoxSizet(const std::vector<char> &buffer,
                                size_t &position) noexcept
{
    size_t first, second;
    CopyFromBuffer(buffer, position, &first, 1);
    CopyFromBuffer(buffer, position, &second, 1);
    return Box<size_t>(first, second);
}

void PrintReadScheduleMap(const WriteScheduleMap &map) noexcept
{
    // <variableName, <reader, <SubFileInfo>>>
    for (const auto &variableNamePair : map)
    {
        std::cout << "{ var = " << variableNamePair.first << " ";
        // <reader, <SubFileInfo>>
        for (const auto &readerPair : variableNamePair.second)
        {
            const size_t readerRank = readerPair.first;
            std::cout << "{ reader = " << readerPair.first << " ";
            const std::vector<SubFileInfo> &sfis = readerPair.second;
            for (const auto &sfi : sfis)
            {
                std::cout << "<";
                PrintSubFileInfo(sfi);
                std::cout << "> ";
            }
            std::cout << "} ";
        }
        std::cout << "} ";
    }
}

void PrintSubFileInfo(const SubFileInfo &sfi) noexcept
{
    std::cout << "(block=";
    PrintBox(sfi.BlockBox);
    std::cout << ", intersection=";
    PrintBox(sfi.IntersectionBox);
    std::cout << ", seeks=";
    PrintBox(sfi.Seeks);
    std::cout << ")";
}

void PrintBox(const Box<Dims> &box) noexcept
{
    std::cout << "[";
    PrintDims(box.first);
    std::cout << ",";
    PrintDims(box.second);
    std::cout << "]";
}

void PrintBox(const Box<size_t> &box) noexcept
{
    std::cout << "{" << box.first << "," << box.second << "}";
}

void PrintDims(const Dims &dims) noexcept
{
    std::cout << "{";
    for (int i = 0; i < dims.size(); i++)
    {
        std::cout << dims[i];
        if (i < dims.size() - 1)
            std::cout << ",";
    }
    std::cout << "}";
}

} // end namespace insitumpi

} // end namespace adios2
<|endoftext|>


<commit_before>#include "SuffixMatchManager.hpp"
#include <document-manager/DocumentManager.h>

#include <boost/filesystem.hpp>
#include <glog/logging.h>

namespace sf1r
{

SuffixMatchManager::SuffixMatchManager(
        const std::string& homePath,
        const std::string& property,
        boost::shared_ptr<DocumentManager>& document_manager)
    : fm_index_path_(homePath + "/" + property + ".fm_idx")
    , property_(property)
    , document_manager_(document_manager)
    , fmi_(new FMIndexType(32))
{
    if (!boost::filesystem::exists(homePath))
    {
        boost::filesystem::create_directories(homePath);
    }
    else
    {
        std::ifstream ifs(fm_index_path_.c_str());
        if (ifs) fmi_->load(ifs);
    }
}

SuffixMatchManager::~SuffixMatchManager()
{
}

void SuffixMatchManager::buildCollection()
{
    FMIndexType* new_fmi = new FMIndexType(32);

    size_t last_docid = fmi_->docCount();
    if (last_docid)
    {
        std::vector<uint16_t> orig_text;
        std::vector<uint32_t> del_docid_list;
        document_manager_->getDeletedDocIdList(del_docid_list);
        fmi_->reconstructText(del_docid_list, orig_text);
        new_fmi->setOrigText(orig_text);
    }

    for (size_t i = last_docid + 1; i <= document_manager_->getMaxDocId(); ++i)
    {
        if (i % 100000 == 0)
        {
            LOG(INFO) << "inserted docs: " << i;
        }
        Document doc;
        document_manager_->getDocument(i, doc);
        Document::property_const_iterator it = doc.findProperty(property_);
        if (it == doc.propertyEnd())
        {
            new_fmi->addDoc(NULL, 0);
            continue;
        }

        const izenelib::util::UString& text = it->second.get<UString>();
        new_fmi->addDoc(text.data(), text.length());

    }
    LOG(INFO) << "inserted docs: " << document_manager_->getMaxDocId();

    LOG(INFO) << "building fm-index";
    new_fmi->build();

    {
        WriteLock lock(mutex_);
        fmi_.reset(new_fmi);
    }

    std::ofstream ofs(fm_index_path_.c_str());
    fmi_->save(ofs);
}

size_t SuffixMatchManager::longestSuffixMatch(const izenelib::util::UString& pattern, size_t max_docs, std::vector<uint32_t>& docid_list, std::vector<float>& score_list) const
{
    std::vector<std::pair<size_t, size_t> >match_ranges;
    std::vector<size_t> doclen_list;
    size_t max_match;

    {
        ReadLock lock(mutex_);
        if ((max_match = fmi_->longestSuffixMatch(pattern.data(), pattern.length(), match_ranges)) == 0)
            return 0;

        fmi_->getMatchedDocIdList(match_ranges, max_docs, docid_list, doclen_list);
    }

    score_list.resize(doclen_list.size());
    for (size_t i = 0; i < doclen_list.size(); ++i)
    {
        score_list[i] = float(max_match) / float(doclen_list[i]);
    }

    size_t total_match = 0;
    for (size_t i = 0; i < match_ranges.size(); ++i)
    {
        total_match += match_ranges[i].second - match_ranges[i].first;
    }

    return total_match;
}

}
<commit_msg>fix segfault<commit_after>#include "SuffixMatchManager.hpp"
#include <document-manager/DocumentManager.h>

#include <boost/filesystem.hpp>
#include <glog/logging.h>

namespace sf1r
{

SuffixMatchManager::SuffixMatchManager(
        const std::string& homePath,
        const std::string& property,
        boost::shared_ptr<DocumentManager>& document_manager)
    : fm_index_path_(homePath + "/" + property + ".fm_idx")
    , property_(property)
    , document_manager_(document_manager)
    , fmi_(new FMIndexType(32))
{
    if (!boost::filesystem::exists(homePath))
    {
        boost::filesystem::create_directories(homePath);
    }
    else
    {
        std::ifstream ifs(fm_index_path_.c_str());
        if (ifs) fmi_->load(ifs);
    }
}

SuffixMatchManager::~SuffixMatchManager()
{
}

void SuffixMatchManager::buildCollection()
{
    FMIndexType* new_fmi = new FMIndexType(32);

    size_t last_docid = fmi_->docCount();
    if (last_docid)
    {
        std::vector<uint16_t> orig_text;
        std::vector<uint32_t> del_docid_list;
        document_manager_->getDeletedDocIdList(del_docid_list);
        fmi_->reconstructText(del_docid_list, orig_text);
        new_fmi->setOrigText(orig_text);
    }

    for (size_t i = last_docid + 1; i <= document_manager_->getMaxDocId(); ++i)
    {
        if (i % 100000 == 0)
        {
            LOG(INFO) << "inserted docs: " << i;
        }
        Document doc;
        document_manager_->getDocument(i, doc);
        Document::property_const_iterator it = doc.findProperty(property_);
        if (it == doc.propertyEnd())
        {
            new_fmi->addDoc(NULL, 0);
            continue;
        }

        const izenelib::util::UString& text = it->second.get<UString>();
        new_fmi->addDoc(text.data(), text.length());

    }
    LOG(INFO) << "inserted docs: " << document_manager_->getMaxDocId();

    LOG(INFO) << "building fm-index";
    new_fmi->build();

    {
        WriteLock lock(mutex_);
        fmi_.reset(new_fmi);
    }

    std::ofstream ofs(fm_index_path_.c_str());
    fmi_->save(ofs);
}

size_t SuffixMatchManager::longestSuffixMatch(const izenelib::util::UString& pattern, size_t max_docs, std::vector<uint32_t>& docid_list, std::vector<float>& score_list) const
{
    if (!fmi_) return 0;

    std::vector<std::pair<size_t, size_t> >match_ranges;
    std::vector<size_t> doclen_list;
    size_t max_match;

    {
        ReadLock lock(mutex_);
        if ((max_match = fmi_->longestSuffixMatch(pattern.data(), pattern.length(), match_ranges)) == 0)
            return 0;

        fmi_->getMatchedDocIdList(match_ranges, max_docs, docid_list, doclen_list);
    }

    score_list.resize(doclen_list.size());
    for (size_t i = 0; i < doclen_list.size(); ++i)
    {
        score_list[i] = float(max_match) / float(doclen_list[i]);
    }

    size_t total_match = 0;
    for (size_t i = 0; i < match_ranges.size(); ++i)
    {
        total_match += match_ranges[i].second - match_ranges[i].first;
    }

    return total_match;
}

}
<|endoftext|>


<commit_before>#include <GL/glew.h>
#include <GLFW/glfw3.h>
#define GLM_FORCE_RADIANS
#include <glm/glm.hpp>
#include <iostream>
#include <iomanip>
#include <math.h>
#include <stdio.h>
#include <stdlib.h>
#include <nanovg.h>
#define NANOVG_GL3_IMPLEMENTATION
#include <nanovg_gl.h>

#include "gfx/Renderer.hpp"
#include "gfx/SceneGraph.hpp"
#include "gfx/Entity.hpp"
#include "gfx/VertexArray.hpp"
#include "gfx/SkyBox.hpp"
#include "gfx/importer/ObjLoader.hpp"
#include "gfx/lights/Light.hpp"
#include "gfx/lights/PointLight.hpp"
#include "gfx/lights/DirLight.hpp"
#include "gfx/shader/SimpleShader.hpp"

#include "input/Actions.hpp"
#include "input/Input.hpp"

#include "ui/DebugText.hpp"
#include "DebugUtils.h"

void glDebugCallback(GLenum source, GLenum type, GLuint id, GLenum severity, GLsizei length, const GLchar *msg, void *data) {
  std::cout <<  "debug call: " << msg << std::endl;
}

glm::vec3 get_random_color() {
  auto r = ((rand() % 100)/100.0);
  auto g = ((rand() % 100)/100.0);
  auto b = ((rand() % 100)/100.0);
  return glm::normalize(glm::vec3{r, g, b});
}

glm::vec3 pos;
glm::vec3 scale;
std::vector<std::pair<glm::vec3, glm::vec3>> rain;
void draw_rain(Renderer* r) {
  static float speed = 0.75;
  //if((rand() % 100)/100.0 < 0.5) {
  //for(int i = 0; i < 5; i++) {
  {
    auto x = ((rand() % 100)/100.0) * -34 + 14;
    auto y = 22 + pos.y;
    auto z = ((rand() % 100)/100.0) * -10 + 3.1;
    glm::vec3 from = glm::vec3(x,y,z);
    glm::vec3 to = from + glm::vec3{5, -16, 5};
    rain.push_back(std::make_pair(from, to));
  }
  std::vector<std::pair<glm::vec3, glm::vec3>> new_rain;
  for(auto& rain_pos: rain) {
    auto& from = std::get<0>(rain_pos);
    auto& to = std::get<1>(rain_pos);
    auto dir = glm::normalize(to - from);
    from += dir * speed;
    to += dir * speed;
    if(from.y > 0) {
      new_rain.push_back(rain_pos);
    }
  }
  r->draw_lines(rain, {0.1, 0.1, 0.1});
  rain = new_rain;
}

static void error_callback(int error, const char* description)
{
  fputs(description, stderr);
}

float step = 0.01f;
int selected_light = -1;

static void scroll_callback(GLFWwindow* window, double x_offset, double y_offset) {
  step = fmax(0.01f, fmin(step * pow(1.5, y_offset), 5));
}

double target_x = 0;
double target_y = 0;
double current_x = 0;
double current_y = 0;
double MOUSE_X_SENSITIVITY = 20;
double MOUSE_Y_SENSITIVITY = 20;
double PANNING_SPEED = 15;
void rotate_camera(Camera& camera) {
  if(Input::is_mouse_locked()) {
    const auto x = Input::get_mouse_dx() / MOUSE_X_SENSITIVITY;
    const auto y = Input::get_mouse_dy() / MOUSE_Y_SENSITIVITY;
    target_x += x;
    target_y += y;
    auto dx = (target_x - current_x) / PANNING_SPEED;
    auto dy = (target_y - current_y) / PANNING_SPEED;
    if(fabs(dx) < 0.001 && fabs(dy) < 0.001) {
      return;
    }
    current_x += dx + copysign(0.01, dx);
    current_y += dy + copysign(0.01, dy);
    camera.rotate(-dx, camera.up);
    camera.rotate(dy, glm::cross(camera.up, camera.dir));
  } else {
    target_x = current_x;
    target_y = current_y;
  }
}

int width = 1920;
int height = 1080;
int main(int argc, char ** argv) {
  glfwSetErrorCallback(error_callback);
  if (!glfwInit())
    exit(EXIT_FAILURE);

  bool fullscreen = false;
  for(int i = 0; i < argc; i++) {
    //std::cout << argv[i] << "\n";
    if(!strcmp(argv[i], "-w")) {
      width = atoi(argv[i+1]);
    }
    if(!strcmp(argv[i], "-h")) {
      height = atoi(argv[i+1]);
    }
    if(!strcmp(argv[i], "-f")) {
      fullscreen = true;
    }

  }
  std::cout << "w: " << width << "\nheight: " << height << "\n";
  glfwWindowHint(GLFW_CONTEXT_VERSION_MAJOR, 4);
  glfwWindowHint(GLFW_CONTEXT_VERSION_MINOR, 2);
  glfwWindowHint(GLFW_OPENGL_PROFILE, GLFW_OPENGL_CORE_PROFILE);
  glfwWindowHint( GLFW_OPENGL_DEBUG_CONTEXT, true );

  GLFWwindow* window = nullptr;
  if(fullscreen) {
    const auto& monitor = glfwGetPrimaryMonitor();
    const GLFWvidmode* mode = glfwGetVideoMode(monitor);
    glfwWindowHint(GLFW_RED_BITS, mode->redBits);
    glfwWindowHint(GLFW_GREEN_BITS, mode->greenBits);
    glfwWindowHint(GLFW_BLUE_BITS, mode->blueBits);
    glfwWindowHint(GLFW_REFRESH_RATE, mode->refreshRate);
    window = glfwCreateWindow(mode->width, mode->height, "Hondo", monitor, NULL);
  } else {
    window = glfwCreateWindow(width, height, "Hondo", NULL, NULL);
  }

  if (!window) {
    glfwTerminate();
    exit(EXIT_FAILURE);
  }
  glfwMakeContextCurrent(window);

  glewExperimental = GL_TRUE;
  GLenum err = glewInit();
  //To discard error 1280 from glewInit().
  glGetError();
  if(GLEW_OK != err)
    std::cout << glewGetErrorString(err) << std::endl;

  {
    const GLubyte* renderer = glGetString(GL_RENDERER);
    const GLubyte* version = glGetString(GL_VERSION);
    printf("Renderer: %s\n", renderer);
    printf("Version: %s\n", version);
    glfwSetCursorPosCallback(window, Input::cursor_pos_callback);
    glfwSetKeyCallback(window, Input::key_callback);
    glfwSetScrollCallback(window, scroll_callback);
    Input::set_active_window(window);
  }
  Renderer renderer(width, height);

  struct NVGcontext* vg = nvgCreateGL3(NVG_ANTIALIAS | NVG_DEBUG);
  glDebugMessageCallback(glDebugCallback, NULL );
  auto& camera = renderer.get_camera();

  Input::on(Actions::Forward, [&]() {
    camera.move_forward(step);
  });

  Input::on(Actions::Backward, [&]() {
    camera.move_forward(-step);
  });

  Input::on(Actions::Left, [&]() {
    camera.move_right(-step);
  });

  Input::on(Actions::Right, [&]() {
    camera.move_right(step);
  });

  Input::on(Actions::Up, [&]() {
    camera.translate({0, step, 0});
  });

  Input::on(Actions::Down, [&]() {
    camera.translate({0, -step, 0});
  });

  Input::on(GLFW_KEY_Z, [&] {
  renderer.toggle_wireframe();
  });

  Input::on(GLFW_KEY_G, []() {
    Input::lock_mouse();
  });
  ObjLoader loader;
  loader.preload_file("assets/sponza-minus.obj");
  loader.preload_file("assets/Cube.obj");
  loader.preload_file("assets/sphere.obj");
  loader.preload_file("assets/SkyDome16.obj");
  loader.load_files();
  Mesh skydome_mesh = loader.get_meshes("assets/SkyDome16.obj")[0];
  Mesh cube_mesh = loader.get_meshes("assets/Cube.obj")[0];
  Mesh sphere_mesh = loader.get_meshes("assets/sphere.obj")[0];
  std::vector<Mesh> sponza_meshes = loader.get_meshes("assets/sponza-minus.obj");

  auto pl2 = std::make_shared<DirLight>(glm::vec3{-1, -1, -1}, glm::vec3{1, 1, 1});
  pl2->ambient_intensity = 0.15f;
  //pl2->set_casts_shadow(true);
  pl2->diffuse_intensity = 0.2f;
  renderer.add_light(pl2);

  SceneGraph scene;
  Entity sponza = scene.create_entity();
  for(auto& sponza_mesh: sponza_meshes) {
    scene.create_entity(sponza, RenderObject(sponza_mesh));
  }
  scene.scale(sponza, {0.05, 0.05, 0.05});

  Entity cube = scene.create_entity(RenderObject(cube_mesh));
  scene.scale(cube, glm::vec3{0.5, 0.5, 0.5});
  scene.translate(cube, glm::vec3{0, 2, 0});
  Entity sphere1 = scene.create_entity(cube, RenderObject(sphere_mesh));
  scene.translate(sphere1, glm::vec3{2, 0, 0});

  Entity sphere2 = scene.create_entity(cube, RenderObject(sphere_mesh));
  scene.translate(sphere2, glm::vec3{-2, 0, 0});
  Entity sphere3 = scene.create_entity(cube, RenderObject(sphere_mesh));
  scene.translate(sphere3, glm::vec3{0, 0, 2});
  Entity sphere6 = scene.create_entity(sphere3, RenderObject(sphere_mesh));
  scene.translate(sphere6, glm::vec3{0, 2, 0});
  scene.scale(sphere6, glm::vec3{0.5, 0.5, 0.5});

  Entity sphere4 = scene.create_entity(cube, RenderObject(sphere_mesh));
  scene.translate(sphere4, glm::vec3{0, 0, -2});
  Entity sphere5 = scene.create_entity(sphere4, RenderObject(sphere_mesh));
  scene.translate(sphere5, glm::vec3{0, 2, 0});
  scene.scale(sphere5, glm::vec3{0.5, 0.5, 0.5});

  renderer.set_skybox(RenderObject(skydome_mesh));

  Input::on(GLFW_KEY_I, [&] {
    renderer.get_shown_light()->translate({ 0.01f, 0, 0});
  }, true);
  Input::on(GLFW_KEY_K, [&] {
    renderer.get_shown_light()->translate({-0.01f, 0, 0});
  }, true);
  Input::on(GLFW_KEY_J, [&] {
    renderer.get_shown_light()->translate({0, 0,  0.01f});
  }, true);
  Input::on(GLFW_KEY_L, [&] {
    renderer.get_shown_light()->translate({0, 0, -0.01f});
  }, true);
  Input::on(GLFW_KEY_O, [&] {
    renderer.get_shown_light()->translate({0,  0.01f, 0});
  }, true);
  Input::on(GLFW_KEY_U, [&] {
    renderer.get_shown_light()->translate({0, -0.01f, 0});
  }, true);
  Input::on(GLFW_KEY_PAGE_UP, [&] {
    if(selected_light < renderer.light_count()) {
      renderer.show_single_light(++selected_light);
    }
  }, false);

  Input::on(GLFW_KEY_DELETE, [&] {
    renderer.clear_lights();
  }, false);
  Input::on(GLFW_KEY_PAGE_DOWN, [&] {
    if(selected_light > 0) {
      renderer.show_single_light(--selected_light);
    }
  }, false);
  Input::on(GLFW_KEY_1, [&] {
      renderer.toggle_shadow_map();
  }, false);
  Input::on(GLFW_KEY_P, [&] {
      if(Input::is_alt_down()) {
        auto light = std::make_shared<PointLight>(camera.pos, get_random_color());
        light->set_casts_shadow(false);
        light->ambient_intensity = 0.0f;
        light->diffuse_intensity = 0.2f;
        renderer.add_light(light);
      } else {
        auto light = std::make_shared<SpotLight>(camera.pos, camera.dir, glm::vec3{1,1,1});
        light->set_casts_shadow(true);
        light->ambient_intensity = 0.0f;
        light->diffuse_intensity = 2.0f;
        renderer.add_light(light);
      }
      std::cout << "Amount of lights: " << renderer.light_count() << "\n";
  }, false);
  Input::on(GLFW_KEY_RIGHT_CONTROL, [&] {
      renderer.show_single_light(-1);
  }, false);
  bool draw_main = true;
  Input::on(GLFW_KEY_T, [&] {
      draw_main = !draw_main;
  }, false);
  Input::on(GLFW_KEY_Q, [&] {
      step = fmax(0.01f, fmin(step * pow(1.5, -1), 5));
  }, false);
  Input::on(GLFW_KEY_E, [&] {
      step = fmax(0.01f, fmin(step * pow(1.5, 1), 5));
  }, false);

  nvgCreateFont(vg, "sans", "/usr/share/fonts/truetype/freefont/FreeSans.ttf");

  DebugText::set_context(vg);

  camera.translate({0, 2, 0});
  while (!glfwWindowShouldClose(window)) {
    glfwPollEvents();
    Input::handle_input();
    rotate_camera(camera);
    renderer.render(scene);
    if(draw_main) {
      scene.rotate(cube, 0.01, glm::vec3{1, 1, 0});
      scene.rotate(sphere1, -0.01, glm::vec3{0, 1, 0});
      scene.rotate(sphere2, -0.01, glm::vec3{0, 1, 0});
      scene.rotate(sphere3, -0.01, glm::vec3{0, 0, 1});
      scene.rotate(sphere4,  0.01, glm::vec3{0, 0, 1});
    }

    std::stringstream str;
    str << std::fixed << std::setprecision(1) << calcFPS(1.0);
    DebugText::set_value("fps", str.str().c_str());

    nvgBeginFrame(vg, width, height, 1);
    DebugText::draw();
    nvgEndFrame(vg);

    glfwSwapBuffers(window);

    Input::reset_delta();
    checkForGlError();
  }
  glfwDestroyWindow(window);
  glfwTerminate();
  return 0;
}

<commit_msg>Changed default screen size, fixed fullscreen<commit_after>#include <GL/glew.h>
#include <GLFW/glfw3.h>
#define GLM_FORCE_RADIANS
#include <glm/glm.hpp>
#include <iostream>
#include <iomanip>
#include <math.h>
#include <stdio.h>
#include <stdlib.h>
#include <nanovg.h>
#define NANOVG_GL3_IMPLEMENTATION
#include <nanovg_gl.h>

#include "gfx/Renderer.hpp"
#include "gfx/SceneGraph.hpp"
#include "gfx/Entity.hpp"
#include "gfx/VertexArray.hpp"
#include "gfx/SkyBox.hpp"
#include "gfx/importer/ObjLoader.hpp"
#include "gfx/lights/Light.hpp"
#include "gfx/lights/PointLight.hpp"
#include "gfx/lights/DirLight.hpp"
#include "gfx/shader/SimpleShader.hpp"

#include "input/Actions.hpp"
#include "input/Input.hpp"

#include "ui/DebugText.hpp"
#include "DebugUtils.h"

void glDebugCallback(GLenum source, GLenum type, GLuint id, GLenum severity, GLsizei length, const GLchar *msg, void *data) {
  std::cout <<  "debug call: " << msg << std::endl;
}

glm::vec3 get_random_color() {
  auto r = ((rand() % 100)/100.0);
  auto g = ((rand() % 100)/100.0);
  auto b = ((rand() % 100)/100.0);
  return glm::normalize(glm::vec3{r, g, b});
}

glm::vec3 pos;
glm::vec3 scale;
std::vector<std::pair<glm::vec3, glm::vec3>> rain;
void draw_rain(Renderer* r) {
  static float speed = 0.75;
  //if((rand() % 100)/100.0 < 0.5) {
  //for(int i = 0; i < 5; i++) {
  {
    auto x = ((rand() % 100)/100.0) * -34 + 14;
    auto y = 22 + pos.y;
    auto z = ((rand() % 100)/100.0) * -10 + 3.1;
    glm::vec3 from = glm::vec3(x,y,z);
    glm::vec3 to = from + glm::vec3{5, -16, 5};
    rain.push_back(std::make_pair(from, to));
  }
  std::vector<std::pair<glm::vec3, glm::vec3>> new_rain;
  for(auto& rain_pos: rain) {
    auto& from = std::get<0>(rain_pos);
    auto& to = std::get<1>(rain_pos);
    auto dir = glm::normalize(to - from);
    from += dir * speed;
    to += dir * speed;
    if(from.y > 0) {
      new_rain.push_back(rain_pos);
    }
  }
  r->draw_lines(rain, {0.1, 0.1, 0.1});
  rain = new_rain;
}

static void error_callback(int error, const char* description)
{
  fputs(description, stderr);
}

float step = 0.01f;
int selected_light = -1;

static void scroll_callback(GLFWwindow* window, double x_offset, double y_offset) {
  step = fmax(0.01f, fmin(step * pow(1.5, y_offset), 5));
}

double target_x = 0;
double target_y = 0;
double current_x = 0;
double current_y = 0;
double MOUSE_X_SENSITIVITY = 20;
double MOUSE_Y_SENSITIVITY = 20;
double PANNING_SPEED = 15;
void rotate_camera(Camera& camera) {
  if(Input::is_mouse_locked()) {
    const auto x = Input::get_mouse_dx() / MOUSE_X_SENSITIVITY;
    const auto y = Input::get_mouse_dy() / MOUSE_Y_SENSITIVITY;
    target_x += x;
    target_y += y;
    auto dx = (target_x - current_x) / PANNING_SPEED;
    auto dy = (target_y - current_y) / PANNING_SPEED;
    if(fabs(dx) < 0.001 && fabs(dy) < 0.001) {
      return;
    }
    current_x += dx + copysign(0.01, dx);
    current_y += dy + copysign(0.01, dy);
    camera.rotate(-dx, camera.up);
    camera.rotate(dy, glm::cross(camera.up, camera.dir));
  } else {
    target_x = current_x;
    target_y = current_y;
  }
}

int width = 1024;
int height = 768;
int main(int argc, char ** argv) {
  glfwSetErrorCallback(error_callback);
  if (!glfwInit())
    exit(EXIT_FAILURE);

  bool fullscreen = false;
  for(int i = 0; i < argc; i++) {
    //std::cout << argv[i] << "\n";
    if(!strcmp(argv[i], "-w")) {
      width = atoi(argv[i+1]);
    }
    if(!strcmp(argv[i], "-h")) {
      height = atoi(argv[i+1]);
    }
    if(!strcmp(argv[i], "-f")) {
      fullscreen = true;
    }

  }
  std::cout << "w: " << width << "\nheight: " << height << "\n";
  glfwWindowHint(GLFW_CONTEXT_VERSION_MAJOR, 4);
  glfwWindowHint(GLFW_CONTEXT_VERSION_MINOR, 2);
  glfwWindowHint(GLFW_OPENGL_PROFILE, GLFW_OPENGL_CORE_PROFILE);
  glfwWindowHint( GLFW_OPENGL_DEBUG_CONTEXT, true );

  GLFWwindow* window = nullptr;
  if(fullscreen) {
    const auto& monitor = glfwGetPrimaryMonitor();
    const GLFWvidmode* mode = glfwGetVideoMode(monitor);
    glfwWindowHint(GLFW_RED_BITS, mode->redBits);
    glfwWindowHint(GLFW_GREEN_BITS, mode->greenBits);
    glfwWindowHint(GLFW_BLUE_BITS, mode->blueBits);
    glfwWindowHint(GLFW_REFRESH_RATE, mode->refreshRate);
    glfwWindowHint(GLFW_SAMPLES, 8);
    window = glfwCreateWindow(mode->width, mode->height, "Hondo", monitor, NULL);
    width = mode->width;
    height = mode->height;
  } else {
    window = glfwCreateWindow(width, height, "Hondo", NULL, NULL);
  }

  if (!window) {
    glfwTerminate();
    exit(EXIT_FAILURE);
  }
  glfwMakeContextCurrent(window);

  glewExperimental = GL_TRUE;
  GLenum err = glewInit();
  //To discard error 1280 from glewInit().
  glGetError();
  if(GLEW_OK != err)
    std::cout << glewGetErrorString(err) << std::endl;

  {
    const GLubyte* renderer = glGetString(GL_RENDERER);
    const GLubyte* version = glGetString(GL_VERSION);
    printf("Renderer: %s\n", renderer);
    printf("Version: %s\n", version);
    glfwSetCursorPosCallback(window, Input::cursor_pos_callback);
    glfwSetKeyCallback(window, Input::key_callback);
    glfwSetScrollCallback(window, scroll_callback);
    Input::set_active_window(window);
  }
  Renderer renderer(width, height);

  struct NVGcontext* vg = nvgCreateGL3(NVG_ANTIALIAS | NVG_DEBUG);
  glDebugMessageCallback(glDebugCallback, NULL );
  auto& camera = renderer.get_camera();

  Input::on(Actions::Forward, [&]() {
    camera.move_forward(step);
  });

  Input::on(Actions::Backward, [&]() {
    camera.move_forward(-step);
  });

  Input::on(Actions::Left, [&]() {
    camera.move_right(-step);
  });

  Input::on(Actions::Right, [&]() {
    camera.move_right(step);
  });

  Input::on(Actions::Up, [&]() {
    camera.translate({0, step, 0});
  });

  Input::on(Actions::Down, [&]() {
    camera.translate({0, -step, 0});
  });

  Input::on(GLFW_KEY_Z, [&] {
  renderer.toggle_wireframe();
  });

  Input::on(GLFW_KEY_G, []() {
    Input::lock_mouse();
  });
  ObjLoader loader;
  loader.preload_file("assets/sponza-minus.obj");
  loader.preload_file("assets/Cube.obj");
  loader.preload_file("assets/sphere.obj");
  loader.preload_file("assets/SkyDome16.obj");
  loader.load_files();
  Mesh skydome_mesh = loader.get_meshes("assets/SkyDome16.obj")[0];
  Mesh cube_mesh = loader.get_meshes("assets/Cube.obj")[0];
  Mesh sphere_mesh = loader.get_meshes("assets/sphere.obj")[0];
  std::vector<Mesh> sponza_meshes = loader.get_meshes("assets/sponza-minus.obj");

  auto pl2 = std::make_shared<DirLight>(glm::vec3{-1, -1, -1}, glm::vec3{1, 1, 1});
  pl2->ambient_intensity = 0.15f;
  //pl2->set_casts_shadow(true);
  pl2->diffuse_intensity = 0.2f;
  renderer.add_light(pl2);

  SceneGraph scene;
  Entity sponza = scene.create_entity();
  for(auto& sponza_mesh: sponza_meshes) {
    scene.create_entity(sponza, RenderObject(sponza_mesh));
  }
  scene.scale(sponza, {0.05, 0.05, 0.05});

  Entity cube = scene.create_entity(RenderObject(cube_mesh));
  scene.scale(cube, glm::vec3{0.5, 0.5, 0.5});
  scene.translate(cube, glm::vec3{0, 2, 0});
  Entity sphere1 = scene.create_entity(cube, RenderObject(sphere_mesh));
  scene.translate(sphere1, glm::vec3{2, 0, 0});

  Entity sphere2 = scene.create_entity(cube, RenderObject(sphere_mesh));
  scene.translate(sphere2, glm::vec3{-2, 0, 0});
  Entity sphere3 = scene.create_entity(cube, RenderObject(sphere_mesh));
  scene.translate(sphere3, glm::vec3{0, 0, 2});
  Entity sphere6 = scene.create_entity(sphere3, RenderObject(sphere_mesh));
  scene.translate(sphere6, glm::vec3{0, 2, 0});
  scene.scale(sphere6, glm::vec3{0.5, 0.5, 0.5});

  Entity sphere4 = scene.create_entity(cube, RenderObject(sphere_mesh));
  scene.translate(sphere4, glm::vec3{0, 0, -2});
  Entity sphere5 = scene.create_entity(sphere4, RenderObject(sphere_mesh));
  scene.translate(sphere5, glm::vec3{0, 2, 0});
  scene.scale(sphere5, glm::vec3{0.5, 0.5, 0.5});

  renderer.set_skybox(RenderObject(skydome_mesh));

  Input::on(GLFW_KEY_I, [&] {
    renderer.get_shown_light()->translate({ 0.01f, 0, 0});
  }, true);
  Input::on(GLFW_KEY_K, [&] {
    renderer.get_shown_light()->translate({-0.01f, 0, 0});
  }, true);
  Input::on(GLFW_KEY_J, [&] {
    renderer.get_shown_light()->translate({0, 0,  0.01f});
  }, true);
  Input::on(GLFW_KEY_L, [&] {
    renderer.get_shown_light()->translate({0, 0, -0.01f});
  }, true);
  Input::on(GLFW_KEY_O, [&] {
    renderer.get_shown_light()->translate({0,  0.01f, 0});
  }, true);
  Input::on(GLFW_KEY_U, [&] {
    renderer.get_shown_light()->translate({0, -0.01f, 0});
  }, true);
  Input::on(GLFW_KEY_PAGE_UP, [&] {
    if(selected_light < renderer.light_count()) {
      renderer.show_single_light(++selected_light);
    }
  }, false);

  Input::on(GLFW_KEY_DELETE, [&] {
    renderer.clear_lights();
  }, false);
  Input::on(GLFW_KEY_PAGE_DOWN, [&] {
    if(selected_light > 0) {
      renderer.show_single_light(--selected_light);
    }
  }, false);
  Input::on(GLFW_KEY_1, [&] {
      renderer.toggle_shadow_map();
  }, false);
  Input::on(GLFW_KEY_P, [&] {
      if(Input::is_alt_down()) {
        auto light = std::make_shared<PointLight>(camera.pos, get_random_color());
        light->set_casts_shadow(false);
        light->ambient_intensity = 0.0f;
        light->diffuse_intensity = 0.2f;
        renderer.add_light(light);
      } else {
        auto light = std::make_shared<SpotLight>(camera.pos, camera.dir, glm::vec3{1,1,1});
        light->set_casts_shadow(true);
        light->ambient_intensity = 0.0f;
        light->diffuse_intensity = 2.0f;
        renderer.add_light(light);
      }
      std::cout << "Amount of lights: " << renderer.light_count() << "\n";
  }, false);
  Input::on(GLFW_KEY_RIGHT_CONTROL, [&] {
      renderer.show_single_light(-1);
  }, false);
  bool draw_main = true;
  Input::on(GLFW_KEY_T, [&] {
      draw_main = !draw_main;
  }, false);
  Input::on(GLFW_KEY_Q, [&] {
      step = fmax(0.01f, fmin(step * pow(1.5, -1), 5));
  }, false);
  Input::on(GLFW_KEY_E, [&] {
      step = fmax(0.01f, fmin(step * pow(1.5, 1), 5));
  }, false);

  nvgCreateFont(vg, "sans", "/usr/share/fonts/truetype/freefont/FreeSans.ttf");

  DebugText::set_context(vg);

  camera.translate({0, 2, 0});
  while (!glfwWindowShouldClose(window)) {
    glfwPollEvents();
    Input::handle_input();
    rotate_camera(camera);
    renderer.render(scene);
    if(draw_main) {
      scene.rotate(cube, 0.01, glm::vec3{1, 1, 0});
      scene.rotate(sphere1, -0.01, glm::vec3{0, 1, 0});
      scene.rotate(sphere2, -0.01, glm::vec3{0, 1, 0});
      scene.rotate(sphere3, -0.01, glm::vec3{0, 0, 1});
      scene.rotate(sphere4,  0.01, glm::vec3{0, 0, 1});
    }

    std::stringstream str;
    str << std::fixed << std::setprecision(1) << calcFPS(1.0);
    DebugText::set_value("fps", str.str().c_str());

    nvgBeginFrame(vg, width, height, 1);
    DebugText::draw();
    nvgEndFrame(vg);

    glfwSwapBuffers(window);

    Input::reset_delta();
    checkForGlError();
  }
  glfwDestroyWindow(window);
  glfwTerminate();
  return 0;
}

<|endoftext|>


<commit_before>#include <algorithm>
#include <cassert>
#include <chrono>
#include <cstddef>
#include <cstdint>
#include <exception>
#include <functional>
#include <iostream>
#include <memory>
#include <random>
#include <string>

#include "chainerx/array.h"
#include "chainerx/array_index.h"
#include "chainerx/backprop_mode.h"
#include "chainerx/backward.h"
#include "chainerx/device_id.h"
#include "chainerx/dtype.h"
#include "chainerx/routines/creation.h"
#include "chainerx/routines/linalg.h"
#include "chainerx/routines/manipulation.h"
#include "chainerx/routines/math.h"
#include "chainerx/shape.h"
#include "chainerx/slice.h"
#include "mnist.h"

chainerx::Array GetRandomArray(std::mt19937& gen, std::normal_distribution<float>& dist, const chainerx::Shape& shape) {
    int64_t n = shape.GetTotalSize();
    std::shared_ptr<float> data{new float[n], std::default_delete<float[]>{}};
    std::generate_n(data.get(), n, [&dist, &gen]() { return dist(gen); });
    return chainerx::FromContiguousHostData(
            shape, chainerx::TypeToDtype<float>, static_cast<std::shared_ptr<void>>(data), chainerx::GetDefaultDevice());
}

class Model {
public:
    Model(int64_t n_in, int64_t n_hidden, int64_t n_out, int64_t n_layers)
        : n_in_{n_in}, n_hidden_{n_hidden}, n_out_{n_out}, n_layers_{n_layers} {};

    chainerx::Array operator()(const chainerx::Array& x) {
        chainerx::Array h = x;
        for (int64_t i = 0; i < n_layers_; ++i) {
            h = chainerx::Dot(h, params_[i * 2]) + params_[i * 2 + 1];
            if (i != n_layers_ - 1) {
                h = chainerx::Maximum(0, h);
            }
        }
        return h;
    }

    const std::vector<chainerx::Array>& params() { return params_; }

    void Initialize(std::mt19937& gen, std::normal_distribution<float>& dist) {
        params_.clear();

        for (int64_t i = 0; i < n_layers_; ++i) {
            int64_t n_in = i == 0 ? n_in_ : n_hidden_;
            int64_t n_out = i == n_layers_ - 1 ? n_out_ : n_hidden_;
            params_.emplace_back(GetRandomArray(gen, dist, {n_in, n_out}));
            params_.emplace_back(chainerx::Zeros({n_out}, chainerx::TypeToDtype<float>));
        }

        for (const chainerx::Array& param : params_) {
            param.RequireGrad();
        }
    }

private:
    int64_t n_in_;
    int64_t n_hidden_;
    int64_t n_out_;
    int64_t n_layers_;
    std::vector<chainerx::Array> params_;
};

chainerx::Array SoftmaxCrossEntropy(const chainerx::Array& y, const chainerx::Array& t, bool normalize = true) {
    chainerx::Array score = chainerx::LogSoftmax(y, 1);
    chainerx::Array mask =
            (t.At({chainerx::Slice{}, chainerx::NewAxis{}}) == chainerx::Arange(score.shape()[1], t.dtype())).AsType(score.dtype());
    if (normalize) {
        return -(score * mask).Sum() / y.shape()[0];
    }
    return -(score * mask).Sum();
}

void RunWithDefaultDevice(int64_t epochs, int64_t batch_size, int64_t n_hidden, int64_t n_layers, float lr, const std::string& mnist_root) {
    // Read the MNIST dataset.
    chainerx::Array train_x = ReadMnistImages(mnist_root + "train-images-idx3-ubyte");
    chainerx::Array train_t = ReadMnistLabels(mnist_root + "train-labels-idx1-ubyte");
    chainerx::Array test_x = ReadMnistImages(mnist_root + "t10k-images-idx3-ubyte");
    chainerx::Array test_t = ReadMnistLabels(mnist_root + "t10k-labels-idx1-ubyte");

    train_x = train_x.AsType(chainerx::Dtype::kFloat32) / 255.f;
    train_t = train_t.AsType(chainerx::Dtype::kInt32);
    test_x = test_x.AsType(chainerx::Dtype::kFloat32) / 255.f;
    test_t = test_t.AsType(chainerx::Dtype::kInt32);

    int64_t n_train = train_x.shape().front();
    int64_t n_test = test_x.shape().front();
    chainerx::Array train_indices = chainerx::Arange(n_train, chainerx::Dtype::kInt64);

    // Initialize the model with random parameters.
    Model model{train_x.shape()[1], n_hidden, 10, n_layers};
    std::random_device rd{};
    std::mt19937 gen{rd()};
    std::normal_distribution<float> dist{0.f, 0.05f};
    model.Initialize(gen, dist);

    auto start = std::chrono::high_resolution_clock::now();

    for (int64_t epoch = 0; epoch < epochs; ++epoch) {
        // The underlying data of the indices' Array is contiguous so we can use std::shuffle to randomize the order.
        assert(train_indices.IsContiguous());
        std::shuffle(
                reinterpret_cast<int64_t*>(train_indices.raw_data()), reinterpret_cast<int64_t*>(train_indices.raw_data()) + n_train, gen);
        for (int64_t i = 0; i < n_train; i += batch_size) {
            chainerx::Array indices = train_indices.At({chainerx::Slice{i, i + batch_size}});
            chainerx::Array x = train_x.Take(indices, 0);
            chainerx::Array t = train_t.Take(indices, 0);

            chainerx::Backward(SoftmaxCrossEntropy(model(x), t));

            // Vanilla SGD.
            for (const chainerx::Array& p : model.params()) {
                chainerx::Array p_as_grad_stopped = p.AsGradStopped();
                p_as_grad_stopped -= p.GetGrad()->AsGradStopped() * lr;
                p.ClearGrad();
            }
        }

        // Evaluate.
        {
            chainerx::NoBackpropModeScope scope{};

            chainerx::Array loss = chainerx::Zeros({}, chainerx::Dtype::kFloat32);
            chainerx::Array acc = chainerx::Zeros({}, chainerx::Dtype::kFloat32);

            for (int64_t i = 0; i < n_test; i += batch_size) {
                std::vector<chainerx::ArrayIndex> indices{chainerx::Slice{i, i + batch_size}};
                chainerx::Array x = test_x.At(indices);
                chainerx::Array t = test_t.At(indices);
                chainerx::Array y = model(x);
                loss += SoftmaxCrossEntropy(y, t, false);
                acc += (y.ArgMax(1).AsType(t.dtype()) == t).Sum().AsType(acc.dtype());
            }

            std::cout << "epoch: " << epoch << " loss=" << chainerx::AsScalar(loss / n_test)
                      << " accuracy=" << chainerx::AsScalar(acc / n_test)
                      << " elapsed_time=" << std::chrono::duration<double>{std::chrono::high_resolution_clock::now() - start}.count()
                      << std::endl;
        }
    }
}

int main(int argc, char** argv) {
    int64_t epochs{20};
    int64_t batch_size{100};
    int64_t n_hidden{1000};
    int64_t n_layers{3};
    float lr{0.01};
    std::string device_name{"native"};
    std::string mnist_root{"./"};

    for (int i = 1; i < argc; i += 2) {
        std::string arg = argv[i];
        if (arg == "--epoch") {
            epochs = std::atoi(argv[i + 1]);
        } else if (arg == "--batchsize") {
            batch_size = std::atoi(argv[i + 1]);
        } else if (arg == "--n-hidden") {
            n_hidden = std::atoi(argv[i + 1]);
        } else if (arg == "--n-layers") {
            n_layers = std::atoi(argv[i + 1]);
        } else if (arg == "--lr") {
            lr = std::atof(argv[i + 1]);
        } else if (arg == "--device") {
            device_name = argv[i + 1];
        } else if (arg == "--data") {
            mnist_root = argv[i + 1];
        } else {
            throw std::runtime_error("Unknown argument: " + arg);
        }
    }

    if (mnist_root.back() != '/') {
        mnist_root += "/";
    }

    chainerx::Context ctx{};
    chainerx::SetDefaultContext(&ctx);
    chainerx::Device& device = ctx.GetDevice(device_name);
    chainerx::SetDefaultDevice(&device);

    std::cout << "Epochs: " << epochs << std::endl;
    std::cout << "Minibatch size: " << batch_size << std::endl;
    std::cout << "Hidden neurons: " << n_hidden << std::endl;
    std::cout << "Layers: " << n_layers << std::endl;
    std::cout << "Learning rate: " << lr << std::endl;
    std::cout << "Device: " << device.name() << std::endl;
    std::cout << "MNIST root: " << mnist_root << std::endl;

    RunWithDefaultDevice(epochs, batch_size, n_hidden, n_layers, lr, mnist_root);
}
<commit_msg>Align variable names with Python example<commit_after>#include <algorithm>
#include <cassert>
#include <chrono>
#include <cstddef>
#include <cstdint>
#include <exception>
#include <functional>
#include <iostream>
#include <memory>
#include <random>
#include <string>

#include "chainerx/array.h"
#include "chainerx/array_index.h"
#include "chainerx/backprop_mode.h"
#include "chainerx/backward.h"
#include "chainerx/device_id.h"
#include "chainerx/dtype.h"
#include "chainerx/routines/creation.h"
#include "chainerx/routines/linalg.h"
#include "chainerx/routines/manipulation.h"
#include "chainerx/routines/math.h"
#include "chainerx/shape.h"
#include "chainerx/slice.h"
#include "mnist.h"

chainerx::Array GetRandomArray(std::mt19937& gen, std::normal_distribution<float>& dist, const chainerx::Shape& shape) {
    int64_t n = shape.GetTotalSize();
    std::shared_ptr<float> data{new float[n], std::default_delete<float[]>{}};
    std::generate_n(data.get(), n, [&dist, &gen]() { return dist(gen); });
    return chainerx::FromContiguousHostData(
            shape, chainerx::TypeToDtype<float>, static_cast<std::shared_ptr<void>>(data), chainerx::GetDefaultDevice());
}

class Model {
public:
    Model(int64_t n_in, int64_t n_hidden, int64_t n_out, int64_t n_layers)
        : n_in_{n_in}, n_hidden_{n_hidden}, n_out_{n_out}, n_layers_{n_layers} {};

    chainerx::Array operator()(const chainerx::Array& x) {
        chainerx::Array h = x;
        for (int64_t i = 0; i < n_layers_; ++i) {
            h = chainerx::Dot(h, params_[i * 2]) + params_[i * 2 + 1];
            if (i != n_layers_ - 1) {
                h = chainerx::Maximum(0, h);
            }
        }
        return h;
    }

    const std::vector<chainerx::Array>& params() { return params_; }

    void Initialize(std::mt19937& gen, std::normal_distribution<float>& dist) {
        params_.clear();

        for (int64_t i = 0; i < n_layers_; ++i) {
            int64_t n_in = i == 0 ? n_in_ : n_hidden_;
            int64_t n_out = i == n_layers_ - 1 ? n_out_ : n_hidden_;
            params_.emplace_back(GetRandomArray(gen, dist, {n_in, n_out}));
            params_.emplace_back(chainerx::Zeros({n_out}, chainerx::TypeToDtype<float>));
        }

        for (const chainerx::Array& param : params_) {
            param.RequireGrad();
        }
    }

private:
    int64_t n_in_;
    int64_t n_hidden_;
    int64_t n_out_;
    int64_t n_layers_;
    std::vector<chainerx::Array> params_;
};

chainerx::Array SoftmaxCrossEntropy(const chainerx::Array& y, const chainerx::Array& t, bool normalize = true) {
    chainerx::Array score = chainerx::LogSoftmax(y, 1);
    chainerx::Array mask =
            (t.At({chainerx::Slice{}, chainerx::NewAxis{}}) == chainerx::Arange(score.shape()[1], t.dtype())).AsType(score.dtype());
    if (normalize) {
        return -(score * mask).Sum() / y.shape()[0];
    }
    return -(score * mask).Sum();
}

void RunWithDefaultDevice(int64_t epochs, int64_t batch_size, int64_t n_hidden, int64_t n_layers, float lr, const std::string& mnist_root) {
    // Read the MNIST dataset.
    chainerx::Array train_x = ReadMnistImages(mnist_root + "train-images-idx3-ubyte");
    chainerx::Array train_t = ReadMnistLabels(mnist_root + "train-labels-idx1-ubyte");
    chainerx::Array test_x = ReadMnistImages(mnist_root + "t10k-images-idx3-ubyte");
    chainerx::Array test_t = ReadMnistLabels(mnist_root + "t10k-labels-idx1-ubyte");

    train_x = train_x.AsType(chainerx::Dtype::kFloat32) / 255.f;
    train_t = train_t.AsType(chainerx::Dtype::kInt32);
    test_x = test_x.AsType(chainerx::Dtype::kFloat32) / 255.f;
    test_t = test_t.AsType(chainerx::Dtype::kInt32);

    int64_t n_train = train_x.shape().front();
    int64_t n_test = test_x.shape().front();
    chainerx::Array train_indices = chainerx::Arange(n_train, chainerx::Dtype::kInt64);

    // Initialize the model with random parameters.
    Model model{train_x.shape()[1], n_hidden, 10, n_layers};
    std::random_device rd{};
    std::mt19937 gen{rd()};
    std::normal_distribution<float> dist{0.f, 0.05f};
    model.Initialize(gen, dist);

    auto start = std::chrono::high_resolution_clock::now();

    for (int64_t epoch = 0; epoch < epochs; ++epoch) {
        // The underlying data of the indices' Array is contiguous so we can use std::shuffle to randomize the order.
        assert(train_indices.IsContiguous());
        std::shuffle(
                reinterpret_cast<int64_t*>(train_indices.raw_data()), reinterpret_cast<int64_t*>(train_indices.raw_data()) + n_train, gen);
        for (int64_t i = 0; i < n_train; i += batch_size) {
            chainerx::Array indices = train_indices.At({chainerx::Slice{i, i + batch_size}});
            chainerx::Array x = train_x.Take(indices, 0);
            chainerx::Array t = train_t.Take(indices, 0);

            chainerx::Backward(SoftmaxCrossEntropy(model(x), t));

            // Vanilla SGD.
            for (const chainerx::Array& param : model.params()) {
                chainerx::Array p = param.AsGradStopped();
                p -= param.GetGrad()->AsGradStopped() * lr;
                param.ClearGrad();
            }
        }

        // Evaluate.
        {
            chainerx::NoBackpropModeScope scope{};

            chainerx::Array loss = chainerx::Zeros({}, chainerx::Dtype::kFloat32);
            chainerx::Array acc = chainerx::Zeros({}, chainerx::Dtype::kFloat32);

            for (int64_t i = 0; i < n_test; i += batch_size) {
                std::vector<chainerx::ArrayIndex> indices{chainerx::Slice{i, i + batch_size}};
                chainerx::Array x = test_x.At(indices);
                chainerx::Array t = test_t.At(indices);
                chainerx::Array y = model(x);
                loss += SoftmaxCrossEntropy(y, t, false);
                acc += (y.ArgMax(1).AsType(t.dtype()) == t).Sum().AsType(acc.dtype());
            }

            std::cout << "epoch: " << epoch << " loss=" << chainerx::AsScalar(loss / n_test)
                      << " accuracy=" << chainerx::AsScalar(acc / n_test)
                      << " elapsed_time=" << std::chrono::duration<double>{std::chrono::high_resolution_clock::now() - start}.count()
                      << std::endl;
        }
    }
}

int main(int argc, char** argv) {
    int64_t epochs{20};
    int64_t batch_size{100};
    int64_t n_hidden{1000};
    int64_t n_layers{3};
    float lr{0.01};
    std::string device_name{"native"};
    std::string mnist_root{"./"};

    for (int i = 1; i < argc; i += 2) {
        std::string arg = argv[i];
        if (arg == "--epoch") {
            epochs = std::atoi(argv[i + 1]);
        } else if (arg == "--batchsize") {
            batch_size = std::atoi(argv[i + 1]);
        } else if (arg == "--unit") {
            n_hidden = std::atoi(argv[i + 1]);
        } else if (arg == "--layer") {
            n_layers = std::atoi(argv[i + 1]);
        } else if (arg == "--lr") {
            lr = std::atof(argv[i + 1]);
        } else if (arg == "--device") {
            device_name = argv[i + 1];
        } else if (arg == "--data") {
            mnist_root = argv[i + 1];
        } else {
            throw std::runtime_error("Unknown argument: " + arg);
        }
    }

    if (mnist_root.back() != '/') {
        mnist_root += "/";
    }

    chainerx::Context ctx{};
    chainerx::SetDefaultContext(&ctx);
    chainerx::Device& device = ctx.GetDevice(device_name);
    chainerx::SetDefaultDevice(&device);

    std::cout << "Epochs: " << epochs << std::endl;
    std::cout << "Minibatch size: " << batch_size << std::endl;
    std::cout << "Hidden neurons: " << n_hidden << std::endl;
    std::cout << "Layers: " << n_layers << std::endl;
    std::cout << "Learning rate: " << lr << std::endl;
    std::cout << "Device: " << device.name() << std::endl;
    std::cout << "MNIST root: " << mnist_root << std::endl;

    RunWithDefaultDevice(epochs, batch_size, n_hidden, n_layers, lr, mnist_root);
}
<|endoftext|>


<commit_before>#include "foo_musicbrainz.h"
#include "Metadata.h"
#include "Release.h"
#include "RequestThread.h"

using namespace foo_musicbrainz;

RequestThread::RequestThread(Query *query, HWND window, ReleaseList *mbc) : query(query), window(window), mbc(mbc) {}

// TODO: abort checks, progress
void RequestThread::run(threaded_process_status &p_status, abort_callback &p_abort) {
	try {
		Metadata *metadata = query->perform();
		auto release = metadata->extract_release();
		delete metadata;
		if (release == nullptr) {
			throw NotFound();
		}
		mbc->add(release);
		ShowWindow(window, SW_SHOW);
	} catch (exception_aborted) {
		PostMessage(window, WM_CLOSE, 0, 0);
	} catch (GenericException e) {
		PostMessage(window, WM_CLOSE, 0, 0);
		popup_message::g_show(e.what(), COMPONENT_TITLE, popup_message::icon_error);
	}
	delete query;
}
<commit_msg>RequestThread: releases are now collected from all the possible XML locations<commit_after>#include "foo_musicbrainz.h"
#include "Metadata.h"
#include "Release.h"
#include "RequestThread.h"

using namespace foo_musicbrainz;

RequestThread::RequestThread(Query *query, HWND window, ReleaseList *mbc) : query(query), window(window), mbc(mbc) {}

// TODO: abort checks, progress
void RequestThread::run(threaded_process_status &p_status, abort_callback &p_abort) {
	try {
		// Extracting releases from all possible locations:
		// /metadata/release
		// /metadata/release-list/release
		// /metadata/discid/release-list/release
		auto metadata = query->perform();
		auto release = metadata->extract_release();
		if (release != nullptr) {
			mbc->add(release);
		} else {
			auto release_list = metadata->get_release_list();
			if (release_list == nullptr) {
				auto discid = metadata->get_discid();
				if (discid != nullptr) {
					release_list = discid->get_release_list();
				}
			}
			if (release_list != nullptr) {
				while (release_list->count() > 0) {
					mbc->add(release_list->extract(0));
				}
			}
		}

		delete metadata;
		if (mbc->count() == 0) {
			throw NotFound();
		}
		ShowWindow(window, SW_SHOW);
	} catch (exception_aborted) {
		PostMessage(window, WM_CLOSE, 0, 0);
	} catch (GenericException e) {
		PostMessage(window, WM_CLOSE, 0, 0);
		popup_message::g_show(e.what(), COMPONENT_TITLE, popup_message::icon_error);
	}
	delete query;
}
<|endoftext|>


<commit_before>// Copyright (c) 2012 The Chromium Authors. All rights reserved.
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file.

#include "chrome/browser/autofill/autofill_download.h"

#include <algorithm>
#include <ostream>
#include <vector>

#include "base/logging.h"
#include "base/rand_util.h"
#include "base/stl_util.h"
#include "base/string_util.h"
#include "chrome/browser/autofill/autofill_metrics.h"
#include "chrome/browser/autofill/autofill_xml_parser.h"
#include "chrome/browser/autofill/form_structure.h"
#include "chrome/browser/api/prefs/pref_service_base.h"
#include "chrome/common/pref_names.h"
#include "content/public/browser/browser_context.h"
#include "googleurl/src/gurl.h"
#include "net/base/load_flags.h"
#include "net/http/http_response_headers.h"
#include "net/url_request/url_fetcher.h"
#include "third_party/libjingle/source/talk/xmllite/xmlparser.h"

using content::BrowserContext;

namespace {
const char kAutofillQueryServerRequestUrl[] =
    "https://clients1.google.com/tbproxy/af/query?client=";
const char kAutofillUploadServerRequestUrl[] =
    "https://clients1.google.com/tbproxy/af/upload?client=";
const char kAutofillQueryServerNameStartInHeader[] = "GFE/";

#if defined(GOOGLE_CHROME_BUILD)
const char kClientName[] = "Google Chrome";
#else
const char kClientName[] = "Chromium";
#endif  // defined(GOOGLE_CHROME_BUILD)

const size_t kMaxFormCacheSize = 16;
};

struct AutofillDownloadManager::FormRequestData {
  std::vector<std::string> form_signatures;
  AutofillRequestType request_type;
};

AutofillDownloadManager::AutofillDownloadManager(BrowserContext* context,
                                                 Observer* observer)
    : browser_context_(context),
      observer_(observer),
      max_form_cache_size_(kMaxFormCacheSize),
      next_query_request_(base::Time::Now()),
      next_upload_request_(base::Time::Now()),
      positive_upload_rate_(0),
      negative_upload_rate_(0),
      fetcher_id_for_unittest_(0) {
  DCHECK(observer_);
  PrefServiceBase* preferences =
      PrefServiceBase::FromBrowserContext(browser_context_);
  positive_upload_rate_ =
      preferences->GetDouble(prefs::kAutofillPositiveUploadRate);
  negative_upload_rate_ =
      preferences->GetDouble(prefs::kAutofillNegativeUploadRate);
}

AutofillDownloadManager::~AutofillDownloadManager() {
  STLDeleteContainerPairFirstPointers(url_fetchers_.begin(),
                                      url_fetchers_.end());
}

bool AutofillDownloadManager::StartQueryRequest(
    const std::vector<FormStructure*>& forms,
    const AutofillMetrics& metric_logger) {
  if (next_query_request_ > base::Time::Now()) {
    // We are in back-off mode: do not do the request.
    return false;
  }
  std::string form_xml;
  FormRequestData request_data;
  if (!FormStructure::EncodeQueryRequest(forms, &request_data.form_signatures,
                                         &form_xml)) {
    return false;
  }

  request_data.request_type = AutofillDownloadManager::REQUEST_QUERY;
  metric_logger.LogServerQueryMetric(AutofillMetrics::QUERY_SENT);

  std::string query_data;
  if (CheckCacheForQueryRequest(request_data.form_signatures, &query_data)) {
    DVLOG(1) << "AutofillDownloadManager: query request has been retrieved from"
             << "the cache";
    observer_->OnLoadedServerPredictions(query_data);
    return true;
  }

  return StartRequest(form_xml, request_data);
}

bool AutofillDownloadManager::StartUploadRequest(
    const FormStructure& form,
    bool form_was_autofilled,
    const FieldTypeSet& available_field_types) {
  if (next_upload_request_ > base::Time::Now()) {
    // We are in back-off mode: do not do the request.
    DVLOG(1) << "AutofillDownloadManager: Upload request is throttled.";
    return false;
  }

  // Flip a coin to see if we should upload this form.
  double upload_rate = form_was_autofilled ? GetPositiveUploadRate() :
                                             GetNegativeUploadRate();
  if (form.upload_required() == UPLOAD_NOT_REQUIRED ||
      (form.upload_required() == USE_UPLOAD_RATES &&
       base::RandDouble() > upload_rate)) {
    DVLOG(1) << "AutofillDownloadManager: Upload request is ignored.";
    // If we ever need notification that upload was skipped, add it here.
    return false;
  }

  std::string form_xml;
  if (!form.EncodeUploadRequest(available_field_types, form_was_autofilled,
                                &form_xml))
    return false;

  FormRequestData request_data;
  request_data.form_signatures.push_back(form.FormSignature());
  request_data.request_type = AutofillDownloadManager::REQUEST_UPLOAD;

  return StartRequest(form_xml, request_data);
}

double AutofillDownloadManager::GetPositiveUploadRate() const {
  return positive_upload_rate_;
}

double AutofillDownloadManager::GetNegativeUploadRate() const {
  return negative_upload_rate_;
}

void AutofillDownloadManager::SetPositiveUploadRate(double rate) {
  if (rate == positive_upload_rate_)
    return;
  positive_upload_rate_ = rate;
  DCHECK_GE(rate, 0.0);
  DCHECK_LE(rate, 1.0);
  PrefServiceBase* preferences = PrefServiceBase::FromBrowserContext(
      browser_context_);
  preferences->SetDouble(prefs::kAutofillPositiveUploadRate, rate);
}

void AutofillDownloadManager::SetNegativeUploadRate(double rate) {
  if (rate == negative_upload_rate_)
    return;
  negative_upload_rate_ = rate;
  DCHECK_GE(rate, 0.0);
  DCHECK_LE(rate, 1.0);
  PrefServiceBase* preferences = PrefServiceBase::FromBrowserContext(
      browser_context_);
  preferences->SetDouble(prefs::kAutofillNegativeUploadRate, rate);
}

bool AutofillDownloadManager::StartRequest(
    const std::string& form_xml,
    const FormRequestData& request_data) {
  net::URLRequestContextGetter* request_context =
      browser_context_->GetRequestContext();
  DCHECK(request_context);
  std::string request_url;
  if (request_data.request_type == AutofillDownloadManager::REQUEST_QUERY)
    request_url = kAutofillQueryServerRequestUrl;
  else
    request_url = kAutofillUploadServerRequestUrl;
  request_url += kClientName;

  // Id is ignored for regular chrome, in unit test id's for fake fetcher
  // factory will be 0, 1, 2, ...
  net::URLFetcher* fetcher = net::URLFetcher::Create(
      fetcher_id_for_unittest_++, GURL(request_url), net::URLFetcher::POST,
      this);
  url_fetchers_[fetcher] = request_data;
  fetcher->SetAutomaticallyRetryOn5xx(false);
  fetcher->SetRequestContext(request_context);
  fetcher->SetUploadData("text/plain", form_xml);
  fetcher->SetLoadFlags(net::LOAD_DO_NOT_SAVE_COOKIES |
                        net::LOAD_DO_NOT_SEND_COOKIES);
  fetcher->Start();
  return true;
}

void AutofillDownloadManager::CacheQueryRequest(
    const std::vector<std::string>& forms_in_query,
    const std::string& query_data) {
  std::string signature = GetCombinedSignature(forms_in_query);
  for (QueryRequestCache::iterator it = cached_forms_.begin();
       it != cached_forms_.end(); ++it) {
    if (it->first == signature) {
      // We hit the cache, move to the first position and return.
      std::pair<std::string, std::string> data = *it;
      cached_forms_.erase(it);
      cached_forms_.push_front(data);
      return;
    }
  }
  std::pair<std::string, std::string> data;
  data.first = signature;
  data.second = query_data;
  cached_forms_.push_front(data);
  while (cached_forms_.size() > max_form_cache_size_)
    cached_forms_.pop_back();
}

bool AutofillDownloadManager::CheckCacheForQueryRequest(
    const std::vector<std::string>& forms_in_query,
    std::string* query_data) const {
  std::string signature = GetCombinedSignature(forms_in_query);
  for (QueryRequestCache::const_iterator it = cached_forms_.begin();
       it != cached_forms_.end(); ++it) {
    if (it->first == signature) {
      // We hit the cache, fill the data and return.
      *query_data = it->second;
      return true;
    }
  }
  return false;
}

std::string AutofillDownloadManager::GetCombinedSignature(
    const std::vector<std::string>& forms_in_query) const {
  size_t total_size = forms_in_query.size();
  for (size_t i = 0; i < forms_in_query.size(); ++i)
    total_size += forms_in_query[i].length();
  std::string signature;

  signature.reserve(total_size);

  for (size_t i = 0; i < forms_in_query.size(); ++i) {
    if (i)
      signature.append(",");
    signature.append(forms_in_query[i]);
  }
  return signature;
}

void AutofillDownloadManager::OnURLFetchComplete(
    const net::URLFetcher* source) {
  std::map<net::URLFetcher *, FormRequestData>::iterator it =
      url_fetchers_.find(const_cast<net::URLFetcher*>(source));
  if (it == url_fetchers_.end()) {
    // Looks like crash on Mac is possibly caused with callback entering here
    // with unknown fetcher when network is refreshed.
    return;
  }
  std::string type_of_request(
      it->second.request_type == AutofillDownloadManager::REQUEST_QUERY ?
          "query" : "upload");
  const int kHttpResponseOk = 200;
  const int kHttpInternalServerError = 500;
  const int kHttpBadGateway = 502;
  const int kHttpServiceUnavailable = 503;

  CHECK(it->second.form_signatures.size());
  if (source->GetResponseCode() != kHttpResponseOk) {
    bool back_off = false;
    std::string server_header;
    switch (source->GetResponseCode()) {
      case kHttpBadGateway:
        if (!source->GetResponseHeaders()->EnumerateHeader(NULL, "server",
                                                           &server_header) ||
            StartsWithASCII(server_header.c_str(),
                            kAutofillQueryServerNameStartInHeader,
                            false) != 0)
          break;
        // Bad gateway was received from Autofill servers. Fall through to back
        // off.
      case kHttpInternalServerError:
      case kHttpServiceUnavailable:
        back_off = true;
        break;
    }

    if (back_off) {
      base::Time back_off_time(base::Time::Now() + source->GetBackoffDelay());
      if (it->second.request_type == AutofillDownloadManager::REQUEST_QUERY) {
        next_query_request_ = back_off_time;
      } else {
        next_upload_request_ = back_off_time;
      }
    }

    DVLOG(1) << "AutofillDownloadManager: " << type_of_request
             << " request has failed with response "
             << source->GetResponseCode();
    observer_->OnServerRequestError(it->second.form_signatures[0],
                                    it->second.request_type,
                                    source->GetResponseCode());
  } else {
    DVLOG(1) << "AutofillDownloadManager: " << type_of_request
             << " request has succeeded";
    std::string response_body;
    source->GetResponseAsString(&response_body);
    if (it->second.request_type == AutofillDownloadManager::REQUEST_QUERY) {
      CacheQueryRequest(it->second.form_signatures, response_body);
      observer_->OnLoadedServerPredictions(response_body);
    } else {
      double new_positive_upload_rate = 0;
      double new_negative_upload_rate = 0;
      AutofillUploadXmlParser parse_handler(&new_positive_upload_rate,
                                            &new_negative_upload_rate);
      buzz::XmlParser parser(&parse_handler);
      parser.Parse(response_body.data(), response_body.length(), true);
      if (parse_handler.succeeded()) {
        SetPositiveUploadRate(new_positive_upload_rate);
        SetNegativeUploadRate(new_negative_upload_rate);
      }

      observer_->OnUploadedPossibleFieldTypes();
    }
  }
  delete it->first;
  url_fetchers_.erase(it);
}
<commit_msg>Add logging of autofill upload request.<commit_after>// Copyright (c) 2012 The Chromium Authors. All rights reserved.
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file.

#include "chrome/browser/autofill/autofill_download.h"

#include <algorithm>
#include <ostream>
#include <vector>

#include "base/logging.h"
#include "base/rand_util.h"
#include "base/stl_util.h"
#include "base/string_util.h"
#include "chrome/browser/autofill/autofill_metrics.h"
#include "chrome/browser/autofill/autofill_xml_parser.h"
#include "chrome/browser/autofill/form_structure.h"
#include "chrome/browser/api/prefs/pref_service_base.h"
#include "chrome/common/pref_names.h"
#include "content/public/browser/browser_context.h"
#include "googleurl/src/gurl.h"
#include "net/base/load_flags.h"
#include "net/http/http_response_headers.h"
#include "net/url_request/url_fetcher.h"
#include "third_party/libjingle/source/talk/xmllite/xmlparser.h"

using content::BrowserContext;

namespace {
const char kAutofillQueryServerRequestUrl[] =
    "https://clients1.google.com/tbproxy/af/query?client=";
const char kAutofillUploadServerRequestUrl[] =
    "https://clients1.google.com/tbproxy/af/upload?client=";
const char kAutofillQueryServerNameStartInHeader[] = "GFE/";

#if defined(GOOGLE_CHROME_BUILD)
const char kClientName[] = "Google Chrome";
#else
const char kClientName[] = "Chromium";
#endif  // defined(GOOGLE_CHROME_BUILD)

const size_t kMaxFormCacheSize = 16;

// Log the contents of the upload request
static void LogUploadRequest(const GURL& url, const std::string& signature,
                             const std::string& form_xml) {
  VLOG(2) << url;
  VLOG(2) << signature;
  VLOG(2) << form_xml;
}

};

struct AutofillDownloadManager::FormRequestData {
  std::vector<std::string> form_signatures;
  AutofillRequestType request_type;
};

AutofillDownloadManager::AutofillDownloadManager(BrowserContext* context,
                                                 Observer* observer)
    : browser_context_(context),
      observer_(observer),
      max_form_cache_size_(kMaxFormCacheSize),
      next_query_request_(base::Time::Now()),
      next_upload_request_(base::Time::Now()),
      positive_upload_rate_(0),
      negative_upload_rate_(0),
      fetcher_id_for_unittest_(0) {
  DCHECK(observer_);
  PrefServiceBase* preferences =
      PrefServiceBase::FromBrowserContext(browser_context_);
  positive_upload_rate_ =
      preferences->GetDouble(prefs::kAutofillPositiveUploadRate);
  negative_upload_rate_ =
      preferences->GetDouble(prefs::kAutofillNegativeUploadRate);
}

AutofillDownloadManager::~AutofillDownloadManager() {
  STLDeleteContainerPairFirstPointers(url_fetchers_.begin(),
                                      url_fetchers_.end());
}

bool AutofillDownloadManager::StartQueryRequest(
    const std::vector<FormStructure*>& forms,
    const AutofillMetrics& metric_logger) {
  if (next_query_request_ > base::Time::Now()) {
    // We are in back-off mode: do not do the request.
    return false;
  }
  std::string form_xml;
  FormRequestData request_data;
  if (!FormStructure::EncodeQueryRequest(forms, &request_data.form_signatures,
                                         &form_xml)) {
    return false;
  }

  request_data.request_type = AutofillDownloadManager::REQUEST_QUERY;
  metric_logger.LogServerQueryMetric(AutofillMetrics::QUERY_SENT);

  std::string query_data;
  if (CheckCacheForQueryRequest(request_data.form_signatures, &query_data)) {
    DVLOG(1) << "AutofillDownloadManager: query request has been retrieved from"
             << "the cache";
    observer_->OnLoadedServerPredictions(query_data);
    return true;
  }

  return StartRequest(form_xml, request_data);
}

bool AutofillDownloadManager::StartUploadRequest(
    const FormStructure& form,
    bool form_was_autofilled,
    const FieldTypeSet& available_field_types) {
  if (next_upload_request_ > base::Time::Now()) {
    // We are in back-off mode: do not do the request.
    DVLOG(1) << "AutofillDownloadManager: Upload request is throttled.";
    return false;
  }

  // Flip a coin to see if we should upload this form.
  double upload_rate = form_was_autofilled ? GetPositiveUploadRate() :
                                             GetNegativeUploadRate();
  if (form.upload_required() == UPLOAD_NOT_REQUIRED ||
      (form.upload_required() == USE_UPLOAD_RATES &&
       base::RandDouble() > upload_rate)) {
    DVLOG(1) << "AutofillDownloadManager: Upload request is ignored.";
    // If we ever need notification that upload was skipped, add it here.
    return false;
  }

  std::string form_xml;
  if (!form.EncodeUploadRequest(available_field_types, form_was_autofilled,
                                &form_xml))
    return false;

  LogUploadRequest(form.source_url(), form.FormSignature(), form_xml);

  FormRequestData request_data;
  request_data.form_signatures.push_back(form.FormSignature());
  request_data.request_type = AutofillDownloadManager::REQUEST_UPLOAD;

  return StartRequest(form_xml, request_data);
}

double AutofillDownloadManager::GetPositiveUploadRate() const {
  return positive_upload_rate_;
}

double AutofillDownloadManager::GetNegativeUploadRate() const {
  return negative_upload_rate_;
}

void AutofillDownloadManager::SetPositiveUploadRate(double rate) {
  if (rate == positive_upload_rate_)
    return;
  positive_upload_rate_ = rate;
  DCHECK_GE(rate, 0.0);
  DCHECK_LE(rate, 1.0);
  PrefServiceBase* preferences = PrefServiceBase::FromBrowserContext(
      browser_context_);
  preferences->SetDouble(prefs::kAutofillPositiveUploadRate, rate);
}

void AutofillDownloadManager::SetNegativeUploadRate(double rate) {
  if (rate == negative_upload_rate_)
    return;
  negative_upload_rate_ = rate;
  DCHECK_GE(rate, 0.0);
  DCHECK_LE(rate, 1.0);
  PrefServiceBase* preferences = PrefServiceBase::FromBrowserContext(
      browser_context_);
  preferences->SetDouble(prefs::kAutofillNegativeUploadRate, rate);
}

bool AutofillDownloadManager::StartRequest(
    const std::string& form_xml,
    const FormRequestData& request_data) {
  net::URLRequestContextGetter* request_context =
      browser_context_->GetRequestContext();
  DCHECK(request_context);
  std::string request_url;
  if (request_data.request_type == AutofillDownloadManager::REQUEST_QUERY)
    request_url = kAutofillQueryServerRequestUrl;
  else
    request_url = kAutofillUploadServerRequestUrl;
  request_url += kClientName;

  // Id is ignored for regular chrome, in unit test id's for fake fetcher
  // factory will be 0, 1, 2, ...
  net::URLFetcher* fetcher = net::URLFetcher::Create(
      fetcher_id_for_unittest_++, GURL(request_url), net::URLFetcher::POST,
      this);
  url_fetchers_[fetcher] = request_data;
  fetcher->SetAutomaticallyRetryOn5xx(false);
  fetcher->SetRequestContext(request_context);
  fetcher->SetUploadData("text/plain", form_xml);
  fetcher->SetLoadFlags(net::LOAD_DO_NOT_SAVE_COOKIES |
                        net::LOAD_DO_NOT_SEND_COOKIES);
  fetcher->Start();
  return true;
}

void AutofillDownloadManager::CacheQueryRequest(
    const std::vector<std::string>& forms_in_query,
    const std::string& query_data) {
  std::string signature = GetCombinedSignature(forms_in_query);
  for (QueryRequestCache::iterator it = cached_forms_.begin();
       it != cached_forms_.end(); ++it) {
    if (it->first == signature) {
      // We hit the cache, move to the first position and return.
      std::pair<std::string, std::string> data = *it;
      cached_forms_.erase(it);
      cached_forms_.push_front(data);
      return;
    }
  }
  std::pair<std::string, std::string> data;
  data.first = signature;
  data.second = query_data;
  cached_forms_.push_front(data);
  while (cached_forms_.size() > max_form_cache_size_)
    cached_forms_.pop_back();
}

bool AutofillDownloadManager::CheckCacheForQueryRequest(
    const std::vector<std::string>& forms_in_query,
    std::string* query_data) const {
  std::string signature = GetCombinedSignature(forms_in_query);
  for (QueryRequestCache::const_iterator it = cached_forms_.begin();
       it != cached_forms_.end(); ++it) {
    if (it->first == signature) {
      // We hit the cache, fill the data and return.
      *query_data = it->second;
      return true;
    }
  }
  return false;
}

std::string AutofillDownloadManager::GetCombinedSignature(
    const std::vector<std::string>& forms_in_query) const {
  size_t total_size = forms_in_query.size();
  for (size_t i = 0; i < forms_in_query.size(); ++i)
    total_size += forms_in_query[i].length();
  std::string signature;

  signature.reserve(total_size);

  for (size_t i = 0; i < forms_in_query.size(); ++i) {
    if (i)
      signature.append(",");
    signature.append(forms_in_query[i]);
  }
  return signature;
}

void AutofillDownloadManager::OnURLFetchComplete(
    const net::URLFetcher* source) {
  std::map<net::URLFetcher *, FormRequestData>::iterator it =
      url_fetchers_.find(const_cast<net::URLFetcher*>(source));
  if (it == url_fetchers_.end()) {
    // Looks like crash on Mac is possibly caused with callback entering here
    // with unknown fetcher when network is refreshed.
    return;
  }
  std::string type_of_request(
      it->second.request_type == AutofillDownloadManager::REQUEST_QUERY ?
          "query" : "upload");
  const int kHttpResponseOk = 200;
  const int kHttpInternalServerError = 500;
  const int kHttpBadGateway = 502;
  const int kHttpServiceUnavailable = 503;

  CHECK(it->second.form_signatures.size());
  if (source->GetResponseCode() != kHttpResponseOk) {
    bool back_off = false;
    std::string server_header;
    switch (source->GetResponseCode()) {
      case kHttpBadGateway:
        if (!source->GetResponseHeaders()->EnumerateHeader(NULL, "server",
                                                           &server_header) ||
            StartsWithASCII(server_header.c_str(),
                            kAutofillQueryServerNameStartInHeader,
                            false) != 0)
          break;
        // Bad gateway was received from Autofill servers. Fall through to back
        // off.
      case kHttpInternalServerError:
      case kHttpServiceUnavailable:
        back_off = true;
        break;
    }

    if (back_off) {
      base::Time back_off_time(base::Time::Now() + source->GetBackoffDelay());
      if (it->second.request_type == AutofillDownloadManager::REQUEST_QUERY) {
        next_query_request_ = back_off_time;
      } else {
        next_upload_request_ = back_off_time;
      }
    }

    DVLOG(1) << "AutofillDownloadManager: " << type_of_request
             << " request has failed with response "
             << source->GetResponseCode();
    observer_->OnServerRequestError(it->second.form_signatures[0],
                                    it->second.request_type,
                                    source->GetResponseCode());
  } else {
    DVLOG(1) << "AutofillDownloadManager: " << type_of_request
             << " request has succeeded";
    std::string response_body;
    source->GetResponseAsString(&response_body);
    if (it->second.request_type == AutofillDownloadManager::REQUEST_QUERY) {
      CacheQueryRequest(it->second.form_signatures, response_body);
      observer_->OnLoadedServerPredictions(response_body);
    } else {
      double new_positive_upload_rate = 0;
      double new_negative_upload_rate = 0;
      AutofillUploadXmlParser parse_handler(&new_positive_upload_rate,
                                            &new_negative_upload_rate);
      buzz::XmlParser parser(&parse_handler);
      parser.Parse(response_body.data(), response_body.length(), true);
      if (parse_handler.succeeded()) {
        SetPositiveUploadRate(new_positive_upload_rate);
        SetNegativeUploadRate(new_negative_upload_rate);
      }

      observer_->OnUploadedPossibleFieldTypes();
    }
  }
  delete it->first;
  url_fetchers_.erase(it);
}
<|endoftext|>