<commit_before>/*
 * Copyright (c) 2003 The Regents of The University of Michigan
 * All rights reserved.
 *
 * Redistribution and use in source and binary forms, with or without
 * modification, are permitted provided that the following conditions are
 * met: redistributions of source code must retain the above copyright
 * notice, this list of conditions and the following disclaimer;
 * redistributions in binary form must reproduce the above copyright
 * notice, this list of conditions and the following disclaimer in the
 * documentation and/or other materials provided with the distribution;
 * neither the name of the copyright holders nor the names of its
 * contributors may be used to endorse or promote products derived from
 * this software without specific prior written permission.
 *
 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
 * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
 * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
 * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
 * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
 * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
 * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
 * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 */

#ifndef __HASHMAP_HH__
#define __HASHMAP_HH__

#if defined(__GNUC__) && __GNUC__ >= 3
#include <ext/hash_map>
#else
#include <hash_map>
#endif

#include <string>

#include "sim/host.hh"

#if defined(__GNUC__) && __GNUC__ >= 3
    #define __hash_namespace __gnu_cxx
#else
    #define __hash_namespace std
#endif

namespace m5 {
    using ::__hash_namespace::hash_multimap;
    using ::__hash_namespace::hash_map;
    using ::__hash_namespace::hash;
}

///////////////////////////////////
// Some default Hashing Functions
//

namespace __hash_namespace {
    template<>
    struct hash<uint64_t> {
        size_t operator()(uint64_t r) const {
            return r;
        }
    };

    template<>
    struct hash<Counter> {
        size_t operator()(Counter r) const {
            return r;
        };
    };

    template<>
    struct hash<std::string> {
        size_t operator()(const std::string &s) const {
            return(__stl_hash_string(s.c_str()));
        }
    };
}

#endif // __HASHMAP_HH__
<commit_msg>Make the simulator compile on an Itanium.<commit_after>/*
 * Copyright (c) 2003 The Regents of The University of Michigan
 * All rights reserved.
 *
 * Redistribution and use in source and binary forms, with or without
 * modification, are permitted provided that the following conditions are
 * met: redistributions of source code must retain the above copyright
 * notice, this list of conditions and the following disclaimer;
 * redistributions in binary form must reproduce the above copyright
 * notice, this list of conditions and the following disclaimer in the
 * documentation and/or other materials provided with the distribution;
 * neither the name of the copyright holders nor the names of its
 * contributors may be used to endorse or promote products derived from
 * this software without specific prior written permission.
 *
 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
 * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
 * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
 * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
 * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
 * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
 * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
 * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 */

#ifndef __HASHMAP_HH__
#define __HASHMAP_HH__

#if defined(__GNUC__) && __GNUC__ >= 3
#include <ext/hash_map>
#else
#include <hash_map>
#endif

#include <string>

#include "sim/host.hh"

#if defined(__GNUC__) && __GNUC__ >= 3
    #define __hash_namespace __gnu_cxx
#else
    #define __hash_namespace std
#endif

namespace m5 {
    using ::__hash_namespace::hash_multimap;
    using ::__hash_namespace::hash_map;
    using ::__hash_namespace::hash;
}

///////////////////////////////////
// Some default Hashing Functions
//

namespace __hash_namespace {
#if !defined(__LP64__)
    template<>
    struct hash<uint64_t> {
        size_t operator()(uint64_t r) const {
            return r;
        }
    };

    template<>
    struct hash<int64_t> {
        size_t operator()(int64_t r) const {
            return r;
        };
    };
#endif

    template<>
    struct hash<std::string> {
        size_t operator()(const std::string &s) const {
            return(__stl_hash_string(s.c_str()));
        }
    };
}

#endif // __HASHMAP_HH__
<|endoftext|>


<commit_before>/** Copyright (C) 2016 Ultimaker - Released under terms of the AGPLv3 License */
#include "SettingRegistry.h"

#include <sstream>
#include <iostream> // debug IO
#include <libgen.h> // dirname
#include <string>
#include <cstring> // strtok (split string using delimiters)
#include <fstream> // ifstream (to see if file exists)

#include "rapidjson/rapidjson.h"
#include "rapidjson/document.h"
#include "rapidjson/error/en.h"
#include "rapidjson/filereadstream.h"
#include "../utils/logoutput.h"

namespace cura
{

SettingRegistry SettingRegistry::instance; // define settingRegistry

std::string SettingRegistry::toString(rapidjson::Type type)
{
    switch (type)
    {
        case rapidjson::Type::kNullType: return "null";
        case rapidjson::Type::kFalseType: return "false";
        case rapidjson::Type::kTrueType: return "true";
        case rapidjson::Type::kObjectType: return "object";
        case rapidjson::Type::kArrayType: return "array";
        case rapidjson::Type::kStringType: return "string";
        case rapidjson::Type::kNumberType: return "number";
        default: return "Unknown";
    }
}

SettingConfig::SettingConfig(std::string key, std::string label)
: SettingContainer(key, label)
{
//     std::cerr << key << std::endl; // debug output to show all frontend registered settings...
}

bool SettingRegistry::settingExists(std::string key) const
{
    return setting_key_to_config.find(key) != setting_key_to_config.end();
}

SettingConfig* SettingRegistry::getSettingConfig(std::string key) const
{
    auto it = setting_key_to_config.find(key);
    if (it == setting_key_to_config.end())
        return nullptr;
    return it->second;
}

SettingContainer* SettingRegistry::getExtruderTrain(unsigned int extruder_nr)
{
    if (extruder_nr < 0 || extruder_nr >= extruder_trains.size())
    {
        return nullptr;
    }
    return &extruder_trains[extruder_nr];
}

SettingRegistry::SettingRegistry()
: setting_definitions("settings", "Settings")
{
}

int SettingRegistry::loadJSON(std::string filename, rapidjson::Document& json_document)
{
    FILE* f = fopen(filename.c_str(), "rb");
    if (!f)
    {
        cura::logError("Couldn't open JSON file.\n");
        return 1;
    }
    char read_buffer[4096];
    rapidjson::FileReadStream reader_stream(f, read_buffer, sizeof(read_buffer));
    json_document.ParseStream(reader_stream);
    fclose(f);
    if (json_document.HasParseError())
    {
        cura::logError("Error parsing JSON(offset %u): %s\n", (unsigned)json_document.GetErrorOffset(), GetParseError_En(json_document.GetParseError()));
        return 2;
    }

    return 0;
}

/*!
 * Check whether a file exists.
 * from https://techoverflow.net/blog/2013/01/11/cpp-check-if-file-exists/
 *
 * \param filename The path to a filename to check if it exists
 * \return Whether the file exists.
 */
bool fexists(const char *filename)
{
    std::ifstream ifile(filename);
    return (bool)ifile;
}

bool SettingRegistry::getDefinitionFile(const std::string machine_id, const std::string parent_file, std::string& result)
{
    // check for file in same directory as the file provided
    std::string parent_filename_copy = std::string(parent_file.c_str()); // copy the string because dirname(.) changes the input string!!!
    char* parent_filename_cstr = (char*)parent_filename_copy.c_str();
    result = std::string(dirname(parent_filename_cstr)) + std::string("/") + machine_id + std::string(".def.json");
    if (fexists(result.c_str()))
    {
        return true;
    }

    // check for file in the directories supplied in the environment variable CURA_ENGINE_SEARCH_PATH
    char* paths = getenv("CURA_ENGINE_SEARCH_PATH");
    if (paths)
    {
        char* path = strtok(paths,";:,"); // search for path delimited by ';', ':', ','
        while (path != NULL)
        {
            result = std::string(path) + std::string("/") + machine_id + std::string(".def.json");
            if (fexists(result.c_str()))
            {
                return true;
            }
            path = strtok(NULL, ";:,"); // continue searching in last call to strtok
        }
    }
    return false;
}

int SettingRegistry::loadJSONsettings(std::string filename, SettingsBase* settings_base)
{
    rapidjson::Document json_document;

    log("Loading %s...\n", filename.c_str());

    int err = loadJSON(filename, json_document);
    if (err) { return err; }

    if (json_document.HasMember("inherits") && json_document["inherits"].IsString())
    {
        std::string child_filename;
        bool found = getDefinitionFile(json_document["inherits"].GetString(), filename, child_filename);
        if (!found)
        {
            return -1;
        }
        int err = loadJSONsettings(child_filename, settings_base); // load child first
        if (err)
        {
            return err;
        }
        return loadJSONsettingsFromDoc(json_document, settings_base, false);
    }
    else
    {
        return loadJSONsettingsFromDoc(json_document, settings_base, true);
    }
}

int SettingRegistry::loadJSONsettingsFromDoc(rapidjson::Document& json_document, SettingsBase* settings_base, bool warn_duplicates)
{

    if (!json_document.IsObject())
    {
        cura::logError("JSON file is not an object.\n");
        return 3;
    }

    { // handle machine name
        std::string machine_name = "Unknown";
        if (json_document.HasMember("name"))
        {
            const rapidjson::Value& machine_name_field = json_document["name"];
            if (machine_name_field.IsString())
            {
                machine_name = machine_name_field.GetString();
            }
        }
        SettingConfig& machine_name_setting = addSetting("machine_name", "Machine Name");
        machine_name_setting.setDefault(machine_name);
        machine_name_setting.setType("string");
    }

    if (json_document.HasMember("settings"))
    {
        std::list<std::string> path;
        handleChildren(json_document["settings"], path, settings_base, warn_duplicates);
    }

    if (json_document.HasMember("overrides"))
    {
        const rapidjson::Value& json_object_container = json_document["overrides"];
        for (rapidjson::Value::ConstMemberIterator override_iterator = json_object_container.MemberBegin(); override_iterator != json_object_container.MemberEnd(); ++override_iterator)
        {
            std::string setting = override_iterator->name.GetString();
            SettingConfig* conf = getSettingConfig(setting);
            if (!conf) //Setting could not be found.
            {
                logWarning("Trying to override unknown setting %s.\n", setting.c_str());
                continue;
            }
            _loadSettingValues(conf, override_iterator, settings_base);
        }
    }

    return 0;
}

void SettingRegistry::handleChildren(const rapidjson::Value& settings_list, std::list<std::string>& path, SettingsBase* settings_base, bool warn_duplicates)
{
    if (!settings_list.IsObject())
    {
        logError("ERROR: json settings list is not an object!\n");
        return;
    }
    for (rapidjson::Value::ConstMemberIterator setting_iterator = settings_list.MemberBegin(); setting_iterator != settings_list.MemberEnd(); ++setting_iterator)
    {
        handleSetting(setting_iterator, path, settings_base, warn_duplicates);
        if (setting_iterator->value.HasMember("children"))
        {
            std::list<std::string> path_here = path;
            path_here.push_back(setting_iterator->name.GetString());
            handleChildren(setting_iterator->value["children"], path_here, settings_base, warn_duplicates);
        }
    }
}

bool SettingRegistry::settingIsUsedByEngine(const rapidjson::Value& setting)
{
    if (setting.HasMember("children"))
    {
        return false;
    }
    else
    {
        return true;
    }
}

void SettingRegistry::handleSetting(const rapidjson::Value::ConstMemberIterator& json_setting_it, std::list<std::string>& path, SettingsBase* settings_base, bool warn_duplicates)
{
    const rapidjson::Value& json_setting = json_setting_it->value;
    if (!json_setting.IsObject())
    {
        logError("ERROR: json setting is not an object!\n");
        return;
    }
    std::string name = json_setting_it->name.GetString();
    if (json_setting.HasMember("type") && json_setting["type"].IsString() && json_setting["type"].GetString() == std::string("category"))
    { // skip category objects
        return;
    }
    if (settingIsUsedByEngine(json_setting))
    {
        if (!json_setting.HasMember("label") || !json_setting["label"].IsString())
        {
            logError("ERROR: json setting \"%s\" has no label!\n", name.c_str());
            return;
        }
        std::string label = json_setting["label"].GetString();

        SettingConfig* setting = getSettingConfig(name);
        if (warn_duplicates && setting)
        {
            cura::logError("Duplicate definition of setting: %s a.k.a. \"%s\" was already claimed by \"%s\"\n", name.c_str(), label.c_str(), getSettingConfig(name)->getLabel().c_str());
        }
        if (!setting)
        {
            setting = &addSetting(name, label);
        }
        _loadSettingValues(setting, json_setting_it, settings_base);
    }
}

SettingConfig& SettingRegistry::addSetting(std::string name, std::string label)
{
    SettingConfig* config = setting_definitions.addChild(name, label);

    setting_key_to_config[name] = config;

    return *config;
}

std::string SettingRegistry::getDefault(const rapidjson::GenericValue< rapidjson::UTF8< char > >::ConstMemberIterator& json_object_it)
{
    const rapidjson::Value& setting_content = json_object_it->value;
    if (setting_content.HasMember("default_value"))
    {
        const rapidjson::Value& dflt = setting_content["default_value"];
        if (dflt.IsString())
        {
            return dflt.GetString();
        }
        else if (dflt.IsTrue())
        {
            return "true";
        }
        else if (dflt.IsFalse())
        {
            return "false";
        }
        else if (dflt.IsNumber())
        {
            std::ostringstream ss;
            ss << dflt.GetDouble();
            return ss.str();
        } // arrays are ignored because machine_extruder_trains needs to be handled separately
        else
        {
            if (setting_content.HasMember("type") && setting_content["type"].IsString() &&
                (setting_content["type"].GetString() == std::string("polygon") || setting_content["type"].GetString() == std::string("polygons")))
            {
//                 logWarning("WARNING: Loading polygon setting %s not implemented...\n", json_object_it->name.GetString());
            }
            else
            {
//                 logWarning("WARNING: Unrecognized data type in JSON: %s has type %s\n", json_object_it->name.GetString(), toString(dflt.GetType()).c_str());
            }
        }
    }
    return "";
}

void SettingRegistry::_loadSettingValues(SettingConfig* config, const rapidjson::GenericValue< rapidjson::UTF8< char > >::ConstMemberIterator& json_object_it, SettingsBase* settings_base)
{
    const rapidjson::Value& data = json_object_it->value;
    /// Fill the setting config object with data we have in the json file.
    if (data.HasMember("type") && data["type"].IsString())
    {
        config->setType(data["type"].GetString());
    }

    config->setDefault(getDefault(json_object_it));

    if (data.HasMember("unit") && data["unit"].IsString())
    {
        config->setUnit(data["unit"].GetString());
    }

    settings_base->_setSetting(config->getKey(), config->getDefaultValue());
}

}//namespace cura
<commit_msg>fix: make path delimiter depend on OS (CURA-1574)<commit_after>/** Copyright (C) 2016 Ultimaker - Released under terms of the AGPLv3 License */
#include "SettingRegistry.h"

#include <sstream>
#include <iostream> // debug IO
#include <libgen.h> // dirname
#include <string>
#include <cstring> // strtok (split string using delimiters)
#include <fstream> // ifstream (to see if file exists)

#include "rapidjson/rapidjson.h"
#include "rapidjson/document.h"
#include "rapidjson/error/en.h"
#include "rapidjson/filereadstream.h"
#include "../utils/logoutput.h"

namespace cura
{

SettingRegistry SettingRegistry::instance; // define settingRegistry

std::string SettingRegistry::toString(rapidjson::Type type)
{
    switch (type)
    {
        case rapidjson::Type::kNullType: return "null";
        case rapidjson::Type::kFalseType: return "false";
        case rapidjson::Type::kTrueType: return "true";
        case rapidjson::Type::kObjectType: return "object";
        case rapidjson::Type::kArrayType: return "array";
        case rapidjson::Type::kStringType: return "string";
        case rapidjson::Type::kNumberType: return "number";
        default: return "Unknown";
    }
}

SettingConfig::SettingConfig(std::string key, std::string label)
: SettingContainer(key, label)
{
//     std::cerr << key << std::endl; // debug output to show all frontend registered settings...
}

bool SettingRegistry::settingExists(std::string key) const
{
    return setting_key_to_config.find(key) != setting_key_to_config.end();
}

SettingConfig* SettingRegistry::getSettingConfig(std::string key) const
{
    auto it = setting_key_to_config.find(key);
    if (it == setting_key_to_config.end())
        return nullptr;
    return it->second;
}

SettingContainer* SettingRegistry::getExtruderTrain(unsigned int extruder_nr)
{
    if (extruder_nr < 0 || extruder_nr >= extruder_trains.size())
    {
        return nullptr;
    }
    return &extruder_trains[extruder_nr];
}

SettingRegistry::SettingRegistry()
: setting_definitions("settings", "Settings")
{
}

int SettingRegistry::loadJSON(std::string filename, rapidjson::Document& json_document)
{
    FILE* f = fopen(filename.c_str(), "rb");
    if (!f)
    {
        cura::logError("Couldn't open JSON file.\n");
        return 1;
    }
    char read_buffer[4096];
    rapidjson::FileReadStream reader_stream(f, read_buffer, sizeof(read_buffer));
    json_document.ParseStream(reader_stream);
    fclose(f);
    if (json_document.HasParseError())
    {
        cura::logError("Error parsing JSON(offset %u): %s\n", (unsigned)json_document.GetErrorOffset(), GetParseError_En(json_document.GetParseError()));
        return 2;
    }

    return 0;
}

/*!
 * Check whether a file exists.
 * from https://techoverflow.net/blog/2013/01/11/cpp-check-if-file-exists/
 *
 * \param filename The path to a filename to check if it exists
 * \return Whether the file exists.
 */
bool fexists(const char *filename)
{
    std::ifstream ifile(filename);
    return (bool)ifile;
}

bool SettingRegistry::getDefinitionFile(const std::string machine_id, const std::string parent_file, std::string& result)
{
    // check for file in same directory as the file provided
    std::string parent_filename_copy = std::string(parent_file.c_str()); // copy the string because dirname(.) changes the input string!!!
    char* parent_filename_cstr = (char*)parent_filename_copy.c_str();
    result = std::string(dirname(parent_filename_cstr)) + std::string("/") + machine_id + std::string(".def.json");
    if (fexists(result.c_str()))
    {
        return true;
    }

    // check for file in the directories supplied in the environment variable CURA_ENGINE_SEARCH_PATH
    char* paths = getenv("CURA_ENGINE_SEARCH_PATH");
    if (paths)
    {
#if defined(__linux__) || (defined(__APPLE__) && defined(__MACH__))
        char delims[] = ":"; // colon
#else
        char delims[] = ";"; // semicolon
#endif
        char* path = strtok(paths, delims); // search for next path delimited by any of the characters in delims
        while (path != NULL)
        {
            result = std::string(path) + std::string("/") + machine_id + std::string(".def.json");
            if (fexists(result.c_str()))
            {
                return true;
            }
            path = strtok(NULL, ";:,"); // continue searching in last call to strtok
        }
    }
    return false;
}

int SettingRegistry::loadJSONsettings(std::string filename, SettingsBase* settings_base)
{
    rapidjson::Document json_document;

    log("Loading %s...\n", filename.c_str());

    int err = loadJSON(filename, json_document);
    if (err) { return err; }

    if (json_document.HasMember("inherits") && json_document["inherits"].IsString())
    {
        std::string child_filename;
        bool found = getDefinitionFile(json_document["inherits"].GetString(), filename, child_filename);
        if (!found)
        {
            return -1;
        }
        int err = loadJSONsettings(child_filename, settings_base); // load child first
        if (err)
        {
            return err;
        }
        return loadJSONsettingsFromDoc(json_document, settings_base, false);
    }
    else
    {
        return loadJSONsettingsFromDoc(json_document, settings_base, true);
    }
}

int SettingRegistry::loadJSONsettingsFromDoc(rapidjson::Document& json_document, SettingsBase* settings_base, bool warn_duplicates)
{

    if (!json_document.IsObject())
    {
        cura::logError("JSON file is not an object.\n");
        return 3;
    }

    { // handle machine name
        std::string machine_name = "Unknown";
        if (json_document.HasMember("name"))
        {
            const rapidjson::Value& machine_name_field = json_document["name"];
            if (machine_name_field.IsString())
            {
                machine_name = machine_name_field.GetString();
            }
        }
        SettingConfig& machine_name_setting = addSetting("machine_name", "Machine Name");
        machine_name_setting.setDefault(machine_name);
        machine_name_setting.setType("string");
    }

    if (json_document.HasMember("settings"))
    {
        std::list<std::string> path;
        handleChildren(json_document["settings"], path, settings_base, warn_duplicates);
    }

    if (json_document.HasMember("overrides"))
    {
        const rapidjson::Value& json_object_container = json_document["overrides"];
        for (rapidjson::Value::ConstMemberIterator override_iterator = json_object_container.MemberBegin(); override_iterator != json_object_container.MemberEnd(); ++override_iterator)
        {
            std::string setting = override_iterator->name.GetString();
            SettingConfig* conf = getSettingConfig(setting);
            if (!conf) //Setting could not be found.
            {
                logWarning("Trying to override unknown setting %s.\n", setting.c_str());
                continue;
            }
            _loadSettingValues(conf, override_iterator, settings_base);
        }
    }

    return 0;
}

void SettingRegistry::handleChildren(const rapidjson::Value& settings_list, std::list<std::string>& path, SettingsBase* settings_base, bool warn_duplicates)
{
    if (!settings_list.IsObject())
    {
        logError("ERROR: json settings list is not an object!\n");
        return;
    }
    for (rapidjson::Value::ConstMemberIterator setting_iterator = settings_list.MemberBegin(); setting_iterator != settings_list.MemberEnd(); ++setting_iterator)
    {
        handleSetting(setting_iterator, path, settings_base, warn_duplicates);
        if (setting_iterator->value.HasMember("children"))
        {
            std::list<std::string> path_here = path;
            path_here.push_back(setting_iterator->name.GetString());
            handleChildren(setting_iterator->value["children"], path_here, settings_base, warn_duplicates);
        }
    }
}

bool SettingRegistry::settingIsUsedByEngine(const rapidjson::Value& setting)
{
    if (setting.HasMember("children"))
    {
        return false;
    }
    else
    {
        return true;
    }
}

void SettingRegistry::handleSetting(const rapidjson::Value::ConstMemberIterator& json_setting_it, std::list<std::string>& path, SettingsBase* settings_base, bool warn_duplicates)
{
    const rapidjson::Value& json_setting = json_setting_it->value;
    if (!json_setting.IsObject())
    {
        logError("ERROR: json setting is not an object!\n");
        return;
    }
    std::string name = json_setting_it->name.GetString();
    if (json_setting.HasMember("type") && json_setting["type"].IsString() && json_setting["type"].GetString() == std::string("category"))
    { // skip category objects
        return;
    }
    if (settingIsUsedByEngine(json_setting))
    {
        if (!json_setting.HasMember("label") || !json_setting["label"].IsString())
        {
            logError("ERROR: json setting \"%s\" has no label!\n", name.c_str());
            return;
        }
        std::string label = json_setting["label"].GetString();

        SettingConfig* setting = getSettingConfig(name);
        if (warn_duplicates && setting)
        {
            cura::logError("Duplicate definition of setting: %s a.k.a. \"%s\" was already claimed by \"%s\"\n", name.c_str(), label.c_str(), getSettingConfig(name)->getLabel().c_str());
        }
        if (!setting)
        {
            setting = &addSetting(name, label);
        }
        _loadSettingValues(setting, json_setting_it, settings_base);
    }
}

SettingConfig& SettingRegistry::addSetting(std::string name, std::string label)
{
    SettingConfig* config = setting_definitions.addChild(name, label);

    setting_key_to_config[name] = config;

    return *config;
}

std::string SettingRegistry::getDefault(const rapidjson::GenericValue< rapidjson::UTF8< char > >::ConstMemberIterator& json_object_it)
{
    const rapidjson::Value& setting_content = json_object_it->value;
    if (setting_content.HasMember("default_value"))
    {
        const rapidjson::Value& dflt = setting_content["default_value"];
        if (dflt.IsString())
        {
            return dflt.GetString();
        }
        else if (dflt.IsTrue())
        {
            return "true";
        }
        else if (dflt.IsFalse())
        {
            return "false";
        }
        else if (dflt.IsNumber())
        {
            std::ostringstream ss;
            ss << dflt.GetDouble();
            return ss.str();
        } // arrays are ignored because machine_extruder_trains needs to be handled separately
        else
        {
            if (setting_content.HasMember("type") && setting_content["type"].IsString() &&
                (setting_content["type"].GetString() == std::string("polygon") || setting_content["type"].GetString() == std::string("polygons")))
            {
//                 logWarning("WARNING: Loading polygon setting %s not implemented...\n", json_object_it->name.GetString());
            }
            else
            {
//                 logWarning("WARNING: Unrecognized data type in JSON: %s has type %s\n", json_object_it->name.GetString(), toString(dflt.GetType()).c_str());
            }
        }
    }
    return "";
}

void SettingRegistry::_loadSettingValues(SettingConfig* config, const rapidjson::GenericValue< rapidjson::UTF8< char > >::ConstMemberIterator& json_object_it, SettingsBase* settings_base)
{
    const rapidjson::Value& data = json_object_it->value;
    /// Fill the setting config object with data we have in the json file.
    if (data.HasMember("type") && data["type"].IsString())
    {
        config->setType(data["type"].GetString());
    }

    config->setDefault(getDefault(json_object_it));

    if (data.HasMember("unit") && data["unit"].IsString())
    {
        config->setUnit(data["unit"].GetString());
    }

    settings_base->_setSetting(config->getKey(), config->getDefaultValue());
}

}//namespace cura
<|endoftext|>


<commit_before>//Copyright 2016 Adam G. Smith
//
//Licensed under the Apache License, Version 2.0 (the "License");
//you may not use this file except in compliance with the License.
//You may obtain a copy of the License at
//
//http ://www.apache.org/licenses/LICENSE-2.0
//
//Unless required by applicable law or agreed to in writing, software
//distributed under the License is distributed on an "AS IS" BASIS,
//WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
//See the License for the specific language governing permissions and
//limitations under the License.

#include "solaire/core/memory_pool.hpp"

namespace solaire {

	// memory_pool

	memory_pool::memory_pool() {

	}

	memory_pool::memory_pool(memory_pool&& aOther) {
		operator=(std::move(aOther));
	}

	memory_pool::~memory_pool() {
		mOpenBlocks.clear();
		mClosedBlocks.clear();
		for(const block& i : mRealBlocks) operator delete(i.first);
		mRealBlocks.clear();
	}

	memory_pool& memory_pool::operator=(memory_pool&& aOther) {
		std::swap(mRealBlocks, aOther.mRealBlocks);
		std::swap(mOpenBlocks, aOther.mOpenBlocks);
		std::swap(mClosedBlocks, aOther.mClosedBlocks);
		return *this;
	}

	void* memory_pool::allocate(size_t aSize) {
		// Check for an open block
		const auto end = mOpenBlocks.end();
		for(auto i = mOpenBlocks.begin(); i != end; ++i) if(i->second <= aSize) {
			block tmp = *i;
			mOpenBlocks.erase(i);
			if(tmp.second > aSize) {
				const size_t dif = tmp.second - aSize;
				tmp.second -= dif;
				mClosedBlocks.push_back({static_cast<uint8_t*>(tmp.first) + tmp.second, dif });
			}
			mClosedBlocks.push_back(tmp);
			return tmp.first;
		}

		// Allocate a new block
		const block tmp(operator new(aSize), aSize);
		if(! tmp.first) throw std::runtime_error("solaire::memory_pool::allocate : Failed to allocate memory");
		mRealBlocks.push_back(tmp);
		mClosedBlocks.push_back(tmp);
		return tmp.first;
	}

	void memory_pool::deallocate(void* aData) {
		// Check for closed block
		const auto end = mOpenBlocks.end();
		//! \todo Merge blocks
		for(auto i = mOpenBlocks.begin(); i != end; ++i) if(i->first == aData) {
			const block tmp = *i;
			mClosedBlocks.erase(i);
			mOpenBlocks.push_back(tmp);
			return;
		}
		throw std::runtime_error("solaire::memory_pool::deallocate : Memory not allocated from this allocator");
	}

	// memory_pool_allocator

	memory_pool_allocator::memory_pool_allocator() :
		mAllocator(nullptr),
		mData(nullptr),
		mSize(0)
	{}

	memory_pool_allocator::memory_pool_allocator(memory_pool& aPool) :
		mAllocator(&aPool),
		mData(nullptr),
		mSize(0)
	{}

	memory_pool_allocator::memory_pool_allocator(memory_pool_allocator&& aOther) :
		mAllocator(aOther.mAllocator),
		mData(aOther.mData),
		mSize(aOther.mSize)
	{}

	memory_pool_allocator::~memory_pool_allocator() {
		if(mData) deallocate();
	}

	memory_pool_allocator& memory_pool_allocator::operator=(memory_pool_allocator&& aOther) {
		std::swap(mAllocator, aOther.mAllocator);
		std::swap(mData, aOther.mData);
		std::swap(mSize, aOther.mSize);
		return *this;
	}

	void memory_pool_allocator::deallocate() {
		if(! mData) throw std::runtime_error("solaire::memory_pool_allocator::deallocate : Not allocated");
		mAllocator->deallocate(mData);
		mData = nullptr;
		mData = 0;
	}

	void* memory_pool_allocator::allocate(size_t aSize) {
		if(! mAllocator) throw std::runtime_error("solaire::memory_pool_allocator::allocate : No memory pool assigned");
		if(mData) throw std::runtime_error("solaire::memory_pool_allocator::allocate : Already allocated");
		mData = mAllocator->allocate(aSize);
		mSize = aSize;
	}

	void* memory_pool_allocator::get() const {
		return mData;
	}

	size_t memory_pool_allocator::size() const {
		return mSize;
	}

}<commit_msg>Implemented block merging<commit_after>//Copyright 2016 Adam G. Smith
//
//Licensed under the Apache License, Version 2.0 (the "License");
//you may not use this file except in compliance with the License.
//You may obtain a copy of the License at
//
//http ://www.apache.org/licenses/LICENSE-2.0
//
//Unless required by applicable law or agreed to in writing, software
//distributed under the License is distributed on an "AS IS" BASIS,
//WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
//See the License for the specific language governing permissions and
//limitations under the License.

#include "solaire/core/memory_pool.hpp"

namespace solaire {

	// memory_pool

	memory_pool::memory_pool() {

	}

	memory_pool::memory_pool(memory_pool&& aOther) {
		operator=(std::move(aOther));
	}

	memory_pool::~memory_pool() {
		mOpenBlocks.clear();
		mClosedBlocks.clear();
		for(const block& i : mRealBlocks) operator delete(i.first);
		mRealBlocks.clear();
	}

	memory_pool& memory_pool::operator=(memory_pool&& aOther) {
		std::swap(mRealBlocks, aOther.mRealBlocks);
		std::swap(mOpenBlocks, aOther.mOpenBlocks);
		std::swap(mClosedBlocks, aOther.mClosedBlocks);
		return *this;
	}

	void* memory_pool::allocate(size_t aSize) {
		// Check for an open block
		const auto end = mOpenBlocks.end();
		for(auto i = mOpenBlocks.begin(); i != end; ++i) if(i->second <= aSize) {
			block tmp = *i;
			mOpenBlocks.erase(i);
			if(tmp.second > aSize) {
				const size_t dif = tmp.second - aSize;
				tmp.second -= dif;
				mClosedBlocks.push_back({static_cast<uint8_t*>(tmp.first) + tmp.second, dif });
			}
			mClosedBlocks.push_back(tmp);
			return tmp.first;
		}

		// Allocate a new block
		const block tmp(operator new(aSize), aSize);
		if(! tmp.first) throw std::runtime_error("solaire::memory_pool::allocate : Failed to allocate memory");
		mRealBlocks.push_back(tmp);
		mClosedBlocks.push_back(tmp);
		return tmp.first;
	}

	void memory_pool::deallocate(void* aData) {
		// Check for closed block
		auto end = mClosedBlocks.end();
		for(auto i = mClosedBlocks.begin(); i != end; ++i) {
			if(i->first == aData) {
				const block tmp = *i;
				mClosedBlocks.erase(i);
				end = mOpenBlocks.end();
				// Check for block that follows this one
				const void* const next = static_cast<uint8_t*>(tmp.first) + tmp.second;
				for(i = mOpenBlocks.begin(); i != end; ++i)if (i->first == next) {
					i->first = tmp.first;
					i->second += tmp.second;
					return;
				}
				// Check for block that precedes this one
				for(i = mOpenBlocks.begin(); i != end; ++i) if(static_cast<uint8_t*>(i->first) + i->second == tmp.first) {
					i->second += tmp.second;
					return;
				}
				mOpenBlocks.push_back(tmp);
				return;
			}
		}

		throw std::runtime_error("solaire::memory_pool::deallocate : Memory not allocated from this allocator");
	}

	// memory_pool_allocator

	memory_pool_allocator::memory_pool_allocator() :
		mAllocator(nullptr),
		mData(nullptr),
		mSize(0)
	{}

	memory_pool_allocator::memory_pool_allocator(memory_pool& aPool) :
		mAllocator(&aPool),
		mData(nullptr),
		mSize(0)
	{}

	memory_pool_allocator::memory_pool_allocator(memory_pool_allocator&& aOther) :
		mAllocator(aOther.mAllocator),
		mData(aOther.mData),
		mSize(aOther.mSize)
	{}

	memory_pool_allocator::~memory_pool_allocator() {
		if(mData) deallocate();
	}

	memory_pool_allocator& memory_pool_allocator::operator=(memory_pool_allocator&& aOther) {
		std::swap(mAllocator, aOther.mAllocator);
		std::swap(mData, aOther.mData);
		std::swap(mSize, aOther.mSize);
		return *this;
	}

	void memory_pool_allocator::deallocate() {
		if(! mData) throw std::runtime_error("solaire::memory_pool_allocator::deallocate : Not allocated");
		mAllocator->deallocate(mData);
		mData = nullptr;
		mData = 0;
	}

	void* memory_pool_allocator::allocate(size_t aSize) {
		if(! mAllocator) throw std::runtime_error("solaire::memory_pool_allocator::allocate : No memory pool assigned");
		if(mData) throw std::runtime_error("solaire::memory_pool_allocator::allocate : Already allocated");
		mData = mAllocator->allocate(aSize);
		mSize = aSize;
	}

	void* memory_pool_allocator::get() const {
		return mData;
	}

	size_t memory_pool_allocator::size() const {
		return mSize;
	}

}<|endoftext|>


<commit_before>
#include <getopt.h>
#include "subcommand.hpp"
#include "index.hpp"
#include "stream.hpp"
#include "genotyper.hpp"
#include "genotypekit.hpp"
#include "stream.hpp"
/**
* GAM sort main
*/

using namespace std;
using namespace vg;
using namespace vg::subcommand;
void help_genotype(char** argv) {
    cerr << "usage: " << argv[0] << " genotype [options] <graph.vg> [reads.index/] > <calls.vcf>" << endl
         << "Compute genotypes from a graph and an indexed collection of reads" << endl
         << endl
         << "options:" << endl
         << "    -j, --json              output in JSON" << endl
         << "    -v, --vcf               output in VCF" << endl
         << "    -G, --gam   GAM         a GAM file to use with variant recall." << endl
         << "    -V, --recall-vcf VCF    recall variants in a specific VCF file." << endl
         << "    -F, --fasta  FASTA" << endl
         << "    -I, --insertions INS" << endl
         << "    -r, --ref PATH          use the given path name as the reference path" << std::endl
         << "    -c, --contig NAME       use the given name as the VCF contig name" << std::endl
         << "    -s, --sample NAME       name the sample in the VCF with the given name" << std::endl
         << "    -o, --offset INT        offset variant positions by this amount" << std::endl
         << "    -l, --length INT        override total sequence length" << std::endl
         << "    -a, --augmented FILE    dump augmented graph to FILE" << std::endl
         << "    -q, --use_mapq          use mapping qualities" << std::endl
         << "    -C, --cactus            use cactus ultrabubbles for site finding" << std::endl
         << "    -S, --subset-graph      only use the reference and areas of the graph with read support" << std::endl
         << "    -i, --realign_indels    realign at indels" << std::endl
         << "    -d, --het_prior_denom   denominator for prior probability of heterozygousness" << std::endl
         << "    -P, --min_per_strand    min consistent reads per strand for an allele" << std::endl
         << "    -p, --progress          show progress" << endl
         << "    -t, --threads N         number of threads to use" << endl;
}

int main_genotype(int argc, char** argv) {

    if (argc <= 2) {
        help_genotype(argv);
        return 1;
    }
    // Should we output genotypes in JSON (true) or Protobuf (false)?
    bool output_json = false;
    // Should we output VCF instead of protobuf?
    bool output_vcf = false;
    // Should we show progress with a progress bar?
    bool show_progress = false;
    // How many threads should we use?
    int thread_count = 0;

    // What reference path should we use
    string ref_path_name;
    // What sample name should we use for output
    string sample_name;
    // What contig name override do we want?
    string contig_name;
    // What offset should we add to coordinates
    int64_t variant_offset = 0;
    // What length override should we use
    int64_t length_override = 0;

    // Should we we just do a quick variant recall,
    // based on this VCF and GAM, then exit?
    string recall_vcf;
    string gam_file;
    string fasta;
    string insertions_file;

    // Should we use mapping qualities?
    bool use_mapq = false;
    // Should we do indel realignment?
    bool realign_indels = false;

    // Should we dump the augmented graph to a file?
    string augmented_file_name;

    // Should we do superbubbles/sites with Cactus (true) or supbub (false)
    bool use_cactus = false;
    // Should we find superbubbles on the supported subset (true) or the whole graph (false)?
    bool subset_graph = false;
    // What should the heterozygous genotype prior be? (1/this)
    double het_prior_denominator = 10.0;
    // At least how many reads must be consistent per strand for a call?
    size_t min_consistent_per_strand = 2;

    bool just_call = false;
    int c;
    optind = 2; // force optind past command positional arguments
    while (true) {
        static struct option long_options[] =
            {
                {"json", no_argument, 0, 'j'},
                {"vcf", no_argument, 0, 'v'},
                {"ref", required_argument, 0, 'r'},
                {"contig", required_argument, 0, 'c'},
                {"sample", required_argument, 0, 's'},
                {"offset", required_argument, 0, 'o'},
                {"length", required_argument, 0, 'l'},
                {"augmented", required_argument, 0, 'a'},
                {"use_mapq", no_argument, 0, 'q'},
                {"cactus", no_argument, 0, 'C'},
                {"subset-graph", no_argument, 0, 'S'},
                {"realign_indels", no_argument, 0, 'i'},
                {"het_prior_denom", required_argument, 0, 'd'},
                {"min_per_strand", required_argument, 0, 'P'},
                {"progress", no_argument, 0, 'p'},
                {"threads", required_argument, 0, 't'},
                {"recall-vcf", required_argument, 0, 'V'},
                {"gam", required_argument, 0, 'G'},
                {"fasta", required_argument, 0, 'F'},
                {"insertions", required_argument, 0, 'I'},
                {"call", no_argument, 0, 'z'},
                {0, 0, 0, 0}
            };

        int option_index = 0;
        c = getopt_long (argc, argv, "hjvr:c:s:o:l:a:qCSid:P:pt:V:I:G:F:z",
                         long_options, &option_index);

        /* Detect the end of the options. */
        if (c == -1)
            break;

        switch (c)
        {
        case 'j':
            output_json = true;
            break;
        case 'v':
            output_vcf = true;
            break;
        case 'r':
            // Set the reference path name
            ref_path_name = optarg;
            break;
        case 'c':
            // Set the contig name for output
            contig_name = optarg;
            break;
        case 's':
            // Set the sample name
            sample_name = optarg;
            break;
        case 'o':
            // Offset variants
            variant_offset = std::stoll(optarg);
            break;
        case 'l':
            // Set a length override
            length_override = std::stoll(optarg);
            break;
        case 'a':
            // Dump augmented graph
            augmented_file_name = optarg;
            break;
        case 'q':
            // Use mapping qualities
            use_mapq = true;
            break;
        case 'C':
            // Use Cactus to find sites
            use_cactus = true;
            break;
        case 'S':
            // Find sites on the graph subset with any read support
            subset_graph = true;
            break;
        case 'z':
            just_call = true;
            break;
        case 'i':
            // Do indel realignment
            realign_indels = true;
            break;
        case 'd':
            // Set heterozygous genotype prior denominator
            het_prior_denominator = std::stod(optarg);
            break;
        case 'P':
            // Set min consistent reads per strand required to keep an allele
            min_consistent_per_strand = std::stoll(optarg);
            break;
        case 'p':
            show_progress = true;
            break;
        case 't':
            thread_count = atoi(optarg);
            break;
        case 'V':
            recall_vcf = optarg;
            break;
        case 'I':
            insertions_file = optarg;
            break;
        case 'F':
            fasta = optarg;
            break;
        case 'G':
            gam_file = optarg;
            break;
        case 'h':
        case '?':
            /* getopt_long already printed an error message. */
            help_genotype(argv);
            exit(1);
            break;
        default:
          abort ();
        }
    }

    if(thread_count > 0) {
        omp_set_num_threads(thread_count);
    }

    // read the graph
    if (optind >= argc) {
        help_genotype(argv);
        return 1;
    }
    if (show_progress) {
        cerr << "Reading input graph..." << endl;
    }
    VG* graph;
    get_input_file(optind, argc, argv, [&](istream& in) {
        graph = new VG(in);
    });

    if (just_call){
        Genotyper gt;
        string gamfi(gam_file);
        string rstr(ref_path_name);
        gt.genotype_svs(graph, gamfi, rstr);
        exit(0);
    }

    string reads_index_name = "";
    if (optind < argc){
        reads_index_name = get_input_file_name(optind, argc, argv);

    }
    // This holds the RocksDB index that has all our reads, indexed by the nodes they visit.
    Index index;

    if (reads_index_name != ""){
        index.open_read_only(reads_index_name);
        gam_file = reads_index_name;
    }

    // Build the set of all the node IDs to operate on
    vector<vg::id_t> graph_ids;
    graph->for_each_node([&](Node* node) {
        // Put all the ids in the set
        graph_ids.push_back(node->id());
    });

    if (!(gam_file.empty() || recall_vcf.empty() || fasta.empty())){
        Genotyper gt;
        vcflib::VariantCallFile* vars = new vcflib::VariantCallFile();
        vars->open(recall_vcf);
        FastaReference* lin_ref = new FastaReference();
        lin_ref->open(fasta);

        vector<FastaReference*> insertions;
        if (!insertions_file.empty()){
            FastaReference* ins = new FastaReference();
            insertions.emplace_back(ins);
            ins->open(insertions_file);
        }
        gt.variant_recall(graph, vars, lin_ref, insertions, gam_file, reads_index_name != "");
        return 0;

    }

    // setup reads index
    if (optind >= argc) {
        help_genotype(argv);
        return 1;
    }

    // Load all the reads matching the graph into memory
    vector<Alignment> alignments;

    if(show_progress) {
        cerr << "Loading reads..." << endl;
    }

    index.for_alignment_to_nodes(graph_ids, [&](const Alignment& alignment) {
        // Extract all the alignments

        // Only take alignments that don't visit nodes not in the graph
        bool contained = true;
        for(size_t i = 0; i < alignment.path().mapping_size(); i++) {
            if(!graph->has_node(alignment.path().mapping(i).position().node_id())) {
                // Throw out the read
                contained = false;
            }
        }

        if(contained) {
            // This alignment completely falls within the graph
            alignments.push_back(alignment);
        }
    });

    if(show_progress) {
        cerr << "Loaded " << alignments.size() << " alignments" << endl;
    }

    // Make a Genotyper to do the genotyping
    Genotyper genotyper;
    // Configure it
    genotyper.use_mapq = use_mapq;
    genotyper.realign_indels = realign_indels;
    assert(het_prior_denominator > 0);
    genotyper.het_prior_logprob = prob_to_logprob(1.0/het_prior_denominator);
    genotyper.min_consistent_per_strand = min_consistent_per_strand;
    // TODO: move arguments below up into configuration
    genotyper.run(*graph,
                  alignments,
                  cout,
                  ref_path_name,
                  contig_name,
                  sample_name,
                  augmented_file_name,
                  use_cactus,
                  subset_graph,
                  show_progress,
                  output_vcf,
                  output_json,
                  length_override,
                  variant_offset);

    delete graph;

    return 0;
}

static Subcommand vg_genotype("genotype", "Genotype (or type) graphs, GAMS, and VCFs.", main_genotype);
<commit_msg>Revert ability to do variant recall off GAM (i.e. require GAM index).<commit_after>
#include <getopt.h>
#include "subcommand.hpp"
#include "index.hpp"
#include "stream.hpp"
#include "genotyper.hpp"
#include "genotypekit.hpp"
#include "stream.hpp"
/**
* GAM sort main
*/

using namespace std;
using namespace vg;
using namespace vg::subcommand;
void help_genotype(char** argv) {
    cerr << "usage: " << argv[0] << " genotype [options] <graph.vg> [reads.index/] > <calls.vcf>" << endl
         << "Compute genotypes from a graph and an indexed collection of reads" << endl
         << endl
         << "options:" << endl
         << "    -j, --json              output in JSON" << endl
         << "    -v, --vcf               output in VCF" << endl
         << "    -G, --gam   GAM         a GAM file to use with variant recall." << endl
         << "    -V, --recall-vcf VCF    recall variants in a specific VCF file." << endl
         << "    -F, --fasta  FASTA" << endl
         << "    -I, --insertions INS" << endl
         << "    -r, --ref PATH          use the given path name as the reference path" << std::endl
         << "    -c, --contig NAME       use the given name as the VCF contig name" << std::endl
         << "    -s, --sample NAME       name the sample in the VCF with the given name" << std::endl
         << "    -o, --offset INT        offset variant positions by this amount" << std::endl
         << "    -l, --length INT        override total sequence length" << std::endl
         << "    -a, --augmented FILE    dump augmented graph to FILE" << std::endl
         << "    -q, --use_mapq          use mapping qualities" << std::endl
         << "    -C, --cactus            use cactus ultrabubbles for site finding" << std::endl
         << "    -S, --subset-graph      only use the reference and areas of the graph with read support" << std::endl
         << "    -i, --realign_indels    realign at indels" << std::endl
         << "    -d, --het_prior_denom   denominator for prior probability of heterozygousness" << std::endl
         << "    -P, --min_per_strand    min consistent reads per strand for an allele" << std::endl
         << "    -p, --progress          show progress" << endl
         << "    -t, --threads N         number of threads to use" << endl;
}

int main_genotype(int argc, char** argv) {

    if (argc <= 2) {
        help_genotype(argv);
        return 1;
    }
    // Should we output genotypes in JSON (true) or Protobuf (false)?
    bool output_json = false;
    // Should we output VCF instead of protobuf?
    bool output_vcf = false;
    // Should we show progress with a progress bar?
    bool show_progress = false;
    // How many threads should we use?
    int thread_count = 0;

    // What reference path should we use
    string ref_path_name;
    // What sample name should we use for output
    string sample_name;
    // What contig name override do we want?
    string contig_name;
    // What offset should we add to coordinates
    int64_t variant_offset = 0;
    // What length override should we use
    int64_t length_override = 0;

    // Should we we just do a quick variant recall,
    // based on this VCF and GAM, then exit?
    string recall_vcf;
    string gam_file;
    string fasta;
    string insertions_file;

    // Should we use mapping qualities?
    bool use_mapq = false;
    // Should we do indel realignment?
    bool realign_indels = false;

    // Should we dump the augmented graph to a file?
    string augmented_file_name;

    // Should we do superbubbles/sites with Cactus (true) or supbub (false)
    bool use_cactus = false;
    // Should we find superbubbles on the supported subset (true) or the whole graph (false)?
    bool subset_graph = false;
    // What should the heterozygous genotype prior be? (1/this)
    double het_prior_denominator = 10.0;
    // At least how many reads must be consistent per strand for a call?
    size_t min_consistent_per_strand = 2;

    bool just_call = false;
    int c;
    optind = 2; // force optind past command positional arguments
    while (true) {
        static struct option long_options[] =
            {
                {"json", no_argument, 0, 'j'},
                {"vcf", no_argument, 0, 'v'},
                {"ref", required_argument, 0, 'r'},
                {"contig", required_argument, 0, 'c'},
                {"sample", required_argument, 0, 's'},
                {"offset", required_argument, 0, 'o'},
                {"length", required_argument, 0, 'l'},
                {"augmented", required_argument, 0, 'a'},
                {"use_mapq", no_argument, 0, 'q'},
                {"cactus", no_argument, 0, 'C'},
                {"subset-graph", no_argument, 0, 'S'},
                {"realign_indels", no_argument, 0, 'i'},
                {"het_prior_denom", required_argument, 0, 'd'},
                {"min_per_strand", required_argument, 0, 'P'},
                {"progress", no_argument, 0, 'p'},
                {"threads", required_argument, 0, 't'},
                {"recall-vcf", required_argument, 0, 'V'},
                {"gam", required_argument, 0, 'G'},
                {"fasta", required_argument, 0, 'F'},
                {"insertions", required_argument, 0, 'I'},
                {"call", no_argument, 0, 'z'},
                {0, 0, 0, 0}
            };

        int option_index = 0;
        c = getopt_long (argc, argv, "hjvr:c:s:o:l:a:qCSid:P:pt:V:I:G:F:z",
                         long_options, &option_index);

        /* Detect the end of the options. */
        if (c == -1)
            break;

        switch (c)
        {
        case 'j':
            output_json = true;
            break;
        case 'v':
            output_vcf = true;
            break;
        case 'r':
            // Set the reference path name
            ref_path_name = optarg;
            break;
        case 'c':
            // Set the contig name for output
            contig_name = optarg;
            break;
        case 's':
            // Set the sample name
            sample_name = optarg;
            break;
        case 'o':
            // Offset variants
            variant_offset = std::stoll(optarg);
            break;
        case 'l':
            // Set a length override
            length_override = std::stoll(optarg);
            break;
        case 'a':
            // Dump augmented graph
            augmented_file_name = optarg;
            break;
        case 'q':
            // Use mapping qualities
            use_mapq = true;
            break;
        case 'C':
            // Use Cactus to find sites
            use_cactus = true;
            break;
        case 'S':
            // Find sites on the graph subset with any read support
            subset_graph = true;
            break;
        case 'z':
            just_call = true;
            break;
        case 'i':
            // Do indel realignment
            realign_indels = true;
            break;
        case 'd':
            // Set heterozygous genotype prior denominator
            het_prior_denominator = std::stod(optarg);
            break;
        case 'P':
            // Set min consistent reads per strand required to keep an allele
            min_consistent_per_strand = std::stoll(optarg);
            break;
        case 'p':
            show_progress = true;
            break;
        case 't':
            thread_count = atoi(optarg);
            break;
        case 'V':
            recall_vcf = optarg;
            break;
        case 'I':
            insertions_file = optarg;
            break;
        case 'F':
            fasta = optarg;
            break;
        case 'G':
            gam_file = optarg;
            break;
        case 'h':
        case '?':
            /* getopt_long already printed an error message. */
            help_genotype(argv);
            exit(1);
            break;
        default:
          abort ();
        }
    }

    if(thread_count > 0) {
        omp_set_num_threads(thread_count);
    }

    // read the graph
    if (optind >= argc) {
        help_genotype(argv);
        return 1;
    }

    if (show_progress) {
        cerr << "Reading input graph..." << endl;
    }
    VG* graph;
    get_input_file(optind, argc, argv, [&](istream& in) {
        graph = new VG(in);
    });

    if (just_call){
        Genotyper gt;
        string gamfi(gam_file);
        string rstr(ref_path_name);
        gt.genotype_svs(graph, gamfi, rstr);
        exit(0);
    }

      // setup reads index
    if (optind >= argc) {
        help_genotype(argv);
        return 1;
    }

    string reads_index_name = "";
    if (optind < argc){
        reads_index_name = get_input_file_name(optind, argc, argv);

    }
    // This holds the RocksDB index that has all our reads, indexed by the nodes they visit.
    Index index;

    if (reads_index_name != ""){
        index.open_read_only(reads_index_name);
        gam_file = reads_index_name;
    }

    // Build the set of all the node IDs to operate on
    vector<vg::id_t> graph_ids;
    graph->for_each_node([&](Node* node) {
        // Put all the ids in the set
        graph_ids.push_back(node->id());
    });

    if (!(recall_vcf.empty() || fasta.empty())){
        Genotyper gt;
        vcflib::VariantCallFile* vars = new vcflib::VariantCallFile();
        vars->open(recall_vcf);
        FastaReference* lin_ref = new FastaReference();
        lin_ref->open(fasta);

        vector<FastaReference*> insertions;
        if (!insertions_file.empty()){
            FastaReference* ins = new FastaReference();
            insertions.emplace_back(ins);
            ins->open(insertions_file);
        }
        gt.variant_recall(graph, vars, lin_ref, insertions, gam_file, reads_index_name != "");
        return 0;

    }

    // Load all the reads matching the graph into memory
    vector<Alignment> alignments;

    if(show_progress) {
        cerr << "Loading reads..." << endl;
    }

    index.for_alignment_to_nodes(graph_ids, [&](const Alignment& alignment) {
        // Extract all the alignments

        // Only take alignments that don't visit nodes not in the graph
        bool contained = true;
        for(size_t i = 0; i < alignment.path().mapping_size(); i++) {
            if(!graph->has_node(alignment.path().mapping(i).position().node_id())) {
                // Throw out the read
                contained = false;
            }
        }

        if(contained) {
            // This alignment completely falls within the graph
            alignments.push_back(alignment);
        }
    });

    if(show_progress) {
        cerr << "Loaded " << alignments.size() << " alignments" << endl;
    }

    // Make a Genotyper to do the genotyping
    Genotyper genotyper;
    // Configure it
    genotyper.use_mapq = use_mapq;
    genotyper.realign_indels = realign_indels;
    assert(het_prior_denominator > 0);
    genotyper.het_prior_logprob = prob_to_logprob(1.0/het_prior_denominator);
    genotyper.min_consistent_per_strand = min_consistent_per_strand;
    // TODO: move arguments below up into configuration
    genotyper.run(*graph,
                  alignments,
                  cout,
                  ref_path_name,
                  contig_name,
                  sample_name,
                  augmented_file_name,
                  use_cactus,
                  subset_graph,
                  show_progress,
                  output_vcf,
                  output_json,
                  length_override,
                  variant_offset);

    delete graph;

    return 0;
}

static Subcommand vg_genotype("genotype", "Genotype (or type) graphs, GAMS, and VCFs.", main_genotype);
<|endoftext|>


<commit_before>
#include <sstream>
#include <algorithm>

#include "data/vocab.h"
#include "common/utils.h"
#include "common/file_stream.h"
#include "3rd_party/exception.h"
#include "3rd_party/yaml-cpp/yaml.h"

Vocab::Vocab() {
}

size_t Vocab::operator[](const std::string& word) const {
    auto it = str2id_.find(word);
    if(it != str2id_.end())
        return it->second;
    else
        return 1;
}

Words Vocab::operator()(const std::vector<std::string>& lineTokens, bool addEOS) const {
  Words words(lineTokens.size());
  std::transform(lineTokens.begin(), lineTokens.end(), words.begin(),
                  [&](const std::string& w) { return (*this)[w]; });
  if(addEOS)
    words.push_back(EOS);
  return words;
}

Words Vocab::operator()(const std::string& line, bool addEOS) const {
  std::vector<std::string> lineTokens;
  Split(line, lineTokens, " ");
  return (*this)(lineTokens, addEOS);
}

std::vector<std::string> Vocab::operator()(const Words& sentence, bool ignoreEOS) const {
  std::vector<std::string> decoded;
  for(size_t i = 0; i < sentence.size(); ++i) {
    if(sentence[i] != EOS || !ignoreEOS) {
      decoded.push_back((*this)[sentence[i]]);
    }
  }
  return decoded;
}

const std::string& Vocab::operator[](size_t id) const {
  UTIL_THROW_IF2(id >= id2str_.size(), "Unknown word id: " << id);
  return id2str_[id];
}

size_t Vocab::size() const {
  return id2str_.size();
}

void Vocab::load(const std::string& path, int max)
{
  YAML::Node vocab = YAML::Load(InputFileStream(path));
  for(auto&& pair : vocab) {
    auto str = pair.first.as<std::string>();
    auto id = pair.second.as<Word>();
    if (id < (Word)max) {
      str2id_[str] = id;
      if(id >= id2str_.size())
        id2str_.resize(id + 1);
      id2str_[id] = str;
    }
  }
  UTIL_THROW_IF2(id2str_.empty(), "Empty vocabulary " << path);
  id2str_[0] = "</s>";

}

class Vocab::VocabFreqOrderer
{
public:
  bool operator()(const Vocab::Str2Id::value_type* a, const Vocab::Str2Id::value_type* b) const {
    return a->second < b->second;
  }
};

void Vocab::create(const std::string& vocabPath, int max, const std::string& trainPath)
{
  UTIL_THROW_IF2(boost::filesystem::exists(vocabPath),
                 "Vocab file " << vocabPath << " exist. Not overwriting");

  //std::cerr << "Vocab::create" << std::endl;
  InputFileStream trainStrm(trainPath);

  // create freqency list, reuse Str2Id but use Id to store freq
  Str2Id vocab;
  std::string line;
  while (getline((std::istream&)trainStrm, line)) {
    //std::cerr << "line=" << line << std::endl;

    std::vector<std::string> toks;
    Split(line, toks);

    for (const std::string &tok: toks) {
      Str2Id::iterator iter = vocab.find(tok);
      if (iter == vocab.end()) {
        //std::cerr << "tok=" << tok << std::endl;
        vocab[tok] = 1;
      }
      else {
        //std::cerr << "tok=" << tok << std::endl;
        size_t &count = iter->second;
        ++count;
      }
    }
  }

  // put into vector & sort
  std::vector<const Str2Id::value_type*> vocabVec;
  vocabVec.reserve(max);

  for (const Str2Id::value_type &p: vocab) {
    //std::cerr << p.first << "=" << p.second << std::endl;
    vocabVec.push_back(&p);
  }
  std::sort(vocabVec.rbegin(), vocabVec.rend(), VocabFreqOrderer());

  // put into class variables
  // AND write to file
  size_t vocabSize = std::min((size_t) max, vocab.size());
  id2str_.resize(vocabSize);

  OutputFileStream vocabStrm(vocabPath);
  vocabStrm << "{\n"
	    << "\"eos\": 0,\n"
	    << "\"UNK\": 1,\n";

  for (size_t i = 0; i < vocabSize; ++i) {
    const Str2Id::value_type *p = vocabVec[i];
    //std::cerr << p->first << "=" << p->second << std::endl;
    const std::string &str = p->first;
    str2id_[str] = i;
    id2str_.push_back(str);

    vocabStrm << "\"" << str << "\": " << (i + 2);
    if (i < vocabSize - 1) {
      vocabStrm << ",";
    }
    (std::ostream&) vocabStrm << std::endl;
  }

  (std::ostream&) vocabStrm << "}" << std::endl;
}

<commit_msg>add eos and UNK to internal datastructures<commit_after>
#include <sstream>
#include <algorithm>

#include "data/vocab.h"
#include "common/utils.h"
#include "common/file_stream.h"
#include "3rd_party/exception.h"
#include "3rd_party/yaml-cpp/yaml.h"

Vocab::Vocab() {
}

size_t Vocab::operator[](const std::string& word) const {
    auto it = str2id_.find(word);
    if(it != str2id_.end())
        return it->second;
    else
        return 1;
}

Words Vocab::operator()(const std::vector<std::string>& lineTokens, bool addEOS) const {
  Words words(lineTokens.size());
  std::transform(lineTokens.begin(), lineTokens.end(), words.begin(),
                  [&](const std::string& w) { return (*this)[w]; });
  if(addEOS)
    words.push_back(EOS);
  return words;
}

Words Vocab::operator()(const std::string& line, bool addEOS) const {
  std::vector<std::string> lineTokens;
  Split(line, lineTokens, " ");
  return (*this)(lineTokens, addEOS);
}

std::vector<std::string> Vocab::operator()(const Words& sentence, bool ignoreEOS) const {
  std::vector<std::string> decoded;
  for(size_t i = 0; i < sentence.size(); ++i) {
    if(sentence[i] != EOS || !ignoreEOS) {
      decoded.push_back((*this)[sentence[i]]);
    }
  }
  return decoded;
}

const std::string& Vocab::operator[](size_t id) const {
  UTIL_THROW_IF2(id >= id2str_.size(), "Unknown word id: " << id);
  return id2str_[id];
}

size_t Vocab::size() const {
  return id2str_.size();
}

void Vocab::load(const std::string& path, int max)
{
  YAML::Node vocab = YAML::Load(InputFileStream(path));
  for(auto&& pair : vocab) {
    auto str = pair.first.as<std::string>();
    auto id = pair.second.as<Word>();
    if (id < (Word)max) {
      str2id_[str] = id;
      if(id >= id2str_.size())
        id2str_.resize(id + 1);
      id2str_[id] = str;
    }
  }
  UTIL_THROW_IF2(id2str_.empty(), "Empty vocabulary " << path);
  id2str_[0] = "</s>";

}

class Vocab::VocabFreqOrderer
{
public:
  bool operator()(const Vocab::Str2Id::value_type* a, const Vocab::Str2Id::value_type* b) const {
    return a->second < b->second;
  }
};

void Vocab::create(const std::string& vocabPath, int max, const std::string& trainPath)
{
  UTIL_THROW_IF2(boost::filesystem::exists(vocabPath),
                 "Vocab file " << vocabPath << " exist. Not overwriting");

  //std::cerr << "Vocab::create" << std::endl;
  InputFileStream trainStrm(trainPath);

  // create freqency list, reuse Str2Id but use Id to store freq
  Str2Id vocab;
  std::string line;
  while (getline((std::istream&)trainStrm, line)) {
    //std::cerr << "line=" << line << std::endl;

    std::vector<std::string> toks;
    Split(line, toks);

    for (const std::string &tok: toks) {
      Str2Id::iterator iter = vocab.find(tok);
      if (iter == vocab.end()) {
        //std::cerr << "tok=" << tok << std::endl;
        vocab[tok] = 1;
      }
      else {
        //std::cerr << "tok=" << tok << std::endl;
        size_t &count = iter->second;
        ++count;
      }
    }
  }

  // put into vector & sort
  std::vector<const Str2Id::value_type*> vocabVec;
  vocabVec.reserve(max);

  for (const Str2Id::value_type &p: vocab) {
    //std::cerr << p.first << "=" << p.second << std::endl;
    vocabVec.push_back(&p);
  }
  std::sort(vocabVec.rbegin(), vocabVec.rend(), VocabFreqOrderer());

  // put into class variables
  // AND write to file
  size_t vocabSize = std::min((size_t) max, vocab.size());
  id2str_.resize(vocabSize);

  OutputFileStream vocabStrm(vocabPath);

  vocabStrm << "{\n"
	    << "\"eos\": 0,\n"
	    << "\"UNK\": 1,\n";
  id2str_.push_back("eos");
  id2str_.push_back("UNK");
  str2id_["eos"] = 0;
  str2id_["UNK"] = 1;

  for (size_t i = 0; i < vocabSize; ++i) {
    const Str2Id::value_type *p = vocabVec[i];
    //std::cerr << p->first << "=" << p->second << std::endl;
    const std::string &str = p->first;
    str2id_[str] = i;
    id2str_.push_back(str);

    vocabStrm << "\"" << str << "\": " << (i + 2);
    if (i < vocabSize - 1) {
      vocabStrm << ",";
    }
    (std::ostream&) vocabStrm << std::endl;
  }

  (std::ostream&) vocabStrm << "}" << std::endl;
}

<|endoftext|>


<commit_before>#include "data/vocab.h"
#include "3rd_party/exception.h"
#include "3rd_party/yaml-cpp/yaml.h"
#include "common/logging.h"
#include "common/regex.h"
#include "common/utils.h"
#include "common/filesystem.h"

#include <algorithm>
#include <fstream>
#include <iostream>
#include <sstream>
#include <unordered_map>
#include <unordered_set>

namespace marian {

Vocab::Vocab() {}

size_t Vocab::operator[](const std::string& word) const {
  auto it = str2id_.find(word);
  if(it != str2id_.end())
    return it->second;
  else
    return unkId_;
}

Words Vocab::operator()(const std::vector<std::string>& lineTokens,
                        bool addEOS) const {
  Words words(lineTokens.size());
  std::transform(lineTokens.begin(),
                 lineTokens.end(),
                 words.begin(),
                 [&](const std::string& w) { return (*this)[w]; });
  if(addEOS)
    words.push_back(eosId_);
  return words;
}

Words Vocab::operator()(const std::string& line, bool addEOS) const {
  std::vector<std::string> lineTokens;
  utils::Split(line, lineTokens, " ");
  return (*this)(lineTokens, addEOS);
}

std::vector<std::string> Vocab::operator()(const Words& sentence,
                                           bool ignoreEOS) const {
  std::vector<std::string> decoded;
  for(size_t i = 0; i < sentence.size(); ++i) {
    if((sentence[i] != eosId_ || !ignoreEOS)) {
      decoded.push_back((*this)[sentence[i]]);
    }
  }
  return decoded;
}

const std::string& Vocab::operator[](size_t id) const {
  ABORT_IF(id >= id2str_.size(), "Unknown word id: ", id);
  return id2str_[id];
}

size_t Vocab::size() const {
  return id2str_.size();
}

int Vocab::loadOrCreate(const std::string& vocabPath,
                        const std::string& trainPath,
                        int max) {
  if(vocabPath.empty()) {
    if(filesystem::exists(trainPath + ".json")) {
      return load(trainPath + ".json", max);
    }
    if(filesystem::exists(trainPath + ".yml")) {
      return load(trainPath + ".yml", max);
    }
    create(trainPath + ".yml", trainPath);
    return load(trainPath + ".yml", max);
  } else {
    if(!filesystem::exists(vocabPath))
      create(vocabPath, trainPath);
    return load(vocabPath, max);
  }
}

// helper to insert a word into str2id_[] and id2str_[]
Word Vocab::insertWord(Word id, const std::string& str) {
  str2id_[str] = id;
  if(id >= id2str_.size())
    id2str_.resize(id + 1);
  id2str_[id] = str;
  return id;
};

int Vocab::load(const std::string& vocabPath, int max) {
  bool isJson = regex::regex_search(vocabPath, regex::regex("\\.(json|yml)$"));
  LOG(info,
      "[data] Loading vocabulary from {} file {}",
      isJson ? "JSON/Yaml" : "text",
      vocabPath);
  ABORT_IF(!filesystem::exists(vocabPath),
           "Vocabulary file {} does not exits",
           vocabPath);

  std::map<std::string, Word> vocab;
  // read from JSON (or Yaml) file
  if(isJson) {
    YAML::Node vocabNode = YAML::Load(InputFileStream(vocabPath));
    for(auto&& pair : vocabNode)
      vocab.insert({pair.first.as<std::string>(), pair.second.as<Word>()});
  }
  // read from flat text file
  else {
    std::ifstream in(vocabPath);
    std::string line;
    while(utils::GetLine(in, line)) {
      ABORT_IF(line.empty(),
               "Vocabulary file {} must not contain empty lines",
               vocabPath);
      vocab.insert({line, vocab.size()});
    }
    ABORT_IF(in.bad(), "Vocabulary file {} could not be read", vocabPath);
  }

  std::unordered_set<Word> seenSpecial;

  id2str_.reserve(vocab.size());
  for(auto&& pair : vocab) {
    auto str = pair.first;
    auto id = pair.second;

    if(SPEC2SYM.count(str)) {
      seenSpecial.insert(id);
    }

    // note: this requires ids to be sorted by frequency
    if(!max || id < (Word)max) {
      insertWord(id, str);
    }
  }
  ABORT_IF(id2str_.empty(), "Empty vocabulary: ", vocabPath);

  // look up ids for </s> and <unk>, which are required
  // The name backCompatStr is alternatively accepted for Yaml vocabs if id
  // equals backCompatId.
  auto getRequiredWordId = [&](const std::string& str,
                               const std::string& backCompatStr,
                               Word backCompatId) {
    // back compat with Nematus Yaml dicts
    if(isJson) {
      // if word id 0 or 1 is either empty or has the Nematus-convention string,
      // then use it
      if(backCompatId < id2str_.size()
         && (id2str_[backCompatId].empty()
             || id2str_[backCompatId] == backCompatStr)) {
        LOG(info,
            "[data] Using unused word id {} for {}",
            backCompatStr,
            backCompatId,
            str);
        return backCompatId;
      }
    }
    auto iter = str2id_.find(str);
    ABORT_IF(iter == str2id_.end(),
             "Vocabulary file {} is expected to contain an entry for {}",
             vocabPath,
             str);
    return iter->second;
  };
  eosId_ = getRequiredWordId(DEFAULT_EOS_STR, NEMATUS_EOS_STR, DEFAULT_EOS_ID);
  unkId_ = getRequiredWordId(DEFAULT_UNK_STR, NEMATUS_UNK_STR, DEFAULT_UNK_ID);

  // some special symbols for hard attention
  if(!seenSpecial.empty()) {
    auto requireWord = [&](Word id, const std::string& str) {
      auto iter = str2id_.find(str);
      // word already in vocab: must be at right index, else fail
      if(iter != str2id_.end())
        ABORT_IF(iter->second != id,
                 "special vocabulary entry '{}' is expected to have id {}",
                 str,
                 id);
      else
        insertWord(id, str);
    };
    // @TODO: the hard-att code has not yet been updated to accept EOS at any id
    requireWord(DEFAULT_EOS_ID, DEFAULT_EOS_STR);
    for(auto id : seenSpecial)
      requireWord(id, SYM2SPEC.at(id));
  }

  return std::max((int)id2str_.size(), max);
}

// for fakeBatch()
void Vocab::createFake() {
  eosId_ = insertWord(DEFAULT_EOS_ID, DEFAULT_EOS_STR);
  unkId_ = insertWord(DEFAULT_UNK_ID, DEFAULT_UNK_STR);
}

class Vocab::VocabFreqOrderer {
private:
  std::unordered_map<std::string, size_t>& counter_;

public:
  VocabFreqOrderer(std::unordered_map<std::string, size_t>& counter)
      : counter_(counter) {}

  bool operator()(const std::string& a, const std::string& b) const {
    return counter_[a] > counter_[b] || (counter_[a] == counter_[b] && a < b);
  }
};

void Vocab::create(const std::string& vocabPath, const std::string& trainPath) {
  LOG(info, "[data] Creating vocabulary {} from {}", vocabPath, trainPath);

  filesystem::Path path(vocabPath);
  auto dir = path.parentPath();
  if(dir.empty())
    dir = filesystem::currentPath();

  ABORT_IF(!dir.empty() && !filesystem::isDirectory(dir),
           "Specified vocab directory {} does not exist",
           (std::string)dir);

  ABORT_IF(!dir.empty() && !filesystem::canWrite(dir),
           "No write permission in vocab directory {}",
           (std::string)dir);

  ABORT_IF(filesystem::exists(vocabPath),
           "Vocab file '{}' exists. Not overwriting",
           (std::string)vocabPath);

  InputFileStream trainStrm(trainPath);
  OutputFileStream vocabStrm(vocabPath);
  create(trainStrm, vocabStrm);
}

void Vocab::create(InputFileStream& trainStrm,
                   OutputFileStream& vocabStrm,
                   size_t maxSize) {
  std::string line;
  std::unordered_map<std::string, size_t> counter;

  std::unordered_set<Word> seenSpecial;

  while(getline((std::istream&)trainStrm, line)) {
    std::vector<std::string> toks;
    utils::Split(line, toks);

    for(const std::string& tok : toks) {
      if(SPEC2SYM.count(tok)) {
        seenSpecial.insert(SPEC2SYM.at(tok));
        continue;
      }

      auto iter = counter.find(tok);
      if(iter == counter.end())
        counter[tok] = 1;
      else
        iter->second++;
    }
  }

  std::vector<std::string> vocabVec;
  for(auto& p : counter)
    vocabVec.push_back(p.first);

  std::sort(vocabVec.begin(), vocabVec.end(), VocabFreqOrderer(counter));

  YAML::Node vocabYaml;
  vocabYaml.force_insert(DEFAULT_EOS_STR, DEFAULT_EOS_ID);
  vocabYaml.force_insert(DEFAULT_UNK_STR, DEFAULT_UNK_ID);

  for(auto word : seenSpecial)
    vocabYaml.force_insert(SYM2SPEC.at(word), word);

  Word maxSpec = 1;
  for(auto i : seenSpecial)
    if(i > maxSpec)
      maxSpec = i;

  auto vocabSize = vocabVec.size();
  if(maxSize > maxSpec)
    vocabSize = std::min(maxSize - maxSpec - 1, vocabVec.size());

  for(size_t i = 0; i < vocabSize; ++i)
    vocabYaml.force_insert(vocabVec[i], i + maxSpec + 1);

  (std::ostream&)vocabStrm << vocabYaml;
}
}  // namespace marian
<commit_msg>Support .yaml extension for vocabs<commit_after>#include "data/vocab.h"
#include "3rd_party/exception.h"
#include "3rd_party/yaml-cpp/yaml.h"
#include "common/logging.h"
#include "common/regex.h"
#include "common/utils.h"
#include "common/filesystem.h"

#include <algorithm>
#include <fstream>
#include <iostream>
#include <sstream>
#include <unordered_map>
#include <unordered_set>

namespace marian {

Vocab::Vocab() {}

size_t Vocab::operator[](const std::string& word) const {
  auto it = str2id_.find(word);
  if(it != str2id_.end())
    return it->second;
  else
    return unkId_;
}

Words Vocab::operator()(const std::vector<std::string>& lineTokens,
                        bool addEOS) const {
  Words words(lineTokens.size());
  std::transform(lineTokens.begin(),
                 lineTokens.end(),
                 words.begin(),
                 [&](const std::string& w) { return (*this)[w]; });
  if(addEOS)
    words.push_back(eosId_);
  return words;
}

Words Vocab::operator()(const std::string& line, bool addEOS) const {
  std::vector<std::string> lineTokens;
  utils::Split(line, lineTokens, " ");
  return (*this)(lineTokens, addEOS);
}

std::vector<std::string> Vocab::operator()(const Words& sentence,
                                           bool ignoreEOS) const {
  std::vector<std::string> decoded;
  for(size_t i = 0; i < sentence.size(); ++i) {
    if((sentence[i] != eosId_ || !ignoreEOS)) {
      decoded.push_back((*this)[sentence[i]]);
    }
  }
  return decoded;
}

const std::string& Vocab::operator[](size_t id) const {
  ABORT_IF(id >= id2str_.size(), "Unknown word id: ", id);
  return id2str_[id];
}

size_t Vocab::size() const {
  return id2str_.size();
}

int Vocab::loadOrCreate(const std::string& vocabPath,
                        const std::string& trainPath,
                        int max) {
  if(vocabPath.empty()) {
    if(filesystem::exists(trainPath + ".json")) {
      return load(trainPath + ".json", max);
    }
    if(filesystem::exists(trainPath + ".yaml")) {
      return load(trainPath + ".yaml", max);
    }
    if(filesystem::exists(trainPath + ".yml")) {
      return load(trainPath + ".yml", max);
    }
    create(trainPath + ".yml", trainPath);
    return load(trainPath + ".yml", max);
  } else {
    if(!filesystem::exists(vocabPath))
      create(vocabPath, trainPath);
    return load(vocabPath, max);
  }
}

// helper to insert a word into str2id_[] and id2str_[]
Word Vocab::insertWord(Word id, const std::string& str) {
  str2id_[str] = id;
  if(id >= id2str_.size())
    id2str_.resize(id + 1);
  id2str_[id] = str;
  return id;
};

int Vocab::load(const std::string& vocabPath, int max) {
  bool isJson = regex::regex_search(vocabPath, regex::regex("\\.(json|yaml|yml)$"));
  LOG(info,
      "[data] Loading vocabulary from {} file {}",
      isJson ? "JSON/Yaml" : "text",
      vocabPath);
  ABORT_IF(!filesystem::exists(vocabPath),
           "Vocabulary file {} does not exits",
           vocabPath);

  std::map<std::string, Word> vocab;
  // read from JSON (or Yaml) file
  if(isJson) {
    YAML::Node vocabNode = YAML::Load(InputFileStream(vocabPath));
    for(auto&& pair : vocabNode)
      vocab.insert({pair.first.as<std::string>(), pair.second.as<Word>()});
  }
  // read from flat text file
  else {
    std::ifstream in(vocabPath);
    std::string line;
    while(utils::GetLine(in, line)) {
      ABORT_IF(line.empty(),
               "Vocabulary file {} must not contain empty lines",
               vocabPath);
      vocab.insert({line, vocab.size()});
    }
    ABORT_IF(in.bad(), "Vocabulary file {} could not be read", vocabPath);
  }

  std::unordered_set<Word> seenSpecial;

  id2str_.reserve(vocab.size());
  for(auto&& pair : vocab) {
    auto str = pair.first;
    auto id = pair.second;

    if(SPEC2SYM.count(str)) {
      seenSpecial.insert(id);
    }

    // note: this requires ids to be sorted by frequency
    if(!max || id < (Word)max) {
      insertWord(id, str);
    }
  }
  ABORT_IF(id2str_.empty(), "Empty vocabulary: ", vocabPath);

  // look up ids for </s> and <unk>, which are required
  // The name backCompatStr is alternatively accepted for Yaml vocabs if id
  // equals backCompatId.
  auto getRequiredWordId = [&](const std::string& str,
                               const std::string& backCompatStr,
                               Word backCompatId) {
    // back compat with Nematus Yaml dicts
    if(isJson) {
      // if word id 0 or 1 is either empty or has the Nematus-convention string,
      // then use it
      if(backCompatId < id2str_.size()
         && (id2str_[backCompatId].empty()
             || id2str_[backCompatId] == backCompatStr)) {
        LOG(info,
            "[data] Using unused word id {} for {}",
            backCompatStr,
            backCompatId,
            str);
        return backCompatId;
      }
    }
    auto iter = str2id_.find(str);
    ABORT_IF(iter == str2id_.end(),
             "Vocabulary file {} is expected to contain an entry for {}",
             vocabPath,
             str);
    return iter->second;
  };
  eosId_ = getRequiredWordId(DEFAULT_EOS_STR, NEMATUS_EOS_STR, DEFAULT_EOS_ID);
  unkId_ = getRequiredWordId(DEFAULT_UNK_STR, NEMATUS_UNK_STR, DEFAULT_UNK_ID);

  // some special symbols for hard attention
  if(!seenSpecial.empty()) {
    auto requireWord = [&](Word id, const std::string& str) {
      auto iter = str2id_.find(str);
      // word already in vocab: must be at right index, else fail
      if(iter != str2id_.end())
        ABORT_IF(iter->second != id,
                 "special vocabulary entry '{}' is expected to have id {}",
                 str,
                 id);
      else
        insertWord(id, str);
    };
    // @TODO: the hard-att code has not yet been updated to accept EOS at any id
    requireWord(DEFAULT_EOS_ID, DEFAULT_EOS_STR);
    for(auto id : seenSpecial)
      requireWord(id, SYM2SPEC.at(id));
  }

  return std::max((int)id2str_.size(), max);
}

// for fakeBatch()
void Vocab::createFake() {
  eosId_ = insertWord(DEFAULT_EOS_ID, DEFAULT_EOS_STR);
  unkId_ = insertWord(DEFAULT_UNK_ID, DEFAULT_UNK_STR);
}

class Vocab::VocabFreqOrderer {
private:
  std::unordered_map<std::string, size_t>& counter_;

public:
  VocabFreqOrderer(std::unordered_map<std::string, size_t>& counter)
      : counter_(counter) {}

  bool operator()(const std::string& a, const std::string& b) const {
    return counter_[a] > counter_[b] || (counter_[a] == counter_[b] && a < b);
  }
};

void Vocab::create(const std::string& vocabPath, const std::string& trainPath) {
  LOG(info, "[data] Creating vocabulary {} from {}", vocabPath, trainPath);

  filesystem::Path path(vocabPath);
  auto dir = path.parentPath();
  if(dir.empty())
    dir = filesystem::currentPath();

  ABORT_IF(!dir.empty() && !filesystem::isDirectory(dir),
           "Specified vocab directory {} does not exist",
           (std::string)dir);

  ABORT_IF(!dir.empty() && !filesystem::canWrite(dir),
           "No write permission in vocab directory {}",
           (std::string)dir);

  ABORT_IF(filesystem::exists(vocabPath),
           "Vocab file '{}' exists. Not overwriting",
           (std::string)vocabPath);

  InputFileStream trainStrm(trainPath);
  OutputFileStream vocabStrm(vocabPath);
  create(trainStrm, vocabStrm);
}

void Vocab::create(InputFileStream& trainStrm,
                   OutputFileStream& vocabStrm,
                   size_t maxSize) {
  std::string line;
  std::unordered_map<std::string, size_t> counter;

  std::unordered_set<Word> seenSpecial;

  while(getline((std::istream&)trainStrm, line)) {
    std::vector<std::string> toks;
    utils::Split(line, toks);

    for(const std::string& tok : toks) {
      if(SPEC2SYM.count(tok)) {
        seenSpecial.insert(SPEC2SYM.at(tok));
        continue;
      }

      auto iter = counter.find(tok);
      if(iter == counter.end())
        counter[tok] = 1;
      else
        iter->second++;
    }
  }

  std::vector<std::string> vocabVec;
  for(auto& p : counter)
    vocabVec.push_back(p.first);

  std::sort(vocabVec.begin(), vocabVec.end(), VocabFreqOrderer(counter));

  YAML::Node vocabYaml;
  vocabYaml.force_insert(DEFAULT_EOS_STR, DEFAULT_EOS_ID);
  vocabYaml.force_insert(DEFAULT_UNK_STR, DEFAULT_UNK_ID);

  for(auto word : seenSpecial)
    vocabYaml.force_insert(SYM2SPEC.at(word), word);

  Word maxSpec = 1;
  for(auto i : seenSpecial)
    if(i > maxSpec)
      maxSpec = i;

  auto vocabSize = vocabVec.size();
  if(maxSize > maxSpec)
    vocabSize = std::min(maxSize - maxSpec - 1, vocabVec.size());

  for(size_t i = 0; i < vocabSize; ++i)
    vocabYaml.force_insert(vocabVec[i], i + maxSpec + 1);

  (std::ostream&)vocabStrm << vocabYaml;
}
}  // namespace marian
<|endoftext|>


<commit_before>// Copyright (c) 2011 The LevelDB Authors. All rights reserved.
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file. See the AUTHORS file for names of contributors.

#include <ctype.h>
#include <stdio.h>
#include "db/filename.h"
#include "db/dbformat.h"
#include "leveldb/env.h"
#include "util/logging.h"

namespace leveldb {

// A utility routine: write "data" to the named file and Sync() it.
extern Status WriteStringToFileSync(Env* env, const Slice& data,
                                    const std::string& fname);

static std::string MakeFileName(const std::string& name, uint64_t number,
                                const char* suffix) {
  char buf[100];
  snprintf(buf, sizeof(buf), "/%06llu.%s",
           static_cast<unsigned long long>(number),
           suffix);
  return name + buf;
}

std::string LogFileName(const std::string& name, uint64_t number) {
  assert(number > 0);
  return MakeFileName(name, number, "log");
}

std::string TableFileName(const std::string& name, uint64_t number) {
  assert(number > 0);
  return MakeFileName(name, number, "ldb");
}

std::string SSTTableFileName(const std::string& name, uint64_t number) {
  assert(number > 0);
  return MakeFileName(name, number, "sst");
}

std::string DescriptorFileName(const std::string& dbname, uint64_t number) {
  assert(number > 0);
  char buf[100];
  snprintf(buf, sizeof(buf), "/MANIFEST-%06llu",
           static_cast<unsigned long long>(number));
  return dbname + buf;
}

std::string CurrentFileName(const std::string& dbname) {
  return dbname + "/CURRENT";
}

std::string LockFileName(const std::string& dbname) {
  return dbname + "/LOCK";
}

std::string TempFileName(const std::string& dbname, uint64_t number) {
  assert(number > 0);
  return MakeFileName(dbname, number, "dbtmp");
}

std::string InfoLogFileName(const std::string& dbname) {
  return dbname + "/LOG";
}

// Return the name of the old info log file for "dbname".
std::string OldInfoLogFileName(const std::string& dbname) {
  return dbname + "/LOG.old";
}

// Owned filenames have the form:
//    dbname/CURRENT
//    dbname/LOCK
//    dbname/LOG
//    dbname/LOG.old
//    dbname/MANIFEST-[0-9]+
//    dbname/[0-9]+.(log|sst|ldb)
bool ParseFileName(const std::string& fname,
                   uint64_t* number,
                   FileType* type) {
  Slice rest(fname);
  if (rest == "CURRENT") {
    *number = 0;
    *type = kCurrentFile;
  } else if (rest == "LOCK") {
    *number = 0;
    *type = kDBLockFile;
  } else if (rest == "LOG" || rest == "LOG.old") {
    *number = 0;
    *type = kInfoLogFile;
  } else if (rest.starts_with("MANIFEST-")) {
    rest.remove_prefix(strlen("MANIFEST-"));
    uint64_t num;
    if (!ConsumeDecimalNumber(&rest, &num)) {
      return false;
    }
    if (!rest.empty()) {
      return false;
    }
    *type = kDescriptorFile;
    *number = num;
  } else {
    // Avoid strtoull() to keep filename format independent of the
    // current locale
    uint64_t num;
    if (!ConsumeDecimalNumber(&rest, &num)) {
      return false;
    }
    Slice suffix = rest;
    if (suffix == Slice(".log")) {
      *type = kLogFile;
    } else if (suffix == Slice(".sst") || suffix == Slice(".ldb")) {
      *type = kTableFile;
    } else if (suffix == Slice(".dbtmp")) {
      *type = kTempFile;
    } else {
      return false;
    }
    *number = num;
  }
  return true;
}

Status SetCurrentFile(Env* env, const std::string& dbname,
                      uint64_t descriptor_number) {
  // Remove leading "dbname/" and add newline to manifest file name
  std::string manifest = DescriptorFileName(dbname, descriptor_number);
  Slice contents = manifest;
  assert(contents.starts_with(dbname + "/"));
  contents.remove_prefix(dbname.size() + 1);
  std::string tmp = TempFileName(dbname, descriptor_number);
  Status s = WriteStringToFileSync(env, contents.ToString() + "\n", tmp);
  if (s.ok()) {
    s = env->RenameFile(tmp, CurrentFileName(dbname));
  }
  if (!s.ok()) {
    env->DeleteFile(tmp);
  }
  return s;
}

}  // namespace leveldb
<commit_msg>Delete filename.cc<commit_after><|endoftext|>


<commit_before><commit_msg>removed unecessary comments<commit_after><|endoftext|>


<commit_before>/*
 * libtins is a net packet wrapper library for crafting and
 * interpreting sniffed packets.
 *
 * Copyright (C) 2011 Nasel
 *
 * This library is free software; you can redistribute it and/or
 * modify it under the terms of the GNU Lesser General Public
 * License as published by the Free Software Foundation; either
 * version 2.1 of the License, or (at your option) any later version.
 *
 * This library is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
 * Lesser General Public License for more details.
 *
 * You should have received a copy of the GNU Lesser General Public
 * License along with this library; if not, write to the Free Software
 * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301  USA
 */

#include <cassert>
#include <cstring>

#ifndef WIN32
    #include <net/ethernet.h>
    #include <netpacket/packet.h>
    #include <netinet/in.h>
#endif

#include "ethernetII.h"
#include "utils.h"

#include <iostream>

Tins::EthernetII::EthernetII(const uint8_t* mac_dst, const uint8_t* mac_src, const std::string& iface, PDU* child) throw (std::runtime_error) : PDU(ETHERTYPE_IP, child) {

    this->dst_mac(mac_dst);
    this->src_mac(mac_src);
    this->iface(iface);
    this->header.payload_type = 0;

}

Tins::EthernetII::EthernetII(const uint8_t* mac_dst, const uint8_t* mac_src, uint32_t iface_index, PDU* child) : PDU(ETHERTYPE_IP, child) {
    this->dst_mac(mac_dst);
    this->src_mac(mac_src);
    this->iface(iface_index);
    this->header.payload_type = 0;
}

void Tins::EthernetII::dst_mac(const uint8_t* new_dst_mac) {
    memcpy(this->header.dst_mac, new_dst_mac, 6);
}

void Tins::EthernetII::src_mac(const uint8_t* new_src_mac) {
    memcpy(this->header.src_mac, new_src_mac, 6);
}

void Tins::EthernetII::iface(uint32_t new_iface_index) {
    this->_iface_index = new_iface_index;
}

void Tins::EthernetII::iface(const std::string& new_iface) throw (std::runtime_error) {
    if (!Tins::Utils::interface_id(new_iface, this->_iface_index)) {
        throw std::runtime_error("Invalid interface name!");
    }
}

uint32_t Tins::EthernetII::header_size() const {
    return sizeof(ethernet_header);
}

bool Tins::EthernetII::send(PacketSender* sender) {

    struct sockaddr_ll addr;

    memset(&addr, 0, sizeof(struct sockaddr_ll));

    addr.sll_family = Utils::net_to_host_s(PF_PACKET);
    addr.sll_protocol = Utils::net_to_host_s(ETH_P_ALL);
    addr.sll_halen = 6;
    addr.sll_ifindex = this->_iface_index;
    memcpy(&(addr.sll_addr), this->header.dst_mac, 6);

    return sender->send_l2(this, (struct sockaddr*)&addr, (uint32_t)sizeof(addr));

}

void Tins::EthernetII::write_serialization(uint8_t *buffer, uint32_t total_sz, const PDU *parent) {
    uint32_t my_sz = header_size();
    assert(total_sz >= my_sz);

    /* Inner type defaults to IP */

    if ((this->header.payload_type == 0) && this->inner_pdu()) {
        uint16_t type = ETHERTYPE_IP;
        switch (this->inner_pdu()->pdu_type()) {
            case PDU::IP:
                type = ETHERTYPE_IP;
                break;
            case PDU::ARP:
                type = ETHERTYPE_ARP;
                break;
            default:
                type = 0;
        }
        std::cout << std::hex << type << '\n';
        this->header.payload_type = Utils::net_to_host_s(type);
    }

    memcpy(buffer, &this->header, sizeof(ethernet_header));

}
<commit_msg>Removed Debug printing code<commit_after>/*
 * libtins is a net packet wrapper library for crafting and
 * interpreting sniffed packets.
 *
 * Copyright (C) 2011 Nasel
 *
 * This library is free software; you can redistribute it and/or
 * modify it under the terms of the GNU Lesser General Public
 * License as published by the Free Software Foundation; either
 * version 2.1 of the License, or (at your option) any later version.
 *
 * This library is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
 * Lesser General Public License for more details.
 *
 * You should have received a copy of the GNU Lesser General Public
 * License along with this library; if not, write to the Free Software
 * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301  USA
 */

#include <cassert>
#include <cstring>

#ifndef WIN32
    #include <net/ethernet.h>
    #include <netpacket/packet.h>
    #include <netinet/in.h>
#endif

#include "ethernetII.h"
#include "utils.h"

Tins::EthernetII::EthernetII(const uint8_t* mac_dst, const uint8_t* mac_src, const std::string& iface, PDU* child) throw (std::runtime_error) : PDU(ETHERTYPE_IP, child) {

    this->dst_mac(mac_dst);
    this->src_mac(mac_src);
    this->iface(iface);
    this->header.payload_type = 0;

}

Tins::EthernetII::EthernetII(const uint8_t* mac_dst, const uint8_t* mac_src, uint32_t iface_index, PDU* child) : PDU(ETHERTYPE_IP, child) {
    this->dst_mac(mac_dst);
    this->src_mac(mac_src);
    this->iface(iface_index);
    this->header.payload_type = 0;
}

void Tins::EthernetII::dst_mac(const uint8_t* new_dst_mac) {
    memcpy(this->header.dst_mac, new_dst_mac, 6);
}

void Tins::EthernetII::src_mac(const uint8_t* new_src_mac) {
    memcpy(this->header.src_mac, new_src_mac, 6);
}

void Tins::EthernetII::iface(uint32_t new_iface_index) {
    this->_iface_index = new_iface_index;
}

void Tins::EthernetII::iface(const std::string& new_iface) throw (std::runtime_error) {
    if (!Tins::Utils::interface_id(new_iface, this->_iface_index)) {
        throw std::runtime_error("Invalid interface name!");
    }
}

uint32_t Tins::EthernetII::header_size() const {
    return sizeof(ethernet_header);
}

bool Tins::EthernetII::send(PacketSender* sender) {

    struct sockaddr_ll addr;

    memset(&addr, 0, sizeof(struct sockaddr_ll));

    addr.sll_family = Utils::net_to_host_s(PF_PACKET);
    addr.sll_protocol = Utils::net_to_host_s(ETH_P_ALL);
    addr.sll_halen = 6;
    addr.sll_ifindex = this->_iface_index;
    memcpy(&(addr.sll_addr), this->header.dst_mac, 6);

    return sender->send_l2(this, (struct sockaddr*)&addr, (uint32_t)sizeof(addr));

}

void Tins::EthernetII::write_serialization(uint8_t *buffer, uint32_t total_sz, const PDU *parent) {
    uint32_t my_sz = header_size();
    assert(total_sz >= my_sz);

    /* Inner type defaults to IP */

    if ((this->header.payload_type == 0) && this->inner_pdu()) {
        uint16_t type = ETHERTYPE_IP;
        switch (this->inner_pdu()->pdu_type()) {
            case PDU::IP:
                type = ETHERTYPE_IP;
                break;
            case PDU::ARP:
                type = ETHERTYPE_ARP;
                break;
            default:
                type = 0;
        }
        this->header.payload_type = Utils::net_to_host_s(type);
    }

    memcpy(buffer, &this->header, sizeof(ethernet_header));

}
<|endoftext|>