<commit_before>/*********************************************************************************
 *
 * Inviwo - Interactive Visualization Workshop
 *
 * Copyright (c) 2018-2019 Inviwo Foundation
 * All rights reserved.
 *
 * Redistribution and use in source and binary forms, with or without
 * modification, are permitted provided that the following conditions are met:
 *
 * 1. Redistributions of source code must retain the above copyright notice, this
 * list of conditions and the following disclaimer.
 * 2. Redistributions in binary form must reproduce the above copyright notice,
 * this list of conditions and the following disclaimer in the documentation
 * and/or other materials provided with the distribution.
 *
 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
 * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
 * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR
 * ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
 * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
 * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
 * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
 * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 *
 *********************************************************************************/

#include <modules/webbrowser/webbrowsermodule.h>
#include <modules/webbrowser/processors/webbrowserprocessor.h>
#include <modules/webbrowser/webbrowserapp.h>

#include <modules/json/io/json/boolpropertyjsonconverter.h>
#include <modules/json/io/json/buttonpropertyjsonconverter.h>
#include <modules/json/io/json/directorypropertyjsonconverter.h>
#include <modules/json/io/json/filepropertyjsonconverter.h>
#include <modules/json/io/json/minmaxpropertyjsonconverter.h>
#include <modules/json/io/json/optionpropertyjsonconverter.h>
#include <modules/json/io/json/ordinalpropertyjsonconverter.h>
#include <modules/json/io/json/templatepropertyjsonconverter.h>

#include <inviwo/dataframe/io/json/dataframepropertyjsonconverter.h>

#include <inviwo/core/util/filesystem.h>
#include <inviwo/core/util/settings/systemsettings.h>

#include <modules/opengl/shader/shadermanager.h>

// Autogenerated
#include <modules/webbrowser/shader_resources.h>

#include <warn/push>
#include <warn/ignore/all>
#include "include/cef_app.h"
#include "include/cef_command_line.h"
#include "include/cef_parser.h"
#include <warn/pop>

namespace inviwo {

struct OrdinalCEFWidgetReghelper {
    template <typename T>
    auto operator()(WebBrowserModule& m) {
        using PropertyType = OrdinalProperty<T>;
        m.registerPropertyWidgetCEF<PropertyWidgetCEF, PropertyType>();
    }
};

struct MinMaxCEFWidgetReghelper {
    template <typename T>
    auto operator()(WebBrowserModule& m) {
        using PropertyType = MinMaxProperty<T>;
        m.registerPropertyWidgetCEF<PropertyWidgetCEF, PropertyType>();
    }
};

struct OptionCEFWidgetReghelper {
    template <typename T>
    auto operator()(WebBrowserModule& m) {
        using PropertyType = TemplateOptionProperty<T>;
        m.registerPropertyWidgetCEF<PropertyWidgetCEF, PropertyType>();
    }
};

WebBrowserModule::WebBrowserModule(InviwoApplication* app)
    : InviwoModule(app, "WebBrowser")
    // Call 60 times per second
    , doChromiumWork_(Timer::Milliseconds(1000 / 60), []() { CefDoMessageLoopWork(); }) {

    // Register widgets
    registerPropertyWidgetCEF<PropertyWidgetCEF, BoolProperty>();
    registerPropertyWidgetCEF<PropertyWidgetCEF, ButtonProperty>();
    registerPropertyWidgetCEF<PropertyWidgetCEF, FileProperty>();
    registerPropertyWidgetCEF<PropertyWidgetCEF, DirectoryProperty>();
    registerPropertyWidgetCEF<PropertyWidgetCEF, StringProperty>();

    registerPropertyWidgetCEF<PropertyWidgetCEF, DataFrameColumnProperty>();

    // Register ordinal property widgets
    using OrdinalTypes =
        std::tuple<float, vec2, vec3, vec4, mat2, mat3, mat4, double, dvec2, dvec3, dvec4, dmat2,
                   dmat3, dmat4, int, ivec2, ivec3, ivec4, glm::i64, unsigned int, uvec2, uvec3,
                   uvec4, size_t, size2_t, size3_t, size4_t, glm::fquat, glm::dquat>;

    using ScalarTypes = std::tuple<float, double, int, glm::i64, size_t>;
    util::for_each_type<OrdinalTypes>{}(OrdinalCEFWidgetReghelper{}, *this);

    // Register MinMaxProperty widgets
    util::for_each_type<ScalarTypes>{}(MinMaxCEFWidgetReghelper{}, *this);

    // Register option property widgets
    using OptionTypes = std::tuple<unsigned int, int, size_t, float, double, std::string>;
    util::for_each_type<OptionTypes>{}(OptionCEFWidgetReghelper{}, *this);

    if (!app->getSystemSettings().enablePickingProperty_) {
        LogInfo(
            "Enabling picking system setting since it is required for interaction "
            "(View->Settings->System settings->Enable picking).");
        app->getSystemSettings().enablePickingProperty_.set(true);
    }
    // CEF initialization
    // Specify the path for the sub-process executable.
    auto exeExtension = filesystem::getFileExtension(filesystem::getExecutablePath());
    // Assume that inviwo_web_helper is next to the main executable
    auto exeDirectory = filesystem::getFileDirectory(filesystem::getExecutablePath());

    auto locale = app->getUILocale().name();
    if (locale == "C") {
        // Crash when default locale "C" is used. Reproduce with GLFWMinimum application
        locale = std::locale("en_US").name();
    }

    void* sandbox_info = NULL;  // Windows specific

#ifdef __APPLE__  // Mac specific

    // Find CEF framework and helper app in
    // exe.app/Contents/Frameworks directory first
    auto cefParentDir = filesystem::getCanonicalPath(exeDirectory + std::string("/.."));
    auto frameworkDirectory = cefParentDir + "/Frameworks/Chromium Embedded Framework.framework";
    auto frameworkPath = frameworkDirectory + "/Chromium Embedded Framework";
    // Load the CEF framework library at runtime instead of linking directly
    // as required by the macOS sandbox implementation.
    if (!cefLib_.LoadInMain()) {
        throw ModuleInitException("Could not find Chromium Embedded Framework.framework: " +
                                  frameworkPath);
    }

    CefMainArgs args(app->getCommandLineParser().getARGC(), app->getCommandLineParser().getARGV());
    CefSettings settings;
    // CefString(&settings.framework_dir_path).FromASCII((frameworkDirectory).c_str());
    // Crashes if not set and non-default locale is used
    CefString(&settings.locales_dir_path)
        .FromASCII((frameworkDirectory + std::string("/Resources")).c_str());
    CefString(&settings.resources_dir_path)
        .FromASCII((frameworkDirectory + std::string("/Resources")).c_str());
    // Locale returns "en_US.UFT8" but "en.UTF8" is needed by CEF
    auto startErasePos = locale.find('_');
    if (startErasePos != std::string::npos) {
        locale.erase(startErasePos, locale.find('.') - startErasePos);
    }

#else
    CefMainArgs args;
    CefSettings settings;
    // Non-mac systems uses a single helper executable so here we can specify name
    auto subProcessExecutable = exeDirectory + "/cef_web_helper." + exeExtension;
        if (!filesystem::fileExists(subProcessExecutable)) {
        throw ModuleInitException("Could not find web helper executable:" + subProcessExecutable);
    }

    // Necessary to run helpers in separate sub-processes
    // Needed since we do not want to edit the "main" function
    CefString(&settings.browser_subprocess_path).FromASCII(subProcessExecutable.c_str());
#endif

#ifdef WIN32
    // Enable High-DPI support on Windows 7 or newer.
    CefEnableHighDPISupport();
#if defined(CEF_USE_SANDBOX)
    // Manage the life span of the sandbox information object. This is necessary
    // for sandbox support on Windows. See cef_sandbox_win.h for complete details.
    CefScopedSandboxInfo scoped_sandbox;
    sandbox_info = scoped_sandbox.sandbox_info();
#endif
#endif
    // When generating projects with CMake the CEF_USE_SANDBOX value will be defined
    // automatically. Pass -DUSE_SANDBOX=OFF to the CMake command-line to disable
    // use of the sandbox.
#if !defined(CEF_USE_SANDBOX)
    settings.no_sandbox = true;
#endif
    // checkout detailed settings options
    // http://magpcss.org/ceforum/apidocs/projects/%28default%29/_cef_settings_t.html nearly all
    // the settings can be set via args too.
    settings.multi_threaded_message_loop = false;  // not supported, except windows

    // We want to use off-screen rendering
    settings.windowless_rendering_enabled = true;

    // Let the Inviwo application (Qt/GLFW) handle operating system event processing
    // instead of CEF. Setting external message pump to false will cause mouse events
    // to be processed in CefDoMessageLoopWork() instead of in the Qt application loop.
    settings.external_message_pump = true;

    CefString(&settings.locale).FromASCII(locale.c_str());

    // Optional implementation of the CefApp interface.
    CefRefPtr<WebBrowserApp> browserApp(new WebBrowserApp);

    bool result = CefInitialize(args, settings, browserApp, sandbox_info);

    if (!result) {
        throw ModuleInitException("Failed to initialize Chromium Embedded Framework");
    }

    // Add a directory to the search path of the Shadermanager
    webbrowser::addShaderResources(ShaderManager::getPtr(), {getPath(ModulePath::GLSL)});
    // ShaderManager::getPtr()->addShaderSearchPath(getPath(ModulePath::GLSL));

    // Register objects that can be shared with the rest of inviwo here:

    // Processors
    registerProcessor<WebBrowserProcessor>();

    doChromiumWork_.start();
}

WebBrowserModule::~WebBrowserModule() {
    // Stop message pumping and make sure that app has finished processing before CefShutdown
    doChromiumWork_.stop();
    app_->waitForPool();
    CefShutdown();
}

std::string WebBrowserModule::getDataURI(const std::string& data, const std::string& mime_type) {
    return "data:" + mime_type + ";base64," +
           CefURIEncode(CefBase64Encode(data.data(), data.size()), false).ToString();
}

std::string WebBrowserModule::getCefErrorString(cef_errorcode_t code) {
    switch (code) {
        case ERR_NONE:
            return "ERR_NONE";
        case ERR_FAILED:
            return "ERR_FAILED";
        case ERR_ABORTED:
            return "ERR_ABORTED";
        case ERR_INVALID_ARGUMENT:
            return "ERR_INVALID_ARGUMENT";
        case ERR_INVALID_HANDLE:
            return "ERR_INVALID_HANDLE";
        case ERR_FILE_NOT_FOUND:
            return "ERR_FILE_NOT_FOUND";
        case ERR_TIMED_OUT:
            return "ERR_TIMED_OUT";
        case ERR_FILE_TOO_BIG:
            return "ERR_FILE_TOO_BIG";
        case ERR_UNEXPECTED:
            return "ERR_UNEXPECTED";
        case ERR_ACCESS_DENIED:
            return "ERR_ACCESS_DENIED";
        case ERR_NOT_IMPLEMENTED:
            return "ERR_NOT_IMPLEMENTED";
        case ERR_CONNECTION_CLOSED:
            return "ERR_CONNECTION_CLOSED";
        case ERR_CONNECTION_RESET:
            return "ERR_CONNECTION_RESET";
        case ERR_CONNECTION_REFUSED:
            return "ERR_CONNECTION_REFUSED";
        case ERR_CONNECTION_ABORTED:
            return "ERR_CONNECTION_ABORTED";
        case ERR_CONNECTION_FAILED:
            return "ERR_CONNECTION_FAILED";
        case ERR_NAME_NOT_RESOLVED:
            return "ERR_NAME_NOT_RESOLVED";
        case ERR_INTERNET_DISCONNECTED:
            return "ERR_INTERNET_DISCONNECTED";
        case ERR_SSL_PROTOCOL_ERROR:
            return "ERR_SSL_PROTOCOL_ERROR";
        case ERR_ADDRESS_INVALID:
            return "ERR_ADDRESS_INVALID";
        case ERR_ADDRESS_UNREACHABLE:
            return "ERR_ADDRESS_UNREACHABLE";
        case ERR_SSL_CLIENT_AUTH_CERT_NEEDED:
            return "ERR_SSL_CLIENT_AUTH_CERT_NEEDED";
        case ERR_TUNNEL_CONNECTION_FAILED:
            return "ERR_TUNNEL_CONNECTION_FAILED";
        case ERR_NO_SSL_VERSIONS_ENABLED:
            return "ERR_NO_SSL_VERSIONS_ENABLED";
        case ERR_SSL_VERSION_OR_CIPHER_MISMATCH:
            return "ERR_SSL_VERSION_OR_CIPHER_MISMATCH";
        case ERR_SSL_RENEGOTIATION_REQUESTED:
            return "ERR_SSL_RENEGOTIATION_REQUESTED";
        case ERR_CERT_COMMON_NAME_INVALID:
            return "ERR_CERT_COMMON_NAME_INVALID";
        case ERR_CERT_DATE_INVALID:
            return "ERR_CERT_DATE_INVALID";
        case ERR_CERT_AUTHORITY_INVALID:
            return "ERR_CERT_AUTHORITY_INVALID";
        case ERR_CERT_CONTAINS_ERRORS:
            return "ERR_CERT_CONTAINS_ERRORS";
        case ERR_CERT_NO_REVOCATION_MECHANISM:
            return "ERR_CERT_NO_REVOCATION_MECHANISM";
        case ERR_CERT_UNABLE_TO_CHECK_REVOCATION:
            return "ERR_CERT_UNABLE_TO_CHECK_REVOCATION";
        case ERR_CERT_REVOKED:
            return "ERR_CERT_REVOKED";
        case ERR_CERT_INVALID:
            return "ERR_CERT_INVALID";
        case ERR_CERT_END:
            return "ERR_CERT_END";
        case ERR_INVALID_URL:
            return "ERR_INVALID_URL";
        case ERR_DISALLOWED_URL_SCHEME:
            return "ERR_DISALLOWED_URL_SCHEME";
        case ERR_UNKNOWN_URL_SCHEME:
            return "ERR_UNKNOWN_URL_SCHEME";
        case ERR_TOO_MANY_REDIRECTS:
            return "ERR_TOO_MANY_REDIRECTS";
        case ERR_UNSAFE_REDIRECT:
            return "ERR_UNSAFE_REDIRECT";
        case ERR_UNSAFE_PORT:
            return "ERR_UNSAFE_PORT";
        case ERR_INVALID_RESPONSE:
            return "ERR_INVALID_RESPONSE";
        case ERR_INVALID_CHUNKED_ENCODING:
            return "ERR_INVALID_CHUNKED_ENCODING";
        case ERR_METHOD_NOT_SUPPORTED:
            return "ERR_METHOD_NOT_SUPPORTED";
        case ERR_UNEXPECTED_PROXY_AUTH:
            return "ERR_UNEXPECTED_PROXY_AUTH";
        case ERR_EMPTY_RESPONSE:
            return "ERR_EMPTY_RESPONSE";
        case ERR_RESPONSE_HEADERS_TOO_BIG:
            return "ERR_RESPONSE_HEADERS_TOO_BIG";
        case ERR_CACHE_MISS:
            return "ERR_CACHE_MISS";
        case ERR_INSECURE_RESPONSE:
            return "ERR_INSECURE_RESPONSE";
        default:
            return "UNKNOWN";
    }
}
}  // namespace inviwo
<commit_msg>Jenkins: Format fixes<commit_after>/*********************************************************************************
 *
 * Inviwo - Interactive Visualization Workshop
 *
 * Copyright (c) 2018-2019 Inviwo Foundation
 * All rights reserved.
 *
 * Redistribution and use in source and binary forms, with or without
 * modification, are permitted provided that the following conditions are met:
 *
 * 1. Redistributions of source code must retain the above copyright notice, this
 * list of conditions and the following disclaimer.
 * 2. Redistributions in binary form must reproduce the above copyright notice,
 * this list of conditions and the following disclaimer in the documentation
 * and/or other materials provided with the distribution.
 *
 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
 * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
 * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR
 * ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
 * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
 * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
 * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
 * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 *
 *********************************************************************************/

#include <modules/webbrowser/webbrowsermodule.h>
#include <modules/webbrowser/processors/webbrowserprocessor.h>
#include <modules/webbrowser/webbrowserapp.h>

#include <modules/json/io/json/boolpropertyjsonconverter.h>
#include <modules/json/io/json/buttonpropertyjsonconverter.h>
#include <modules/json/io/json/directorypropertyjsonconverter.h>
#include <modules/json/io/json/filepropertyjsonconverter.h>
#include <modules/json/io/json/minmaxpropertyjsonconverter.h>
#include <modules/json/io/json/optionpropertyjsonconverter.h>
#include <modules/json/io/json/ordinalpropertyjsonconverter.h>
#include <modules/json/io/json/templatepropertyjsonconverter.h>

#include <inviwo/dataframe/io/json/dataframepropertyjsonconverter.h>

#include <inviwo/core/util/filesystem.h>
#include <inviwo/core/util/settings/systemsettings.h>

#include <modules/opengl/shader/shadermanager.h>

// Autogenerated
#include <modules/webbrowser/shader_resources.h>

#include <warn/push>
#include <warn/ignore/all>
#include "include/cef_app.h"
#include "include/cef_command_line.h"
#include "include/cef_parser.h"
#include <warn/pop>

namespace inviwo {

struct OrdinalCEFWidgetReghelper {
    template <typename T>
    auto operator()(WebBrowserModule& m) {
        using PropertyType = OrdinalProperty<T>;
        m.registerPropertyWidgetCEF<PropertyWidgetCEF, PropertyType>();
    }
};

struct MinMaxCEFWidgetReghelper {
    template <typename T>
    auto operator()(WebBrowserModule& m) {
        using PropertyType = MinMaxProperty<T>;
        m.registerPropertyWidgetCEF<PropertyWidgetCEF, PropertyType>();
    }
};

struct OptionCEFWidgetReghelper {
    template <typename T>
    auto operator()(WebBrowserModule& m) {
        using PropertyType = TemplateOptionProperty<T>;
        m.registerPropertyWidgetCEF<PropertyWidgetCEF, PropertyType>();
    }
};

WebBrowserModule::WebBrowserModule(InviwoApplication* app)
    : InviwoModule(app, "WebBrowser")
    // Call 60 times per second
    , doChromiumWork_(Timer::Milliseconds(1000 / 60), []() { CefDoMessageLoopWork(); }) {

    // Register widgets
    registerPropertyWidgetCEF<PropertyWidgetCEF, BoolProperty>();
    registerPropertyWidgetCEF<PropertyWidgetCEF, ButtonProperty>();
    registerPropertyWidgetCEF<PropertyWidgetCEF, FileProperty>();
    registerPropertyWidgetCEF<PropertyWidgetCEF, DirectoryProperty>();
    registerPropertyWidgetCEF<PropertyWidgetCEF, StringProperty>();

    registerPropertyWidgetCEF<PropertyWidgetCEF, DataFrameColumnProperty>();

    // Register ordinal property widgets
    using OrdinalTypes =
        std::tuple<float, vec2, vec3, vec4, mat2, mat3, mat4, double, dvec2, dvec3, dvec4, dmat2,
                   dmat3, dmat4, int, ivec2, ivec3, ivec4, glm::i64, unsigned int, uvec2, uvec3,
                   uvec4, size_t, size2_t, size3_t, size4_t, glm::fquat, glm::dquat>;

    using ScalarTypes = std::tuple<float, double, int, glm::i64, size_t>;
    util::for_each_type<OrdinalTypes>{}(OrdinalCEFWidgetReghelper{}, *this);

    // Register MinMaxProperty widgets
    util::for_each_type<ScalarTypes>{}(MinMaxCEFWidgetReghelper{}, *this);

    // Register option property widgets
    using OptionTypes = std::tuple<unsigned int, int, size_t, float, double, std::string>;
    util::for_each_type<OptionTypes>{}(OptionCEFWidgetReghelper{}, *this);

    if (!app->getSystemSettings().enablePickingProperty_) {
        LogInfo(
            "Enabling picking system setting since it is required for interaction "
            "(View->Settings->System settings->Enable picking).");
        app->getSystemSettings().enablePickingProperty_.set(true);
    }
    // CEF initialization
    // Specify the path for the sub-process executable.
    auto exeExtension = filesystem::getFileExtension(filesystem::getExecutablePath());
    // Assume that inviwo_web_helper is next to the main executable
    auto exeDirectory = filesystem::getFileDirectory(filesystem::getExecutablePath());

    auto locale = app->getUILocale().name();
    if (locale == "C") {
        // Crash when default locale "C" is used. Reproduce with GLFWMinimum application
        locale = std::locale("en_US").name();
    }

    void* sandbox_info = NULL;  // Windows specific

#ifdef __APPLE__  // Mac specific

    // Find CEF framework and helper app in
    // exe.app/Contents/Frameworks directory first
    auto cefParentDir = filesystem::getCanonicalPath(exeDirectory + std::string("/.."));
    auto frameworkDirectory = cefParentDir + "/Frameworks/Chromium Embedded Framework.framework";
    auto frameworkPath = frameworkDirectory + "/Chromium Embedded Framework";
    // Load the CEF framework library at runtime instead of linking directly
    // as required by the macOS sandbox implementation.
    if (!cefLib_.LoadInMain()) {
        throw ModuleInitException("Could not find Chromium Embedded Framework.framework: " +
                                  frameworkPath);
    }

    CefMainArgs args(app->getCommandLineParser().getARGC(), app->getCommandLineParser().getARGV());
    CefSettings settings;
    // CefString(&settings.framework_dir_path).FromASCII((frameworkDirectory).c_str());
    // Crashes if not set and non-default locale is used
    CefString(&settings.locales_dir_path)
        .FromASCII((frameworkDirectory + std::string("/Resources")).c_str());
    CefString(&settings.resources_dir_path)
        .FromASCII((frameworkDirectory + std::string("/Resources")).c_str());
    // Locale returns "en_US.UFT8" but "en.UTF8" is needed by CEF
    auto startErasePos = locale.find('_');
    if (startErasePos != std::string::npos) {
        locale.erase(startErasePos, locale.find('.') - startErasePos);
    }

#else
    CefMainArgs args;
    CefSettings settings;
    // Non-mac systems uses a single helper executable so here we can specify name
    auto subProcessExecutable = exeDirectory + "/cef_web_helper." + exeExtension;
    if (!filesystem::fileExists(subProcessExecutable)) {
        throw ModuleInitException("Could not find web helper executable:" + subProcessExecutable);
    }

    // Necessary to run helpers in separate sub-processes
    // Needed since we do not want to edit the "main" function
    CefString(&settings.browser_subprocess_path).FromASCII(subProcessExecutable.c_str());
#endif

#ifdef WIN32
    // Enable High-DPI support on Windows 7 or newer.
    CefEnableHighDPISupport();
#if defined(CEF_USE_SANDBOX)
    // Manage the life span of the sandbox information object. This is necessary
    // for sandbox support on Windows. See cef_sandbox_win.h for complete details.
    CefScopedSandboxInfo scoped_sandbox;
    sandbox_info = scoped_sandbox.sandbox_info();
#endif
#endif
    // When generating projects with CMake the CEF_USE_SANDBOX value will be defined
    // automatically. Pass -DUSE_SANDBOX=OFF to the CMake command-line to disable
    // use of the sandbox.
#if !defined(CEF_USE_SANDBOX)
    settings.no_sandbox = true;
#endif
    // checkout detailed settings options
    // http://magpcss.org/ceforum/apidocs/projects/%28default%29/_cef_settings_t.html nearly all
    // the settings can be set via args too.
    settings.multi_threaded_message_loop = false;  // not supported, except windows

    // We want to use off-screen rendering
    settings.windowless_rendering_enabled = true;

    // Let the Inviwo application (Qt/GLFW) handle operating system event processing
    // instead of CEF. Setting external message pump to false will cause mouse events
    // to be processed in CefDoMessageLoopWork() instead of in the Qt application loop.
    settings.external_message_pump = true;

    CefString(&settings.locale).FromASCII(locale.c_str());

    // Optional implementation of the CefApp interface.
    CefRefPtr<WebBrowserApp> browserApp(new WebBrowserApp);

    bool result = CefInitialize(args, settings, browserApp, sandbox_info);

    if (!result) {
        throw ModuleInitException("Failed to initialize Chromium Embedded Framework");
    }

    // Add a directory to the search path of the Shadermanager
    webbrowser::addShaderResources(ShaderManager::getPtr(), {getPath(ModulePath::GLSL)});
    // ShaderManager::getPtr()->addShaderSearchPath(getPath(ModulePath::GLSL));

    // Register objects that can be shared with the rest of inviwo here:

    // Processors
    registerProcessor<WebBrowserProcessor>();

    doChromiumWork_.start();
}

WebBrowserModule::~WebBrowserModule() {
    // Stop message pumping and make sure that app has finished processing before CefShutdown
    doChromiumWork_.stop();
    app_->waitForPool();
    CefShutdown();
}

std::string WebBrowserModule::getDataURI(const std::string& data, const std::string& mime_type) {
    return "data:" + mime_type + ";base64," +
           CefURIEncode(CefBase64Encode(data.data(), data.size()), false).ToString();
}

std::string WebBrowserModule::getCefErrorString(cef_errorcode_t code) {
    switch (code) {
        case ERR_NONE:
            return "ERR_NONE";
        case ERR_FAILED:
            return "ERR_FAILED";
        case ERR_ABORTED:
            return "ERR_ABORTED";
        case ERR_INVALID_ARGUMENT:
            return "ERR_INVALID_ARGUMENT";
        case ERR_INVALID_HANDLE:
            return "ERR_INVALID_HANDLE";
        case ERR_FILE_NOT_FOUND:
            return "ERR_FILE_NOT_FOUND";
        case ERR_TIMED_OUT:
            return "ERR_TIMED_OUT";
        case ERR_FILE_TOO_BIG:
            return "ERR_FILE_TOO_BIG";
        case ERR_UNEXPECTED:
            return "ERR_UNEXPECTED";
        case ERR_ACCESS_DENIED:
            return "ERR_ACCESS_DENIED";
        case ERR_NOT_IMPLEMENTED:
            return "ERR_NOT_IMPLEMENTED";
        case ERR_CONNECTION_CLOSED:
            return "ERR_CONNECTION_CLOSED";
        case ERR_CONNECTION_RESET:
            return "ERR_CONNECTION_RESET";
        case ERR_CONNECTION_REFUSED:
            return "ERR_CONNECTION_REFUSED";
        case ERR_CONNECTION_ABORTED:
            return "ERR_CONNECTION_ABORTED";
        case ERR_CONNECTION_FAILED:
            return "ERR_CONNECTION_FAILED";
        case ERR_NAME_NOT_RESOLVED:
            return "ERR_NAME_NOT_RESOLVED";
        case ERR_INTERNET_DISCONNECTED:
            return "ERR_INTERNET_DISCONNECTED";
        case ERR_SSL_PROTOCOL_ERROR:
            return "ERR_SSL_PROTOCOL_ERROR";
        case ERR_ADDRESS_INVALID:
            return "ERR_ADDRESS_INVALID";
        case ERR_ADDRESS_UNREACHABLE:
            return "ERR_ADDRESS_UNREACHABLE";
        case ERR_SSL_CLIENT_AUTH_CERT_NEEDED:
            return "ERR_SSL_CLIENT_AUTH_CERT_NEEDED";
        case ERR_TUNNEL_CONNECTION_FAILED:
            return "ERR_TUNNEL_CONNECTION_FAILED";
        case ERR_NO_SSL_VERSIONS_ENABLED:
            return "ERR_NO_SSL_VERSIONS_ENABLED";
        case ERR_SSL_VERSION_OR_CIPHER_MISMATCH:
            return "ERR_SSL_VERSION_OR_CIPHER_MISMATCH";
        case ERR_SSL_RENEGOTIATION_REQUESTED:
            return "ERR_SSL_RENEGOTIATION_REQUESTED";
        case ERR_CERT_COMMON_NAME_INVALID:
            return "ERR_CERT_COMMON_NAME_INVALID";
        case ERR_CERT_DATE_INVALID:
            return "ERR_CERT_DATE_INVALID";
        case ERR_CERT_AUTHORITY_INVALID:
            return "ERR_CERT_AUTHORITY_INVALID";
        case ERR_CERT_CONTAINS_ERRORS:
            return "ERR_CERT_CONTAINS_ERRORS";
        case ERR_CERT_NO_REVOCATION_MECHANISM:
            return "ERR_CERT_NO_REVOCATION_MECHANISM";
        case ERR_CERT_UNABLE_TO_CHECK_REVOCATION:
            return "ERR_CERT_UNABLE_TO_CHECK_REVOCATION";
        case ERR_CERT_REVOKED:
            return "ERR_CERT_REVOKED";
        case ERR_CERT_INVALID:
            return "ERR_CERT_INVALID";
        case ERR_CERT_END:
            return "ERR_CERT_END";
        case ERR_INVALID_URL:
            return "ERR_INVALID_URL";
        case ERR_DISALLOWED_URL_SCHEME:
            return "ERR_DISALLOWED_URL_SCHEME";
        case ERR_UNKNOWN_URL_SCHEME:
            return "ERR_UNKNOWN_URL_SCHEME";
        case ERR_TOO_MANY_REDIRECTS:
            return "ERR_TOO_MANY_REDIRECTS";
        case ERR_UNSAFE_REDIRECT:
            return "ERR_UNSAFE_REDIRECT";
        case ERR_UNSAFE_PORT:
            return "ERR_UNSAFE_PORT";
        case ERR_INVALID_RESPONSE:
            return "ERR_INVALID_RESPONSE";
        case ERR_INVALID_CHUNKED_ENCODING:
            return "ERR_INVALID_CHUNKED_ENCODING";
        case ERR_METHOD_NOT_SUPPORTED:
            return "ERR_METHOD_NOT_SUPPORTED";
        case ERR_UNEXPECTED_PROXY_AUTH:
            return "ERR_UNEXPECTED_PROXY_AUTH";
        case ERR_EMPTY_RESPONSE:
            return "ERR_EMPTY_RESPONSE";
        case ERR_RESPONSE_HEADERS_TOO_BIG:
            return "ERR_RESPONSE_HEADERS_TOO_BIG";
        case ERR_CACHE_MISS:
            return "ERR_CACHE_MISS";
        case ERR_INSECURE_RESPONSE:
            return "ERR_INSECURE_RESPONSE";
        default:
            return "UNKNOWN";
    }
}
}  // namespace inviwo
<|endoftext|>


<commit_before>// Copyright (c) 2011-present, Facebook, Inc.  All rights reserved.
//  This source code is licensed under both the GPLv2 (found in the
//  COPYING file in the root directory) and Apache 2.0 License
//  (found in the LICENSE.Apache file in the root directory).
//
//
// The goal of this tool is to be a simple stress test with focus on catching:
// * bugs in compaction/flush processes, especially the ones that cause
// assertion errors
// * bugs in the code that deletes obsolete files
//
// There are two parts of the test:
// * write_stress, a binary that writes to the database
// * write_stress_runner.py, a script that invokes and kills write_stress
//
// Here are some interesting parts of write_stress:
// * Runs with very high concurrency of compactions and flushes (32 threads
// total) and tries to create a huge amount of small files
// * The keys written to the database are not uniformly distributed -- there is
// a 3-character prefix that mutates occasionally (in prefix mutator thread), in
// such a way that the first character mutates slower than second, which mutates
// slower than third character. That way, the compaction stress tests some
// interesting compaction features like trivial moves and bottommost level
// calculation
// * There is a thread that creates an iterator, holds it for couple of seconds
// and then iterates over all keys. This is supposed to test RocksDB's abilities
// to keep the files alive when there are references to them.
// * Some writes trigger WAL sync. This is stress testing our WAL sync code.
// * At the end of the run, we make sure that we didn't leak any of the sst
// files
//
// write_stress_runner.py changes the mode in which we run write_stress and also
// kills and restarts it. There are some interesting characteristics:
// * At the beginning we divide the full test runtime into smaller parts --
// shorter runtimes (couple of seconds) and longer runtimes (100, 1000) seconds
// * The first time we run write_stress, we destroy the old DB. Every next time
// during the test, we use the same DB.
// * We can run in kill mode or clean-restart mode. Kill mode kills the
// write_stress violently.
// * We can run in mode where delete_obsolete_files_with_fullscan is true or
// false
// * We can run with low_open_files mode turned on or off. When it's turned on,
// we configure table cache to only hold a couple of files -- that way we need
// to reopen files every time we access them.
//
// Another goal was to create a stress test without a lot of parameters. So
// tools/write_stress_runner.py should only take one parameter -- runtime_sec
// and it should figure out everything else on its own.

#include <cstdio>

#ifndef GFLAGS
int main() {
  fprintf(stderr, "Please install gflags to run rocksdb tools\n");
  return 1;
}
#else

#include <gflags/gflags.h>

#define __STDC_FORMAT_MACROS
#include <inttypes.h>
#include <atomic>
#include <random>
#include <set>
#include <string>
#include <thread>

#include "port/port.h"
#include "rocksdb/db.h"
#include "rocksdb/env.h"
#include "rocksdb/options.h"
#include "rocksdb/slice.h"
#include "util/filename.h"

using GFLAGS::ParseCommandLineFlags;
using GFLAGS::RegisterFlagValidator;
using GFLAGS::SetUsageMessage;

DEFINE_int32(key_size, 10, "Key size");
DEFINE_int32(value_size, 100, "Value size");
DEFINE_string(db, "", "Use the db with the following name.");
DEFINE_bool(destroy_db, true,
            "Destroy the existing DB before running the test");

DEFINE_int32(runtime_sec, 10 * 60, "How long are we running for, in seconds");
DEFINE_int32(seed, 139, "Random seed");

DEFINE_double(prefix_mutate_period_sec, 1.0,
              "How often are we going to mutate the prefix");
DEFINE_double(first_char_mutate_probability, 0.1,
              "How likely are we to mutate the first char every period");
DEFINE_double(second_char_mutate_probability, 0.2,
              "How likely are we to mutate the second char every period");
DEFINE_double(third_char_mutate_probability, 0.5,
              "How likely are we to mutate the third char every period");

DEFINE_int32(iterator_hold_sec, 5,
             "How long will the iterator hold files before it gets destroyed");

DEFINE_double(sync_probability, 0.01, "How often are we syncing writes");
DEFINE_bool(delete_obsolete_files_with_fullscan, false,
            "If true, we delete obsolete files after each compaction/flush "
            "using GetChildren() API");
DEFINE_bool(low_open_files_mode, false,
            "If true, we set max_open_files to 20, so that every file access "
            "needs to reopen it");

namespace rocksdb {

static const int kPrefixSize = 3;

class WriteStress {
 public:
  WriteStress() : stop_(false) {
    // initialize key_prefix
    for (int i = 0; i < kPrefixSize; ++i) {
      key_prefix_[i].store('a');
    }

    // Choose a location for the test database if none given with --db=<path>
    if (FLAGS_db.empty()) {
      std::string default_db_path;
      Env::Default()->GetTestDirectory(&default_db_path);
      default_db_path += "/write_stress";
      FLAGS_db = default_db_path;
    }

    Options options;
    if (FLAGS_destroy_db) {
      DestroyDB(FLAGS_db, options);  // ignore
    }

    // make the LSM tree deep, so that we have many concurrent flushes and
    // compactions
    options.create_if_missing = true;
    options.write_buffer_size = 256 * 1024;              // 256k
    options.max_bytes_for_level_base = 1 * 1024 * 1204;  // 1MB
    options.target_file_size_base = 100 * 1204;          // 100k
    options.max_write_buffer_number = 16;
    options.max_background_compactions = 16;
    options.max_background_flushes = 16;
    options.max_open_files = FLAGS_low_open_files_mode ? 20 : -1;
    if (FLAGS_delete_obsolete_files_with_fullscan) {
      options.delete_obsolete_files_period_micros = 0;
    }

    // open DB
    DB* db;
    Status s = DB::Open(options, FLAGS_db, &db);
    if (!s.ok()) {
      fprintf(stderr, "Can't open database: %s\n", s.ToString().c_str());
      std::abort();
    }
    db_.reset(db);
  }

  void WriteThread() {
    std::mt19937 rng(static_cast<unsigned int>(FLAGS_seed));
    std::uniform_real_distribution<double> dist(0, 1);

    auto random_string = [](std::mt19937& r, int len) {
      std::uniform_int_distribution<int> char_dist('a', 'z');
      std::string ret;
      for (int i = 0; i < len; ++i) {
        ret += char_dist(r);
      }
      return ret;
    };

    while (!stop_.load(std::memory_order_relaxed)) {
      std::string prefix;
      prefix.resize(kPrefixSize);
      for (int i = 0; i < kPrefixSize; ++i) {
        prefix[i] = key_prefix_[i].load(std::memory_order_relaxed);
      }
      auto key = prefix + random_string(rng, FLAGS_key_size - kPrefixSize);
      auto value = random_string(rng, FLAGS_value_size);
      WriteOptions woptions;
      woptions.sync = dist(rng) < FLAGS_sync_probability;
      auto s = db_->Put(woptions, key, value);
      if (!s.ok()) {
        fprintf(stderr, "Write to DB failed: %s\n", s.ToString().c_str());
        std::abort();
      }
    }
  }

  void IteratorHoldThread() {
    while (!stop_.load(std::memory_order_relaxed)) {
      std::unique_ptr<Iterator> iterator(db_->NewIterator(ReadOptions()));
      Env::Default()->SleepForMicroseconds(FLAGS_iterator_hold_sec * 1000 *
                                           1000LL);
      for (iterator->SeekToFirst(); iterator->Valid(); iterator->Next()) {
      }
      if (!iterator->status().ok()) {
        fprintf(stderr, "Iterator statuts not OK: %s\n",
                iterator->status().ToString().c_str());
        std::abort();
      }
    }
  }

  void PrefixMutatorThread() {
    std::mt19937 rng(static_cast<unsigned int>(FLAGS_seed));
    std::uniform_real_distribution<double> dist(0, 1);
    std::uniform_int_distribution<int> char_dist('a', 'z');
    while (!stop_.load(std::memory_order_relaxed)) {
      Env::Default()->SleepForMicroseconds(static_cast<int>(
                                           FLAGS_prefix_mutate_period_sec *
                                           1000 * 1000LL));
      if (dist(rng) < FLAGS_first_char_mutate_probability) {
        key_prefix_[0].store(char_dist(rng), std::memory_order_relaxed);
      }
      if (dist(rng) < FLAGS_second_char_mutate_probability) {
        key_prefix_[1].store(char_dist(rng), std::memory_order_relaxed);
      }
      if (dist(rng) < FLAGS_third_char_mutate_probability) {
        key_prefix_[2].store(char_dist(rng), std::memory_order_relaxed);
      }
    }
  }

  int Run() {
    threads_.emplace_back([&]() { WriteThread(); });
    threads_.emplace_back([&]() { PrefixMutatorThread(); });
    threads_.emplace_back([&]() { IteratorHoldThread(); });

    if (FLAGS_runtime_sec == -1) {
      // infinite runtime, until we get killed
      while (true) {
        Env::Default()->SleepForMicroseconds(1000 * 1000);
      }
    }

    Env::Default()->SleepForMicroseconds(FLAGS_runtime_sec * 1000 * 1000);

    stop_.store(true, std::memory_order_relaxed);
    for (auto& t : threads_) {
      t.join();
    }
    threads_.clear();

// Skip checking for leaked files in ROCKSDB_LITE since we don't have access to
// function GetLiveFilesMetaData
#ifndef ROCKSDB_LITE
    // let's see if we leaked some files
    db_->PauseBackgroundWork();
    std::vector<LiveFileMetaData> metadata;
    db_->GetLiveFilesMetaData(&metadata);
    std::set<uint64_t> sst_file_numbers;
    for (const auto& file : metadata) {
      uint64_t number;
      FileType type;
      if (!ParseFileName(file.name, &number, "LOG", &type)) {
        continue;
      }
      if (type == kTableFile) {
        sst_file_numbers.insert(number);
      }
    }

    std::vector<std::string> children;
    Env::Default()->GetChildren(FLAGS_db, &children);
    for (const auto& child : children) {
      uint64_t number;
      FileType type;
      if (!ParseFileName(child, &number, "LOG", &type)) {
        continue;
      }
      if (type == kTableFile) {
        if (sst_file_numbers.find(number) == sst_file_numbers.end()) {
          fprintf(stderr,
                  "Found a table file in DB path that should have been "
                  "deleted: %s\n",
                  child.c_str());
          std::abort();
        }
      }
    }
    db_->ContinueBackgroundWork();
#endif  // !ROCKSDB_LITE

    return 0;
  }

 private:
  // each key is prepended with this prefix. we occasionally change it. third
  // letter is changed more frequently than second, which is changed more
  // frequently than the first one.
  std::atomic<char> key_prefix_[kPrefixSize];
  std::atomic<bool> stop_;
  std::vector<port::Thread> threads_;
  std::unique_ptr<DB> db_;
};

}  // namespace rocksdb

int main(int argc, char** argv) {
  SetUsageMessage(std::string("\nUSAGE:\n") + std::string(argv[0]) +
                  " [OPTIONS]...");
  ParseCommandLineFlags(&argc, &argv, true);
  rocksdb::WriteStress write_stress;
  return write_stress.Run();
}

#endif  // GFLAGS
<commit_msg>tools/write_stress.cc: Correct "1204" typos.<commit_after>// Copyright (c) 2011-present, Facebook, Inc.  All rights reserved.
//  This source code is licensed under both the GPLv2 (found in the
//  COPYING file in the root directory) and Apache 2.0 License
//  (found in the LICENSE.Apache file in the root directory).
//
//
// The goal of this tool is to be a simple stress test with focus on catching:
// * bugs in compaction/flush processes, especially the ones that cause
// assertion errors
// * bugs in the code that deletes obsolete files
//
// There are two parts of the test:
// * write_stress, a binary that writes to the database
// * write_stress_runner.py, a script that invokes and kills write_stress
//
// Here are some interesting parts of write_stress:
// * Runs with very high concurrency of compactions and flushes (32 threads
// total) and tries to create a huge amount of small files
// * The keys written to the database are not uniformly distributed -- there is
// a 3-character prefix that mutates occasionally (in prefix mutator thread), in
// such a way that the first character mutates slower than second, which mutates
// slower than third character. That way, the compaction stress tests some
// interesting compaction features like trivial moves and bottommost level
// calculation
// * There is a thread that creates an iterator, holds it for couple of seconds
// and then iterates over all keys. This is supposed to test RocksDB's abilities
// to keep the files alive when there are references to them.
// * Some writes trigger WAL sync. This is stress testing our WAL sync code.
// * At the end of the run, we make sure that we didn't leak any of the sst
// files
//
// write_stress_runner.py changes the mode in which we run write_stress and also
// kills and restarts it. There are some interesting characteristics:
// * At the beginning we divide the full test runtime into smaller parts --
// shorter runtimes (couple of seconds) and longer runtimes (100, 1000) seconds
// * The first time we run write_stress, we destroy the old DB. Every next time
// during the test, we use the same DB.
// * We can run in kill mode or clean-restart mode. Kill mode kills the
// write_stress violently.
// * We can run in mode where delete_obsolete_files_with_fullscan is true or
// false
// * We can run with low_open_files mode turned on or off. When it's turned on,
// we configure table cache to only hold a couple of files -- that way we need
// to reopen files every time we access them.
//
// Another goal was to create a stress test without a lot of parameters. So
// tools/write_stress_runner.py should only take one parameter -- runtime_sec
// and it should figure out everything else on its own.

#include <cstdio>

#ifndef GFLAGS
int main() {
  fprintf(stderr, "Please install gflags to run rocksdb tools\n");
  return 1;
}
#else

#include <gflags/gflags.h>

#define __STDC_FORMAT_MACROS
#include <inttypes.h>
#include <atomic>
#include <random>
#include <set>
#include <string>
#include <thread>

#include "port/port.h"
#include "rocksdb/db.h"
#include "rocksdb/env.h"
#include "rocksdb/options.h"
#include "rocksdb/slice.h"
#include "util/filename.h"

using GFLAGS::ParseCommandLineFlags;
using GFLAGS::RegisterFlagValidator;
using GFLAGS::SetUsageMessage;

DEFINE_int32(key_size, 10, "Key size");
DEFINE_int32(value_size, 100, "Value size");
DEFINE_string(db, "", "Use the db with the following name.");
DEFINE_bool(destroy_db, true,
            "Destroy the existing DB before running the test");

DEFINE_int32(runtime_sec, 10 * 60, "How long are we running for, in seconds");
DEFINE_int32(seed, 139, "Random seed");

DEFINE_double(prefix_mutate_period_sec, 1.0,
              "How often are we going to mutate the prefix");
DEFINE_double(first_char_mutate_probability, 0.1,
              "How likely are we to mutate the first char every period");
DEFINE_double(second_char_mutate_probability, 0.2,
              "How likely are we to mutate the second char every period");
DEFINE_double(third_char_mutate_probability, 0.5,
              "How likely are we to mutate the third char every period");

DEFINE_int32(iterator_hold_sec, 5,
             "How long will the iterator hold files before it gets destroyed");

DEFINE_double(sync_probability, 0.01, "How often are we syncing writes");
DEFINE_bool(delete_obsolete_files_with_fullscan, false,
            "If true, we delete obsolete files after each compaction/flush "
            "using GetChildren() API");
DEFINE_bool(low_open_files_mode, false,
            "If true, we set max_open_files to 20, so that every file access "
            "needs to reopen it");

namespace rocksdb {

static const int kPrefixSize = 3;

class WriteStress {
 public:
  WriteStress() : stop_(false) {
    // initialize key_prefix
    for (int i = 0; i < kPrefixSize; ++i) {
      key_prefix_[i].store('a');
    }

    // Choose a location for the test database if none given with --db=<path>
    if (FLAGS_db.empty()) {
      std::string default_db_path;
      Env::Default()->GetTestDirectory(&default_db_path);
      default_db_path += "/write_stress";
      FLAGS_db = default_db_path;
    }

    Options options;
    if (FLAGS_destroy_db) {
      DestroyDB(FLAGS_db, options);  // ignore
    }

    // make the LSM tree deep, so that we have many concurrent flushes and
    // compactions
    options.create_if_missing = true;
    options.write_buffer_size = 256 * 1024;              // 256k
    options.max_bytes_for_level_base = 1 * 1024 * 1024;  // 1MB
    options.target_file_size_base = 100 * 1024;          // 100k
    options.max_write_buffer_number = 16;
    options.max_background_compactions = 16;
    options.max_background_flushes = 16;
    options.max_open_files = FLAGS_low_open_files_mode ? 20 : -1;
    if (FLAGS_delete_obsolete_files_with_fullscan) {
      options.delete_obsolete_files_period_micros = 0;
    }

    // open DB
    DB* db;
    Status s = DB::Open(options, FLAGS_db, &db);
    if (!s.ok()) {
      fprintf(stderr, "Can't open database: %s\n", s.ToString().c_str());
      std::abort();
    }
    db_.reset(db);
  }

  void WriteThread() {
    std::mt19937 rng(static_cast<unsigned int>(FLAGS_seed));
    std::uniform_real_distribution<double> dist(0, 1);

    auto random_string = [](std::mt19937& r, int len) {
      std::uniform_int_distribution<int> char_dist('a', 'z');
      std::string ret;
      for (int i = 0; i < len; ++i) {
        ret += char_dist(r);
      }
      return ret;
    };

    while (!stop_.load(std::memory_order_relaxed)) {
      std::string prefix;
      prefix.resize(kPrefixSize);
      for (int i = 0; i < kPrefixSize; ++i) {
        prefix[i] = key_prefix_[i].load(std::memory_order_relaxed);
      }
      auto key = prefix + random_string(rng, FLAGS_key_size - kPrefixSize);
      auto value = random_string(rng, FLAGS_value_size);
      WriteOptions woptions;
      woptions.sync = dist(rng) < FLAGS_sync_probability;
      auto s = db_->Put(woptions, key, value);
      if (!s.ok()) {
        fprintf(stderr, "Write to DB failed: %s\n", s.ToString().c_str());
        std::abort();
      }
    }
  }

  void IteratorHoldThread() {
    while (!stop_.load(std::memory_order_relaxed)) {
      std::unique_ptr<Iterator> iterator(db_->NewIterator(ReadOptions()));
      Env::Default()->SleepForMicroseconds(FLAGS_iterator_hold_sec * 1000 *
                                           1000LL);
      for (iterator->SeekToFirst(); iterator->Valid(); iterator->Next()) {
      }
      if (!iterator->status().ok()) {
        fprintf(stderr, "Iterator statuts not OK: %s\n",
                iterator->status().ToString().c_str());
        std::abort();
      }
    }
  }

  void PrefixMutatorThread() {
    std::mt19937 rng(static_cast<unsigned int>(FLAGS_seed));
    std::uniform_real_distribution<double> dist(0, 1);
    std::uniform_int_distribution<int> char_dist('a', 'z');
    while (!stop_.load(std::memory_order_relaxed)) {
      Env::Default()->SleepForMicroseconds(static_cast<int>(
                                           FLAGS_prefix_mutate_period_sec *
                                           1000 * 1000LL));
      if (dist(rng) < FLAGS_first_char_mutate_probability) {
        key_prefix_[0].store(char_dist(rng), std::memory_order_relaxed);
      }
      if (dist(rng) < FLAGS_second_char_mutate_probability) {
        key_prefix_[1].store(char_dist(rng), std::memory_order_relaxed);
      }
      if (dist(rng) < FLAGS_third_char_mutate_probability) {
        key_prefix_[2].store(char_dist(rng), std::memory_order_relaxed);
      }
    }
  }

  int Run() {
    threads_.emplace_back([&]() { WriteThread(); });
    threads_.emplace_back([&]() { PrefixMutatorThread(); });
    threads_.emplace_back([&]() { IteratorHoldThread(); });

    if (FLAGS_runtime_sec == -1) {
      // infinite runtime, until we get killed
      while (true) {
        Env::Default()->SleepForMicroseconds(1000 * 1000);
      }
    }

    Env::Default()->SleepForMicroseconds(FLAGS_runtime_sec * 1000 * 1000);

    stop_.store(true, std::memory_order_relaxed);
    for (auto& t : threads_) {
      t.join();
    }
    threads_.clear();

// Skip checking for leaked files in ROCKSDB_LITE since we don't have access to
// function GetLiveFilesMetaData
#ifndef ROCKSDB_LITE
    // let's see if we leaked some files
    db_->PauseBackgroundWork();
    std::vector<LiveFileMetaData> metadata;
    db_->GetLiveFilesMetaData(&metadata);
    std::set<uint64_t> sst_file_numbers;
    for (const auto& file : metadata) {
      uint64_t number;
      FileType type;
      if (!ParseFileName(file.name, &number, "LOG", &type)) {
        continue;
      }
      if (type == kTableFile) {
        sst_file_numbers.insert(number);
      }
    }

    std::vector<std::string> children;
    Env::Default()->GetChildren(FLAGS_db, &children);
    for (const auto& child : children) {
      uint64_t number;
      FileType type;
      if (!ParseFileName(child, &number, "LOG", &type)) {
        continue;
      }
      if (type == kTableFile) {
        if (sst_file_numbers.find(number) == sst_file_numbers.end()) {
          fprintf(stderr,
                  "Found a table file in DB path that should have been "
                  "deleted: %s\n",
                  child.c_str());
          std::abort();
        }
      }
    }
    db_->ContinueBackgroundWork();
#endif  // !ROCKSDB_LITE

    return 0;
  }

 private:
  // each key is prepended with this prefix. we occasionally change it. third
  // letter is changed more frequently than second, which is changed more
  // frequently than the first one.
  std::atomic<char> key_prefix_[kPrefixSize];
  std::atomic<bool> stop_;
  std::vector<port::Thread> threads_;
  std::unique_ptr<DB> db_;
};

}  // namespace rocksdb

int main(int argc, char** argv) {
  SetUsageMessage(std::string("\nUSAGE:\n") + std::string(argv[0]) +
                  " [OPTIONS]...");
  ParseCommandLineFlags(&argc, &argv, true);
  rocksdb::WriteStress write_stress;
  return write_stress.Run();
}

#endif  // GFLAGS
<|endoftext|>


<commit_before>/*
 * HyperbolicSpace.cpp
 *
 *  Created on: 20.05.2014
 *      Author: Moritz v. Looz (moritz.looz-corswarem@kit.edu)
 */

#include <assert.h>
#include <cmath>

#include "HyperbolicSpace.h"
#include "../auxiliary/Log.h"

using std::abs;
using std::max;

namespace NetworKit {

HyperbolicSpace::HyperbolicSpace() {

}

HyperbolicSpace::~HyperbolicSpace() {

}

/**
 * This distance measure is taken from the Poincaré disc model.
 */
double HyperbolicSpace::poincareMetric(double phi_a, double  r_a, double phi_b, double r_b) {
	assert(r_a < 1);
	assert(r_b < 1);
	return poincareMetric(polarToCartesian(phi_a, r_a), polarToCartesian(phi_b, r_b));
}

double HyperbolicSpace::poincareMetric(Point2D<double> a, Point2D<double> b) {
	double result = acosh( 1 + 2*a.squaredDistance(b) / ((1 - a.squaredLength())*(1 - b.squaredLength())));
	assert(result >= 0);
	return result;
}

void HyperbolicSpace::fillPoints(vector<double> &angles, vector<double> &radii, double stretch, double alpha) {
	uint64_t n = radii.size();
	double R = stretch*hyperbolicAreaToRadius(n);
	fillPoints(angles, radii, 0, 2*M_PI, 0, R, alpha);
}

void HyperbolicSpace::fillPoints(vector<double> &angles, vector<double> &radii, double minPhi, double maxPhi, double minR, double maxR, double alpha) {
	uint64_t n = radii.size();
	assert(angles.size() == n);

	double mincdf = cosh(alpha*minR);
	double maxcdf = cosh(alpha*maxR);
	std::uniform_real_distribution<double> phidist{minPhi, maxPhi};
	std::uniform_real_distribution<double> rdist{mincdf, maxcdf};
	double r = hyperbolicRadiusToEuclidean(maxR);

	for (uint64_t i = 0; i < n; i++) {
		angles[i] = phidist(Aux::Random::getURNG());
		/**
		 * for the radial coordinate distribution, I took the probability density from Greedy Forwarding in Dynamic Scale-Free Networks Embedded in Hyperbolic Metric Spaces
		 * f (r) = sinh r/(cosh R − 1)
		 * \int sinh = cosh+const
		 */
		double random = rdist(Aux::Random::getURNG());
		double radius = (acosh(random)/alpha);
		assert(radius < maxR);
		//now translate into coordinates of Poincaré disc
		radii[i] = hyperbolicRadiusToEuclidean(radius);
		assert(radii[i] < r);
	}
}

Point2D<double> HyperbolicSpace::polarToCartesian(double phi, double r) {
	return Point2D<double>(r*cos(phi), r*sin(phi));
}

std::map<index, Point<float> > HyperbolicSpace::polarToCartesian(vector<double> &angles, vector<double> &radii) {
	count n = angles.size();
	assert(radii.size() == n);
	std::map<index, Point<float> > result;
	for (index i = 0; i < angles.size(); i++) {
		Point2D<double> coord = HyperbolicSpace::polarToCartesian(angles[i], radii[i]);
		Point<float> temp(coord[0], coord[1]);
		result.insert(std::make_pair(i, temp));
	}
	return result;
}

void HyperbolicSpace::cartesianToPolar(Point2D<double> a, double &phi, double &r) {
	r = a.length();
	if (r == 0) phi = 0;
	else if (a[1] >= 0){
		phi = acos(a[0]/ r);
	} else {
		phi = -acos(a[0] / r);
	}
	if (phi < 0) phi += 2*M_PI;
}

void HyperbolicSpace::getEuclideanCircle(Point2D<double> hyperbolicCenter, double hyperbolicRadius, Point2D<double> &euclideanCenter, double &euclideanRadius) {
	double phi_h, r_h;
	HyperbolicSpace::cartesianToPolar(hyperbolicCenter, phi_h, r_h);
	double r_c;
	HyperbolicSpace::getEuclideanCircle(r_h, hyperbolicRadius, r_c, euclideanRadius);
	euclideanCenter = HyperbolicSpace::polarToCartesian(phi_h, r_c);
}

void HyperbolicSpace::getEuclideanCircle(double r_h, double hyperbolicRadius, double &radialCoordOfEuclideanCenter, double &euclideanRadius) {
	double a = cosh(hyperbolicRadius)-1;
	double b = 1-(r_h*r_h);
	radialCoordOfEuclideanCenter = (2*r_h)/(b*a+2);
	euclideanRadius = sqrt(radialCoordOfEuclideanCenter*radialCoordOfEuclideanCenter - (2*r_h*r_h - b*a)/(b*a+2));
}

double HyperbolicSpace::hyperbolicRadiusToEuclidean(double hyperbolicRadius) {
	double ch = cosh(hyperbolicRadius);
	return sqrt((ch-1)/(ch+1));
}

double HyperbolicSpace::EuclideanRadiusToHyperbolic(double euclideanRadius) {
	double eusq = euclideanRadius*euclideanRadius;
	double result = acosh( 1 + 2*eusq / ((1 - eusq)));
	return result;
}

double HyperbolicSpace::maxRinSlice(double minPhi, double maxPhi, double phi_c, double r_c, double euRadius) {
	double maxCos = max(cos(abs(minPhi - phi_c)), cos(abs(maxPhi - phi_c)));
	//double mirrorAngle;
	//if (phi_c >= M_PI) mirrorAngle = phi_c - M_PI;
	//else mirrorAngle = phi_c + M_PI;

	if (minPhi < phi_c && phi_c < maxPhi) maxCos = 1;
	//applying law of cosines here
	double p = r_c*maxCos;
	double maxR = p + sqrt(p*p - r_c*r_c + euRadius*euRadius);
	return maxR;
}

double HyperbolicSpace::hyperbolicSpaceInEuclideanCircle(double r_c, double d_c,
		double r_max) {
	double result = 0;
	assert(r_c >= 0);
	assert(d_c >= 0);
	assert(r_c <= r_max);
	double min = r_c - d_c;
	double max = std::min(r_c+d_c, r_max);

	if (d_c > r_c) {
		//the query circle overlaps the origin

		if (d_c - r_c < r_max) {
			//remaining query circle is smaller than the disk representation
			result += 2*M_PI*(cosh(EuclideanRadiusToHyperbolic(d_c-r_c))-1);//adding small circle around origin
		} else {
			result += 2*M_PI*(cosh(EuclideanRadiusToHyperbolic(r_max))-1);//adding small circle around origin
		}
		assert(result <= 2*M_PI*(cosh(EuclideanRadiusToHyperbolic(r_max))-1));
		min = std::nextafter(d_c-r_c, std::numeric_limits<double>::max());//correcting integral start to exclude circle
	}

	/**
	 * Now, the integral.
	 * It is 4\int_{min}^{max} \text{acos}(\frac{r_c^2-d_c^2+r^2}{2r_c\cdot r})  \cdot \frac{1}{1-r^2} \cdot (\sinh (\text{acosh}( 1 + 2\frac{r^2}{1 - r^2})))\,dr
	 * This solution was computed by WolframAlpha
	 */

	if (max < min) return result;

	auto realpart = [](double r, double d, double c) {
		double result = acos((c*c-d*d+r*r) / (2*c*r)) / (r*r-1);
		return result;
	};

	auto firstlogpart = [](double r, double d, double c) {
		double s = (c*c-d*d);
		//double denominator = r*r*s*s;
		double rsqs = r*r+s;
		double real = -2*s*sqrt(4*c*c*r*r-rsqs*rsqs);
		double imag = -4*c*c*r*r+2*s*r*r+2*s*s;
		return atan2(imag, real)/2;
	};

	auto secondlogpart = [](double r, double d, double c) {
		double s = (c*c-d*d);
		double rsqs = r*r+s;
		//double denominator = (r*r-1)*(s-1);
		double real = sqrt(4*c*c*r*r-rsqs*rsqs);
		double imag = 2*c*c*(r*r+1)-(s+1)*rsqs;
		imag = imag / sqrt((s+1)*(s+1)-(4*c*c));
		return (s-1)*atan2(imag, real)/(2*sqrt((s+1)*(s+1)-(4*c*c)));
	};

	double lower = -realpart(min, d_c, r_c) -firstlogpart(min, d_c, r_c) + secondlogpart(min, d_c, r_c);
	double upper = -realpart(max, d_c, r_c) -firstlogpart(max, d_c, r_c) + secondlogpart(max, d_c, r_c);
	result += 4*(upper - lower);
	return result;
}
}
<commit_msg>replaced assertion with silent correction<commit_after>/*
 * HyperbolicSpace.cpp
 *
 *  Created on: 20.05.2014
 *      Author: Moritz v. Looz (moritz.looz-corswarem@kit.edu)
 */

#include <assert.h>
#include <cmath>

#include "HyperbolicSpace.h"
#include "../auxiliary/Log.h"

using std::abs;
using std::max;

namespace NetworKit {

HyperbolicSpace::HyperbolicSpace() {

}

HyperbolicSpace::~HyperbolicSpace() {

}

/**
 * This distance measure is taken from the Poincaré disc model.
 */
double HyperbolicSpace::poincareMetric(double phi_a, double  r_a, double phi_b, double r_b) {
	assert(r_a < 1);
	assert(r_b < 1);
	return poincareMetric(polarToCartesian(phi_a, r_a), polarToCartesian(phi_b, r_b));
}

double HyperbolicSpace::poincareMetric(Point2D<double> a, Point2D<double> b) {
	double result = acosh( 1 + 2*a.squaredDistance(b) / ((1 - a.squaredLength())*(1 - b.squaredLength())));
	assert(result >= 0);
	return result;
}

void HyperbolicSpace::fillPoints(vector<double> &angles, vector<double> &radii, double stretch, double alpha) {
	uint64_t n = radii.size();
	double R = stretch*hyperbolicAreaToRadius(n);
	fillPoints(angles, radii, 0, 2*M_PI, 0, R, alpha);
}

void HyperbolicSpace::fillPoints(vector<double> &angles, vector<double> &radii, double minPhi, double maxPhi, double minR, double maxR, double alpha) {
	uint64_t n = radii.size();
	assert(angles.size() == n);

	double mincdf = cosh(alpha*minR);
	double maxcdf = cosh(alpha*maxR);
	std::uniform_real_distribution<double> phidist{minPhi, maxPhi};
	std::uniform_real_distribution<double> rdist{mincdf, maxcdf};
	double r = hyperbolicRadiusToEuclidean(maxR);

	for (uint64_t i = 0; i < n; i++) {
		angles[i] = phidist(Aux::Random::getURNG());
		/**
		 * for the radial coordinate distribution, I took the probability density from Greedy Forwarding in Dynamic Scale-Free Networks Embedded in Hyperbolic Metric Spaces
		 * f (r) = sinh r/(cosh R − 1)
		 * \int sinh = cosh+const
		 */
		double random = rdist(Aux::Random::getURNG());
		double radius = (acosh(random)/alpha);
		//assert(radius < maxR);
		//now translate into coordinates of Poincaré disc
		radii[i] = hyperbolicRadiusToEuclidean(radius);
		assert(radii[i] <= r);
		if (radii[i] == r) radii[i] = std::nextafter(radii[i], 0);

	}
}

Point2D<double> HyperbolicSpace::polarToCartesian(double phi, double r) {
	return Point2D<double>(r*cos(phi), r*sin(phi));
}

std::map<index, Point<float> > HyperbolicSpace::polarToCartesian(vector<double> &angles, vector<double> &radii) {
	count n = angles.size();
	assert(radii.size() == n);
	std::map<index, Point<float> > result;
	for (index i = 0; i < angles.size(); i++) {
		Point2D<double> coord = HyperbolicSpace::polarToCartesian(angles[i], radii[i]);
		Point<float> temp(coord[0], coord[1]);
		result.insert(std::make_pair(i, temp));
	}
	return result;
}

void HyperbolicSpace::cartesianToPolar(Point2D<double> a, double &phi, double &r) {
	r = a.length();
	if (r == 0) phi = 0;
	else if (a[1] >= 0){
		phi = acos(a[0]/ r);
	} else {
		phi = -acos(a[0] / r);
	}
	if (phi < 0) phi += 2*M_PI;
}

void HyperbolicSpace::getEuclideanCircle(Point2D<double> hyperbolicCenter, double hyperbolicRadius, Point2D<double> &euclideanCenter, double &euclideanRadius) {
	double phi_h, r_h;
	HyperbolicSpace::cartesianToPolar(hyperbolicCenter, phi_h, r_h);
	double r_c;
	HyperbolicSpace::getEuclideanCircle(r_h, hyperbolicRadius, r_c, euclideanRadius);
	euclideanCenter = HyperbolicSpace::polarToCartesian(phi_h, r_c);
}

void HyperbolicSpace::getEuclideanCircle(double r_h, double hyperbolicRadius, double &radialCoordOfEuclideanCenter, double &euclideanRadius) {
	double a = cosh(hyperbolicRadius)-1;
	double b = 1-(r_h*r_h);
	radialCoordOfEuclideanCenter = (2*r_h)/(b*a+2);
	euclideanRadius = sqrt(radialCoordOfEuclideanCenter*radialCoordOfEuclideanCenter - (2*r_h*r_h - b*a)/(b*a+2));
}

double HyperbolicSpace::hyperbolicRadiusToEuclidean(double hyperbolicRadius) {
	double ch = cosh(hyperbolicRadius);
	return sqrt((ch-1)/(ch+1));
}

double HyperbolicSpace::EuclideanRadiusToHyperbolic(double euclideanRadius) {
	double eusq = euclideanRadius*euclideanRadius;
	double result = acosh( 1 + 2*eusq / ((1 - eusq)));
	return result;
}

double HyperbolicSpace::maxRinSlice(double minPhi, double maxPhi, double phi_c, double r_c, double euRadius) {
	double maxCos = max(cos(abs(minPhi - phi_c)), cos(abs(maxPhi - phi_c)));
	//double mirrorAngle;
	//if (phi_c >= M_PI) mirrorAngle = phi_c - M_PI;
	//else mirrorAngle = phi_c + M_PI;

	if (minPhi < phi_c && phi_c < maxPhi) maxCos = 1;
	//applying law of cosines here
	double p = r_c*maxCos;
	double maxR = p + sqrt(p*p - r_c*r_c + euRadius*euRadius);
	return maxR;
}

double HyperbolicSpace::hyperbolicSpaceInEuclideanCircle(double r_c, double d_c,
		double r_max) {
	double result = 0;
	assert(r_c >= 0);
	assert(d_c >= 0);
	assert(r_c <= r_max);
	double min = r_c - d_c;
	double max = std::min(r_c+d_c, r_max);

	if (d_c > r_c) {
		//the query circle overlaps the origin

		if (d_c - r_c < r_max) {
			//remaining query circle is smaller than the disk representation
			result += 2*M_PI*(cosh(EuclideanRadiusToHyperbolic(d_c-r_c))-1);//adding small circle around origin
		} else {
			result += 2*M_PI*(cosh(EuclideanRadiusToHyperbolic(r_max))-1);//adding small circle around origin
		}
		assert(result <= 2*M_PI*(cosh(EuclideanRadiusToHyperbolic(r_max))-1));
		min = std::nextafter(d_c-r_c, std::numeric_limits<double>::max());//correcting integral start to exclude circle
	}

	/**
	 * Now, the integral.
	 * It is 4\int_{min}^{max} \text{acos}(\frac{r_c^2-d_c^2+r^2}{2r_c\cdot r})  \cdot \frac{1}{1-r^2} \cdot (\sinh (\text{acosh}( 1 + 2\frac{r^2}{1 - r^2})))\,dr
	 * This solution was computed by WolframAlpha
	 */

	if (max < min) return result;

	auto realpart = [](double r, double d, double c) {
		double result = acos((c*c-d*d+r*r) / (2*c*r)) / (r*r-1);
		return result;
	};

	auto firstlogpart = [](double r, double d, double c) {
		double s = (c*c-d*d);
		//double denominator = r*r*s*s;
		double rsqs = r*r+s;
		double real = -2*s*sqrt(4*c*c*r*r-rsqs*rsqs);
		double imag = -4*c*c*r*r+2*s*r*r+2*s*s;
		return atan2(imag, real)/2;
	};

	auto secondlogpart = [](double r, double d, double c) {
		double s = (c*c-d*d);
		double rsqs = r*r+s;
		//double denominator = (r*r-1)*(s-1);
		double real = sqrt(4*c*c*r*r-rsqs*rsqs);
		double imag = 2*c*c*(r*r+1)-(s+1)*rsqs;
		imag = imag / sqrt((s+1)*(s+1)-(4*c*c));
		return (s-1)*atan2(imag, real)/(2*sqrt((s+1)*(s+1)-(4*c*c)));
	};

	double lower = -realpart(min, d_c, r_c) -firstlogpart(min, d_c, r_c) + secondlogpart(min, d_c, r_c);
	double upper = -realpart(max, d_c, r_c) -firstlogpart(max, d_c, r_c) + secondlogpart(max, d_c, r_c);
	result += 4*(upper - lower);
	return result;
}
}
<|endoftext|>


<commit_before>/*
  Logger - a simple logger for Qt-based applications.
  Copyright (C) 2011 Bjørn Øivind Bjørnsen

  This library is free software; you can redistribute it and/or
  modify it under the terms of the GNU Lesser General Public
  License as published by the Free Software Foundation; either
  version 2.1 of the License, or (at your option) any later version.

  This library is distributed in the hope that it will be useful,
  but WITHOUT ANY WARRANTY; without even the implied warranty of
  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
  Lesser General Public License for more details.

  You should have received a copy of the GNU Lesser General Public
  License along with this library; if not, write to the Free Software
  Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301  USA
  */

/**
  @file

  Implementation of Logger.
  */
#include "logger.h"

// for Indent
#include "debug.h"

#include <iostream>

#include <QMutexLocker>
#include <QSettings>
#include <QCoreApplication>
#include <QDir>
#include <QFile>
#include <QIODevice>
#include <QTextStream>
#include <QTime>

Logger* Logger::_instance = 0;
QMutex Logger::m_creationalMutex;
QMutex Logger::m_operationalMutex;
#ifdef Q_OS_LINUX
//                          Grey   White  Brown    Red
QString Logger::col[4] = { "01;30", "1", "00;33", "01;31" };
#endif

Logger* Logger::instance() throw()
{
    QMutexLocker locker(&m_creationalMutex);
    if(_instance == 0) {
        _instance = new Logger();
    }
    return _instance;
}

void Logger::close()
{
    delete _instance;
    _instance = 0;
}

void Logger::log(LogLevel level, QString message) throw()
{
    if(!logFile.isOpen())
        return;

    QMutexLocker locker(&m_operationalMutex);

    // do we bother with opening the stream and logging?
    if(level < m_logThreshold)
        return;

    // shall we limit the logfile?
    if(m_logLimit) {
        if(m_linesLogged >= m_logLimit) {
            // truncate the log file
            if(!logFile.resize(0))
                return;
            m_linesLogged = 0;
        }
    }

    ++m_linesLogged;

    QTextStream stream(&logFile);

    // print timestamp
    QString output = "[" + QTime::currentTime().toString() + "] ";
    switch(level) {
    case DEBUG:
        output += "[DEBUG]    " + QString(Debug::Indent::getIndent(), ' ');
        break;
    case INFO:
        output += "[INFO]     ";
        break;
    case WARNING:
        output += "[WARNING]  ";
        break;
    case CRITICAL:
        output += "[CRITICAL] ";
        break;
    case NONE:
        // don't log, fall through
    default:
        break;
    }

    output += message + "\n";
    stream << output;
#ifdef Q_OS_LINUX
    if(!qgetenv("LOG_COLOR").isEmpty()) {
        output.prepend("\x1b[" + col[level] + "m");
        output.append("\x1b[00;39m");
    }
#endif
    if(m_logToConsole)
        std::cerr << qPrintable(output);
}

void Logger::setLogPath(QString dir, QString filename)
{
    QMutexLocker locker(&m_operationalMutex);

    QDir logDir;
    logDir.setPath(dir);
    if(!logDir.exists())
        logDir.mkpath(dir);

    if(logFile.isOpen())
        logFile.close();

    logFile.setFileName(dir + QDir::separator() + filename);
    logFile.open(QIODevice::ReadWrite | QIODevice::Text | QIODevice::Truncate);

    // store the settings.
    QSettings settings;

    settings.setValue("Log/log_path", dir);
    settings.setValue("Log/log_filename", filename);
}

QString Logger::logPath() const
{
    QSettings settings;
    return settings.value("Log/log_path").toString();
}

QString Logger::logFilename() const
{
    QSettings settings;
    return settings.value("Log/log_filename").toString();
}

void Logger::setLogThreshold(LogLevel level)
{
    m_logThreshold = level;

    // store the setting
    QSettings s;
    QVariant value;

    switch(m_logThreshold) {
    case DEBUG:
        value = "DEBUG";
        break;
    case INFO:
        value = "INFO";
        break;
    case WARNING:
        value = "WARNING";
        break;
    case CRITICAL:
        value = "CRITICAL";
        break;
    case NONE:
        value = "NONE";
        break;
    }

    s.setValue("Log/log_threshold", value);
}

LogLevel Logger::logThreshold() const
{
    return m_logThreshold;
}

void Logger::setLogToConsole(bool enabled)
{
    m_logToConsole = enabled;
}

bool Logger::logToConsole() const
{
    return m_logToConsole;
}

void Logger::setLogLimit(int numLines)
{
    m_logLimit = numLines;
}

int Logger::logLimit() const
{
    return m_logLimit;
}

void Logger::logMessageHandler(QtMsgType type, const char *msg)
{
    switch(type)
    {
    case QtDebugMsg:
        instance()->log(DEBUG, QString(msg));
        break;

    case QtWarningMsg:
        instance()->log(WARNING, QString(msg));
        break;

    case QtCriticalMsg:
        instance()->log(CRITICAL, QString(msg));
        break;

    case QtFatalMsg:
        instance()->log(CRITICAL, QString(msg));
        // Fatal error, kill the application
        QCoreApplication::quit();
        break;
    }
}

Logger::Logger() throw()
{
    // open file(s) for logging, set logging threshold
    QSettings settings;
    QString path;
    QString filename;
    QString threshold;
    // log to console by default
    m_logToConsole = true;
    // do not limit the logfile by default
    m_logLimit = 0;
    m_linesLogged = 0;
    m_logThreshold = NONE;
    // by default the path is a dot-folder under the users home directory.
    path = settings.value("Log/log_path", QVariant(QString("%1%2.%3").arg(QDir::homePath(), QDir::separator(), QCoreApplication::applicationName()))).toString();
    // default filename is <application name>.log
    filename = settings.value("Log/log_filename", QVariant(QString("%1.log").arg(QCoreApplication::applicationName()))).toString();
    // default threshold is WARNING
    threshold = settings.value("Log/log_threshold", "WARNING").toString();

    setLogPath(path, filename);

    // set log threshold
    QString comp[5] = { "DEBUG", "INFO", "WARNING", "CRITICAL", "NONE" };
    for(int i = 0; i < 5; i++) {
        if(threshold == comp[i]) {
            m_logThreshold = (LogLevel)i;
            break;
        }
    }

    // Setup the qMsgHandler
    oldHandler = qInstallMsgHandler(Logger::logMessageHandler);
}

Logger::~Logger() throw()
{
    qDebug() << "Closing logger.";
    if(logFile.isOpen())
        logFile.close();

    qInstallMsgHandler(oldHandler);
}

<commit_msg>Logger: remove printout.<commit_after>/*
  Logger - a simple logger for Qt-based applications.
  Copyright (C) 2011 Bjørn Øivind Bjørnsen

  This library is free software; you can redistribute it and/or
  modify it under the terms of the GNU Lesser General Public
  License as published by the Free Software Foundation; either
  version 2.1 of the License, or (at your option) any later version.

  This library is distributed in the hope that it will be useful,
  but WITHOUT ANY WARRANTY; without even the implied warranty of
  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
  Lesser General Public License for more details.

  You should have received a copy of the GNU Lesser General Public
  License along with this library; if not, write to the Free Software
  Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301  USA
  */

/**
  @file

  Implementation of Logger.
  */
#include "logger.h"

// for Indent
#include "debug.h"

#include <iostream>

#include <QMutexLocker>
#include <QSettings>
#include <QCoreApplication>
#include <QDir>
#include <QFile>
#include <QIODevice>
#include <QTextStream>
#include <QTime>

Logger* Logger::_instance = 0;
QMutex Logger::m_creationalMutex;
QMutex Logger::m_operationalMutex;
#ifdef Q_OS_LINUX
//                          Grey   White  Brown    Red
QString Logger::col[4] = { "01;30", "1", "00;33", "01;31" };
#endif

Logger* Logger::instance() throw()
{
    QMutexLocker locker(&m_creationalMutex);
    if(_instance == 0) {
        _instance = new Logger();
    }
    return _instance;
}

void Logger::close()
{
    delete _instance;
    _instance = 0;
}

void Logger::log(LogLevel level, QString message) throw()
{
    if(!logFile.isOpen())
        return;

    QMutexLocker locker(&m_operationalMutex);

    // do we bother with opening the stream and logging?
    if(level < m_logThreshold)
        return;

    // shall we limit the logfile?
    if(m_logLimit) {
        if(m_linesLogged >= m_logLimit) {
            // truncate the log file
            if(!logFile.resize(0))
                return;
            m_linesLogged = 0;
        }
    }

    ++m_linesLogged;

    QTextStream stream(&logFile);

    // print timestamp
    QString output = "[" + QTime::currentTime().toString() + "] ";
    switch(level) {
    case DEBUG:
        output += "[DEBUG]    " + QString(Debug::Indent::getIndent(), ' ');
        break;
    case INFO:
        output += "[INFO]     ";
        break;
    case WARNING:
        output += "[WARNING]  ";
        break;
    case CRITICAL:
        output += "[CRITICAL] ";
        break;
    case NONE:
        // don't log, fall through
    default:
        break;
    }

    output += message + "\n";
    stream << output;
#ifdef Q_OS_LINUX
    if(!qgetenv("LOG_COLOR").isEmpty()) {
        output.prepend("\x1b[" + col[level] + "m");
        output.append("\x1b[00;39m");
    }
#endif
    if(m_logToConsole)
        std::cerr << qPrintable(output);
}

void Logger::setLogPath(QString dir, QString filename)
{
    QMutexLocker locker(&m_operationalMutex);

    QDir logDir;
    logDir.setPath(dir);
    if(!logDir.exists())
        logDir.mkpath(dir);

    if(logFile.isOpen())
        logFile.close();

    logFile.setFileName(dir + QDir::separator() + filename);
    logFile.open(QIODevice::ReadWrite | QIODevice::Text | QIODevice::Truncate);

    // store the settings.
    QSettings settings;

    settings.setValue("Log/log_path", dir);
    settings.setValue("Log/log_filename", filename);
}

QString Logger::logPath() const
{
    QSettings settings;
    return settings.value("Log/log_path").toString();
}

QString Logger::logFilename() const
{
    QSettings settings;
    return settings.value("Log/log_filename").toString();
}

void Logger::setLogThreshold(LogLevel level)
{
    m_logThreshold = level;

    // store the setting
    QSettings s;
    QVariant value;

    switch(m_logThreshold) {
    case DEBUG:
        value = "DEBUG";
        break;
    case INFO:
        value = "INFO";
        break;
    case WARNING:
        value = "WARNING";
        break;
    case CRITICAL:
        value = "CRITICAL";
        break;
    case NONE:
        value = "NONE";
        break;
    }

    s.setValue("Log/log_threshold", value);
}

LogLevel Logger::logThreshold() const
{
    return m_logThreshold;
}

void Logger::setLogToConsole(bool enabled)
{
    m_logToConsole = enabled;
}

bool Logger::logToConsole() const
{
    return m_logToConsole;
}

void Logger::setLogLimit(int numLines)
{
    m_logLimit = numLines;
}

int Logger::logLimit() const
{
    return m_logLimit;
}

void Logger::logMessageHandler(QtMsgType type, const char *msg)
{
    switch(type)
    {
    case QtDebugMsg:
        instance()->log(DEBUG, QString(msg));
        break;

    case QtWarningMsg:
        instance()->log(WARNING, QString(msg));
        break;

    case QtCriticalMsg:
        instance()->log(CRITICAL, QString(msg));
        break;

    case QtFatalMsg:
        instance()->log(CRITICAL, QString(msg));
        // Fatal error, kill the application
        QCoreApplication::quit();
        break;
    }
}

Logger::Logger() throw()
{
    // open file(s) for logging, set logging threshold
    QSettings settings;
    QString path;
    QString filename;
    QString threshold;
    // log to console by default
    m_logToConsole = true;
    // do not limit the logfile by default
    m_logLimit = 0;
    m_linesLogged = 0;
    m_logThreshold = NONE;
    // by default the path is a dot-folder under the users home directory.
    path = settings.value("Log/log_path", QVariant(QString("%1%2.%3").arg(QDir::homePath(), QDir::separator(), QCoreApplication::applicationName()))).toString();
    // default filename is <application name>.log
    filename = settings.value("Log/log_filename", QVariant(QString("%1.log").arg(QCoreApplication::applicationName()))).toString();
    // default threshold is WARNING
    threshold = settings.value("Log/log_threshold", "WARNING").toString();

    setLogPath(path, filename);

    // set log threshold
    QString comp[5] = { "DEBUG", "INFO", "WARNING", "CRITICAL", "NONE" };
    for(int i = 0; i < 5; i++) {
        if(threshold == comp[i]) {
            m_logThreshold = (LogLevel)i;
            break;
        }
    }

    // Setup the qMsgHandler
    oldHandler = qInstallMsgHandler(Logger::logMessageHandler);
}

Logger::~Logger() throw()
{
    if(logFile.isOpen())
        logFile.close();

    qInstallMsgHandler(oldHandler);
}

<|endoftext|>


<commit_before>/**
 * Copyright (C) 2015 Dato, Inc.
 * All rights reserved.
 *
 * This software may be modified and distributed under the terms
 * of the BSD license. See the LICENSE file for details.
 */
#include <unity/extensions/grouped_sframe.hpp>

namespace graphlab {

/// Public methods
void grouped_sframe::group(const gl_sframe &sf, const std::vector<std::string>
    column_names, bool is_grouped) {
  if(m_inited)
    log_and_throw("Group has already been called on this object!");

  // Do our "grouping" if it hasn't already been done
  if(!is_grouped) {
    m_grouped_sf = sf.sort(column_names);
  } else {
    m_grouped_sf = sf;
  }
  m_key_col_names = column_names;

  // Get indices from column names
  std::vector<size_t> col_ids;
  std::unordered_set<size_t> dedup_set;
  for(const auto &i : column_names) {
    auto col_id = sf.column_index(i);
    col_ids.push_back(col_id);
    auto ins_ret = dedup_set.insert(col_id);
    if(!ins_ret.second)
      log_and_throw("Found duplicate column name: " + i);
  }

  // Build the directory of ranges to allow querying of the groups
  // (this is an extra, sequential pass over the data)
  auto sf_range = m_grouped_sf.range_iterator();
  auto iter = sf_range.begin();
  size_t cnt = 0;
  std::vector<flexible_type> prev_elem(col_ids.size());
  std::vector<flexible_type> cur_elem(col_ids.size());
  bool first = true;
  for(; iter != sf_range.end(); ++iter, ++cnt) {
    // Create cur_elem
    int col_cnt = 0;
    for(const auto &i : col_ids) {
      cur_elem[col_cnt] = (*iter)[i];
      ++col_cnt;
    }

    // Check for new group
    if((cur_elem != prev_elem) || first) {
      first = false;
      m_key2range.insert(std::make_pair(cur_elem, m_range_directory.size()));
      m_range_directory.push_back(cnt);
      if(cur_elem.size() == 1)
        m_group_names.push_back(cur_elem[0]);
      else
        m_group_names.push_back(cur_elem);
    }

    prev_elem = cur_elem;
  }

  if(col_ids.size() > 1) {
    m_group_type = flex_type_enum::LIST;
  } else {
    m_group_type = prev_elem[0].get_type();
  }

  m_inited = true;
}

gl_sframe grouped_sframe::get_group(std::vector<flexible_type> key) {
  if(!m_inited) {
    log_and_throw("The 'group' operation needs to occur before getting a "
        "group!");
  }
  //TODO: This is a HUGE hack. From the Python side, a list of ints is turned
  //in to a list of floats unless there's a None in it. We add a None so this
  //doesn't happen and take it away to look up the group. We'll actually fix
  //the problem soon.
  if((key.size() > 1) && (key.back().get_type() == flex_type_enum::UNDEFINED)) {
    key.pop_back();
  }
  auto ret = m_key2range.find(key);
  if(ret == m_key2range.end()) {
    log_and_throw("Group not found!");
  }

  size_t range_dir_idx = ret->second;

  return this->get_group_by_index(range_dir_idx);
}

gl_sarray grouped_sframe::groups() {
  if(!m_inited)
    log_and_throw("The 'group' operation needs to occur before getting the "
        "list of groups!");

  // Construct the SArray of group names if it hasn't been constructed
  if(m_groups_sa.size() == 0) {
    // Determine type
    auto wr = std::make_shared<gl_sarray_writer>(m_group_type,8);
    for(const auto &i : m_group_names) {
      wr->write(i, 0);
    }

    m_groups_sa = wr->close();
  }

  return m_groups_sa;
}

std::vector<std::pair<flexible_type,gl_sframe>>
grouped_sframe::iterator_get_next(size_t len) {
  if(!m_inited)
    log_and_throw("The 'group' operation needs to occur before iteration!");
  if(!m_iterating)
    log_and_throw("Must begin iteration before iterating!");

  std::vector<std::pair<flexible_type,gl_sframe>> ret;

  if(len < 1) {
    return ret;
  } else {
    auto items_left = m_range_directory.size() - m_cur_iterator_idx;
    if(len > items_left) {
      len = items_left;
    }
    ret.resize(len);
  }

  size_t cur_cnt = 0;
  for(; (m_cur_iterator_idx < m_range_directory.size()) && (cur_cnt < len);
      ++m_cur_iterator_idx, ++cur_cnt) {
    auto sf = this->get_group_by_index(m_cur_iterator_idx);
    auto name = m_group_names[m_cur_iterator_idx];
    ret[cur_cnt] = std::make_pair(name, sf);
  }

  if(cur_cnt < len) {
    m_iterating = false;
  }

  return ret;
}

gl_sframe grouped_sframe::group_info() const {
  if (m_group_names.size() == 0) {
    log_and_throw("No groups present. Cannot obtain group info.");
  }

  // Return column names.
  std::vector<std::string> ret_column_names = m_key_col_names;
  ret_column_names.push_back("group_size");
  DASSERT_EQ(ret_column_names.size(), m_key_col_names.size() + 1);

  // Return column types from the first group info.
  DASSERT_TRUE(m_group_names.size() > 1);
  std::vector<flex_type_enum> ret_column_types;
  flexible_type first_key = m_group_names[0];
  flex_type_enum key_type = first_key.get_type();
  if (key_type == flex_type_enum::LIST) {
    for (size_t k = 0; k < first_key.size(); k++) {
      ret_column_types.push_back(first_key.array_at(k).get_type());
    }
  } else {
    ret_column_types.push_back(key_type);
  }
  ret_column_types.push_back(flex_type_enum::INTEGER);
  DASSERT_EQ(ret_column_types.size(), ret_column_names.size());

  // Prepare for writing.
  size_t num_segments = thread::cpu_count();
  gl_sframe_writer writer(ret_column_names, ret_column_types, num_segments);
  size_t range_dir_size = m_range_directory.size();

  // Write the group info.
  in_parallel([&](size_t thread_idx, size_t num_threads) {

    size_t start_idx = range_dir_size * thread_idx / num_threads;
    size_t end_idx = range_dir_size * (thread_idx + 1) / num_threads;

    for (size_t i = start_idx; i < end_idx; i++) {
      size_t range_start = m_range_directory[i];
      size_t range_end = 0;
      if((i + 1) == m_range_directory.size()) {
        range_end = m_grouped_sf.size();
      } else {
        range_end = m_range_directory[i + 1];
      }
      size_t num_rows = range_end - range_start;
      std::vector<flexible_type> vals = m_group_names[i];
      vals.push_back(num_rows);
      DASSERT_EQ(vals.size(), ret_column_names.size());
      writer.write(vals, thread_idx);
    }
    return writer.close();
  });
}

/// Private methods
gl_sframe grouped_sframe::get_group_by_index(size_t range_dir_idx) {
  int64_t range_start = m_range_directory[range_dir_idx];
  int64_t range_end;
  if((range_dir_idx+1) == m_range_directory.size()) {
    range_end = m_grouped_sf.size();
  } else {
    range_end = m_range_directory[range_dir_idx+1];
  }

  return m_grouped_sf[{range_start,range_end}];
}

} // namespace graphlab

using namespace graphlab;

BEGIN_CLASS_REGISTRATION
REGISTER_CLASS(grouped_sframe)
END_CLASS_REGISTRATION
<commit_msg>[GroupedSFrame] Fixed a bug in group-info.<commit_after>/**
 * Copyright (C) 2015 Dato, Inc.
 * All rights reserved.
 *
 * This software may be modified and distributed under the terms
 * of the BSD license. See the LICENSE file for details.
 */
#include <unity/extensions/grouped_sframe.hpp>

namespace graphlab {

/// Public methods
void grouped_sframe::group(const gl_sframe &sf, const std::vector<std::string>
    column_names, bool is_grouped) {
  if(m_inited)
    log_and_throw("Group has already been called on this object!");

  // Do our "grouping" if it hasn't already been done
  if(!is_grouped) {
    m_grouped_sf = sf.sort(column_names);
  } else {
    m_grouped_sf = sf;
  }
  m_key_col_names = column_names;

  // Get indices from column names
  std::vector<size_t> col_ids;
  std::unordered_set<size_t> dedup_set;
  for(const auto &i : column_names) {
    auto col_id = sf.column_index(i);
    col_ids.push_back(col_id);
    auto ins_ret = dedup_set.insert(col_id);
    if(!ins_ret.second)
      log_and_throw("Found duplicate column name: " + i);
  }

  // Build the directory of ranges to allow querying of the groups
  // (this is an extra, sequential pass over the data)
  auto sf_range = m_grouped_sf.range_iterator();
  auto iter = sf_range.begin();
  size_t cnt = 0;
  std::vector<flexible_type> prev_elem(col_ids.size());
  std::vector<flexible_type> cur_elem(col_ids.size());
  bool first = true;
  for(; iter != sf_range.end(); ++iter, ++cnt) {
    // Create cur_elem
    int col_cnt = 0;
    for(const auto &i : col_ids) {
      cur_elem[col_cnt] = (*iter)[i];
      ++col_cnt;
    }

    // Check for new group
    if((cur_elem != prev_elem) || first) {
      first = false;
      m_key2range.insert(std::make_pair(cur_elem, m_range_directory.size()));
      m_range_directory.push_back(cnt);
      if(cur_elem.size() == 1)
        m_group_names.push_back(cur_elem[0]);
      else
        m_group_names.push_back(cur_elem);
    }

    prev_elem = cur_elem;
  }

  if(col_ids.size() > 1) {
    m_group_type = flex_type_enum::LIST;
  } else {
    m_group_type = prev_elem[0].get_type();
  }

  m_inited = true;
}

gl_sframe grouped_sframe::get_group(std::vector<flexible_type> key) {
  if(!m_inited) {
    log_and_throw("The 'group' operation needs to occur before getting a "
        "group!");
  }
  //TODO: This is a HUGE hack. From the Python side, a list of ints is turned
  //in to a list of floats unless there's a None in it. We add a None so this
  //doesn't happen and take it away to look up the group. We'll actually fix
  //the problem soon.
  if((key.size() > 1) && (key.back().get_type() == flex_type_enum::UNDEFINED)) {
    key.pop_back();
  }
  auto ret = m_key2range.find(key);
  if(ret == m_key2range.end()) {
    log_and_throw("Group not found!");
  }

  size_t range_dir_idx = ret->second;

  return this->get_group_by_index(range_dir_idx);
}

gl_sarray grouped_sframe::groups() {
  if(!m_inited)
    log_and_throw("The 'group' operation needs to occur before getting the "
        "list of groups!");

  // Construct the SArray of group names if it hasn't been constructed
  if(m_groups_sa.size() == 0) {
    // Determine type
    auto wr = std::make_shared<gl_sarray_writer>(m_group_type,8);
    for(const auto &i : m_group_names) {
      wr->write(i, 0);
    }

    m_groups_sa = wr->close();
  }

  return m_groups_sa;
}

std::vector<std::pair<flexible_type,gl_sframe>>
grouped_sframe::iterator_get_next(size_t len) {
  if(!m_inited)
    log_and_throw("The 'group' operation needs to occur before iteration!");
  if(!m_iterating)
    log_and_throw("Must begin iteration before iterating!");

  std::vector<std::pair<flexible_type,gl_sframe>> ret;

  if(len < 1) {
    return ret;
  } else {
    auto items_left = m_range_directory.size() - m_cur_iterator_idx;
    if(len > items_left) {
      len = items_left;
    }
    ret.resize(len);
  }

  size_t cur_cnt = 0;
  for(; (m_cur_iterator_idx < m_range_directory.size()) && (cur_cnt < len);
      ++m_cur_iterator_idx, ++cur_cnt) {
    auto sf = this->get_group_by_index(m_cur_iterator_idx);
    auto name = m_group_names[m_cur_iterator_idx];
    ret[cur_cnt] = std::make_pair(name, sf);
  }

  if(cur_cnt < len) {
    m_iterating = false;
  }

  return ret;
}

gl_sframe grouped_sframe::group_info() const {
  if (m_group_names.size() == 0) {
    log_and_throw("No groups present. Cannot obtain group info.");
  }

  // Return column names.
  std::vector<std::string> ret_column_names = m_key_col_names;
  ret_column_names.push_back("group_size");
  DASSERT_EQ(ret_column_names.size(), m_key_col_names.size() + 1);

  // Return column types from the first group info.
  DASSERT_TRUE(m_group_names.size() > 1);
  std::vector<flex_type_enum> ret_column_types;
  flexible_type first_key = m_group_names[0];
  flex_type_enum key_type = first_key.get_type();
  if (key_type == flex_type_enum::LIST) {
    for (size_t k = 0; k < first_key.size(); k++) {
      ret_column_types.push_back(first_key.array_at(k).get_type());
    }
  } else {
    ret_column_types.push_back(key_type);
  }
  ret_column_types.push_back(flex_type_enum::INTEGER);
  DASSERT_EQ(ret_column_types.size(), ret_column_names.size());

  // Prepare for writing.
  size_t num_segments = thread::cpu_count();
  gl_sframe_writer writer(ret_column_names, ret_column_types, num_segments);
  size_t range_dir_size = m_range_directory.size();

  // Write the group info.
  in_parallel([&](size_t thread_idx, size_t num_threads) {

    size_t start_idx = range_dir_size * thread_idx / num_threads;
    size_t end_idx = range_dir_size * (thread_idx + 1) / num_threads;

    for (size_t i = start_idx; i < end_idx; i++) {
      size_t range_start = m_range_directory[i];
      size_t range_end = 0;
      if((i + 1) == m_range_directory.size()) {
        range_end = m_grouped_sf.size();
      } else {
        range_end = m_range_directory[i + 1];
      }
      size_t num_rows = range_end - range_start;
      std::vector<flexible_type> vals = m_group_names[i];
      vals.push_back(num_rows);
      DASSERT_EQ(vals.size(), ret_column_names.size());
      writer.write(vals, thread_idx);
    }
  });
  return writer.close();
}

/// Private methods
gl_sframe grouped_sframe::get_group_by_index(size_t range_dir_idx) {
  int64_t range_start = m_range_directory[range_dir_idx];
  int64_t range_end;
  if((range_dir_idx+1) == m_range_directory.size()) {
    range_end = m_grouped_sf.size();
  } else {
    range_end = m_range_directory[range_dir_idx+1];
  }

  return m_grouped_sf[{range_start,range_end}];
}

} // namespace graphlab

using namespace graphlab;

BEGIN_CLASS_REGISTRATION
REGISTER_CLASS(grouped_sframe)
END_CLASS_REGISTRATION
<|endoftext|>