<commit_before>// Copyright (c) 2011, Steinwurf ApS
// All rights reserved.
//
// Redistribution and use in source and binary forms, with or without
// modification, are permitted provided that the following conditions are met:
//
// 1. Redistributions of source code must retain the above copyright notice, this
//    list of conditions and the following disclaimer.
// 2. Redistributions in binary form must reproduce the above copyright notice,
//    this list of conditions and the following disclaimer in the documentation
//    and/or other materials provided with the distribution.
//
// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
// ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
// WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
// DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR
// ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
// (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
// LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
// ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
// SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
//
// The views and conclusions contained in the software and documentation are those
// of the authors and should not be interpreted as representing official policies,
// either expressed or implied, of the FreeBSD Project.

#include <gtest/gtest.h>
#include <sak/convert_endian.h>

// From test proposed here:
// http://stackoverflow.com/questions/1001307/
bool is_big_endian()
{
    union {
        uint32_t i;
        char c[4];
    } bint = {0x01020304};

    return bint.c[0] == 1;
}

TEST(ConvertEndian, CheckEndian)
{
    EXPECT_TRUE(is_big_endian() == sak::host_endian::big_endian);
}

TEST(ConvertEndian, Convert)
{

    {
        uint8_t data[1];
        uint16_t in  = 0x11U;

        // If the host is big endian the put function
        // should not change the byte order
        sak::big_endian::put8(in, data);
        EXPECT_TRUE(0x11U == data[0]);

        // Neither should the get
        uint8_t out = sak::big_endian::get8(data);
        EXPECT_TRUE(out == in);

    }

    {
        if(sak::host_endian::big_endian)
        {
            uint8_t data[2];
            uint16_t in  = 0x1122U;

            // If the host is big endian the put function
            // should not change the byte order
            sak::big_endian::put16(in, data);
            EXPECT_TRUE(0x11U == data[0]);
            EXPECT_TRUE(0x22U == data[1]);

            // Neither should the get
            uint16_t out = sak::big_endian::get16(data);
            EXPECT_TRUE(out == in);
        }
        else
        {
            uint8_t data[2];
            uint16_t in  = 0x1122U;

            // If the host is little endian the put function
            // should change the byte order
            sak::big_endian::put16(in, data);
            EXPECT_TRUE(0x11U == data[0]);
            EXPECT_TRUE(0x22U == data[1]);

            // Get should swap the value back
            uint16_t out = sak::big_endian::get16(data);
            EXPECT_TRUE(out == in);
        }

    }

    {
        if(sak::host_endian::big_endian)
        {
            uint8_t data[4];
            uint32_t in  = 0x11223344U;

            // If the host is big endian the put function
            // should not change the byte order
            sak::big_endian::put32(in, data);
            EXPECT_TRUE(0x11U == data[0]);
            EXPECT_TRUE(0x22U == data[1]);
            EXPECT_TRUE(0x33U == data[2]);
            EXPECT_TRUE(0x44U == data[3]);

            // Neither should the get
            uint32_t out = sak::big_endian::get32(data);
            EXPECT_TRUE(out == in);
        }
        else
        {
            uint8_t data[4];
            uint32_t in  = 0x11223344U;

            // If the host is little endian the put function
            // should change the byte order
            sak::big_endian::put32(in, data);
            EXPECT_TRUE(0x11U == data[0]);
            EXPECT_TRUE(0x22U == data[1]);
            EXPECT_TRUE(0x33U == data[2]);
            EXPECT_TRUE(0x44U == data[3]);

            // Get should swap the value back
            uint32_t out = sak::big_endian::get32(data);
            EXPECT_TRUE(out == in);
        }

    }

    {
        if(sak::host_endian::big_endian)
        {
            uint8_t data[8];
            uint64_t in  = 0x1122334455667788ULL;

            // If the host is big endian the put function
            // should not change the byte order
            sak::big_endian::put64(in, reinterpret_cast<uint8_t*>(data));
            EXPECT_TRUE(0x11U == data[0]);
            EXPECT_TRUE(0x22U == data[1]);
            EXPECT_TRUE(0x33U == data[2]);
            EXPECT_TRUE(0x44U == data[3]);
            EXPECT_TRUE(0x55U == data[4]);
            EXPECT_TRUE(0x66U == data[5]);
            EXPECT_TRUE(0x77U == data[6]);
            EXPECT_TRUE(0x88U == data[7]);

            // Neither should the get
            uint64_t out = sak::big_endian::get64(data);
            EXPECT_TRUE(out == in);
        }
        else
        {
            uint8_t data[8];
            uint64_t in  = 0x1122334455667788ULL;

            // If the host is little endian the put function
            // should change the byte order
            sak::big_endian::put64(in, data);
            EXPECT_TRUE(0x11U == data[0]);
            EXPECT_TRUE(0x22U == data[1]);
            EXPECT_TRUE(0x33U == data[2]);
            EXPECT_TRUE(0x44U == data[3]);
            EXPECT_TRUE(0x55U == data[4]);
            EXPECT_TRUE(0x66U == data[5]);
            EXPECT_TRUE(0x77U == data[6]);
            EXPECT_TRUE(0x88U == data[7]);

            // Get should swap the value back
            uint64_t out = sak::big_endian::get64(data);
            EXPECT_TRUE(out == in);
        }

    }

}

TEST(ConvertEndian, ConvertTemplate)
{
    {
        if(sak::host_endian::big_endian)
        {
            uint8_t data[2];
            uint16_t in  = 0x1122U;

            // If the host is big endian the put function
            // should not change the byte order
            sak::big_endian::put<uint16_t>(in, data);
            EXPECT_TRUE(0x11U == data[0]);
            EXPECT_TRUE(0x22U == data[1]);

            // Neither should the get
            uint16_t out = sak::big_endian::get<uint16_t>(data);
            EXPECT_TRUE(out == in);
        }
        else
        {
            uint8_t data[2];
            uint16_t in  = 0x1122U;

            // If the host is little endian the put function
            // should change the byte order
            sak::big_endian::put<uint16_t>(in, data);
            EXPECT_TRUE(0x11U == data[0]);
            EXPECT_TRUE(0x22U == data[1]);

            // Get should swap the value back
            uint16_t out = sak::big_endian::get<uint16_t>(data);
            EXPECT_TRUE(out == in);
        }

    }
}

<commit_msg>Updated license to new BSD<commit_after>// Copyright (c) 2012, Steinwurf ApS
// All rights reserved.
//
// Redistribution and use in source and binary forms, with or without
// modification, are permitted provided that the following conditions are met:
//     * Redistributions of source code must retain the above copyright
//       notice, this list of conditions and the following disclaimer.
//     * Redistributions in binary form must reproduce the above copyright
//       notice, this list of conditions and the following disclaimer in the
//       documentation and/or other materials provided with the distribution.
//     * Neither the name of Steinwurf ApS nor the
//       names of its contributors may be used to endorse or promote products
//       derived from this software without specific prior written permission.
//
// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
// ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
// WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
// DISCLAIMED. IN NO EVENT SHALL <COPYRIGHT HOLDER> BE LIABLE FOR ANY
// DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
// (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
// LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
// ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
// SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.

#include <gtest/gtest.h>
#include <sak/convert_endian.h>

// From test proposed here:
// http://stackoverflow.com/questions/1001307/
bool is_big_endian()
{
    union {
        uint32_t i;
        char c[4];
    } bint = {0x01020304};

    return bint.c[0] == 1;
}

TEST(ConvertEndian, CheckEndian)
{
    EXPECT_TRUE(is_big_endian() == sak::host_endian::big_endian);
}

TEST(ConvertEndian, Convert)
{

    {
        uint8_t data[1];
        uint16_t in  = 0x11U;

        // If the host is big endian the put function
        // should not change the byte order
        sak::big_endian::put8(in, data);
        EXPECT_TRUE(0x11U == data[0]);

        // Neither should the get
        uint8_t out = sak::big_endian::get8(data);
        EXPECT_TRUE(out == in);

    }

    {
        if(sak::host_endian::big_endian)
        {
            uint8_t data[2];
            uint16_t in  = 0x1122U;

            // If the host is big endian the put function
            // should not change the byte order
            sak::big_endian::put16(in, data);
            EXPECT_TRUE(0x11U == data[0]);
            EXPECT_TRUE(0x22U == data[1]);

            // Neither should the get
            uint16_t out = sak::big_endian::get16(data);
            EXPECT_TRUE(out == in);
        }
        else
        {
            uint8_t data[2];
            uint16_t in  = 0x1122U;

            // If the host is little endian the put function
            // should change the byte order
            sak::big_endian::put16(in, data);
            EXPECT_TRUE(0x11U == data[0]);
            EXPECT_TRUE(0x22U == data[1]);

            // Get should swap the value back
            uint16_t out = sak::big_endian::get16(data);
            EXPECT_TRUE(out == in);
        }

    }

    {
        if(sak::host_endian::big_endian)
        {
            uint8_t data[4];
            uint32_t in  = 0x11223344U;

            // If the host is big endian the put function
            // should not change the byte order
            sak::big_endian::put32(in, data);
            EXPECT_TRUE(0x11U == data[0]);
            EXPECT_TRUE(0x22U == data[1]);
            EXPECT_TRUE(0x33U == data[2]);
            EXPECT_TRUE(0x44U == data[3]);

            // Neither should the get
            uint32_t out = sak::big_endian::get32(data);
            EXPECT_TRUE(out == in);
        }
        else
        {
            uint8_t data[4];
            uint32_t in  = 0x11223344U;

            // If the host is little endian the put function
            // should change the byte order
            sak::big_endian::put32(in, data);
            EXPECT_TRUE(0x11U == data[0]);
            EXPECT_TRUE(0x22U == data[1]);
            EXPECT_TRUE(0x33U == data[2]);
            EXPECT_TRUE(0x44U == data[3]);

            // Get should swap the value back
            uint32_t out = sak::big_endian::get32(data);
            EXPECT_TRUE(out == in);
        }

    }

    {
        if(sak::host_endian::big_endian)
        {
            uint8_t data[8];
            uint64_t in  = 0x1122334455667788ULL;

            // If the host is big endian the put function
            // should not change the byte order
            sak::big_endian::put64(in, reinterpret_cast<uint8_t*>(data));
            EXPECT_TRUE(0x11U == data[0]);
            EXPECT_TRUE(0x22U == data[1]);
            EXPECT_TRUE(0x33U == data[2]);
            EXPECT_TRUE(0x44U == data[3]);
            EXPECT_TRUE(0x55U == data[4]);
            EXPECT_TRUE(0x66U == data[5]);
            EXPECT_TRUE(0x77U == data[6]);
            EXPECT_TRUE(0x88U == data[7]);

            // Neither should the get
            uint64_t out = sak::big_endian::get64(data);
            EXPECT_TRUE(out == in);
        }
        else
        {
            uint8_t data[8];
            uint64_t in  = 0x1122334455667788ULL;

            // If the host is little endian the put function
            // should change the byte order
            sak::big_endian::put64(in, data);
            EXPECT_TRUE(0x11U == data[0]);
            EXPECT_TRUE(0x22U == data[1]);
            EXPECT_TRUE(0x33U == data[2]);
            EXPECT_TRUE(0x44U == data[3]);
            EXPECT_TRUE(0x55U == data[4]);
            EXPECT_TRUE(0x66U == data[5]);
            EXPECT_TRUE(0x77U == data[6]);
            EXPECT_TRUE(0x88U == data[7]);

            // Get should swap the value back
            uint64_t out = sak::big_endian::get64(data);
            EXPECT_TRUE(out == in);
        }

    }

}

TEST(ConvertEndian, ConvertTemplate)
{
    {
        if(sak::host_endian::big_endian)
        {
            uint8_t data[2];
            uint16_t in  = 0x1122U;

            // If the host is big endian the put function
            // should not change the byte order
            sak::big_endian::put<uint16_t>(in, data);
            EXPECT_TRUE(0x11U == data[0]);
            EXPECT_TRUE(0x22U == data[1]);

            // Neither should the get
            uint16_t out = sak::big_endian::get<uint16_t>(data);
            EXPECT_TRUE(out == in);
        }
        else
        {
            uint8_t data[2];
            uint16_t in  = 0x1122U;

            // If the host is little endian the put function
            // should change the byte order
            sak::big_endian::put<uint16_t>(in, data);
            EXPECT_TRUE(0x11U == data[0]);
            EXPECT_TRUE(0x22U == data[1]);

            // Get should swap the value back
            uint16_t out = sak::big_endian::get<uint16_t>(data);
            EXPECT_TRUE(out == in);
        }

    }
}

<|endoftext|>


<commit_before>#include "erl_nif.h"
#include "CPool.hpp"
#include "stdio.h"

extern "C" {
    static ErlNifResourceType* cpool_node_RESOURCE = NULL;
    static ErlNifResourceType* cpool_RESOURCE = NULL;

    typedef struct {
        CPool *pool;
    } cpool_handle;
    typedef struct {
        CPoolNode *node;
    } cpool_node_handle;

    // Prototypes
    static ERL_NIF_TERM cpool_new(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[]);
    static ERL_NIF_TERM cpool_join(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[]);
    static ERL_NIF_TERM cpool_depart(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[]);
    static ERL_NIF_TERM cpool_next(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[]);

    static ErlNifFunc nif_funcs[] = {
        {"new", 1, cpool_new},
        {"join", 2, cpool_join},
        {"depart", 2, cpool_depart},
        {"next", 2, cpool_next}
    };

    static ERL_NIF_TERM cpool_new(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[]) {
        ERL_NIF_TERM result;
        unsigned long schedulers = 0;

        if (enif_get_ulong(env, argv[0], &schedulers) == 0) {
            return enif_make_badarg(env);
        }

        cpool_handle* handle = (cpool_handle*)enif_alloc_resource(cpool_RESOURCE, sizeof(cpool_handle));
        handle->pool = new CPool(schedulers);

        result = enif_make_resource(env, handle);
        enif_release_resource(handle);
        return enif_make_tuple2(env, enif_make_atom(env, "ok"), result);
    }

    static ERL_NIF_TERM cpool_join(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[]) {
        cpool_handle* handle = NULL;
        cpool_node_handle* node;
        ERL_NIF_TERM result;

        if (enif_get_resource(env, argv[0], cpool_RESOURCE, (void**)&handle) == 0) {
            return enif_make_badarg(env);
        }

        node = (cpool_node_handle*)enif_alloc_resource(cpool_node_RESOURCE, sizeof(cpool_node_handle));
        node->node = handle->pool->Join(argv[1]);
        result  = enif_make_resource(env, (void*)node);
        enif_release_resource((void*)node);

        return result;
    }

    static ERL_NIF_TERM cpool_depart(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[]) {
        ERL_NIF_TERM result;
        cpool_handle* handle_pool = NULL;
        cpool_node_handle* handle_node = NULL;

        if (enif_get_resource(env, argv[0], cpool_RESOURCE, (void**)&handle_pool) == 0) {
            return enif_make_badarg(env);
        }

        if (enif_get_resource(env, argv[1], cpool_node_RESOURCE, (void**)&handle_node) == 0) {
            return enif_make_badarg(env);
        }

        result = handle_pool->pool->Depart(env, handle_node->node);

        return result;
    }

    static ERL_NIF_TERM cpool_next(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[]) {
        ERL_NIF_TERM result;
        cpool_handle* handle = NULL;
        unsigned long iterator;

        if (enif_get_resource(env, argv[0], cpool_RESOURCE, (void**)&handle) == 0) {
            return enif_make_badarg(env);
        }

        if (enif_get_ulong(env, argv[1], &iterator) == 0) {
            return enif_make_badarg(env);
        }

        // Reduce iterator to account for zero indice
        --iterator;
        result = handle->pool->Next(env, iterator);

        return result;
    }

    static void cpool_resource_cleanup(ErlNifEnv* env, void* arg) {
        cpool_handle* handle = (cpool_handle*)arg;
        delete handle->pool;
        handle->pool = NULL;
    }

    static void cpool_node_resource_cleanup(ErlNifEnv* env, void* arg) {
        ((cpool_node_handle*)arg)->node = NULL;
    }

    static int on_load(ErlNifEnv* env, void** priv_data, ERL_NIF_TERM load_info) {
        ErlNifResourceFlags flags = (ErlNifResourceFlags)(ERL_NIF_RT_CREATE | ERL_NIF_RT_TAKEOVER);
        ErlNifResourceType* cpool = enif_open_resource_type(env, NULL, "cpool_resource", &cpool_resource_cleanup, flags, NULL);
        if (cpool == NULL) return -1;
        ErlNifResourceType* cpool_node = enif_open_resource_type(env, NULL, "cpool_node_resource", &cpool_node_resource_cleanup, flags,  NULL);
        if (cpool_node == NULL) return -1;

        cpool_RESOURCE = cpool;
        cpool_node_RESOURCE = cpool_node;

        return 0;
    }

    static int on_upgrade(ErlNifEnv* env, void** old_priv_data, void** priv_data, ERL_NIF_TERM load_info) {
        ErlNifResourceFlags flags = (ErlNifResourceFlags)(ERL_NIF_RT_CREATE | ERL_NIF_RT_TAKEOVER);
        ErlNifResourceType* cpool = enif_open_resource_type(env, NULL, "cpool_resource", &cpool_resource_cleanup, flags, NULL);
        if (cpool == NULL) return -1;
        ErlNifResourceType* cpool_node = enif_open_resource_type(env, NULL, "cpool_node_resource", &cpool_node_resource_cleanup, flags,  NULL);
        if (cpool_node == NULL) return -1;

        cpool_RESOURCE = cpool;
        cpool_node_RESOURCE = cpool_node;

        return 0;
    }

    ERL_NIF_INIT(cpool, nif_funcs, &on_load, NULL, &on_upgrade, NULL);
}
<commit_msg>DEBUG: on_upgrade debug message<commit_after>#include "erl_nif.h"
#include "CPool.hpp"
#include "stdio.h"

extern "C" {
    static ErlNifResourceType* cpool_node_RESOURCE = NULL;
    static ErlNifResourceType* cpool_RESOURCE = NULL;

    typedef struct {
        CPool *pool;
    } cpool_handle;
    typedef struct {
        CPoolNode *node;
    } cpool_node_handle;

    // Prototypes
    static ERL_NIF_TERM cpool_new(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[]);
    static ERL_NIF_TERM cpool_join(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[]);
    static ERL_NIF_TERM cpool_depart(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[]);
    static ERL_NIF_TERM cpool_next(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[]);

    static ErlNifFunc nif_funcs[] = {
        {"new", 1, cpool_new},
        {"join", 2, cpool_join},
        {"depart", 2, cpool_depart},
        {"next", 2, cpool_next}
    };

    static ERL_NIF_TERM cpool_new(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[]) {
        ERL_NIF_TERM result;
        unsigned long schedulers = 0;

        if (enif_get_ulong(env, argv[0], &schedulers) == 0) {
            return enif_make_badarg(env);
        }

        cpool_handle* handle = (cpool_handle*)enif_alloc_resource(cpool_RESOURCE, sizeof(cpool_handle));
        handle->pool = new CPool(schedulers);

        result = enif_make_resource(env, handle);
        enif_release_resource(handle);
        return enif_make_tuple2(env, enif_make_atom(env, "ok"), result);
    }

    static ERL_NIF_TERM cpool_join(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[]) {
        cpool_handle* handle = NULL;
        cpool_node_handle* node;
        ERL_NIF_TERM result;

        if (enif_get_resource(env, argv[0], cpool_RESOURCE, (void**)&handle) == 0) {
            return enif_make_badarg(env);
        }

        node = (cpool_node_handle*)enif_alloc_resource(cpool_node_RESOURCE, sizeof(cpool_node_handle));
        node->node = handle->pool->Join(argv[1]);
        result  = enif_make_resource(env, (void*)node);
        enif_release_resource((void*)node);

        return result;
    }

    static ERL_NIF_TERM cpool_depart(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[]) {
        ERL_NIF_TERM result;
        cpool_handle* handle_pool = NULL;
        cpool_node_handle* handle_node = NULL;

        if (enif_get_resource(env, argv[0], cpool_RESOURCE, (void**)&handle_pool) == 0) {
            return enif_make_badarg(env);
        }

        if (enif_get_resource(env, argv[1], cpool_node_RESOURCE, (void**)&handle_node) == 0) {
            return enif_make_badarg(env);
        }

        result = handle_pool->pool->Depart(env, handle_node->node);

        return result;
    }

    static ERL_NIF_TERM cpool_next(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[]) {
        ERL_NIF_TERM result;
        cpool_handle* handle = NULL;
        unsigned long iterator;

        if (enif_get_resource(env, argv[0], cpool_RESOURCE, (void**)&handle) == 0) {
            return enif_make_badarg(env);
        }

        if (enif_get_ulong(env, argv[1], &iterator) == 0) {
            return enif_make_badarg(env);
        }

        // Reduce iterator to account for zero indice
        --iterator;
        result = handle->pool->Next(env, iterator);

        return result;
    }

    static void cpool_resource_cleanup(ErlNifEnv* env, void* arg) {
        cpool_handle* handle = (cpool_handle*)arg;
        delete handle->pool;
        handle->pool = NULL;
    }

    static void cpool_node_resource_cleanup(ErlNifEnv* env, void* arg) {
        ((cpool_node_handle*)arg)->node = NULL;
    }

    static int on_load(ErlNifEnv* env, void** priv_data, ERL_NIF_TERM load_info) {
        ErlNifResourceFlags flags = (ErlNifResourceFlags)(ERL_NIF_RT_CREATE | ERL_NIF_RT_TAKEOVER);
        ErlNifResourceType* cpool = enif_open_resource_type(env, NULL, "cpool_resource", &cpool_resource_cleanup, flags, NULL);
        if (cpool == NULL) return -1;
        ErlNifResourceType* cpool_node = enif_open_resource_type(env, NULL, "cpool_node_resource", &cpool_node_resource_cleanup, flags,  NULL);
        if (cpool_node == NULL) return -1;

        cpool_RESOURCE = cpool;
        cpool_node_RESOURCE = cpool_node;

        return 0;
    }

    static int on_upgrade(ErlNifEnv* env, void** old_priv_data, void** priv_data, ERL_NIF_TERM load_info) {
        printf("Executing on_upgrade callback.\r\n");
        ErlNifResourceFlags flags = (ErlNifResourceFlags)(ERL_NIF_RT_CREATE | ERL_NIF_RT_TAKEOVER);
        ErlNifResourceType* cpool = enif_open_resource_type(env, NULL, "cpool_resource", &cpool_resource_cleanup, flags, NULL);
        if (cpool == NULL) return -1;
        ErlNifResourceType* cpool_node = enif_open_resource_type(env, NULL, "cpool_node_resource", &cpool_node_resource_cleanup, flags,  NULL);
        if (cpool_node == NULL) return -1;

        cpool_RESOURCE = cpool;
        cpool_node_RESOURCE = cpool_node;

        return 0;
    }

    ERL_NIF_INIT(cpool, nif_funcs, &on_load, NULL, &on_upgrade, NULL);
}
<|endoftext|>


<commit_before>/** \brief Utility for updating SQL schemata etc.
 *  \author Dr. Johannes Ruscheinski (johannes.ruscheinski@uni-tuebingen.de)
 *
 *  \copyright 2019 Universitätsbibliothek Tübingen.  All rights reserved.
 *
 *  This program is free software: you can redistribute it and/or modify
 *  it under the terms of the GNU Affero General Public License as
 *  published by the Free Software Foundation, either version 3 of the
 *  License, or (at your option) any later version.
 *
 *  This program is distributed in the hope that it will be useful,
 *  but WITHOUT ANY WARRANTY; without even the implied warranty of
 *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 *  GNU Affero General Public License for more details.
 *
 *  You should have received a copy of the GNU Affero General Public License
 *  along with this program.  If not, see <http://www.gnu.org/licenses/>.
*/

#include <algorithm>
#include <iostream>
#include <stdexcept>
#include <vector>
#include <cstdio>
#include <cstdlib>
#include "Compiler.h"
#include "FileUtil.h"
#include "DbConnection.h"
#include "StringUtil.h"
#include "util.h"

namespace {

[[noreturn]] void Usage() {
    std::cerr << "Usage: " << ::progname << " [--verbose] update_directory_path\n";
    std::exit(EXIT_FAILURE);
}

void SplitIntoDatabaseTableAndVersion(const std::string &update_filename, std::string * const database, std::string * const table,
                                      unsigned * const version)
{
    std::vector<std::string> parts;
    if (unlikely(StringUtil::Split(update_filename, '.', &parts) != 3))
        LOG_ERROR("failed to split \"" + update_filename + "\" into \"database.table.version\"!");
    *database = parts[0];
    *table    = parts[1];
    *version  = StringUtil::ToUnsigned(parts[2]);
}

// The filenames being compared are assumed to have the "structure database.table.version"
bool FileNameCompare(const std::string &filename1, const std::string &filename2) {
    std::string database1, table1;
    unsigned version1;
    SplitIntoDatabaseTableAndVersion(filename1, &database1, &table1, &version1);

    std::string database2, table2;
    unsigned version2;
    SplitIntoDatabaseTableAndVersion(filename2, &database2, &table2, &version2);

    // Compare database names:
    if (database1 < database2)
        return true;
    if (database1 > database2)
        return false;

    // Compare table names:
    if (table1 < table2)
        return true;
    if (table1 > table2)
        return false;

    return version1 < version2;
}

void LoadAndSortUpdateFilenames(const std::string &directory_path, std::vector<std::string> * const update_filenames) {
    FileUtil::Directory directory(directory_path, "[^.]+\\.[^.]+\\.\\d+");
    for (const auto &entry : directory)
        update_filenames->emplace_back(entry.getName());

    std::sort(update_filenames->begin(), update_filenames->end(), FileNameCompare);
}

void ApplyUpdate(DbConnection * const /*db_connection*/, const std::string &database, const std::string &table, const unsigned version) {
    LOG_INFO("applying update \"" + database + "." + table + "." + std::to_string(version) + "\".");
}

void ApplyUpdates(DbConnection * const db_connection, const std::vector<std::string> &update_filenames) {
    std::string previous_database, previous_table;
    unsigned previous_version;

    for (const auto &update_filename : update_filenames) {
        std::string current_database, current_table;
        unsigned current_version;
        SplitIntoDatabaseTableAndVersion(update_filename, &current_database, &current_table, &current_version);
        if (not previous_database.empty() and (previous_database != current_database or previous_table != current_table))
            ApplyUpdate(db_connection, previous_database, previous_table, previous_version);

        previous_database = current_database;
        previous_table    = current_table;
        previous_version  = current_version;
    }

    if (not previous_database.empty())
        ApplyUpdate(db_connection, previous_database, previous_table, previous_version);
}

} // unnamed namespace

int Main(int argc, char *argv[]) {
    if (argc != 2)
        Usage();

    std::vector<std::string> update_filenames;
    LoadAndSortUpdateFilenames(argv[1], &update_filenames);

    DbConnection db_connection;
    if (not db_connection.tableExists("ub_tools", "table_versions"))
        db_connection.queryOrDie("CREATE TABLE ub_tools.table_versions (version INT UNSIGNED NOT NULL, table_name VARCHAR(64) NOT NULL, "
                                 "UNIQUE(table_name)) CHARACTER SET utf8mb4 COLLATE utf8mb4_bin");

    ApplyUpdates(&db_connection, update_filenames);

    return EXIT_SUCCESS;
}
<commit_msg>Use generic Usage.<commit_after>/** \brief Utility for updating SQL schemata etc.
 *  \author Dr. Johannes Ruscheinski (johannes.ruscheinski@uni-tuebingen.de)
 *
 *  \copyright 2019 Universitätsbibliothek Tübingen.  All rights reserved.
 *
 *  This program is free software: you can redistribute it and/or modify
 *  it under the terms of the GNU Affero General Public License as
 *  published by the Free Software Foundation, either version 3 of the
 *  License, or (at your option) any later version.
 *
 *  This program is distributed in the hope that it will be useful,
 *  but WITHOUT ANY WARRANTY; without even the implied warranty of
 *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 *  GNU Affero General Public License for more details.
 *
 *  You should have received a copy of the GNU Affero General Public License
 *  along with this program.  If not, see <http://www.gnu.org/licenses/>.
*/

#include <algorithm>
#include <iostream>
#include <stdexcept>
#include <vector>
#include <cstdio>
#include <cstdlib>
#include "Compiler.h"
#include "FileUtil.h"
#include "DbConnection.h"
#include "StringUtil.h"
#include "util.h"

namespace {

void SplitIntoDatabaseTableAndVersion(const std::string &update_filename, std::string * const database, std::string * const table,
                                      unsigned * const version)
{
    std::vector<std::string> parts;
    if (unlikely(StringUtil::Split(update_filename, '.', &parts) != 3))
        LOG_ERROR("failed to split \"" + update_filename + "\" into \"database.table.version\"!");
    *database = parts[0];
    *table    = parts[1];
    *version  = StringUtil::ToUnsigned(parts[2]);
}

// The filenames being compared are assumed to have the "structure database.table.version"
bool FileNameCompare(const std::string &filename1, const std::string &filename2) {
    std::string database1, table1;
    unsigned version1;
    SplitIntoDatabaseTableAndVersion(filename1, &database1, &table1, &version1);

    std::string database2, table2;
    unsigned version2;
    SplitIntoDatabaseTableAndVersion(filename2, &database2, &table2, &version2);

    // Compare database names:
    if (database1 < database2)
        return true;
    if (database1 > database2)
        return false;

    // Compare table names:
    if (table1 < table2)
        return true;
    if (table1 > table2)
        return false;

    return version1 < version2;
}

void LoadAndSortUpdateFilenames(const std::string &directory_path, std::vector<std::string> * const update_filenames) {
    FileUtil::Directory directory(directory_path, "[^.]+\\.[^.]+\\.\\d+");
    for (const auto &entry : directory)
        update_filenames->emplace_back(entry.getName());

    std::sort(update_filenames->begin(), update_filenames->end(), FileNameCompare);
}

void ApplyUpdate(DbConnection * const /*db_connection*/, const std::string &database, const std::string &table, const unsigned version) {
    LOG_INFO("applying update \"" + database + "." + table + "." + std::to_string(version) + "\".");
}

void ApplyUpdates(DbConnection * const db_connection, const std::vector<std::string> &update_filenames) {
    std::string previous_database, previous_table;
    unsigned previous_version;

    for (const auto &update_filename : update_filenames) {
        std::string current_database, current_table;
        unsigned current_version;
        SplitIntoDatabaseTableAndVersion(update_filename, &current_database, &current_table, &current_version);
        if (not previous_database.empty() and (previous_database != current_database or previous_table != current_table))
            ApplyUpdate(db_connection, previous_database, previous_table, previous_version);

        previous_database = current_database;
        previous_table    = current_table;
        previous_version  = current_version;
    }

    if (not previous_database.empty())
        ApplyUpdate(db_connection, previous_database, previous_table, previous_version);
}

} // unnamed namespace

int Main(int argc, char *argv[]) {
    if (argc != 2)
        ::Usage("[--verbose] update_directory_path");

    std::vector<std::string> update_filenames;
    LoadAndSortUpdateFilenames(argv[1], &update_filenames);

    DbConnection db_connection;
    if (not db_connection.tableExists("ub_tools", "table_versions"))
        db_connection.queryOrDie("CREATE TABLE ub_tools.table_versions (version INT UNSIGNED NOT NULL, table_name VARCHAR(64) NOT NULL, "
                                 "UNIQUE(table_name)) CHARACTER SET utf8mb4 COLLATE utf8mb4_bin");

    ApplyUpdates(&db_connection, update_filenames);

    return EXIT_SUCCESS;
}
<|endoftext|>


<commit_before>#include <iostream>
#include "bh/bh.hpp"
#include "util/timing.hpp"
#include "util/argparse.hpp"

using namespace std;
using namespace bh;
using namespace argparse;

double monte_carlo_pi(int samples, int iterations)
{
    multi_array<double> x, y, m, c, accu(1);        // Operands
    accu = (double)0.0;                             // Acculumate across iterations
    for(int i=0; i<iterations; ++i) {
        x = random<double>(samples);                // Sample random numbers
        y = random<double>(samples);
        m = as<double>(sqrt(x*x + y*y)<=1.0);    // Model
        c = reduce(m, ADD, 0);                      // Count

        accu += (c*4.0) / (double)samples;          // Approximate
    }
    accu /= (double)iterations;

    return scalar(accu);
}

int main(int argc, char* argv[])
{
    const char usage[] = "usage: ./monte_carlo_pi --size=1000*10 [--verbose]";
    if (2>argc) {
        cout << usage << endl;
        return 1;
    }

    arguments_t args;                               // Parse command-line
    if (!parse_args(argc, argv, args)) {
        cout << "Err: Invalid argument(s)." << endl;
        cout << usage << endl;
        return 1;
    }
    if (2 > args.size.size()) {
        cout << "Err: Not enough arguments." << endl;
        cout << usage << endl;
        return 1;
    }
    if (2 < args.size.size()) {
        cout << "Err: Too many arguments." << endl;
        cout << usage << endl;
        return 1;
    }

    size_t start = sample_time();
    double mcp = monte_carlo_pi(args.size[0], args.size[1]);
    size_t end = sample_time();
                                                    // Output timing
    cout << "{elapsed-time: "<< (end-start)/1000000.0 <<"";
    if (args.verbose) {                             // and values.
        cout << ", \"output\": [";
        cout << mcp;
        cout << "]";
    }
    cout << "}" << endl;

    return 0;
}

<commit_msg>cppb: More fixes...<commit_after>#include <iostream>
#include "bh/bh.hpp"
#include "util/timing.hpp"
#include "util/argparse.hpp"

using namespace std;
using namespace bh;
using namespace argparse;

double monte_carlo_pi(int samples, int iterations)
{
    multi_array<double> x, y, m, c, accu(1);        // Operands
    std::cout << "HMM" << std::endl;
    accu = (double)0.0;                             // Acculumate across iterations
    std::cout << "HMM2" << std::endl;
    for(int i=0; i<iterations; ++i) {
        x = random<double>(samples);                // Sample random numbers
        y = random<double>(samples);
        m = as<double>(sqrt(x*x + y*y)<=1.0);    // Model
        c = reduce(m, ADD, 0);                      // Count

        accu += (c*4.0) / (double)samples;          // Approximate
    }
    accu /= (double)iterations;

    return scalar(accu);
}

int main(int argc, char* argv[])
{
    const char usage[] = "usage: ./monte_carlo_pi --size=1000*10 [--verbose]";
    if (2>argc) {
        cout << usage << endl;
        return 1;
    }

    arguments_t args;                               // Parse command-line
    if (!parse_args(argc, argv, args)) {
        cout << "Err: Invalid argument(s)." << endl;
        cout << usage << endl;
        return 1;
    }
    if (2 > args.size.size()) {
        cout << "Err: Not enough arguments." << endl;
        cout << usage << endl;
        return 1;
    }
    if (2 < args.size.size()) {
        cout << "Err: Too many arguments." << endl;
        cout << usage << endl;
        return 1;
    }

    size_t start = sample_time();
    double mcp = monte_carlo_pi(args.size[0], args.size[1]);
    size_t end = sample_time();
                                                    // Output timing
    cout << "{elapsed-time: "<< (end-start)/1000000.0 <<"";
    if (args.verbose) {                             // and values.
        cout << ", \"output\": [";
        cout << mcp;
        cout << "]";
    }
    cout << "}" << endl;

    return 0;
}

<|endoftext|>


<commit_before>/** \brief Utility for augmenting MARC records with links to a local full-text database.
 *  \author Dr. Johannes Ruscheinski (johannes.ruscheinski@uni-tuebingen.de)
 *
 *  \copyright 2015-2017 Universitätsbibliothek Tübingen.  All rights reserved.
 *
 *  This program is free software: you can redistribute it and/or modify
 *  it under the terms of the GNU Affero General Public License as
 *  published by the Free Software Foundation, either version 3 of the
 *  License, or (at your option) any later version.
 *
 *  This program is distributed in the hope that it will be useful,
 *  but WITHOUT ANY WARRANTY; without even the implied warranty of
 *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 *  GNU Affero General Public License for more details.
 *
 *  You should have received a copy of the GNU Affero General Public License
 *  along with this program.  If not, see <http://www.gnu.org/licenses/>.
 */

#include <iostream>
#include <memory>
#include <vector>
#include <cassert>
#include <cstdio>
#include <cstdlib>
#include "Compiler.h"
#include "DbConnection.h"
#include "DirectoryEntry.h"
#include "Downloader.h"
#include "ExecUtil.h"
#include "FileLocker.h"
#include "FileUtil.h"
#include "FullTextCache.h"
#include "MarcRecord.h"
#include "MarcReader.h"
#include "MarcUtil.h"
#include "MarcWriter.h"
#include "MediaTypeUtil.h"
#include "MiscUtil.h"
#include "PdfUtil.h"
#include "Semaphore.h"
#include "StringUtil.h"
#include "Subfields.h"
#include "TextUtil.h"
#include "util.h"
#include "VuFind.h"

static void Usage() __attribute__((noreturn));

static void Usage() {
    std::cerr << "Usage: " << ::progname << " file_offset marc_input marc_output full_text_db error_log_path\n\n"
              << "       file_offset  Where to start reading a MARC data set from in marc_input.\n\n";
    std::exit(EXIT_FAILURE);
}

// \note Sets "error_message" when it returns false.
bool GetDocumentAndMediaType(const std::string &url, const unsigned timeout, std::string * const document,
                             std::string * const media_type, std::string * const error_message)
{
    Downloader::Params params;
    Downloader downloader(url, params, timeout);
    if (downloader.anErrorOccurred()) {
        *error_message = downloader.getLastErrorMessage();
        return false;
    }

    *document = downloader.getMessageBody();
    *media_type = downloader.getMediaType();
    if (media_type->empty()) {
        *error_message = "failed to determine the media type!";
        return false;
    }

    return true;
}

static const std::map<std::string, std::string> marc_to_tesseract_language_codes_map {
    { "bul", "bul" },
    { "cze", "ces" },
    { "dan", "dan" },
    { "dut", "nld" },
    { "eng", "eng" },
    { "fin", "fin" },
    { "fre", "fra" },
    { "ger", "deu" },
    { "grc", "grc" },
    { "heb", "heb" },
    { "hun", "hun" },
    { "ita", "ita" },
    { "lat", "lat" },
    { "nor", "nor" },
    { "pol", "pol" },
    { "por", "por" },
    { "rus", "rus" },
    { "slv", "slv" },
    { "spa", "spa" },
    { "swe", "swe" },
};

std::string GetTesseractLanguageCode(const MarcRecord &record) {
    const auto map_iter(marc_to_tesseract_language_codes_map.find(
        record.getLanguageCode()));
    return (map_iter == marc_to_tesseract_language_codes_map.cend()) ? "" : map_iter->second;
}

// Checks subfields "3" and "z" to see if they start w/ "Rezension".
bool IsProbablyAReview(const Subfields &subfields) {
    const auto _3_begin_end(subfields.getIterators('3'));
    if (_3_begin_end.first != _3_begin_end.second) {
        if (StringUtil::StartsWith(_3_begin_end.first->value_, "Rezension"))
            return true;
    } else {
        const auto z_begin_end(subfields.getIterators('z'));
        if (z_begin_end.first != z_begin_end.second
            and StringUtil::StartsWith(z_begin_end.first->value_, "Rezension"))
            return true;
    }

    return false;
}

// \return The concatenated contents of all 520$a subfields.
std::string GetTextFrom520a(const MarcRecord &record) {
    std::string concatenated_text;

    std::vector<size_t> field_indices;
    record.getFieldIndices("520", &field_indices);
    for (const auto index : field_indices) {
        const Subfields subfields(record.getFieldData(index));
        if (subfields.hasSubfield('a')) {
            if (not concatenated_text.empty())
                concatenated_text += ' ';
            concatenated_text += subfields.getFirstSubfieldValue('a');
        }
    }

    return concatenated_text;
}

std::string ConvertToPlainText(const std::string &media_type, const std::string &tesseract_language_code,
                               const std::string &document)
{
    if (media_type == "text/plain")
        return document;
    if (media_type == "text/html")
        return TextUtil::ExtractTextFromHtml(document);
    if (StringUtil::StartsWith(media_type, "application/pdf")) {
        if (PdfUtil::PdfDocContainsNoText(document)) {
            std::string extracted_text;
            if (not PdfUtil::GetTextFromImagePDF(document, tesseract_language_code, &extracted_text)) {
                std::cerr << "Warning: Failed to extract text from an image PDF!\n";
                return "";
            }
            return extracted_text;
        }
        return PdfUtil::ExtractText(document);

    }
    logger->warning("in ConvertToPlainText: don't know how to handle media type: " + media_type);
    return "";
}

// Returns true if text has been successfully extracted, else false.
bool ProcessRecord(MarcReader * const marc_reader, const std::string &marc_output_filename,
                   const std::string &full_text_db_path)
{
    MarcRecord record(marc_reader->read());
    const std::string ppn(record.getControlNumber());

    std::string mysql_url;
    VuFind::GetMysqlURL(&mysql_url);
    DbConnection db_connection(mysql_url);

    bool success(false);
    if (not FullTextCache::CacheEntryExpired(&db_connection, full_text_db_path, ppn)) {
        Semaphore semaphore("/full_text_cached_counter", Semaphore::ATTACH);
        ++semaphore;
        success = true;
    } else {
        std::string combined_text(GetTextFrom520a(record));

        std::vector<size_t> _856_field_indices;
        record.getFieldIndices("856", &_856_field_indices);

        constexpr unsigned PER_DOC_TIMEOUT(10000); // in milliseconds

        std::string error_message, url;
        for (const size_t _856_field_index : _856_field_indices) {
            const Subfields _856_subfields(record.getSubfields(_856_field_index));
            if (_856_subfields.getIndicator1() == '7' or not _856_subfields.hasSubfield('u'))
                continue;

            if (IsProbablyAReview(_856_subfields))
                continue;

            url = _856_subfields.getFirstSubfieldValue('u');
            std::string document, media_type;
            if ((not GetDocumentAndMediaType(url, PER_DOC_TIMEOUT, &document, &media_type, &error_message)))
                break;
            else {
                std::string extracted_text(ConvertToPlainText(media_type, GetTesseractLanguageCode(record),
                                                              document));
                if (not extracted_text.empty()) {
                    if (combined_text.empty())
                        combined_text.swap(extracted_text);
                    else
                        combined_text += " " + extracted_text;
                } else {
                    error_message = "failed to extract text from the downloaded document!";
                    break;
                }
            }
        }

        std::string data;
        if (error_message.empty()) {
            url.clear();
            data = "Content-type: text/plain\r\n\r\n" + combined_text;
        }
        FullTextCache::InsertIntoCache(&db_connection, full_text_db_path, ppn, url, data, error_message);

        if (not combined_text.empty()) {
            record.insertSubfield("FUL", 'e', "http://localhost/cgi-bin/full_text_lookup?id=" + ppn);
            success = true;
        }
    }

    // Safely append the modified MARC data to the MARC output file:
    std::unique_ptr<MarcWriter> marc_writer(MarcWriter::Factory(marc_output_filename, MarcWriter::BINARY,
                                                                MarcWriter::APPEND));
    MarcUtil::FileLockedComposeAndWriteRecord(marc_writer.get(), &record);

    return success;
}

int main(int argc, char *argv[]) {
    ::progname = argv[0];

    if (argc != 5)
        Usage();

    long offset;
    if (not StringUtil::ToNumber(argv[1], &offset))
        logger->error("file offset must be a number!");

    std::unique_ptr<MarcReader> marc_reader(MarcReader::Factory(argv[2], MarcReader::BINARY));
    if (not marc_reader->seek(offset, SEEK_SET))
        logger->error("failed to position " + marc_reader->getPath() + " at offset " + std::to_string(offset)
                      + "! (" + std::to_string(errno) + ")");

    try {
        return ProcessRecord(marc_reader.get(), argv[3], argv[4]) ? EXIT_SUCCESS : EXIT_FAILURE;
    } catch (const std::exception &e) {
        logger->error("While reading \"" + marc_reader->getPath() + "\" starting at offset \""
                      + std::string(argv[1]) + "\", caught exception: " + std::string(e.what()));
    }
}
<commit_msg>Fixed a capitalisation problem.<commit_after>/** \brief Utility for augmenting MARC records with links to a local full-text database.
 *  \author Dr. Johannes Ruscheinski (johannes.ruscheinski@uni-tuebingen.de)
 *
 *  \copyright 2015-2017 Universitätsbibliothek Tübingen.  All rights reserved.
 *
 *  This program is free software: you can redistribute it and/or modify
 *  it under the terms of the GNU Affero General Public License as
 *  published by the Free Software Foundation, either version 3 of the
 *  License, or (at your option) any later version.
 *
 *  This program is distributed in the hope that it will be useful,
 *  but WITHOUT ANY WARRANTY; without even the implied warranty of
 *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 *  GNU Affero General Public License for more details.
 *
 *  You should have received a copy of the GNU Affero General Public License
 *  along with this program.  If not, see <http://www.gnu.org/licenses/>.
 */

#include <iostream>
#include <memory>
#include <vector>
#include <cassert>
#include <cstdio>
#include <cstdlib>
#include "Compiler.h"
#include "DbConnection.h"
#include "DirectoryEntry.h"
#include "Downloader.h"
#include "ExecUtil.h"
#include "FileLocker.h"
#include "FileUtil.h"
#include "FullTextCache.h"
#include "MarcRecord.h"
#include "MarcReader.h"
#include "MarcUtil.h"
#include "MarcWriter.h"
#include "MediaTypeUtil.h"
#include "MiscUtil.h"
#include "PdfUtil.h"
#include "Semaphore.h"
#include "StringUtil.h"
#include "Subfields.h"
#include "TextUtil.h"
#include "util.h"
#include "VuFind.h"

static void Usage() __attribute__((noreturn));

static void Usage() {
    std::cerr << "Usage: " << ::progname << " file_offset marc_input marc_output full_text_db error_log_path\n\n"
              << "       file_offset  Where to start reading a MARC data set from in marc_input.\n\n";
    std::exit(EXIT_FAILURE);
}

// \note Sets "error_message" when it returns false.
bool GetDocumentAndMediaType(const std::string &url, const unsigned timeout, std::string * const document,
                             std::string * const media_type, std::string * const error_message)
{
    Downloader::Params params;
    Downloader downloader(url, params, timeout);
    if (downloader.anErrorOccurred()) {
        *error_message = downloader.getLastErrorMessage();
        return false;
    }

    *document = downloader.getMessageBody();
    *media_type = downloader.getMediaType();
    if (media_type->empty()) {
        *error_message = "failed to determine the media type!";
        return false;
    }

    return true;
}

static const std::map<std::string, std::string> marc_to_tesseract_language_codes_map {
    { "bul", "bul" },
    { "cze", "ces" },
    { "dan", "dan" },
    { "dut", "nld" },
    { "eng", "eng" },
    { "fin", "fin" },
    { "fre", "fra" },
    { "ger", "deu" },
    { "grc", "grc" },
    { "heb", "heb" },
    { "hun", "hun" },
    { "ita", "ita" },
    { "lat", "lat" },
    { "nor", "nor" },
    { "pol", "pol" },
    { "por", "por" },
    { "rus", "rus" },
    { "slv", "slv" },
    { "spa", "spa" },
    { "swe", "swe" },
};

std::string GetTesseractLanguageCode(const MarcRecord &record) {
    const auto map_iter(marc_to_tesseract_language_codes_map.find(
        record.getLanguageCode()));
    return (map_iter == marc_to_tesseract_language_codes_map.cend()) ? "" : map_iter->second;
}

// Checks subfields "3" and "z" to see if they start w/ "Rezension".
bool IsProbablyAReview(const Subfields &subfields) {
    const auto _3_begin_end(subfields.getIterators('3'));
    if (_3_begin_end.first != _3_begin_end.second) {
        if (StringUtil::StartsWith(_3_begin_end.first->value_, "Rezension"))
            return true;
    } else {
        const auto z_begin_end(subfields.getIterators('z'));
        if (z_begin_end.first != z_begin_end.second
            and StringUtil::StartsWith(z_begin_end.first->value_, "Rezension"))
            return true;
    }

    return false;
}

// \return The concatenated contents of all 520$a subfields.
std::string GetTextFrom520a(const MarcRecord &record) {
    std::string concatenated_text;

    std::vector<size_t> field_indices;
    record.getFieldIndices("520", &field_indices);
    for (const auto index : field_indices) {
        const Subfields subfields(record.getFieldData(index));
        if (subfields.hasSubfield('a')) {
            if (not concatenated_text.empty())
                concatenated_text += ' ';
            concatenated_text += subfields.getFirstSubfieldValue('a');
        }
    }

    return concatenated_text;
}

std::string ConvertToPlainText(const std::string &media_type, const std::string &tesseract_language_code,
                               const std::string &document)
{
    if (media_type == "text/plain")
        return document;
    if (media_type == "text/html")
        return TextUtil::ExtractTextFromHtml(document);
    if (StringUtil::StartsWith(media_type, "application/pdf")) {
        if (PdfUtil::PdfDocContainsNoText(document)) {
            std::string extracted_text;
            if (not PdfUtil::GetTextFromImagePDF(document, tesseract_language_code, &extracted_text)) {
                std::cerr << "Warning: Failed to extract text from an image PDF!\n";
                return "";
            }
            return extracted_text;
        }
        return PdfUtil::ExtractText(document);

    }
    logger->warning("in ConvertToPlainText: don't know how to handle media type: " + media_type);
    return "";
}

// Returns true if text has been successfully extracted, else false.
bool ProcessRecord(MarcReader * const marc_reader, const std::string &marc_output_filename,
                   const std::string &full_text_db_path)
{
    MarcRecord record(marc_reader->read());
    const std::string ppn(record.getControlNumber());

    std::string mysql_url;
    VuFind::GetMysqlURL(&mysql_url);
    DbConnection db_connection(mysql_url);

    bool success(false);
    if (not FullTextCache::CacheEntryExpired(&db_connection, full_text_db_path, ppn)) {
        Semaphore semaphore("/full_text_cached_counter", Semaphore::ATTACH);
        ++semaphore;
        success = true;
    } else {
        std::string combined_text(GetTextFrom520a(record));

        std::vector<size_t> _856_field_indices;
        record.getFieldIndices("856", &_856_field_indices);

        constexpr unsigned PER_DOC_TIMEOUT(10000); // in milliseconds

        std::string error_message, url;
        for (const size_t _856_field_index : _856_field_indices) {
            const Subfields _856_subfields(record.getSubfields(_856_field_index));
            if (_856_subfields.getIndicator1() == '7' or not _856_subfields.hasSubfield('u'))
                continue;

            if (IsProbablyAReview(_856_subfields))
                continue;

            url = _856_subfields.getFirstSubfieldValue('u');
            std::string document, media_type;
            if ((not GetDocumentAndMediaType(url, PER_DOC_TIMEOUT, &document, &media_type, &error_message)))
                break;
            else {
                std::string extracted_text(ConvertToPlainText(media_type, GetTesseractLanguageCode(record),
                                                              document));
                if (not extracted_text.empty()) {
                    if (combined_text.empty())
                        combined_text.swap(extracted_text);
                    else
                        combined_text += " " + extracted_text;
                } else {
                    error_message = "failed to extract text from the downloaded document!";
                    break;
                }
            }
        }

        std::string data;
        if (error_message.empty()) {
            url.clear();
            data = "Content-Type: text/plain\r\n\r\n" + combined_text;
        }
        FullTextCache::InsertIntoCache(&db_connection, full_text_db_path, ppn, url, data, error_message);

        if (not combined_text.empty()) {
            record.insertSubfield("FUL", 'e', "http://localhost/cgi-bin/full_text_lookup?id=" + ppn);
            success = true;
        }
    }

    // Safely append the modified MARC data to the MARC output file:
    std::unique_ptr<MarcWriter> marc_writer(MarcWriter::Factory(marc_output_filename, MarcWriter::BINARY,
                                                                MarcWriter::APPEND));
    MarcUtil::FileLockedComposeAndWriteRecord(marc_writer.get(), &record);

    return success;
}

int main(int argc, char *argv[]) {
    ::progname = argv[0];

    if (argc != 5)
        Usage();

    long offset;
    if (not StringUtil::ToNumber(argv[1], &offset))
        logger->error("file offset must be a number!");

    std::unique_ptr<MarcReader> marc_reader(MarcReader::Factory(argv[2], MarcReader::BINARY));
    if (not marc_reader->seek(offset, SEEK_SET))
        logger->error("failed to position " + marc_reader->getPath() + " at offset " + std::to_string(offset)
                      + "! (" + std::to_string(errno) + ")");

    try {
        return ProcessRecord(marc_reader.get(), argv[3], argv[4]) ? EXIT_SUCCESS : EXIT_FAILURE;
    } catch (const std::exception &e) {
        logger->error("While reading \"" + marc_reader->getPath() + "\" starting at offset \""
                      + std::string(argv[1]) + "\", caught exception: " + std::string(e.what()));
    }
}
<|endoftext|>


<commit_before>#include "Platform.hpp"
#include "genome.pb.h"
#include "CellComponent.hpp"
#include "ActivityStats.hpp"
#include "Neuron.hpp"
#include "Branch.hpp"
#include "Synapse.hpp"
#include "Base64.hpp"
#include "Brain.hpp"
#include "SimpleProteinEnvironment.hpp"
#include <time.h>

namespace Elysia {
void testTwoConnectedNeurons() {
    ProteinEnvironment *myProteinEnvironment= new SimpleProteinEnvironment();

	Brain *brain= new Brain(myProteinEnvironment);
	FILE *dendriteTree=NULL;
	dendriteTree = fopen("Dendritic_Tree.txt", "w");
	std::vector<Neuron *> createdList;
	int neuronNumber = 2;

	for(int i=0;i<neuronNumber;i++){
        Genome::Gene gene;//FIXME set source and target regions to match the desired behavior
        Genome::TemporalBoundingBox *sourcebb=gene.add_bounds();
        Genome::TemporalBoundingBox *targetbb=gene.add_bounds();
        sourcebb->set_minx(i);
        sourcebb->set_miny(i);
        sourcebb->set_minz(i);

        sourcebb->set_maxx(i);
        sourcebb->set_maxy(i);
        sourcebb->set_maxz(i);

        targetbb->set_minx(i);
        targetbb->set_miny(i);
        targetbb->set_minz(i);

        targetbb->set_maxx(1-i);
        targetbb->set_maxy(1-i);
        targetbb->set_maxz(2);

		Vector3f v;
		v.x = i;
		v.y = i;
		v.z = 1;
		Neuron *n;
		brain->mAllNeurons.insert(n = new Neuron(brain, 2, 3, 4, v,gene));
		createdList.push_back(n);
		}

	for(float i=0;i<neuronNumber;i++){
		Neuron *n = createdList[i];
        n->developSynapse(n->getActivityStats());
        size_t parent;
        parent = 0;
        n->visualizeTree(dendriteTree, parent);
		//const Vector3f &location):  mNeuronLocation(location){));
	}
	for(int j=0; j<10; j++){
		for(float i=0;i<neuronNumber;i++){
			Neuron *n = createdList[i];
			if(j== 0 && i==0){n->activateComponent(*brain,100);}
			//if(j== 2){brain->inactivateNeuron(n);}
			n->tick();
		//const Vector3f &location):  mNeuronLocation(location){));
		}

		brain ->tick();
	}
	fclose(dendriteTree);
    delete brain;
}

//Target is axon, Source is dendrite (?)
Neuron* placeTestNeuron(Brain* brain, float locx, float locy, float locz, float sx, float sy, float sz, float range){
		float random = rand()/(float)RAND_MAX;
		Genome::Gene gene;//FIXME set source and target regions to match the desired behavior
        Genome::TemporalBoundingBox *sourcebb=gene.add_bounds();
        Genome::TemporalBoundingBox *targetbb=gene.add_bounds();
        sourcebb->set_minx(sx-range*random);
        sourcebb->set_miny(sy-range*random);
        sourcebb->set_minz(1);

        sourcebb->set_maxx(sx+range*random);
        sourcebb->set_maxy(sy+range*random);
        sourcebb->set_maxz(1);

        targetbb->set_minx(locx-range);
        targetbb->set_miny(locy-range);
        targetbb->set_minz(1);

        targetbb->set_maxx(locx+range);
        targetbb->set_maxy(locy+range);
        targetbb->set_maxz(1);

		Vector3f v;
		v.x = locx+range*random;
		v.y = locy+range*random;
		v.z = 1;
		Neuron *n;
		srand(time(NULL));
		brain->mAllNeurons.insert(n = new Neuron(brain, 2, 3, 4, v,gene));
		return n;
	}

void testDevelopment(){
	ProteinEnvironment *myProteinEnvironment= new SimpleProteinEnvironment();
	srand(time(NULL));
	Brain *brain= new Brain(myProteinEnvironment);
	FILE *dendriteTree=NULL;
	dendriteTree = fopen("Development_Tree.txt", "w");
	std::vector<Neuron *> createdList;
	int neuronNumber = 4;

	//Region 1
	for(int i=0;i<neuronNumber;i++){
		Neuron* n = placeTestNeuron(brain, 10, 10, 1, 100, 100, 1, 5);
		createdList.push_back(n);
	}
	//Region 2
	for(int i=0;i<neuronNumber;i++){
		Neuron* n = placeTestNeuron(brain, 100, 100, 1, 10, 10, 1, 5);
		createdList.push_back(n);
	}

	for(float i=0;i<2*neuronNumber;i++){
		Neuron *n = createdList[i];
        n->developSynapse(n->getActivityStats());
        size_t parent;
        parent = 0;
        n->visualizeTree(dendriteTree, parent);
		//const Vector3f &location):  mNeuronLocation(location){));
	}
	for(int j=0; j<10; j++){
		for(float i=0;i<neuronNumber;i++){
			Neuron *n = createdList[i];
			if(j==0){n->activateComponent(*brain,100);}
			n->tick();
		//const Vector3f &location):  mNeuronLocation(location){));
		}

		brain ->tick();
	}
	fclose(dendriteTree);
    delete brain;
}

void testResultHelper(const std::vector<std::pair<Elysia::Genome::Effect, float> >&combinedResult, float*grow_leaf_count,float*grow_neuron_count,float*other_count){
   using namespace Elysia::Genome;
    *grow_neuron_count=0;
    *grow_leaf_count=0;
    *other_count=0;
    for (size_t i=0;i<combinedResult.size();++i) {
       switch(combinedResult[i].first) {
         case GROW_NEURON:
           *grow_neuron_count+=combinedResult[i].second;
           break;
         case GROW_LEAF:
           *grow_leaf_count+=combinedResult[i].second;
           break;
         default:
           *other_count+=combinedResult[i].second;
           break;
       }
   }

}

void testProteinEnvironment() {
   using namespace Elysia::Genome;
   Elysia::Genome::Genome twoOverlappingGenes;
   Elysia::Genome::Chromosome *father=twoOverlappingGenes.mutable_fathers();
   Elysia::Genome::Gene firstGene;
   Elysia::Genome::Protein firstProtein;
   firstProtein.set_protein_code(GROW_NEURON);//so we can easily identify where
   firstProtein.set_density(0.125);
   Elysia::Genome::Protein firstAuxProtein;
   firstAuxProtein.set_protein_code(GROW_LEAF);//so we see that they are additive
   firstAuxProtein.set_density(0.25);
   *firstGene.add_external_proteins()=firstProtein;
   assert(firstGene.external_proteins(0).protein_code()==GROW_NEURON);
   *firstGene.add_external_proteins()=firstAuxProtein;
   assert(firstGene.external_proteins(1).protein_code()==GROW_LEAF);
   Elysia::Genome::TemporalBoundingBox firstRegion;
   firstRegion.set_minx(0);
   firstRegion.set_miny(0);
   firstRegion.set_minz(0);
   firstRegion.set_maxx(2);
   firstRegion.set_maxy(2);
   firstRegion.set_maxz(2);

   *firstGene.add_bounds()=firstRegion;

   Elysia::Genome::TemporalBoundingBox firstTargetRegion;
   firstTargetRegion.set_minx(5);
   firstTargetRegion.set_miny(5);
   firstTargetRegion.set_minz(5);
   firstTargetRegion.set_maxx(8);
   firstTargetRegion.set_maxy(8);
   firstTargetRegion.set_maxz(8);

   *firstGene.add_target_region()=firstTargetRegion;
   Elysia::Genome::Gene secondGene;
   Elysia::Genome::Protein secondProtein;
   secondProtein.set_protein_code(GROW_LEAF);
   secondProtein.set_density(0.5);
   *secondGene.add_external_proteins()=secondProtein;
   Elysia::Genome::TemporalBoundingBox secondRegion;
   secondRegion.set_minx(-1);
   secondRegion.set_miny(-1);
   secondRegion.set_minz(-1);
   secondRegion.set_maxx(1);
   secondRegion.set_maxy(1);
   secondRegion.set_maxz(1);
   *secondGene.add_bounds()=secondRegion;

   Elysia::Genome::TemporalBoundingBox secondTargetRegion;
   secondTargetRegion.set_minx(-5);
   secondTargetRegion.set_miny(-5);
   secondTargetRegion.set_minz(-5);
   secondTargetRegion.set_maxx(3);
   secondTargetRegion.set_maxy(3);
   secondTargetRegion.set_maxz(3);

   *secondGene.add_target_region()=secondTargetRegion;

   *father->add_genes()=firstGene;
   *father->add_genes()=secondGene;

   ProteinEnvironment * pe=new SimpleProteinEnvironment;
   pe->initialize(twoOverlappingGenes);
   std::vector<std::pair<Elysia::Genome::Effect, float> > combinedResult=pe->getCompleteProteinDensity(Vector3f(.5,.5,0));
   //check that firstResult matches expectations
   std::vector<std::pair<Elysia::Genome::Effect, float> > firstResult=pe->getCompleteProteinDensity(Vector3f(1.5,1.5,0));
   //check that secondResult matches expectations
   std::vector<std::pair<Elysia::Genome::Effect, float> > secondResult=pe->getCompleteProteinDensity(Vector3f(-.5,-.5,0));
   float grow_leaf_count=0;
   float grow_neuron_count=0;
   float other_count=0;

   testResultHelper(combinedResult,&grow_leaf_count,&grow_neuron_count,&other_count);
   assert(grow_leaf_count==.75);
   assert(grow_neuron_count==.125);
   assert(other_count==0);

   testResultHelper(firstResult,&grow_leaf_count,&grow_neuron_count,&other_count);
   assert(grow_leaf_count==.25);
   assert(grow_neuron_count==.125);
   assert(other_count==0);

   testResultHelper(secondResult,&grow_leaf_count,&grow_neuron_count,&other_count);
   assert(grow_leaf_count==.5);
   assert(grow_neuron_count==0);
   assert(other_count==0);
   //return;//UNCOMMENT TO GET FAILING TEST
   const Gene * grow_neuron_gene = &pe->retrieveGene(Vector3f(0.5,0.5,0),
                                              GROW_NEURON);
   const Gene * combined_grow_leaf_gene = &pe->retrieveGene(Vector3f(0.5,0.5,0),
                                                     GROW_LEAF);
   const Gene * first_grow_leaf_gene = &pe->retrieveGene(Vector3f(1.5,1.5,0),
                                                  GROW_LEAF);
   const Gene * second_grow_leaf_gene = &pe->retrieveGene(Vector3f(-0.5,-0.5,0),
                                                   GROW_LEAF);
   assert(grow_neuron_gene->target_region(0).minx()==5);

   assert(combined_grow_leaf_gene->target_region(0).minx()==5||combined_grow_leaf_gene->target_region(0).minx()==-5);
   assert(first_grow_leaf_gene->target_region(0).minx()==5);
   assert(second_grow_leaf_gene->target_region(0).minx()==-5);

   delete pe;
}

}

int runtest(){
    //Elysia::testTwoConnectedNeurons();
	Elysia::testDevelopment();
    Elysia::testProteinEnvironment();
    if (0) for (int i=0;i<30000;++i) {
        Elysia::Brain b(new Elysia::SimpleProteinEnvironment);
//        usleep(10);
    }
	//getchar();
	return 1;

}
<commit_msg>Cosmetic change, have to commit to merge<commit_after>#include "Platform.hpp"
#include "genome.pb.h"
#include "CellComponent.hpp"
#include "ActivityStats.hpp"
#include "Neuron.hpp"
#include "Branch.hpp"
#include "Synapse.hpp"
#include "Base64.hpp"
#include "Brain.hpp"
#include "SimpleProteinEnvironment.hpp"
#include <time.h>

namespace Elysia {
void testTwoConnectedNeurons() {
    ProteinEnvironment *myProteinEnvironment= new SimpleProteinEnvironment();

	Brain *brain= new Brain(myProteinEnvironment);
	FILE *dendriteTree=NULL;
	dendriteTree = fopen("Dendritic_Tree.txt", "w");
	std::vector<Neuron *> createdList;
	int neuronNumber = 2;

	for(int i=0;i<neuronNumber;i++){
        Genome::Gene gene;//FIXME set source and target regions to match the desired behavior
        Genome::TemporalBoundingBox *sourcebb=gene.add_bounds();
        Genome::TemporalBoundingBox *targetbb=gene.add_bounds();
        sourcebb->set_minx(i);
        sourcebb->set_miny(i);
        sourcebb->set_minz(i);

        sourcebb->set_maxx(i);
        sourcebb->set_maxy(i);
        sourcebb->set_maxz(i);

        targetbb->set_minx(i);
        targetbb->set_miny(i);
        targetbb->set_minz(i);

        targetbb->set_maxx(1-i);
        targetbb->set_maxy(1-i);
        targetbb->set_maxz(2);

		Vector3f v;
		v.x = i;
		v.y = i;
		v.z = 1;
		Neuron *n;
		brain->mAllNeurons.insert(n = new Neuron(brain, 2, 3, 4, v,gene));
		createdList.push_back(n);
		}

	for(float i=0;i<neuronNumber;i++){
		Neuron *n = createdList[i];
        n->developSynapse(n->getActivityStats());
        size_t parent;
        parent = 0;
        n->visualizeTree(dendriteTree, parent);
		//const Vector3f &location):  mNeuronLocation(location){));
	}
	for(int j=0; j<10; j++){
		for(float i=0;i<neuronNumber;i++){
			Neuron *n = createdList[i];
			if(j== 0 && i==0){n->activateComponent(*brain,100);}
			//if(j== 2){brain->inactivateNeuron(n);}
			n->tick();
		//const Vector3f &location):  mNeuronLocation(location){));
		}

		brain ->tick();
	}
	fclose(dendriteTree);
    delete brain;
}

//Target is axon, Source is dendrite
Neuron* placeTestNeuron(Brain* brain, float locx, float locy, float locz, float sx, float sy, float sz, float range){
		float random = rand()/(float)RAND_MAX;
		Genome::Gene gene;//FIXME set source and target regions to match the desired behavior
        Genome::TemporalBoundingBox *sourcebb=gene.add_bounds();
        Genome::TemporalBoundingBox *targetbb=gene.add_bounds();
        sourcebb->set_minx(sx-range*random);
        sourcebb->set_miny(sy-range*random);
        sourcebb->set_minz(1);

        sourcebb->set_maxx(sx+range*random);
        sourcebb->set_maxy(sy+range*random);
        sourcebb->set_maxz(1);

        targetbb->set_minx(locx-range);
        targetbb->set_miny(locy-range);
        targetbb->set_minz(1);

        targetbb->set_maxx(locx+range);
        targetbb->set_maxy(locy+range);
        targetbb->set_maxz(1);

		Vector3f v;
		v.x = locx+range*random;
		v.y = locy+range*random;
		v.z = 1;
		Neuron *n;
		srand(time(NULL));
		brain->mAllNeurons.insert(n = new Neuron(brain, 2, 3, 4, v,gene));
		return n;
	}

void testDevelopment(){
	ProteinEnvironment *myProteinEnvironment= new SimpleProteinEnvironment();
	srand(time(NULL));
	Brain *brain= new Brain(myProteinEnvironment);
	FILE *dendriteTree=NULL;
	dendriteTree = fopen("Development_Tree.txt", "w");
	std::vector<Neuron *> createdList;
	int neuronNumber = 4;

	//Region 1
	for(int i=0;i<neuronNumber;i++){
		Neuron* n = placeTestNeuron(brain, 10, 10, 1, 100, 100, 1, 5);
		createdList.push_back(n);
	}
	//Region 2
	for(int i=0;i<neuronNumber;i++){
		Neuron* n = placeTestNeuron(brain, 100, 100, 1, 10, 10, 1, 5);
		createdList.push_back(n);
	}

	for(float i=0;i<2*neuronNumber;i++){
		Neuron *n = createdList[i];
        n->developSynapse(n->getActivityStats());
        size_t parent;
        parent = 0;
        n->visualizeTree(dendriteTree, parent);
		//const Vector3f &location):  mNeuronLocation(location){));
	}
	for(int j=0; j<10; j++){
		for(float i=0;i<neuronNumber;i++){
			Neuron *n = createdList[i];
			if(j==0){n->activateComponent(*brain,100);}
			n->tick();
		//const Vector3f &location):  mNeuronLocation(location){));
		}

		brain ->tick();
	}
	fclose(dendriteTree);
    delete brain;
}

void testResultHelper(const std::vector<std::pair<Elysia::Genome::Effect, float> >&combinedResult, float*grow_leaf_count,float*grow_neuron_count,float*other_count){
   using namespace Elysia::Genome;
    *grow_neuron_count=0;
    *grow_leaf_count=0;
    *other_count=0;
    for (size_t i=0;i<combinedResult.size();++i) {
       switch(combinedResult[i].first) {
         case GROW_NEURON:
           *grow_neuron_count+=combinedResult[i].second;
           break;
         case GROW_LEAF:
           *grow_leaf_count+=combinedResult[i].second;
           break;
         default:
           *other_count+=combinedResult[i].second;
           break;
       }
   }

}

void testProteinEnvironment() {
   using namespace Elysia::Genome;
   Elysia::Genome::Genome twoOverlappingGenes;
   Elysia::Genome::Chromosome *father=twoOverlappingGenes.mutable_fathers();
   Elysia::Genome::Gene firstGene;
   Elysia::Genome::Protein firstProtein;
   firstProtein.set_protein_code(GROW_NEURON);//so we can easily identify where
   firstProtein.set_density(0.125);
   Elysia::Genome::Protein firstAuxProtein;
   firstAuxProtein.set_protein_code(GROW_LEAF);//so we see that they are additive
   firstAuxProtein.set_density(0.25);
   *firstGene.add_external_proteins()=firstProtein;
   assert(firstGene.external_proteins(0).protein_code()==GROW_NEURON);
   *firstGene.add_external_proteins()=firstAuxProtein;
   assert(firstGene.external_proteins(1).protein_code()==GROW_LEAF);
   Elysia::Genome::TemporalBoundingBox firstRegion;
   firstRegion.set_minx(0);
   firstRegion.set_miny(0);
   firstRegion.set_minz(0);
   firstRegion.set_maxx(2);
   firstRegion.set_maxy(2);
   firstRegion.set_maxz(2);

   *firstGene.add_bounds()=firstRegion;

   Elysia::Genome::TemporalBoundingBox firstTargetRegion;
   firstTargetRegion.set_minx(5);
   firstTargetRegion.set_miny(5);
   firstTargetRegion.set_minz(5);
   firstTargetRegion.set_maxx(8);
   firstTargetRegion.set_maxy(8);
   firstTargetRegion.set_maxz(8);

   *firstGene.add_target_region()=firstTargetRegion;
   Elysia::Genome::Gene secondGene;
   Elysia::Genome::Protein secondProtein;
   secondProtein.set_protein_code(GROW_LEAF);
   secondProtein.set_density(0.5);
   *secondGene.add_external_proteins()=secondProtein;
   Elysia::Genome::TemporalBoundingBox secondRegion;
   secondRegion.set_minx(-1);
   secondRegion.set_miny(-1);
   secondRegion.set_minz(-1);
   secondRegion.set_maxx(1);
   secondRegion.set_maxy(1);
   secondRegion.set_maxz(1);
   *secondGene.add_bounds()=secondRegion;

   Elysia::Genome::TemporalBoundingBox secondTargetRegion;
   secondTargetRegion.set_minx(-5);
   secondTargetRegion.set_miny(-5);
   secondTargetRegion.set_minz(-5);
   secondTargetRegion.set_maxx(3);
   secondTargetRegion.set_maxy(3);
   secondTargetRegion.set_maxz(3);

   *secondGene.add_target_region()=secondTargetRegion;

   *father->add_genes()=firstGene;
   *father->add_genes()=secondGene;

   ProteinEnvironment * pe=new SimpleProteinEnvironment;
   pe->initialize(twoOverlappingGenes);
   std::vector<std::pair<Elysia::Genome::Effect, float> > combinedResult=pe->getCompleteProteinDensity(Vector3f(.5,.5,0));
   //check that firstResult matches expectations
   std::vector<std::pair<Elysia::Genome::Effect, float> > firstResult=pe->getCompleteProteinDensity(Vector3f(1.5,1.5,0));
   //check that secondResult matches expectations
   std::vector<std::pair<Elysia::Genome::Effect, float> > secondResult=pe->getCompleteProteinDensity(Vector3f(-.5,-.5,0));
   float grow_leaf_count=0;
   float grow_neuron_count=0;
   float other_count=0;

   testResultHelper(combinedResult,&grow_leaf_count,&grow_neuron_count,&other_count);
   assert(grow_leaf_count==.75);
   assert(grow_neuron_count==.125);
   assert(other_count==0);

   testResultHelper(firstResult,&grow_leaf_count,&grow_neuron_count,&other_count);
   assert(grow_leaf_count==.25);
   assert(grow_neuron_count==.125);
   assert(other_count==0);

   testResultHelper(secondResult,&grow_leaf_count,&grow_neuron_count,&other_count);
   assert(grow_leaf_count==.5);
   assert(grow_neuron_count==0);
   assert(other_count==0);
   //return;//UNCOMMENT TO GET FAILING TEST
   const Gene * grow_neuron_gene = &pe->retrieveGene(Vector3f(0.5,0.5,0),
                                              GROW_NEURON);
   const Gene * combined_grow_leaf_gene = &pe->retrieveGene(Vector3f(0.5,0.5,0),
                                                     GROW_LEAF);
   const Gene * first_grow_leaf_gene = &pe->retrieveGene(Vector3f(1.5,1.5,0),
                                                  GROW_LEAF);
   const Gene * second_grow_leaf_gene = &pe->retrieveGene(Vector3f(-0.5,-0.5,0),
                                                   GROW_LEAF);
   assert(grow_neuron_gene->target_region(0).minx()==5);

   assert(combined_grow_leaf_gene->target_region(0).minx()==5||combined_grow_leaf_gene->target_region(0).minx()==-5);
   assert(first_grow_leaf_gene->target_region(0).minx()==5);
   assert(second_grow_leaf_gene->target_region(0).minx()==-5);

   delete pe;
}

}

int runtest(){
    //Elysia::testTwoConnectedNeurons();
	Elysia::testDevelopment();
    Elysia::testProteinEnvironment();
    if (0) for (int i=0;i<30000;++i) {
        Elysia::Brain b(new Elysia::SimpleProteinEnvironment);
//        usleep(10);
    }
	//getchar();
	return 1;

}
<|endoftext|>


<commit_before>#include "std_incl.h"
#include "ResultManager.h"
#include "utils.h"

ResultManager::ResultManager(const char *outfile, const char* frameInfoFile, ResultManagerConfig *cfg)
{
	config = *cfg;
	outputFile = outfile;
	this->frameInfoFile = frameInfoFile;

	startFrame = 0;
	lastSaveFrame = 0;
	processedFrames = 0;

	qtrk = 0;

	thread = Threads::Create(ThreadLoop, this);
	quit=false;

	frameResults.resize(100);
	for (int i=0;i<frameResults.size();i++)
		frameResults[i] =new FrameResult(config.numBeads, config.numFrameInfoColumns);

	remove(outfile);
	remove(frameInfoFile);
}

ResultManager::~ResultManager()
{
	quit = true;
	Threads::WaitAndClose(thread);

	DeleteAllElems(frameResults);
}

void ResultManager::StoreResult(LocalizationResult *r)
{
	int index = r->job.frame - startFrame;

	if (index >= frameResults.size()) {
		return; // add errors?
	}

	LocalizationResult scaled = *r;
	// Make roi-centered pos
	scaled.pos = scaled.pos - vector3f( qtrk->cfg.width*0.5f, qtrk->cfg.height*0.5f, 0);
	scaled.pos = ( scaled.pos + config.offset ) * config.scaling;
	FrameResult* fr = frameResults[index];
	fr->results[r->job.zlutIndex] = scaled;
	fr->count++;

	// Advance fullFrames
	frameCountMutex.lock();
	while (processedFrames - startFrame < frameResults.size() && frameResults[processedFrames-startFrame]->count == config.numBeads)
		processedFrames ++;
	frameCountMutex.unlock();
}

void ResultManager::Write()
{
	FILE* f = fopen(outputFile.c_str(), "a");
	FILE* finfo = fopen(frameInfoFile.c_str(), "a");

	resultMutex.lock();
	if (config.binaryOutput) {
		for (uint j=lastSaveFrame; j<processedFrames;j++)
		{
			auto fr = frameResults[j-startFrame];
			fwrite(&j, sizeof(uint), 1, f);
			fwrite(&fr->timestamp, sizeof(double), 1, f);
			fwrite(&fr->timestamp, sizeof(double), 1, finfo);
			for (int i=0;i<config.numBeads;i++)
			{
				LocalizationResult *r = &fr->results[i];
				fwrite(&r->pos, sizeof(vector3f), 1, f);
			}
		}
	}
	else {
		for (uint k=lastSaveFrame; k<processedFrames;k++)
		{
			auto fr = frameResults[k-startFrame];
			fprintf(f,"%d\t%f\t", k, fr->timestamp);
			fprintf(finfo,"%d\t%f\t", k, fr->timestamp);
			for (int i=0;i<config.numFrameInfoColumns;i++)
				fprintf(finfo, "%.5f\t", fr->frameInfo[i]);
			fputs("\n", finfo);
			for (int i=0;i<config.numBeads;i++)
			{
				LocalizationResult *r = &fr->results[i];
				fprintf(f, "%.5f\t%.5f\t%.5f\t", r->pos.x,r->pos.y,r->pos.z);
			}
			fputs("\n", f);
		}
	}

	dbgprintf("Saved frame %d to %d\n", lastSaveFrame, processedFrames);

	fclose(f);
	fclose(finfo);
	frameCountMutex.lock();
	lastSaveFrame = processedFrames;
	frameCountMutex.unlock();

	resultMutex.unlock();
}

void ResultManager::SetTracker(QueuedTracker *qtrk)
{
	trackerMutex.lock();
	this->qtrk = qtrk;
	trackerMutex.unlock();
}

bool ResultManager::Update()
{
	trackerMutex.lock();

	if (!qtrk) {
		trackerMutex.unlock();
		return 0;
	}

	const int NResultBuf = 10;
	LocalizationResult resultbuf[NResultBuf];

	int count = qtrk->PollFinished( resultbuf, NResultBuf );
	trackerMutex.unlock();

	resultMutex.lock();
	for (int i=0;i<count;i++)
		StoreResult(&resultbuf[i]);
	resultMutex.unlock();

	if (processedFrames - lastSaveFrame >= config.writeInterval) {
		Write();
	}

	if (config.maxFramesInMemory>0 && frameResults.size () > config.maxFramesInMemory) {
		int del = frameResults.size()-config.maxFramesInMemory;
		dbgprintf("Removing %d frames from memory\n", del);

		for (int i=0;i<del;i++)
			delete frameResults[i];
		frameResults.erase(frameResults.begin(), frameResults.begin()+del);

		frameCountMutex.lock();
		startFrame += del;
		frameCountMutex.unlock();
	}

	return count>0;
}

void ResultManager::ThreadLoop(void *param)
{
	ResultManager* rm = (ResultManager*)param;

	while(true) {
		if (!rm->Update())
			Threads::Sleep(20);

		if (rm->quit)
			break;
	}
}

int ResultManager::GetBeadPositions(int startFrame, int endFrame, int bead, LocalizationResult* results)
{
	int start = startFrame - this->startFrame;

	if (endFrame > processedFrames)
		endFrame = processedFrames;

	int end = endFrame - this->startFrame;

	if (start < 0) {
		memset(results, 0, sizeof(LocalizationResult) * -start); // we already deleted this data
		start = 0;
	}

	resultMutex.lock();
	for (int i=start;i<end;i++){
		results[i-start] = frameResults[i]->results[bead];
	}
	resultMutex.unlock();

	return end-start;
}

void ResultManager::Flush()
{
	Write();
}

void ResultManager::GetFrameCounters(int* startFrame, int *processedFrames, int *lastSaveFrame, int *capturedFrames)
{
	frameCountMutex.lock();
	if (startFrame) *startFrame = this->startFrame;
	if (processedFrames) *processedFrames = this->processedFrames;
	if (lastSaveFrame) *lastSaveFrame = this->lastSaveFrame;
	frameCountMutex.unlock();

	if (capturedFrames) {
		resultMutex.lock();
		*capturedFrames = this->frameResults.size();
		resultMutex.unlock();
	}
}

int ResultManager::GetResults(LocalizationResult* results, int startFrame, int numFrames)
{
	frameCountMutex.lock();

	if (startFrame >= this->startFrame && numFrames+startFrame <= processedFrames)  {
		resultMutex.lock();
		for (int f=0;f<numFrames;f++) {
			int index = f + startFrame - this->startFrame;
			for (int j=0;j<config.numBeads;j++)
				results[config.numBeads*f+j] = frameResults[index]->results[j];
		}

		resultMutex.unlock();
	}
	frameCountMutex.unlock();

	return numFrames;
}

int ResultManager::StoreFrameInfo(double timestamp, float* columns)
{
	resultMutex.lock();
	auto fr = new FrameResult( config.numBeads, config.numFrameInfoColumns);
	fr->timestamp = timestamp;
	for(int i=0;i<config.numFrameInfoColumns;i++)
		fr->frameInfo[i]=columns[i];
	frameResults.push_back (fr);
	int nfr = frameResults.size();
	resultMutex.unlock();
	return nfr;
}

int ResultManager::GetFrameCount()
{
	resultMutex.lock();
	int nfr = frameResults.size();
	resultMutex.unlock();
	return nfr;
}

<commit_msg>ResultManager: fix tracker being used outside of mutex lock<commit_after>#include "std_incl.h"
#include "ResultManager.h"
#include "utils.h"

ResultManager::ResultManager(const char *outfile, const char* frameInfoFile, ResultManagerConfig *cfg)
{
	config = *cfg;
	outputFile = outfile;
	this->frameInfoFile = frameInfoFile;

	startFrame = 0;
	lastSaveFrame = 0;
	processedFrames = 0;

	qtrk = 0;

	thread = Threads::Create(ThreadLoop, this);
	quit=false;

	frameResults.resize(100);
	for (int i=0;i<frameResults.size();i++)
		frameResults[i] =new FrameResult(config.numBeads, config.numFrameInfoColumns);

	remove(outfile);
	remove(frameInfoFile);
}

ResultManager::~ResultManager()
{
	quit = true;
	Threads::WaitAndClose(thread);

	DeleteAllElems(frameResults);
}

void ResultManager::StoreResult(LocalizationResult *r)
{
	int index = r->job.frame - startFrame;

	if (index >= frameResults.size()) {
		return; // add errors?
	}

	LocalizationResult scaled = *r;
	// Make roi-centered pos
	scaled.pos = scaled.pos - vector3f( qtrk->cfg.width*0.5f, qtrk->cfg.height*0.5f, 0);
	scaled.pos = ( scaled.pos + config.offset ) * config.scaling;
	FrameResult* fr = frameResults[index];
	fr->results[r->job.zlutIndex] = scaled;
	fr->count++;

	// Advance fullFrames
	frameCountMutex.lock();
	while (processedFrames - startFrame < frameResults.size() && frameResults[processedFrames-startFrame]->count == config.numBeads)
		processedFrames ++;
	frameCountMutex.unlock();
}

void ResultManager::Write()
{
	FILE* f = fopen(outputFile.c_str(), "a");
	FILE* finfo = fopen(frameInfoFile.c_str(), "a");

	resultMutex.lock();
	if (config.binaryOutput) {
		for (uint j=lastSaveFrame; j<processedFrames;j++)
		{
			auto fr = frameResults[j-startFrame];
			fwrite(&j, sizeof(uint), 1, f);
			fwrite(&fr->timestamp, sizeof(double), 1, f);
			fwrite(&fr->timestamp, sizeof(double), 1, finfo);
			for (int i=0;i<config.numBeads;i++)
			{
				LocalizationResult *r = &fr->results[i];
				fwrite(&r->pos, sizeof(vector3f), 1, f);
			}
		}
	}
	else {
		for (uint k=lastSaveFrame; k<processedFrames;k++)
		{
			auto fr = frameResults[k-startFrame];
			fprintf(f,"%d\t%f\t", k, fr->timestamp);
			fprintf(finfo,"%d\t%f\t", k, fr->timestamp);
			for (int i=0;i<config.numFrameInfoColumns;i++)
				fprintf(finfo, "%.5f\t", fr->frameInfo[i]);
			fputs("\n", finfo);
			for (int i=0;i<config.numBeads;i++)
			{
				LocalizationResult *r = &fr->results[i];
				fprintf(f, "%.5f\t%.5f\t%.5f\t", r->pos.x,r->pos.y,r->pos.z);
			}
			fputs("\n", f);
		}
	}

	dbgprintf("Saved frame %d to %d\n", lastSaveFrame, processedFrames);

	fclose(f);
	fclose(finfo);
	frameCountMutex.lock();
	lastSaveFrame = processedFrames;
	frameCountMutex.unlock();

	resultMutex.unlock();
}

void ResultManager::SetTracker(QueuedTracker *qtrk)
{
	trackerMutex.lock();
	this->qtrk = qtrk;
	trackerMutex.unlock();
}

bool ResultManager::Update()
{
	trackerMutex.lock();

	if (!qtrk) {
		trackerMutex.unlock();
		return 0;
	}

	const int NResultBuf = 10;
	LocalizationResult resultbuf[NResultBuf];

	int count = qtrk->PollFinished( resultbuf, NResultBuf );

	resultMutex.lock();
	for (int i=0;i<count;i++)
		StoreResult(&resultbuf[i]);
	resultMutex.unlock();

	trackerMutex.unlock();

	if (processedFrames - lastSaveFrame >= config.writeInterval) {
		Write();
	}

	if (config.maxFramesInMemory>0 && frameResults.size () > config.maxFramesInMemory) {
		int del = frameResults.size()-config.maxFramesInMemory;
		dbgprintf("Removing %d frames from memory\n", del);

		for (int i=0;i<del;i++)
			delete frameResults[i];
		frameResults.erase(frameResults.begin(), frameResults.begin()+del);

		frameCountMutex.lock();
		startFrame += del;
		frameCountMutex.unlock();
	}

	return count>0;
}

void ResultManager::ThreadLoop(void *param)
{
	ResultManager* rm = (ResultManager*)param;

	while(true) {
		if (!rm->Update())
			Threads::Sleep(20);

		if (rm->quit)
			break;
	}
}

int ResultManager::GetBeadPositions(int startFrame, int endFrame, int bead, LocalizationResult* results)
{
	int start = startFrame - this->startFrame;

	if (endFrame > processedFrames)
		endFrame = processedFrames;

	int end = endFrame - this->startFrame;

	if (start < 0)
		return 0;

	resultMutex.lock();
	for (int i=start;i<end;i++){
		results[i-start] = frameResults[i]->results[bead];
	}
	resultMutex.unlock();

	return end-start;
}

void ResultManager::Flush()
{
	Write();
}

void ResultManager::GetFrameCounters(int* startFrame, int *processedFrames, int *lastSaveFrame, int *capturedFrames)
{
	frameCountMutex.lock();
	if (startFrame) *startFrame = this->startFrame;
	if (processedFrames) *processedFrames = this->processedFrames;
	if (lastSaveFrame) *lastSaveFrame = this->lastSaveFrame;
	frameCountMutex.unlock();

	if (capturedFrames) {
		resultMutex.lock();
		*capturedFrames = this->frameResults.size();
		resultMutex.unlock();
	}
}

int ResultManager::GetResults(LocalizationResult* results, int startFrame, int numFrames)
{
	frameCountMutex.lock();

	if (startFrame >= this->startFrame && numFrames+startFrame <= processedFrames)  {
		resultMutex.lock();
		for (int f=0;f<numFrames;f++) {
			int index = f + startFrame - this->startFrame;
			for (int j=0;j<config.numBeads;j++)
				results[config.numBeads*f+j] = frameResults[index]->results[j];
		}

		resultMutex.unlock();
	}
	frameCountMutex.unlock();

	return numFrames;
}

int ResultManager::StoreFrameInfo(double timestamp, float* columns)
{
	resultMutex.lock();
	auto fr = new FrameResult( config.numBeads, config.numFrameInfoColumns);
	fr->timestamp = timestamp;
	for(int i=0;i<config.numFrameInfoColumns;i++)
		fr->frameInfo[i]=columns[i];
	frameResults.push_back (fr);
	int nfr = frameResults.size();
	resultMutex.unlock();
	return nfr;
}

int ResultManager::GetFrameCount()
{
	resultMutex.lock();
	int nfr = frameResults.size();
	resultMutex.unlock();
	return nfr;
}

<|endoftext|>


<commit_before><commit_msg>SfxItemPool: replace deque with vector for SfxPoolItem array<commit_after><|endoftext|>


<commit_before>// Copyright (c) 2016 The HellBender Authors. All rights reserved.
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file.

#include "gmock/gmock.h"
#include "SharedLibrary.h"

using namespace ::testing;
using namespace hellbender;

// typedef int (*TheAnswerFunc)();
using TheAnswerFunc = int(*)();

class ASharedLibrary : public Test {
public:
    SharedLibrary sharedlibrary_;

    const std::string anInvalidPath = "foobar.so";
    const std::string anInvalidSymbol = "foo";
    const std::string aValidSymbol = "theAnswer";
    const std::string pathToTestLib = "libTestLib.so";

    SharedLibrary testLib_{pathToTestLib};
};

TEST_F(ASharedLibrary, HasNoLibLoadedWhenDefaultConstructed) {
    ASSERT_THAT(sharedlibrary_.isLoaded(), Eq(false));
}

TEST_F(ASharedLibrary, CanBeConstructedWithPath) {
    SharedLibrary tLib(pathToTestLib);
    ASSERT_THAT(tLib.isLoaded(), Eq(true));
}

#ifdef _WIN32
TEST_F(ASharedLibrary, HasNoSuffixDLLOnWin32) {
    ASSERT_THAT(sharedlibrary_.suffic(), StrEq(".dll"));
}
#else
TEST_F(ASharedLibrary, HasSuffixsoOnUnix) {
    ASSERT_THAT(sharedlibrary_.suffix(), StrEq(".so"));
}
#endif

TEST_F(ASharedLibrary, ThrowsWhenLoadingInvalidPath) {
    ASSERT_THROW(sharedlibrary_.load(anInvalidPath), std::runtime_error);
}

TEST_F(ASharedLibrary, ThrowsWhenSymbolNotFound) {
    ASSERT_THROW(sharedlibrary_.getSymbol(anInvalidSymbol), std::runtime_error);
}

TEST_F(ASharedLibrary, CanCheckIfSymbolExists) {
    ASSERT_THAT(testLib_.hasSymbol(anInvalidSymbol), Eq(false));
    ASSERT_THAT(testLib_.hasSymbol(aValidSymbol), Eq(true));
}

TEST_F(ASharedLibrary, LoadsSymbolByName) {
    TheAnswerFunc answerFunc = (TheAnswerFunc) testLib_.getSymbol("theAnswer");
    ASSERT_THAT(answerFunc(), Eq(42));
}
<commit_msg>shared libs can be unloaded<commit_after>// Copyright (c) 2016 The HellBender Authors. All rights reserved.
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file.

#include "gmock/gmock.h"
#include "SharedLibrary.h"

using namespace ::testing;
using namespace hellbender;

// typedef int (*TheAnswerFunc)();
using TheAnswerFunc = int(*)();

class ASharedLibrary : public Test {
public:
    SharedLibrary sharedlibrary_;

    const std::string anInvalidPath = "foobar.so";
    const std::string anInvalidSymbol = "foo";
    const std::string aValidSymbol = "theAnswer";
    const std::string pathToTestLib = "libTestLib.so";

    SharedLibrary testLib_{pathToTestLib};
};

TEST_F(ASharedLibrary, HasNoLibLoadedWhenDefaultConstructed) {
    ASSERT_THAT(sharedlibrary_.isLoaded(), Eq(false));
}

TEST_F(ASharedLibrary, CanBeConstructedWithPath) {
    SharedLibrary tLib(pathToTestLib);
    ASSERT_THAT(tLib.isLoaded(), Eq(true));
}

TEST_F(ASharedLibrary, CanBeUnloaded) {
    testLib_.unload();
    ASSERT_THAT(testLib_.isLoaded(), Eq(false));
}

#ifdef _WIN32
TEST_F(ASharedLibrary, HasNoSuffixDLLOnWin32) {
    ASSERT_THAT(sharedlibrary_.suffic(), StrEq(".dll"));
}
#else
TEST_F(ASharedLibrary, HasSuffixsoOnUnix) {
    ASSERT_THAT(sharedlibrary_.suffix(), StrEq(".so"));
}
#endif

TEST_F(ASharedLibrary, ThrowsWhenLoadingInvalidPath) {
    ASSERT_THROW(sharedlibrary_.load(anInvalidPath), std::runtime_error);
}

TEST_F(ASharedLibrary, ThrowsWhenSymbolNotFound) {
    ASSERT_THROW(sharedlibrary_.getSymbol(anInvalidSymbol), std::runtime_error);
}

TEST_F(ASharedLibrary, CanCheckIfSymbolExists) {
    ASSERT_THAT(testLib_.hasSymbol(anInvalidSymbol), Eq(false));
    ASSERT_THAT(testLib_.hasSymbol(aValidSymbol), Eq(true));
}

TEST_F(ASharedLibrary, LoadsSymbolByName) {
    TheAnswerFunc answerFunc = (TheAnswerFunc) testLib_.getSymbol(aValidSymbol);
    ASSERT_THAT(answerFunc(), Eq(42));
}
<|endoftext|>