<commit_before>// Copyright (c) 2012, Cornell University
// All rights reserved.
//
// Redistribution and use in source and binary forms, with or without
// modification, are permitted provided that the following conditions are met:
//
//     * Redistributions of source code must retain the above copyright notice,
//       this list of conditions and the following disclaimer.
//     * Redistributions in binary form must reproduce the above copyright
//       notice, this list of conditions and the following disclaimer in the
//       documentation and/or other materials provided with the distribution.
//     * Neither the name of HyperDex nor the names of its contributors may be
//       used to endorse or promote products derived from this software without
//       specific prior written permission.
//
// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
// AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
// ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
// LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
// CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
// SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
// CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
// ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
// POSSIBILITY OF SUCH DAMAGE.

// The purpose of this test is to expose consistency errors.

// C
#include <cstdlib>
#include <stdint.h>

// Popt
#include <popt.h>

// C++
#include <iostream>

// STL
#include <map>
#include <tr1/memory>
#include <vector>

// po6
#include <po6/threads/barrier.h>
#include <po6/threads/mutex.h>
#include <po6/threads/thread.h>

// e
#include <e/guard.h>

// HyperClient
#include <hyperclient.h>

static int64_t window = 128;
static int64_t repetitions = 1024;
static int64_t threads = 16;
static const char* space = "consistency";
static const char* host = "127.0.0.1";
static uint16_t port = 1234;
static std::auto_ptr<po6::threads::barrier> barrier;
static po6::threads::mutex results_lock;
static int done = 0;
static std::map<hyperclient_returncode, uint64_t> failed_puts;
static std::map<hyperclient_returncode, uint64_t> failed_loops;
static std::map<hyperclient_returncode, uint64_t> ops;
static uint64_t failed_writes = 0;
static uint64_t inconsistencies = 0;

extern "C"
{

static struct poptOption popts[] = {
    POPT_AUTOHELP
    {"window-size", 'w', POPT_ARG_LONGLONG, &window, 'w',
        "the number of sequential keys which will be used for the test",
        "keys"},
    {"repetitiions", 'r', POPT_ARG_LONGLONG, &repetitions, 'r',
        "the number of tests which will be run before exiting",
        "number"},
    {"threads", 't', POPT_ARG_LONGLONG, &threads, 't',
        "the number of threads which will check for inconsistencies",
        "number"},
    {"space", 's', POPT_ARG_STRING, &space, 's',
        "the HyperDex space to use",
        "space"},
    {"host", 'h', POPT_ARG_STRING, &host, 'h',
        "the IP address of the coordinator",
        "IP"},
    {"port", 'p', POPT_ARG_SHORT, &port, 'p',
        "the port number of the coordinator",
        "port"},
    POPT_TABLEEND
};

} // extern "C"

static void
writer_thread();
static void
reader_thread();

int
main(int argc, const char* argv[])
{
    poptContext poptcon;
    poptcon = poptGetContext(NULL, argc, (const char**) argv, popts, POPT_CONTEXT_POSIXMEHARDER);
    e::guard g = e::makeguard(poptFreeContext, poptcon);
    g.use_variable();
    int rc;

    while ((rc = poptGetNextOpt(poptcon)) != -1)
    {
        switch (rc)
        {
            case 'w':
                if (window < 0)
                {
                    std::cerr << "window-size must be >= 0" << std::endl;
                    return EXIT_FAILURE;
                }

                break;
            case 'r':
                if (repetitions < 0)
                {
                    std::cerr << "repetitions must be >= 0" << std::endl;
                    return EXIT_FAILURE;
                }

                break;
            case 't':
                if (threads < 0)
                {
                    std::cerr << "threads must be >= 0" << std::endl;
                    return EXIT_FAILURE;
                }

                break;
            case 's':
                break;
            case 'h':
                break;
            case 'p':
                break;
            case POPT_ERROR_NOARG:
            case POPT_ERROR_BADOPT:
            case POPT_ERROR_BADNUMBER:
            case POPT_ERROR_OVERFLOW:
                std::cerr << poptStrerror(rc) << " " << poptBadOption(poptcon, 0) << std::endl;
                return EXIT_FAILURE;
            case POPT_ERROR_OPTSTOODEEP:
            case POPT_ERROR_BADQUOTE:
            case POPT_ERROR_ERRNO:
            default:
                std::cerr << "logic error in argument parsing" << std::endl;
                return EXIT_FAILURE;
        }
    }

    barrier.reset(new po6::threads::barrier(threads + 1));
    po6::threads::thread writer(writer_thread);
    writer.start();
    std::vector<std::tr1::shared_ptr<po6::threads::thread> > readers;

    for (int64_t i = 0; i < threads; ++i)
    {
        std::tr1::shared_ptr<po6::threads::thread> tptr(new po6::threads::thread(reader_thread));
        readers.push_back(tptr);
        tptr->start();
    }

    writer.join();

    bool success = __sync_bool_compare_and_swap(&done, 0, 1);
    assert(success);

    for (int64_t i = 0; i < threads; ++i)
    {
        readers[i]->join();
    }

    po6::threads::mutex::hold hold(&results_lock);
    typedef std::map<hyperclient_returncode, uint64_t>::iterator result_iter_t;
    std::cout << "Failed puts:" << std::endl;

    for (result_iter_t o = failed_puts.begin(); o != failed_puts.end(); ++o)
    {
        std::cout << o->first << "\t" << o->second << std::endl;
    }

    std::cout << std::endl << "Failed loops:" << std::endl;

    for (result_iter_t o = failed_loops.begin(); o != failed_loops.end(); ++o)
    {
        std::cout << o->first << "\t" << o->second << std::endl;
    }

    std::cout << std::endl << "Normal ops:" << std::endl;

    for (result_iter_t o = ops.begin(); o != ops.end(); ++o)
    {
        std::cout << o->first << "\t" << o->second << std::endl;
    }

    std::cout << std::endl << "failed writes:  " << failed_writes << std::endl
              << "Inconsistencies:  " << inconsistencies << std::endl;
    return EXIT_SUCCESS;
}

static void
writer_thread()
{
    std::map<hyperclient_returncode, uint64_t> lfailed_puts;
    std::map<hyperclient_returncode, uint64_t> lfailed_loops;
    std::map<hyperclient_returncode, uint64_t> lops;
    uint64_t lfailed_writes = 0;
    hyperclient cl(host, port);
    bool fail = false;

    for (int64_t i = 0; i < window; ++i)
    {
        int64_t key = htobe64(i);
        int64_t did;
        hyperclient_returncode dstatus;

        const char* keystr = reinterpret_cast<const char*>(&key);
        did = cl.del(space, keystr, sizeof(key), &dstatus);

        if (did < 0)
        {
            std::cerr << "delete failed with " << dstatus << std::endl;
            fail = true;
            break;
        }

        int64_t lid;
        hyperclient_returncode lstatus;
        lid = cl.loop(-1, &lstatus);

        if (lid < 0)
        {
            std::cerr << "loop failed with " << lstatus << std::endl;
            fail = true;
            break;
        }

        assert(lid == did);

        if (dstatus != HYPERCLIENT_SUCCESS && dstatus != HYPERCLIENT_NOTFOUND)
        {
            std::cerr << "delete returned " << dstatus << std::endl;
            fail = true;
            break;
        }
    }

    barrier->wait();

    if (fail)
    {
        std::cerr << "the above errors are fatal" << std::endl;
        return;
    }

    std::cout << "starting the consistency stress test" << std::endl;

    for (int64_t r = 0; r < repetitions; ++r)
    {
        for (int64_t i = 0; i < window; ++i)
        {
            uint64_t count = 0;

            for (count = 0; count < 65536; ++count)
            {
                int64_t key = htobe64(i);
                int64_t val = htobe64(r);
                int64_t pid;
                hyperclient_attribute attr;
                hyperclient_returncode pstatus;

                const char* keystr = reinterpret_cast<const char*>(&key);
                attr.attr = "repetition";
                attr.value = reinterpret_cast<const char*>(&val);
                attr.value_sz = sizeof(val);
                pid = cl.put(space, keystr, sizeof(key), &attr, 1, &pstatus);

                if (pid < 0)
                {
                    ++lfailed_puts[pstatus];
                    continue;
                }

                int64_t lid;
                hyperclient_returncode lstatus;
                lid = cl.loop(-1, &lstatus);

                if (lid < 0)
                {
                    ++lfailed_loops[lstatus];
                    continue;
                }

                assert(lid == pid);
                ++lops[pstatus];
                break;
            }

            if (count == 65536)
            {
                ++lfailed_writes;
            }
        }

        std::cout << "done " << r << "/" << repetitions << std::endl;
    }

    po6::threads::mutex::hold hold(&results_lock);
    typedef std::map<hyperclient_returncode, uint64_t>::iterator result_iter_t;

    for (result_iter_t o = lfailed_puts.begin(); o != lfailed_puts.end(); ++o)
    {
        failed_puts[o->first] += o->second;
    }

    for (result_iter_t o = lfailed_loops.begin(); o != lfailed_loops.end(); ++o)
    {
        failed_loops[o->first] += o->second;
    }

    for (result_iter_t o = lops.begin(); o != lops.end(); ++o)
    {
        ops[o->first] += o->second;
    }

    failed_writes = lfailed_writes;
}

static void
reader_thread()
{
    std::map<hyperclient_returncode, uint64_t> lfailed_puts;
    std::map<hyperclient_returncode, uint64_t> lfailed_loops;
    std::map<hyperclient_returncode, uint64_t> lops;
    uint64_t linconsistencies = 0;
    hyperclient cl(host, port);
    barrier->wait();

    while (!__sync_bool_compare_and_swap(&done, 1, 1))
    {
        int64_t oldval = 0;

        for (int64_t i = window - 1; i >= 0; --i)
        {
            while (true)
            {
                int64_t key = htobe64(i);
                int64_t gid;
                hyperclient_attribute* attrs = NULL;
                size_t attrs_sz = 0;
                hyperclient_returncode gstatus;

                const char* keystr = reinterpret_cast<const char*>(&key);
                gid = cl.get(space, keystr, sizeof(key), &gstatus, &attrs, &attrs_sz);

                if (gid < 0)
                {
                    ++lfailed_puts[gstatus];
                    continue;
                }

                int64_t lid;
                hyperclient_returncode lstatus;
                lid = cl.loop(-1, &lstatus);

                if (lid < 0)
                {
                    ++lfailed_loops[lstatus];
                    continue;
                }

                assert(lid == gid);
                ++lops[gstatus];

                if (gstatus == HYPERCLIENT_SUCCESS)
                {
                    assert(attrs_sz == 1);
                    assert(strcmp(attrs[0].attr, "repetition") == 0);
                    int64_t val = 0;
                    memmove(&val, attrs[0].value, attrs[0].value_sz);
                    val = be64toh(val);

                    if (val < oldval)
                    {
                        ++inconsistencies;
                    }

                    oldval = val;
                }

                if (attrs)
                {
                    hyperclient_destroy_attrs(attrs, attrs_sz);
                }

                break;
            }
        }
    }

    po6::threads::mutex::hold hold(&results_lock);
    typedef std::map<hyperclient_returncode, uint64_t>::iterator result_iter_t;

    for (result_iter_t o = lfailed_puts.begin(); o != lfailed_puts.end(); ++o)
    {
        failed_puts[o->first] += o->second;
    }

    for (result_iter_t o = lfailed_loops.begin(); o != lfailed_loops.end(); ++o)
    {
        failed_loops[o->first] += o->second;
    }

    for (result_iter_t o = lops.begin(); o != lops.end(); ++o)
    {
        ops[o->first] += o->second;
    }

    inconsistencies += linconsistencies;
}
<commit_msg>Use POPT_ARG_LONG for argument types.<commit_after>// Copyright (c) 2012, Cornell University
// All rights reserved.
//
// Redistribution and use in source and binary forms, with or without
// modification, are permitted provided that the following conditions are met:
//
//     * Redistributions of source code must retain the above copyright notice,
//       this list of conditions and the following disclaimer.
//     * Redistributions in binary form must reproduce the above copyright
//       notice, this list of conditions and the following disclaimer in the
//       documentation and/or other materials provided with the distribution.
//     * Neither the name of HyperDex nor the names of its contributors may be
//       used to endorse or promote products derived from this software without
//       specific prior written permission.
//
// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
// AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
// ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
// LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
// CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
// SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
// CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
// ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
// POSSIBILITY OF SUCH DAMAGE.

// The purpose of this test is to expose consistency errors.

// C
#include <cstdlib>
#include <stdint.h>

// Popt
#include <popt.h>

// C++
#include <iostream>

// STL
#include <map>
#include <tr1/memory>
#include <vector>

// po6
#include <po6/threads/barrier.h>
#include <po6/threads/mutex.h>
#include <po6/threads/thread.h>

// e
#include <e/guard.h>

// HyperClient
#include <hyperclient.h>

static int64_t window = 128;
static int64_t repetitions = 1024;
static int64_t threads = 16;
static const char* space = "consistency";
static const char* host = "127.0.0.1";
static uint16_t port = 1234;
static std::auto_ptr<po6::threads::barrier> barrier;
static po6::threads::mutex results_lock;
static int done = 0;
static std::map<hyperclient_returncode, uint64_t> failed_puts;
static std::map<hyperclient_returncode, uint64_t> failed_loops;
static std::map<hyperclient_returncode, uint64_t> ops;
static uint64_t failed_writes = 0;
static uint64_t inconsistencies = 0;

extern "C"
{

static struct poptOption popts[] = {
    POPT_AUTOHELP
    {"window-size", 'w', POPT_ARG_LONG, &window, 'w',
        "the number of sequential keys which will be used for the test",
        "keys"},
    {"repetitiions", 'r', POPT_ARG_LONG, &repetitions, 'r',
        "the number of tests which will be run before exiting",
        "number"},
    {"threads", 't', POPT_ARG_LONG, &threads, 't',
        "the number of threads which will check for inconsistencies",
        "number"},
    {"space", 's', POPT_ARG_STRING, &space, 's',
        "the HyperDex space to use",
        "space"},
    {"host", 'h', POPT_ARG_STRING, &host, 'h',
        "the IP address of the coordinator",
        "IP"},
    {"port", 'p', POPT_ARG_LONG, &port, 'p',
        "the port number of the coordinator",
        "port"},
    POPT_TABLEEND
};

} // extern "C"

static void
writer_thread();
static void
reader_thread();

int
main(int argc, const char* argv[])
{
    poptContext poptcon;
    poptcon = poptGetContext(NULL, argc, (const char**) argv, popts, POPT_CONTEXT_POSIXMEHARDER);
    e::guard g = e::makeguard(poptFreeContext, poptcon);
    g.use_variable();
    int rc;

    while ((rc = poptGetNextOpt(poptcon)) != -1)
    {
        switch (rc)
        {
            case 'w':
                if (window < 0)
                {
                    std::cerr << "window-size must be >= 0" << std::endl;
                    return EXIT_FAILURE;
                }

                break;
            case 'r':
                if (repetitions < 0)
                {
                    std::cerr << "repetitions must be >= 0" << std::endl;
                    return EXIT_FAILURE;
                }

                break;
            case 't':
                if (threads < 0)
                {
                    std::cerr << "threads must be >= 0" << std::endl;
                    return EXIT_FAILURE;
                }

                break;
            case 's':
                break;
            case 'h':
                break;
            case 'p':
                break;
            case POPT_ERROR_NOARG:
            case POPT_ERROR_BADOPT:
            case POPT_ERROR_BADNUMBER:
            case POPT_ERROR_OVERFLOW:
                std::cerr << poptStrerror(rc) << " " << poptBadOption(poptcon, 0) << std::endl;
                return EXIT_FAILURE;
            case POPT_ERROR_OPTSTOODEEP:
            case POPT_ERROR_BADQUOTE:
            case POPT_ERROR_ERRNO:
            default:
                std::cerr << "logic error in argument parsing" << std::endl;
                return EXIT_FAILURE;
        }
    }

    barrier.reset(new po6::threads::barrier(threads + 1));
    po6::threads::thread writer(writer_thread);
    writer.start();
    std::vector<std::tr1::shared_ptr<po6::threads::thread> > readers;

    for (int64_t i = 0; i < threads; ++i)
    {
        std::tr1::shared_ptr<po6::threads::thread> tptr(new po6::threads::thread(reader_thread));
        readers.push_back(tptr);
        tptr->start();
    }

    writer.join();

    bool success = __sync_bool_compare_and_swap(&done, 0, 1);
    assert(success);

    for (int64_t i = 0; i < threads; ++i)
    {
        readers[i]->join();
    }

    po6::threads::mutex::hold hold(&results_lock);
    typedef std::map<hyperclient_returncode, uint64_t>::iterator result_iter_t;
    std::cout << "Failed puts:" << std::endl;

    for (result_iter_t o = failed_puts.begin(); o != failed_puts.end(); ++o)
    {
        std::cout << o->first << "\t" << o->second << std::endl;
    }

    std::cout << std::endl << "Failed loops:" << std::endl;

    for (result_iter_t o = failed_loops.begin(); o != failed_loops.end(); ++o)
    {
        std::cout << o->first << "\t" << o->second << std::endl;
    }

    std::cout << std::endl << "Normal ops:" << std::endl;

    for (result_iter_t o = ops.begin(); o != ops.end(); ++o)
    {
        std::cout << o->first << "\t" << o->second << std::endl;
    }

    std::cout << std::endl << "failed writes:  " << failed_writes << std::endl
              << "Inconsistencies:  " << inconsistencies << std::endl;
    return EXIT_SUCCESS;
}

static void
writer_thread()
{
    std::map<hyperclient_returncode, uint64_t> lfailed_puts;
    std::map<hyperclient_returncode, uint64_t> lfailed_loops;
    std::map<hyperclient_returncode, uint64_t> lops;
    uint64_t lfailed_writes = 0;
    hyperclient cl(host, port);
    bool fail = false;

    for (int64_t i = 0; i < window; ++i)
    {
        int64_t key = htobe64(i);
        int64_t did;
        hyperclient_returncode dstatus;

        const char* keystr = reinterpret_cast<const char*>(&key);
        did = cl.del(space, keystr, sizeof(key), &dstatus);

        if (did < 0)
        {
            std::cerr << "delete failed with " << dstatus << std::endl;
            fail = true;
            break;
        }

        int64_t lid;
        hyperclient_returncode lstatus;
        lid = cl.loop(-1, &lstatus);

        if (lid < 0)
        {
            std::cerr << "loop failed with " << lstatus << std::endl;
            fail = true;
            break;
        }

        assert(lid == did);

        if (dstatus != HYPERCLIENT_SUCCESS && dstatus != HYPERCLIENT_NOTFOUND)
        {
            std::cerr << "delete returned " << dstatus << std::endl;
            fail = true;
            break;
        }
    }

    barrier->wait();

    if (fail)
    {
        std::cerr << "the above errors are fatal" << std::endl;
        return;
    }

    std::cout << "starting the consistency stress test" << std::endl;

    for (int64_t r = 0; r < repetitions; ++r)
    {
        for (int64_t i = 0; i < window; ++i)
        {
            uint64_t count = 0;

            for (count = 0; count < 65536; ++count)
            {
                int64_t key = htobe64(i);
                int64_t val = htobe64(r);
                int64_t pid;
                hyperclient_attribute attr;
                hyperclient_returncode pstatus;

                const char* keystr = reinterpret_cast<const char*>(&key);
                attr.attr = "repetition";
                attr.value = reinterpret_cast<const char*>(&val);
                attr.value_sz = sizeof(val);
                pid = cl.put(space, keystr, sizeof(key), &attr, 1, &pstatus);

                if (pid < 0)
                {
                    ++lfailed_puts[pstatus];
                    continue;
                }

                int64_t lid;
                hyperclient_returncode lstatus;
                lid = cl.loop(-1, &lstatus);

                if (lid < 0)
                {
                    ++lfailed_loops[lstatus];
                    continue;
                }

                assert(lid == pid);
                ++lops[pstatus];
                break;
            }

            if (count == 65536)
            {
                ++lfailed_writes;
            }
        }

        std::cout << "done " << r << "/" << repetitions << std::endl;
    }

    po6::threads::mutex::hold hold(&results_lock);
    typedef std::map<hyperclient_returncode, uint64_t>::iterator result_iter_t;

    for (result_iter_t o = lfailed_puts.begin(); o != lfailed_puts.end(); ++o)
    {
        failed_puts[o->first] += o->second;
    }

    for (result_iter_t o = lfailed_loops.begin(); o != lfailed_loops.end(); ++o)
    {
        failed_loops[o->first] += o->second;
    }

    for (result_iter_t o = lops.begin(); o != lops.end(); ++o)
    {
        ops[o->first] += o->second;
    }

    failed_writes = lfailed_writes;
}

static void
reader_thread()
{
    std::map<hyperclient_returncode, uint64_t> lfailed_puts;
    std::map<hyperclient_returncode, uint64_t> lfailed_loops;
    std::map<hyperclient_returncode, uint64_t> lops;
    uint64_t linconsistencies = 0;
    hyperclient cl(host, port);
    barrier->wait();

    while (!__sync_bool_compare_and_swap(&done, 1, 1))
    {
        int64_t oldval = 0;

        for (int64_t i = window - 1; i >= 0; --i)
        {
            while (true)
            {
                int64_t key = htobe64(i);
                int64_t gid;
                hyperclient_attribute* attrs = NULL;
                size_t attrs_sz = 0;
                hyperclient_returncode gstatus;

                const char* keystr = reinterpret_cast<const char*>(&key);
                gid = cl.get(space, keystr, sizeof(key), &gstatus, &attrs, &attrs_sz);

                if (gid < 0)
                {
                    ++lfailed_puts[gstatus];
                    continue;
                }

                int64_t lid;
                hyperclient_returncode lstatus;
                lid = cl.loop(-1, &lstatus);

                if (lid < 0)
                {
                    ++lfailed_loops[lstatus];
                    continue;
                }

                assert(lid == gid);
                ++lops[gstatus];

                if (gstatus == HYPERCLIENT_SUCCESS)
                {
                    assert(attrs_sz == 1);
                    assert(strcmp(attrs[0].attr, "repetition") == 0);
                    int64_t val = 0;
                    memmove(&val, attrs[0].value, attrs[0].value_sz);
                    val = be64toh(val);

                    if (val < oldval)
                    {
                        ++inconsistencies;
                    }

                    oldval = val;
                }

                if (attrs)
                {
                    hyperclient_destroy_attrs(attrs, attrs_sz);
                }

                break;
            }
        }
    }

    po6::threads::mutex::hold hold(&results_lock);
    typedef std::map<hyperclient_returncode, uint64_t>::iterator result_iter_t;

    for (result_iter_t o = lfailed_puts.begin(); o != lfailed_puts.end(); ++o)
    {
        failed_puts[o->first] += o->second;
    }

    for (result_iter_t o = lfailed_loops.begin(); o != lfailed_loops.end(); ++o)
    {
        failed_loops[o->first] += o->second;
    }

    for (result_iter_t o = lops.begin(); o != lops.end(); ++o)
    {
        ops[o->first] += o->second;
    }

    inconsistencies += linconsistencies;
}
<|endoftext|>


<commit_before>#include "indMatch_utils.hpp"

#include "openMVG/matching/indMatch.hpp"

#include <cereal/archives/portable_binary.hpp>
#include <cereal/types/map.hpp>
#include <cereal/types/utility.hpp>
#include <cereal/types/vector.hpp>

#include "third_party/stlplus3/filesystemSimplified/file_system.hpp"

#include <map>
#include <fstream>
#include <iterator>
#include <string>
#include <vector>

namespace openMVG {
namespace matching {

bool LoadMatchFile(
  PairWiseMatches & matches,
  const std::string & folder,
  const std::string & filename)
{
  if(!stlplus::is_file(stlplus::create_filespec(folder, filename)))
    return false;

  const std::string ext = stlplus::extension_part(filename);
  const std::string filepath = folder + "/" + filename;

  if (ext == "txt")
  {
    std::ifstream stream(filepath.c_str());
    if (!stream.is_open())
      return false;

    // Read from the text file
    // I J
    // #matches count
    // idx idx
    // ...
    size_t I, J, number;
    while (stream >> I >> J >> number)
    {
      std::vector<IndMatch> read_matches(number);
      for (size_t i = 0; i < number; ++i)
      {
        stream >> read_matches[i];
      }
      matches[std::make_pair(I,J)] = std::move(read_matches);
    }
    stream.close();
    return true;
  }
  else if (ext == "bin")
  {
    std::ifstream stream (filepath.c_str(), std::ios::in | std::ios::binary);
    if (!stream.is_open())
      return false;

    cereal::PortableBinaryInputArchive archive(stream);
    PairWiseMatches loadMatches;
    archive(loadMatches);
    stream.close();
    if(matches.empty())
    {
        matches.swap(loadMatches);
    }
    else
    {
        // merge the loaded matches into the output
        for(const auto& v: loadMatches)
        {
            matches[v.first] = v.second;
        }
    }
    return true;
  }
  else
  {
    std::cerr << "Unknown matching file format: " << ext << std::endl;
  }
  return false;
}

void filterMatches(
  PairWiseMatches & matches,
  const std::set<IndexT> & viewsKeys)
{
  matching::PairWiseMatches filteredMatches;
  for (matching::PairWiseMatches::const_iterator iter = matches.begin();
    iter != matches.end();
    ++iter)
  {
    if(viewsKeys.find(iter->first.first) != viewsKeys.end() &&
       viewsKeys.find(iter->first.second) != viewsKeys.end())
    {
      filteredMatches.insert(*iter);
    }
  }
  matches.swap(filteredMatches);
}

bool LoadMatchFilePerImage(
  PairWiseMatches & matches,
  const std::set<IndexT> & viewsKeys,
  const std::string & folder,
  const std::string & basename)
{
  int nbLoadedMatchFiles = 0;
  // Load one match file per image
  for(IndexT idView: viewsKeys)
  {
    const std::string matchFilename = std::to_string(idView) + "." + basename;
    if(!LoadMatchFile(matches, folder, matchFilename))
    {
      std::cerr << "Unable to load match file: " << folder << "/" << matchFilename << std::endl;
      continue;
    }
    ++nbLoadedMatchFiles;
  }
  if( nbLoadedMatchFiles == 0 )
  {
    std::cerr << std::endl << "No matches file loaded in: " << folder << std::endl;
    return false;
  }
  return true;
}

bool Load(
  PairWiseMatches & matches,
  const std::set<IndexT> & viewsKeys,
  const std::string & folder,
  const std::string & mode)
{
  bool res = false;
  const std::string basename = "matches." + mode;
  if(stlplus::is_file(stlplus::create_filespec(folder, basename + ".txt")))
  {
    res = LoadMatchFile(matches, folder, basename + ".txt");
  }
  else if(stlplus::is_file(stlplus::create_filespec(folder, basename + ".bin")))
  {
    res = LoadMatchFile(matches, folder, basename + ".bin");
  }
  else if(!stlplus::folder_wildcard(folder, "*."+basename+".txt", false, true).empty())
  {
    res = LoadMatchFilePerImage(matches, viewsKeys, folder, basename + ".txt");
  }
  else if(!stlplus::folder_wildcard(folder, "*."+basename+".bin", false, true).empty())
  {
    res = LoadMatchFilePerImage(matches, viewsKeys, folder, basename + ".bin");
  }
  if(!res)
    return res;

  if(!viewsKeys.empty())
    filterMatches(matches, viewsKeys);

  return res;
}

class MatchExporter
{
private:
  void saveTxt(
    const std::string & filepath,
    const PairWiseMatches::const_iterator& matchBegin,
    const PairWiseMatches::const_iterator& matchEnd)
  {
    std::ofstream stream(filepath.c_str(), std::ios::out);
    for(PairWiseMatches::const_iterator match = matchBegin;
      match != matchEnd;
      ++match)
    {
      const size_t I = match->first.first;
      const size_t J = match->first.second;
      const std::vector<IndMatch> & pair_matches = match->second;
      stream << I << " " << J << '\n'
             << pair_matches.size() << '\n';

      copy(pair_matches.begin(), pair_matches.end(),
           std::ostream_iterator<IndMatch>(stream, "\n"));
    }
  }
  void saveBinary(
    const std::string & filepath,
    const PairWiseMatches::const_iterator& matchBegin,
    const PairWiseMatches::const_iterator& matchEnd)
  {
    std::ofstream stream(filepath.c_str(), std::ios::out | std::ios::binary);
    cereal::PortableBinaryOutputArchive archive(stream);
    const PairWiseMatches matchesToExport(matchBegin, matchEnd);
    archive(matchesToExport);
    stream.close();
  }

public:
  MatchExporter(
    const PairWiseMatches& matches,
    const std::string& folder,
    const std::string& filename)
    : m_matches(matches)
    , m_directory(folder)
    , m_filename(filename)
    , m_ext(stlplus::extension_part(filename))
  {
  }

  ~MatchExporter()
  {
  }

  void saveGlobalFile()
  {
    const std::string filepath = m_directory + "/" + m_filename;
    if(m_ext == "txt")
    {
      saveTxt(filepath, m_matches.begin(), m_matches.end());
    }
    else if(m_ext == "bin")
    {
      saveBinary(filepath, m_matches.begin(), m_matches.end());
    }
    else
    {
      throw std::runtime_error(std::string("Unknown matching file format: ") + m_ext);
    }
  }

  /// Export matches into separate files, one for each image.
  void saveOneFilePerImage()
  {
    std::set<IndexT> keys;
    std::transform(
        m_matches.begin(), m_matches.end(),
        std::inserter(keys, keys.begin()),
        [](const PairWiseMatches::value_type &v) { return v.first.first; });

    PairWiseMatches::const_iterator matchBegin = m_matches.begin();
    PairWiseMatches::const_iterator matchEnd = m_matches.end();
    for(IndexT key: keys)
    {
      PairWiseMatches::const_iterator match = matchBegin;
      while(match->first.first == key && match != matchEnd)
      {
        ++match;
      }
      const std::string filepath = m_directory + "/" + std::to_string(key) + "." + m_filename;
      std::cout << "Export Matches in " << filepath << std::endl;

      if(m_ext == "txt")
      {
        saveTxt(filepath, matchBegin, match);
      }
      else if(m_ext == "bin")
      {
        saveBinary(filepath, matchBegin, match);
      }
      else
      {
        throw std::runtime_error(std::string("Unknown matching file format: ") + m_ext);
      }
      matchBegin = match;
    }
  }

public:
  const PairWiseMatches& m_matches;
  const std::string m_ext;
  std::string m_directory;
  std::string m_filename;
};

bool Save(
  const PairWiseMatches & matches,
  const std::string & folder,
  const std::string & mode,
  const std::string & extension,
  bool matchFilePerImage)
{
  const std::string filename = "matches." + mode + "." + extension;
  MatchExporter exporter(matches, folder, filename);
  if(matchFilePerImage)
    exporter.saveOneFilePerImage();
  else
    exporter.saveGlobalFile();
  return true;
}

}  // namespace matching
}  // namespace openMVG
<commit_msg>[matching] when using multiple match files, load them in parallel<commit_after>#include "indMatch_utils.hpp"

#include "openMVG/matching/indMatch.hpp"

#include <cereal/archives/portable_binary.hpp>
#include <cereal/types/map.hpp>
#include <cereal/types/utility.hpp>
#include <cereal/types/vector.hpp>

#include "third_party/stlplus3/filesystemSimplified/file_system.hpp"

#include <map>
#include <fstream>
#include <iterator>
#include <string>
#include <vector>

namespace openMVG {
namespace matching {

bool LoadMatchFile(
  PairWiseMatches & matches,
  const std::string & folder,
  const std::string & filename)
{
  if(!stlplus::is_file(stlplus::create_filespec(folder, filename)))
    return false;

  const std::string ext = stlplus::extension_part(filename);
  const std::string filepath = folder + "/" + filename;

  if (ext == "txt")
  {
    std::ifstream stream(filepath.c_str());
    if (!stream.is_open())
      return false;

    // Read from the text file
    // I J
    // #matches count
    // idx idx
    // ...
    size_t I, J, number;
    while (stream >> I >> J >> number)
    {
      std::vector<IndMatch> read_matches(number);
      for (size_t i = 0; i < number; ++i)
      {
        stream >> read_matches[i];
      }
      matches[std::make_pair(I,J)] = std::move(read_matches);
    }
    stream.close();
    return true;
  }
  else if (ext == "bin")
  {
    std::ifstream stream (filepath.c_str(), std::ios::in | std::ios::binary);
    if (!stream.is_open())
      return false;

    cereal::PortableBinaryInputArchive archive(stream);
    PairWiseMatches loadMatches;
    archive(loadMatches);
    stream.close();
    if(matches.empty())
    {
        matches.swap(loadMatches);
    }
    else
    {
        // merge the loaded matches into the output
        for(const auto& v: loadMatches)
        {
            matches[v.first] = v.second;
        }
    }
    return true;
  }
  else
  {
    std::cerr << "Unknown matching file format: " << ext << std::endl;
  }
  return false;
}

void filterMatches(
  PairWiseMatches & matches,
  const std::set<IndexT> & viewsKeys)
{
  matching::PairWiseMatches filteredMatches;
  for (matching::PairWiseMatches::const_iterator iter = matches.begin();
    iter != matches.end();
    ++iter)
  {
    if(viewsKeys.find(iter->first.first) != viewsKeys.end() &&
       viewsKeys.find(iter->first.second) != viewsKeys.end())
    {
      filteredMatches.insert(*iter);
    }
  }
  matches.swap(filteredMatches);
}

bool LoadMatchFilePerImage(
  PairWiseMatches & matches,
  const std::set<IndexT> & viewsKeys,
  const std::string & folder,
  const std::string & basename)
{
  int nbLoadedMatchFiles = 0;
  // Load one match file per image
#ifdef OPENMVG_USE_OPENMP
    #pragma omp parallel for
#endif
  for(size_t i = 0; i < viewsKeys.size(); ++i)
  {
    std::set<IndexT>::const_iterator it = viewsKeys.begin();
    std::advance(it, i);
    const IndexT idView = *it;
    const std::string matchFilename = std::to_string(idView) + "." + basename;
    PairWiseMatches fileMatches;
    if(!LoadMatchFile(fileMatches, folder, matchFilename))
    {
      std::cerr << "Unable to load match file: " << folder << "/" << matchFilename << std::endl;
      continue;
    }
#ifdef OPENMVG_USE_OPENMP
      #pragma omp critical
#endif
    {
      ++nbLoadedMatchFiles;
      // merge the loaded matches into the output
      for(const auto& v: fileMatches)
      {
          matches[v.first] = v.second;
      }
    }
  }
  if( nbLoadedMatchFiles == 0 )
  {
    std::cerr << std::endl << "No matches file loaded in: " << folder << std::endl;
    return false;
  }
  return true;
}

bool Load(
  PairWiseMatches & matches,
  const std::set<IndexT> & viewsKeys,
  const std::string & folder,
  const std::string & mode)
{
  bool res = false;
  const std::string basename = "matches." + mode;
  if(stlplus::is_file(stlplus::create_filespec(folder, basename + ".txt")))
  {
    res = LoadMatchFile(matches, folder, basename + ".txt");
  }
  else if(stlplus::is_file(stlplus::create_filespec(folder, basename + ".bin")))
  {
    res = LoadMatchFile(matches, folder, basename + ".bin");
  }
  else if(!stlplus::folder_wildcard(folder, "*."+basename+".txt", false, true).empty())
  {
    res = LoadMatchFilePerImage(matches, viewsKeys, folder, basename + ".txt");
  }
  else if(!stlplus::folder_wildcard(folder, "*."+basename+".bin", false, true).empty())
  {
    res = LoadMatchFilePerImage(matches, viewsKeys, folder, basename + ".bin");
  }
  if(!res)
    return res;

  if(!viewsKeys.empty())
    filterMatches(matches, viewsKeys);

  return res;
}

class MatchExporter
{
private:
  void saveTxt(
    const std::string & filepath,
    const PairWiseMatches::const_iterator& matchBegin,
    const PairWiseMatches::const_iterator& matchEnd)
  {
    std::ofstream stream(filepath.c_str(), std::ios::out);
    for(PairWiseMatches::const_iterator match = matchBegin;
      match != matchEnd;
      ++match)
    {
      const size_t I = match->first.first;
      const size_t J = match->first.second;
      const std::vector<IndMatch> & pair_matches = match->second;
      stream << I << " " << J << '\n'
             << pair_matches.size() << '\n';

      copy(pair_matches.begin(), pair_matches.end(),
           std::ostream_iterator<IndMatch>(stream, "\n"));
    }
  }
  void saveBinary(
    const std::string & filepath,
    const PairWiseMatches::const_iterator& matchBegin,
    const PairWiseMatches::const_iterator& matchEnd)
  {
    std::ofstream stream(filepath.c_str(), std::ios::out | std::ios::binary);
    cereal::PortableBinaryOutputArchive archive(stream);
    const PairWiseMatches matchesToExport(matchBegin, matchEnd);
    archive(matchesToExport);
    stream.close();
  }

public:
  MatchExporter(
    const PairWiseMatches& matches,
    const std::string& folder,
    const std::string& filename)
    : m_matches(matches)
    , m_directory(folder)
    , m_filename(filename)
    , m_ext(stlplus::extension_part(filename))
  {
  }

  ~MatchExporter()
  {
  }

  void saveGlobalFile()
  {
    const std::string filepath = m_directory + "/" + m_filename;
    if(m_ext == "txt")
    {
      saveTxt(filepath, m_matches.begin(), m_matches.end());
    }
    else if(m_ext == "bin")
    {
      saveBinary(filepath, m_matches.begin(), m_matches.end());
    }
    else
    {
      throw std::runtime_error(std::string("Unknown matching file format: ") + m_ext);
    }
  }

  /// Export matches into separate files, one for each image.
  void saveOneFilePerImage()
  {
    std::set<IndexT> keys;
    std::transform(
        m_matches.begin(), m_matches.end(),
        std::inserter(keys, keys.begin()),
        [](const PairWiseMatches::value_type &v) { return v.first.first; });

    PairWiseMatches::const_iterator matchBegin = m_matches.begin();
    PairWiseMatches::const_iterator matchEnd = m_matches.end();
    for(IndexT key: keys)
    {
      PairWiseMatches::const_iterator match = matchBegin;
      while(match->first.first == key && match != matchEnd)
      {
        ++match;
      }
      const std::string filepath = m_directory + "/" + std::to_string(key) + "." + m_filename;
      std::cout << "Export Matches in " << filepath << std::endl;

      if(m_ext == "txt")
      {
        saveTxt(filepath, matchBegin, match);
      }
      else if(m_ext == "bin")
      {
        saveBinary(filepath, matchBegin, match);
      }
      else
      {
        throw std::runtime_error(std::string("Unknown matching file format: ") + m_ext);
      }
      matchBegin = match;
    }
  }

public:
  const PairWiseMatches& m_matches;
  const std::string m_ext;
  std::string m_directory;
  std::string m_filename;
};

bool Save(
  const PairWiseMatches & matches,
  const std::string & folder,
  const std::string & mode,
  const std::string & extension,
  bool matchFilePerImage)
{
  const std::string filename = "matches." + mode + "." + extension;
  MatchExporter exporter(matches, folder, filename);
  if(matchFilePerImage)
    exporter.saveOneFilePerImage();
  else
    exporter.saveGlobalFile();
  return true;
}

}  // namespace matching
}  // namespace openMVG
<|endoftext|>


<commit_before>/* -LICENSE-START-
** Copyright (c) 2013 Blackmagic Design
**
** Permission is hereby granted, free of charge, to any person or organization
** obtaining a copy of the software and accompanying documentation covered by
** this license (the "Software") to use, reproduce, display, distribute,
** execute, and transmit the Software, and to prepare derivative works of the
** Software, and to permit third-parties to whom the Software is furnished to
** do so, all subject to the following:
**
** The copyright notices in the Software and this entire statement, including
** the above license grant, this restriction and the following disclaimer,
** must be included in all copies of the Software, in whole or in part, and
** all derivative works of the Software, unless such copies or derivative
** works are solely in the form of machine-executable object code generated by
** a source language processor.
**
** THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
** IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
** FITNESS FOR A PARTICULAR PURPOSE, TITLE AND NON-INFRINGEMENT. IN NO EVENT
** SHALL THE COPYRIGHT HOLDERS OR ANYONE DISTRIBUTING THE SOFTWARE BE LIABLE
** FOR ANY DAMAGES OR OTHER LIABILITY, WHETHER IN CONTRACT, TORT OR OTHERWISE,
** ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
** DEALINGS IN THE SOFTWARE.
** -LICENSE-END-
*/

#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include <pthread.h>
#include <unistd.h>
#include <fcntl.h>
#include <csignal>
#include <jni.h>
#include <string>

#include "DeckLinkAPI.h"
#include "Capture.h"
#include "Util.hpp"
#include "us_ihmc_javadecklink_Capture.h"

DeckLinkCaptureDelegate::DeckLinkCaptureDelegate(IDeckLink* decklink, IDeckLinkInput* decklinkInput, JavaVM* vm, jobject obj, jmethodID methodID) :
    m_refCount(1),
    decklink(decklink),
    decklinkInput(decklinkInput),
    vm(vm),
    obj(obj),
    methodID(methodID)
{
    avcodec_register_all();
    codec = avcodec_find_encoder(AV_CODEC_ID_MJPEG);
    c = avcodec_alloc_context3(codec);
    if (!codec) {
        fprintf(stderr, "codec not found\n");
        exit(1);
    }
}

ULONG DeckLinkCaptureDelegate::AddRef(void)
{
	return __sync_add_and_fetch(&m_refCount, 1);
}

ULONG DeckLinkCaptureDelegate::Release(void)
{
	int32_t newRefValue = __sync_sub_and_fetch(&m_refCount, 1);
	if (newRefValue == 0)
	{
		delete this;
		return 0;
	}
	return newRefValue;
}

/*
static void video_encode_example(const char *filename)
{
    int i, ret, x, y, got_output;
    uint8_t endcode[] = { 0, 0, 1, 0xb7 };
    printf("Video encoding\n");

    /* encode 1 second of video *
    for(i=0;i<25;i++) {

    }
    /* get the delayed frames *
    for (got_output = 1; got_output; i++) {
        fflush(stdout);
        ret = avcodec_encode_video2(c, &pkt, NULL, &got_output);
        if (ret < 0) {
            fprintf(stderr, "error encoding frame\n");
            exit(1);
        }
        if (got_output) {
            printf("encoding frame %3d (size=%5d)\n", i, pkt.size);
            fwrite(pkt.data, 1, pkt.size, f);
            av_packet_unref(&pkt);
        }
    }
    /* add sequence end code to have a real mpeg file *
    fwrite(endcode, 1, sizeof(endcode), f);
    printf("\n");
}
*/
HRESULT DeckLinkCaptureDelegate::VideoInputFrameArrived(IDeckLinkVideoInputFrame* videoFrame, IDeckLinkAudioInputPacket* audioFrame)
{
    void* frameBytes;

	// Handle Video Frame
	if (videoFrame)
    {

        JNIEnv* env = getEnv(vm);
        if(env == 0)
        {
            // Cannot throw a runtime exception because we don't have an env
            std::cerr << "Cannot load env" << std::endl;
            return S_OK;
        }

		if (videoFrame->GetFlags() & bmdFrameHasNoInputSource)
		{
            printf("Frame received - No input signal detected\n");

            //env->CallVoidMethod(obj, methodID, false, 0, 0, NULL);
		}
		else
        {

            printf("Frame received - Size: %li bytes\n",
				videoFrame->GetRowBytes() * videoFrame->GetHeight());

				videoFrame->GetBytes(&frameBytes);
                //write(g_videoOutputFile, frameBytes, videoFrame->GetRowBytes() * videoFrame->GetHeight());

                av_init_packet(&pkt);
                pkt.data = NULL;    // packet data will be allocated by the encoder
                pkt.size = 0;

                pictureUYVY->pts = i;
                pictureYUV420->pts = i;

                avpicture_fill((AVPicture*)pictureUYVY, (uint8_t*) frameBytes, AV_PIX_FMT_UYVY422, pictureUYVY->width, pictureUYVY->height);
                sws_scale(img_convert_ctx, pictureUYVY->data, pictureUYVY->linesize, 0, c->height, pictureYUV420->data, pictureYUV420->linesize);

                /* encode the image */
                int got_output;
                int ret = avcodec_encode_video2(c, &pkt, pictureYUV420, &got_output);
                if (ret < 0) {
                    fprintf(stderr, "error encoding frame\n");
                    exit(1);
                }
                if (got_output) {
                    printf("encoding frame %3d (size=%5d)\n", i, pkt.size);
                    fwrite(pkt.data, 1, pkt.size, f);
                    av_free_packet(&pkt); //depreacted, use av_packet_unref(&pkt); after Ubuntu 16.04 comes out

                }

            ++i;
		}

	}

	return S_OK;
}

HRESULT DeckLinkCaptureDelegate::VideoInputFormatChanged(BMDVideoInputFormatChangedEvents events, IDeckLinkDisplayMode *mode, BMDDetectedVideoInputFormatFlags formatFlags)
{
    // This only gets called if bmdVideoInputEnableFormatDetection was set
    // when enabling video input
    HRESULT	result;
    char*	displayModeName = NULL;
    BMDPixelFormat	pixelFormat = bmdFormat8BitYUV;

    JNIEnv* env = getEnv(vm);
    if (formatFlags & bmdDetectedVideoInputRGB444)
    {
        throwRuntimeException(env, "Unsupported input format: RGB444");
        goto bail;
    }

    mode->GetName((const char**)&displayModeName);
    printf("Video format changed to %s %s\n", displayModeName, formatFlags & bmdDetectedVideoInputRGB444 ? "RGB" : "YUV");

    if (displayModeName)
        free(displayModeName);

    if (decklinkInput)
    {
        decklinkInput->StopStreams();

        /* put sample parameters */
        c->qcompress = 0.80;
        /* resolution must be a multiple of two */
        c->width = mode->GetWidth();
        c->height = mode->GetHeight();
        /* frames per second */

        BMDTimeValue numerator;
        BMDTimeScale denumerator;

        mode->GetFrameRate(&numerator, &denumerator);

        c->time_base= (AVRational){(int) numerator, (int) denumerator};
        c->pix_fmt = AV_PIX_FMT_YUVJ420P;
        /* open it */
        if (avcodec_open2(c, codec, NULL) < 0) {
            throwRuntimeException(env, "Could not open codec");
            goto bail;
        }
        f = fopen("test.mp4", "wb");
        if (!f) {
            throwRuntimeException(env, "Could not open file");
            goto bail;
        }

        pictureYUV420 = avcodec_alloc_frame();
       int ret = av_image_alloc(pictureYUV420->data, pictureYUV420->linesize, c->width, c->height,
                             c->pix_fmt, 32);
        if (ret < 0) {
            throwRuntimeException(env, "could not alloc raw picture buffer\n");
            goto bail;
        }
        pictureYUV420->format = c->pix_fmt;
        pictureYUV420->width  = c->width;
        pictureYUV420->height = c->height;

        pictureUYVY = avcodec_alloc_frame();
        pictureYUV420->width = c->width;
        pictureYUV420->height = c->height;
        pictureYUV420->format = AV_PIX_FMT_UYVY422;

        img_convert_ctx = sws_getContext(c->width, c->height,
        PIX_FMT_YUV420P,
        c->width, c->height,
        c->pix_fmt,
        sws_flags, NULL, NULL, NULL);

        result = decklinkInput->EnableVideoInput(mode->GetDisplayMode(), pixelFormat, bmdVideoInputFlagDefault | bmdVideoInputEnableFormatDetection);
        if (result != S_OK)
        {
            throwRuntimeException(env, "Failed to switch to new video mode");
            goto bail;
        }

        decklinkInput->StartStreams();
    }

bail:
    std::cout << "Detected new mode " << mode->GetWidth() << "x" << mode->GetHeight() << std::endl;
	return S_OK;
}

void DeckLinkCaptureDelegate::Stop()
{
    decklinkInput->StopStreams();
    decklinkInput->DisableVideoInput();

    if (decklinkInput != NULL)
    {
        decklinkInput->Release();
        decklinkInput = NULL;
    }

    if (decklink != NULL)
    {
        decklink->Release();
        decklink = NULL;
    }

    JNIEnv* env = getEnv(vm);
    env->DeleteGlobalRef(obj);
    releaseEnv(vm);

    fclose(f);
    avcodec_close(c);
    av_free(c);
    av_freep(&pictureYUV420->data[0]);
    avcodec_free_frame(&pictureYUV420);

}

JNIEXPORT void JNICALL Java_us_ihmc_javadecklink_Capture_stopCaptureNative
  (JNIEnv *, jobject, jlong delegatePtr)
{
    DeckLinkCaptureDelegate* delegate = (DeckLinkCaptureDelegate*) delegatePtr;
    delegate->Stop();
}

JNIEXPORT jlong JNICALL Java_us_ihmc_javadecklink_Capture_startCaptureNative
  (JNIEnv *env, jobject obj, jint idx, jint displayModeId)
{

	IDeckLinkIterator*				deckLinkIterator = NULL;
	IDeckLink*						deckLink = NULL;

	IDeckLinkDisplayModeIterator*	displayModeIterator = NULL;
	IDeckLinkDisplayMode*			displayMode = NULL;
    char*							displayModeName = NULL;
	BMDDisplayModeSupport			displayModeSupported;

	DeckLinkCaptureDelegate*		delegate = NULL;

    HRESULT                                                 result;

    IDeckLinkInput*	g_deckLinkInput = NULL;

    JavaVM* vm;
    JNIassert(env, env->GetJavaVM(&vm) == 0);

    jobject globalObj = env->NewGlobalRef(obj);
    jclass target = env->GetObjectClass(globalObj);
    JNIassert(env, target != NULL);

    jmethodID methodID = env->GetMethodID(target, "receivedFrameFromNative", "(ZIIILjava/nio/ByteBuffer;)V");
    JNIassert(env, methodID != NULL);

	// Get the DeckLink device
	deckLinkIterator = CreateDeckLinkIteratorInstance();
	if (!deckLinkIterator)
	{
		fprintf(stderr, "This application requires the DeckLink drivers installed.\n");
		goto bail;
	}

	while ((result = deckLinkIterator->Next(&deckLink)) == S_OK)
	{
		if (idx == 0)
			break;
		--idx;

		deckLink->Release();
	}

	if (result != S_OK || deckLink == NULL)
	{
        fprintf(stderr, "Unable to get DeckLink device %u\n", idx);
		goto bail;
	}

	// Get the input (capture) interface of the DeckLink device
    result = deckLink->QueryInterface(IID_IDeckLinkInput, (void**)&g_deckLinkInput);
	if (result != S_OK)
		goto bail;

    result = g_deckLinkInput->GetDisplayModeIterator(&displayModeIterator);
	if (result != S_OK)
		goto bail;

    displayModeId = 0;
	while ((result = displayModeIterator->Next(&displayMode)) == S_OK)
	{
        if (displayModeId == 0)
			break;
        --displayModeId;

		displayMode->Release();
	}

	if (result != S_OK || displayMode == NULL)
	{
        fprintf(stderr, "Unable to get display mode %d\n", displayModeId);
		goto bail;
	}

	// Get display mode name
	result = displayMode->GetName((const char**)&displayModeName);
	if (result != S_OK)
	{
		displayModeName = (char *)malloc(32);
        snprintf(displayModeName, 32, "[index %d]", displayModeId);
	}

	// Check display mode is supported with given options
    result = g_deckLinkInput->DoesSupportVideoMode(displayMode->GetDisplayMode(), bmdFormat8BitYUV, bmdVideoInputFlagDefault, &displayModeSupported, NULL);
	if (result != S_OK)
		goto bail;

	if (displayModeSupported == bmdDisplayModeNotSupported)
	{
		fprintf(stderr, "The display mode %s is not supported with the selected pixel format\n", displayModeName);
		goto bail;
	}

	// Configure the capture callback
    delegate = new DeckLinkCaptureDelegate(deckLink, g_deckLinkInput, vm, globalObj, methodID);
	g_deckLinkInput->SetCallback(delegate);

    // Start capturing
    result = g_deckLinkInput->EnableVideoInput(displayMode->GetDisplayMode(), bmdFormat8BitYUV, bmdVideoInputFlagDefault | bmdVideoInputEnableFormatDetection);
    if (result != S_OK)
    {
        fprintf(stderr, "Failed to enable video input. Is another application using the card?\n");
        goto bail;
    }

    result = g_deckLinkInput->StartStreams();
    if (result != S_OK)
        goto bail;

bail:

	if (displayModeName != NULL)
		free(displayModeName);

	if (displayMode != NULL)
		displayMode->Release();

	if (displayModeIterator != NULL)
		displayModeIterator->Release();

    if (deckLinkIterator != NULL)
		deckLinkIterator->Release();

    return (jlong) delegate;
}
<commit_msg>Fixed initialization of software scaler<commit_after>/* -LICENSE-START-
** Copyright (c) 2013 Blackmagic Design
**
** Permission is hereby granted, free of charge, to any person or organization
** obtaining a copy of the software and accompanying documentation covered by
** this license (the "Software") to use, reproduce, display, distribute,
** execute, and transmit the Software, and to prepare derivative works of the
** Software, and to permit third-parties to whom the Software is furnished to
** do so, all subject to the following:
**
** The copyright notices in the Software and this entire statement, including
** the above license grant, this restriction and the following disclaimer,
** must be included in all copies of the Software, in whole or in part, and
** all derivative works of the Software, unless such copies or derivative
** works are solely in the form of machine-executable object code generated by
** a source language processor.
**
** THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
** IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
** FITNESS FOR A PARTICULAR PURPOSE, TITLE AND NON-INFRINGEMENT. IN NO EVENT
** SHALL THE COPYRIGHT HOLDERS OR ANYONE DISTRIBUTING THE SOFTWARE BE LIABLE
** FOR ANY DAMAGES OR OTHER LIABILITY, WHETHER IN CONTRACT, TORT OR OTHERWISE,
** ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
** DEALINGS IN THE SOFTWARE.
** -LICENSE-END-
*/

#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include <pthread.h>
#include <unistd.h>
#include <fcntl.h>
#include <csignal>
#include <jni.h>
#include <string>

#include "DeckLinkAPI.h"
#include "Capture.h"
#include "Util.hpp"
#include "us_ihmc_javadecklink_Capture.h"

DeckLinkCaptureDelegate::DeckLinkCaptureDelegate(IDeckLink* decklink, IDeckLinkInput* decklinkInput, JavaVM* vm, jobject obj, jmethodID methodID) :
    m_refCount(1),
    decklink(decklink),
    decklinkInput(decklinkInput),
    vm(vm),
    obj(obj),
    methodID(methodID)
{
    avcodec_register_all();
    codec = avcodec_find_encoder(AV_CODEC_ID_MJPEG);
    c = avcodec_alloc_context3(codec);
    if (!codec) {
        fprintf(stderr, "codec not found\n");
        exit(1);
    }
}

ULONG DeckLinkCaptureDelegate::AddRef(void)
{
	return __sync_add_and_fetch(&m_refCount, 1);
}

ULONG DeckLinkCaptureDelegate::Release(void)
{
	int32_t newRefValue = __sync_sub_and_fetch(&m_refCount, 1);
	if (newRefValue == 0)
	{
		delete this;
		return 0;
	}
	return newRefValue;
}

/*
static void video_encode_example(const char *filename)
{
    int i, ret, x, y, got_output;
    uint8_t endcode[] = { 0, 0, 1, 0xb7 };
    printf("Video encoding\n");

    /* encode 1 second of video *
    for(i=0;i<25;i++) {

    }
    /* get the delayed frames *
    for (got_output = 1; got_output; i++) {
        fflush(stdout);
        ret = avcodec_encode_video2(c, &pkt, NULL, &got_output);
        if (ret < 0) {
            fprintf(stderr, "error encoding frame\n");
            exit(1);
        }
        if (got_output) {
            printf("encoding frame %3d (size=%5d)\n", i, pkt.size);
            fwrite(pkt.data, 1, pkt.size, f);
            av_packet_unref(&pkt);
        }
    }
    /* add sequence end code to have a real mpeg file *
    fwrite(endcode, 1, sizeof(endcode), f);
    printf("\n");
}
*/
HRESULT DeckLinkCaptureDelegate::VideoInputFrameArrived(IDeckLinkVideoInputFrame* videoFrame, IDeckLinkAudioInputPacket* audioFrame)
{
    void* frameBytes;

	// Handle Video Frame
	if (videoFrame)
    {

        JNIEnv* env = getEnv(vm);
        if(env == 0)
        {
            // Cannot throw a runtime exception because we don't have an env
            std::cerr << "Cannot load env" << std::endl;
            return S_OK;
        }

		if (videoFrame->GetFlags() & bmdFrameHasNoInputSource)
		{
            printf("Frame received - No input signal detected\n");

            //env->CallVoidMethod(obj, methodID, false, 0, 0, NULL);
		}
		else
        {

            printf("Frame received - Size: %li bytes\n",
				videoFrame->GetRowBytes() * videoFrame->GetHeight());

				videoFrame->GetBytes(&frameBytes);
                //write(g_videoOutputFile, frameBytes, videoFrame->GetRowBytes() * videoFrame->GetHeight());

                av_init_packet(&pkt);
                pkt.data = NULL;    // packet data will be allocated by the encoder
                pkt.size = 0;

                pictureUYVY->pts = i;
                pictureYUV420->pts = i;

                avpicture_fill((AVPicture*)pictureUYVY, (uint8_t*) frameBytes, AV_PIX_FMT_UYVY422, pictureUYVY->width, pictureUYVY->height);
                sws_scale(img_convert_ctx, pictureUYVY->data, pictureUYVY->linesize, 0, c->height, pictureYUV420->data, pictureYUV420->linesize);

                /* encode the image */
                int got_output;
                int ret = avcodec_encode_video2(c, &pkt, pictureYUV420, &got_output);
                if (ret < 0) {
                    fprintf(stderr, "error encoding frame\n");
                    exit(1);
                }
                if (got_output) {
                    printf("encoding frame %3d (size=%5d)\n", i, pkt.size);
                    fwrite(pkt.data, 1, pkt.size, f);
                    av_free_packet(&pkt); //depreacted, use av_packet_unref(&pkt); after Ubuntu 16.04 comes out

                }

            ++i;
		}

	}

	return S_OK;
}

HRESULT DeckLinkCaptureDelegate::VideoInputFormatChanged(BMDVideoInputFormatChangedEvents events, IDeckLinkDisplayMode *mode, BMDDetectedVideoInputFormatFlags formatFlags)
{
    // This only gets called if bmdVideoInputEnableFormatDetection was set
    // when enabling video input
    HRESULT	result;
    char*	displayModeName = NULL;
    BMDPixelFormat	pixelFormat = bmdFormat8BitYUV;

    JNIEnv* env = getEnv(vm);
    if (formatFlags & bmdDetectedVideoInputRGB444)
    {
        throwRuntimeException(env, "Unsupported input format: RGB444");
        goto bail;
    }

    mode->GetName((const char**)&displayModeName);
    printf("Video format changed to %s %s\n", displayModeName, formatFlags & bmdDetectedVideoInputRGB444 ? "RGB" : "YUV");

    if (displayModeName)
        free(displayModeName);

    if (decklinkInput)
    {
        decklinkInput->StopStreams();

        /* put sample parameters */
        c->qcompress = 0.80;
        /* resolution must be a multiple of two */
        c->width = mode->GetWidth();
        c->height = mode->GetHeight();
        /* frames per second */

        BMDTimeValue numerator;
        BMDTimeScale denumerator;

        mode->GetFrameRate(&numerator, &denumerator);

        c->time_base= (AVRational){(int) numerator, (int) denumerator};
        c->pix_fmt = AV_PIX_FMT_YUVJ420P;
        /* open it */
        if (avcodec_open2(c, codec, NULL) < 0) {
            throwRuntimeException(env, "Could not open codec");
            goto bail;
        }
        f = fopen("test.mp4", "wb");
        if (!f) {
            throwRuntimeException(env, "Could not open file");
            goto bail;
        }

        pictureYUV420 = avcodec_alloc_frame();
       int ret = av_image_alloc(pictureYUV420->data, pictureYUV420->linesize, c->width, c->height,
                             c->pix_fmt, 32);
        if (ret < 0) {
            throwRuntimeException(env, "could not alloc raw picture buffer\n");
            goto bail;
        }
        pictureYUV420->format = c->pix_fmt;
        pictureYUV420->width  = c->width;
        pictureYUV420->height = c->height;

        pictureUYVY = avcodec_alloc_frame();
        pictureUYVY->width = c->width;
        pictureUYVY->height = c->height;
        pictureUYVY->format = AV_PIX_FMT_UYVY422;

        img_convert_ctx = sws_getContext(c->width, c->height,
        AV_PIX_FMT_UYVY422,
        c->width, c->height,
        c->pix_fmt,
        sws_flags, NULL, NULL, NULL);

        result = decklinkInput->EnableVideoInput(mode->GetDisplayMode(), pixelFormat, bmdVideoInputFlagDefault | bmdVideoInputEnableFormatDetection);
        if (result != S_OK)
        {
            throwRuntimeException(env, "Failed to switch to new video mode");
            goto bail;
        }

        decklinkInput->StartStreams();
    }

bail:
    std::cout << "Detected new mode " << mode->GetWidth() << "x" << mode->GetHeight() << std::endl;
	return S_OK;
}

void DeckLinkCaptureDelegate::Stop()
{
    decklinkInput->StopStreams();
    decklinkInput->DisableVideoInput();

    if (decklinkInput != NULL)
    {
        decklinkInput->Release();
        decklinkInput = NULL;
    }

    if (decklink != NULL)
    {
        decklink->Release();
        decklink = NULL;
    }

    JNIEnv* env = getEnv(vm);
    env->DeleteGlobalRef(obj);
    releaseEnv(vm);

    fclose(f);
    avcodec_close(c);
    av_free(c);
    av_freep(&pictureYUV420->data[0]);
    avcodec_free_frame(&pictureYUV420);

}

JNIEXPORT void JNICALL Java_us_ihmc_javadecklink_Capture_stopCaptureNative
  (JNIEnv *, jobject, jlong delegatePtr)
{
    DeckLinkCaptureDelegate* delegate = (DeckLinkCaptureDelegate*) delegatePtr;
    delegate->Stop();
}

JNIEXPORT jlong JNICALL Java_us_ihmc_javadecklink_Capture_startCaptureNative
  (JNIEnv *env, jobject obj, jint idx, jint displayModeId)
{

	IDeckLinkIterator*				deckLinkIterator = NULL;
	IDeckLink*						deckLink = NULL;

	IDeckLinkDisplayModeIterator*	displayModeIterator = NULL;
	IDeckLinkDisplayMode*			displayMode = NULL;
    char*							displayModeName = NULL;
	BMDDisplayModeSupport			displayModeSupported;

	DeckLinkCaptureDelegate*		delegate = NULL;

    HRESULT                                                 result;

    IDeckLinkInput*	g_deckLinkInput = NULL;

    JavaVM* vm;
    JNIassert(env, env->GetJavaVM(&vm) == 0);

    jobject globalObj = env->NewGlobalRef(obj);
    jclass target = env->GetObjectClass(globalObj);
    JNIassert(env, target != NULL);

    jmethodID methodID = env->GetMethodID(target, "receivedFrameFromNative", "(ZIIILjava/nio/ByteBuffer;)V");
    JNIassert(env, methodID != NULL);

	// Get the DeckLink device
	deckLinkIterator = CreateDeckLinkIteratorInstance();
	if (!deckLinkIterator)
	{
		fprintf(stderr, "This application requires the DeckLink drivers installed.\n");
		goto bail;
	}

	while ((result = deckLinkIterator->Next(&deckLink)) == S_OK)
	{
		if (idx == 0)
			break;
		--idx;

		deckLink->Release();
	}

	if (result != S_OK || deckLink == NULL)
	{
        fprintf(stderr, "Unable to get DeckLink device %u\n", idx);
		goto bail;
	}

	// Get the input (capture) interface of the DeckLink device
    result = deckLink->QueryInterface(IID_IDeckLinkInput, (void**)&g_deckLinkInput);
	if (result != S_OK)
		goto bail;

    result = g_deckLinkInput->GetDisplayModeIterator(&displayModeIterator);
	if (result != S_OK)
		goto bail;

    displayModeId = 0;
	while ((result = displayModeIterator->Next(&displayMode)) == S_OK)
	{
        if (displayModeId == 0)
			break;
        --displayModeId;

		displayMode->Release();
	}

	if (result != S_OK || displayMode == NULL)
	{
        fprintf(stderr, "Unable to get display mode %d\n", displayModeId);
		goto bail;
	}

	// Get display mode name
	result = displayMode->GetName((const char**)&displayModeName);
	if (result != S_OK)
	{
		displayModeName = (char *)malloc(32);
        snprintf(displayModeName, 32, "[index %d]", displayModeId);
	}

	// Check display mode is supported with given options
    result = g_deckLinkInput->DoesSupportVideoMode(displayMode->GetDisplayMode(), bmdFormat8BitYUV, bmdVideoInputFlagDefault, &displayModeSupported, NULL);
	if (result != S_OK)
		goto bail;

	if (displayModeSupported == bmdDisplayModeNotSupported)
	{
		fprintf(stderr, "The display mode %s is not supported with the selected pixel format\n", displayModeName);
		goto bail;
	}

	// Configure the capture callback
    delegate = new DeckLinkCaptureDelegate(deckLink, g_deckLinkInput, vm, globalObj, methodID);
	g_deckLinkInput->SetCallback(delegate);

    // Start capturing
    result = g_deckLinkInput->EnableVideoInput(displayMode->GetDisplayMode(), bmdFormat8BitYUV, bmdVideoInputFlagDefault | bmdVideoInputEnableFormatDetection);
    if (result != S_OK)
    {
        fprintf(stderr, "Failed to enable video input. Is another application using the card?\n");
        goto bail;
    }

    result = g_deckLinkInput->StartStreams();
    if (result != S_OK)
        goto bail;

bail:

	if (displayModeName != NULL)
		free(displayModeName);

	if (displayMode != NULL)
		displayMode->Release();

	if (displayModeIterator != NULL)
		displayModeIterator->Release();

    if (deckLinkIterator != NULL)
		deckLinkIterator->Release();

    return (jlong) delegate;
}
<|endoftext|>


<commit_before>/** \file    delete_unused_local_data.cc
 *  \author  Oliver Obenland
 *
 *  Local data blocks are embedded marc records inside of a record using LOK-Fields.
 *  Each local data block belongs to an institution and is marked by the institution's sigil.
 *  This tool filters for local data blocks of some institutions of the University of Tübingen
 *  and deletes all other local blocks.
 */

/*
    Copyright (C) 2016, Library of the University of Tübingen

    This program is free software: you can redistribute it and/or modify
    it under the terms of the GNU Affero General Public License as
    published by the Free Software Foundation, either version 3 of the
    License, or (at your option) any later version.

    This program is distributed in the hope that it will be useful,
    but WITHOUT ANY WARRANTY; without even the implied warranty of
    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
    GNU Affero General Public License for more details.

    You should have received a copy of the GNU Affero General Public License
    along with this program.  If not, see <http://www.gnu.org/licenses/>.
*/

#include <iostream>
#include "MarcUtil.h"
#include "RegexMatcher.h"
#include "util.h"
#include "XmlWriter.h"

static ssize_t count(0), before_count(0), after_count(0), no_local_data_records_count(0);

void Usage() {
    std::cerr << "Usage: " << ::progname << "  marc_input marc_output\n";
    std::exit(EXIT_FAILURE);
}

bool IsUnusedLocalBlock(const MarcUtil::Record * const record, const std::pair<size_t, size_t> &block_start_and_end) {
    static RegexMatcher *matcher(nullptr);
    std::string err_msg;
    if (unlikely(matcher == nullptr)) {
        matcher = RegexMatcher::RegexMatcherFactory("^.*aDE-21.*$|^.*aDE-21-24.*$|^.*aDE-21-110.*$", &err_msg);
        if (matcher == nullptr)
            Error(err_msg);
    }

    std::string err_msg;
    std::vector<size_t> field_indices;
    record->findFieldsInLocalBlock("852", "??", block_start_and_end, &field_indices);

    const std::vector<std::string> &fields(record->getFields());
    for (const auto field_index : field_indices) {
        const bool matched = matcher->matched(fields[field_index], &err_msg);
        if (not matched and not err_msg.empty())
            Error("Unexpected error while trying to match a field in IsUnusedLocalBlock: " + err_msg);
        if (matched)
            return false;
    }
    return true;
}

void DeleteLocalBlock(MarcUtil::Record * const record, const std::pair<size_t, size_t> &block_start_and_end) {
    for (size_t field_index(block_start_and_end.second - 1); field_index >= block_start_and_end.first; --field_index)
        record->deleteField(field_index);
}

bool ProcessRecord(MarcUtil::Record * const record) {
    std::vector<std::pair<size_t, size_t>> local_block_boundaries;
    ssize_t local_data_count = record->findAllLocalDataBlocks(&local_block_boundaries);
    std::reverse(local_block_boundaries.begin(), local_block_boundaries.end());

    before_count += local_data_count;
    for (const std::pair<size_t, size_t> &block_start_and_end : local_block_boundaries) {
        if (IsUnusedLocalBlock(record, block_start_and_end)) {
            DeleteLocalBlock(record, block_start_and_end);
            --local_data_count;
        }
    }

    after_count += local_data_count;
    return local_data_count != 0;
}

void DeleteUnusedLocalData(File * const input, File * const output) {
    XmlWriter xml_writer(output);
    xml_writer.openTag("marc:collection",
                        { std::make_pair("xmlns:marc", "http://www.loc.gov/MARC21/slim"),
                          std::make_pair("xmlns:xsi", "http://www.w3.org/2001/XMLSchema-instance"),
                          std::make_pair("xsi:schemaLocation", "http://www.loc.gov/standards/marcxml/schema/MARC21slim.xsd")});

    while (MarcUtil::Record record = MarcUtil::Record::XmlFactory(input)) {
        ++count;
        record.setRecordWillBeWrittenAsXml(true);
        ProcessRecord(&record);
        record.write(&xml_writer);
    }

    xml_writer.closeTag();

    std::cerr << ::progname << ": Deleted " << (before_count - after_count) << " of " << before_count << " local data blocks.\n";
    std::cerr << ::progname << ": Deleted " << no_local_data_records_count << " of " << count << "records without local data.\n";
}

int main(int argc, char **argv) {
    ::progname = argv[0];

    if (argc != 3)
        Usage();

    const std::string input_filename(argv[1]);
    File input(input_filename, "r");
    if (not input)
        Error("can't open \"" + input_filename + "\" for reading!");

    const std::string output_filename(argv[2]);
    File output(output_filename, "w");
    if (not output)
        Error("can't open \"" + output_filename + "\" for writing!");

    try {
        DeleteUnusedLocalData(&input, &output);
    } catch (const std::exception &x) {
        Error("caught exception: " + std::string(x.what()));
    }
}
<commit_msg>Fixing compiling bug<commit_after>/** \file    delete_unused_local_data.cc
 *  \author  Oliver Obenland
 *
 *  Local data blocks are embedded marc records inside of a record using LOK-Fields.
 *  Each local data block belongs to an institution and is marked by the institution's sigil.
 *  This tool filters for local data blocks of some institutions of the University of Tübingen
 *  and deletes all other local blocks.
 */

/*
    Copyright (C) 2016, Library of the University of Tübingen

    This program is free software: you can redistribute it and/or modify
    it under the terms of the GNU Affero General Public License as
    published by the Free Software Foundation, either version 3 of the
    License, or (at your option) any later version.

    This program is distributed in the hope that it will be useful,
    but WITHOUT ANY WARRANTY; without even the implied warranty of
    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
    GNU Affero General Public License for more details.

    You should have received a copy of the GNU Affero General Public License
    along with this program.  If not, see <http://www.gnu.org/licenses/>.
*/

#include <iostream>
#include "MarcUtil.h"
#include "RegexMatcher.h"
#include "util.h"
#include "XmlWriter.h"

static ssize_t count(0), before_count(0), after_count(0), no_local_data_records_count(0);

void Usage() {
    std::cerr << "Usage: " << ::progname << "  marc_input marc_output\n";
    std::exit(EXIT_FAILURE);
}

bool IsUnusedLocalBlock(const MarcUtil::Record * const record, const std::pair<size_t, size_t> &block_start_and_end) {
    static RegexMatcher *matcher(nullptr);
    std::string err_msg;
    if (unlikely(matcher == nullptr)) {
        matcher = RegexMatcher::RegexMatcherFactory("^.*aDE-21.*$|^.*aDE-21-24.*$|^.*aDE-21-110.*$", &err_msg);
        if (matcher == nullptr)
            Error(err_msg);
    }

    std::vector<size_t> field_indices;
    record->findFieldsInLocalBlock("852", "??", block_start_and_end, &field_indices);

    const std::vector<std::string> &fields(record->getFields());
    for (const auto field_index : field_indices) {
        const bool matched = matcher->matched(fields[field_index], &err_msg);
        if (not matched and not err_msg.empty())
            Error("Unexpected error while trying to match a field in IsUnusedLocalBlock: " + err_msg);
        if (matched)
            return false;
    }
    return true;
}

void DeleteLocalBlock(MarcUtil::Record * const record, const std::pair<size_t, size_t> &block_start_and_end) {
    for (size_t field_index(block_start_and_end.second - 1); field_index >= block_start_and_end.first; --field_index)
        record->deleteField(field_index);
}

bool ProcessRecord(MarcUtil::Record * const record) {
    std::vector<std::pair<size_t, size_t>> local_block_boundaries;
    ssize_t local_data_count = record->findAllLocalDataBlocks(&local_block_boundaries);
    std::reverse(local_block_boundaries.begin(), local_block_boundaries.end());

    before_count += local_data_count;
    for (const std::pair<size_t, size_t> &block_start_and_end : local_block_boundaries) {
        if (IsUnusedLocalBlock(record, block_start_and_end)) {
            DeleteLocalBlock(record, block_start_and_end);
            --local_data_count;
        }
    }

    after_count += local_data_count;
    return local_data_count != 0;
}

void DeleteUnusedLocalData(File * const input, File * const output) {
    XmlWriter xml_writer(output);
    xml_writer.openTag("marc:collection",
                        { std::make_pair("xmlns:marc", "http://www.loc.gov/MARC21/slim"),
                          std::make_pair("xmlns:xsi", "http://www.w3.org/2001/XMLSchema-instance"),
                          std::make_pair("xsi:schemaLocation", "http://www.loc.gov/standards/marcxml/schema/MARC21slim.xsd")});

    while (MarcUtil::Record record = MarcUtil::Record::XmlFactory(input)) {
        ++count;
        record.setRecordWillBeWrittenAsXml(true);
        ProcessRecord(&record);
        record.write(&xml_writer);
    }

    xml_writer.closeTag();

    std::cerr << ::progname << ": Deleted " << (before_count - after_count) << " of " << before_count << " local data blocks.\n";
    std::cerr << ::progname << ": Deleted " << no_local_data_records_count << " of " << count << "records without local data.\n";
}

int main(int argc, char **argv) {
    ::progname = argv[0];

    if (argc != 3)
        Usage();

    const std::string input_filename(argv[1]);
    File input(input_filename, "r");
    if (not input)
        Error("can't open \"" + input_filename + "\" for reading!");

    const std::string output_filename(argv[2]);
    File output(output_filename, "w");
    if (not output)
        Error("can't open \"" + output_filename + "\" for writing!");

    try {
        DeleteUnusedLocalData(&input, &output);
    } catch (const std::exception &x) {
        Error("caught exception: " + std::string(x.what()));
    }
}
<|endoftext|>


<commit_before>/*
 *  PathResolver.cpp
 *  openc2e
 *
 *  Created by Bryan Donlan
 *  Copyright (c) 2005 Bryan Donlan. All rights reserved.
 *
 *  This library is free software; you can redistribute it and/or
 *  modify it under the terms of the GNU Lesser General Public
 *  License as published by the Free Software Foundation; either
 *  version 2 of the License, or (at your option) any later version.
 *
 *  This library is distributed in the hope that it will be useful,
 *  but WITHOUT ANY WARRANTY; without even the implied warranty of
 *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
 *  Lesser General Public License for more details.
 *
 */

#include "PathResolver.h"

#include <boost/filesystem/path.hpp>
#include <boost/filesystem/operations.hpp>
#include <boost/xpressive/xpressive.hpp>
#include <set>
#include <map>
#include <cctype>
#include <string>
#include <algorithm>
#include <iostream>
#include <sstream>

using std::map;
using std::set;
using std::string;
using namespace boost::filesystem;
namespace x = boost::xpressive;

static set<string> dircache;
static map<string, string> cache;

static bool checkDirCache(path &dir);
static bool doCacheDir(path &dir);

/* C++ is far too verbose for its own good */
static string toLowerCase(string in) {
	transform(in.begin(), in.end(), in.begin(), (int(*)(int))tolower);
	return in;
}

static path lcpath(path &orig) {
	return path(toLowerCase(orig.string()), native);
}

static path lcleaf(path &orig) {
	path br, leaf;
	br = orig.branch_path();
	leaf = path(toLowerCase(orig.leaf()), native);
	return br / leaf;
}

bool resolveFile(path &p) {
	string s = p.string();
	if (!resolveFile(s)) {
		// Maybe something changed underneath us; reset the cache and try again
		cache.clear();
		dircache.clear();
		if (!resolveFile(s))
			return false;
	}
	p = path(s, native);
	return true;
}

bool resolveFile_(string &srcPath) {
	path orig(srcPath, native);
	if (exists(orig))
		return true;

	orig.normalize();
	path dir = orig.branch_path();
	path leaf = path(orig.leaf(), native);

	if (!checkDirCache(dir))
		return false;

	orig = dir / lcpath(leaf);
	string fn = orig.string();

	if (exists(orig)) {
		srcPath = fn;
		return true;
	}

	map<string, string>::iterator i = cache.find(fn);
	if (i == cache.end()) {
		assert(!exists(orig));
		return false;
	}
	srcPath = cache[fn];
	return true;
}

bool resolveFile(std::string &path) {
	std::string orig = path;
	bool res = resolveFile_(path);
#if 0
	std::cerr << orig << " -> ";
	if (!res)
		std::cerr << "(nil)";
	else
		std::cerr << path;
	std::cerr << std::endl;
#endif
	return res;
}

/* If dir is cached, do nothing.
 * If dir exists, cache it.
 * If dir does not exist, see if there's one with different capitalization.
 *
 * If we find a dir, return true. Else, false.
 */
bool checkDirCache(path &dir) {
	if (dir == path())
		dir = path(".");
//	std::cerr << "checkDirCache: " << dir.string() << std::endl;
	if (dircache.end() != dircache.find(dir.string())) {
		return true;
	}
	if (exists(dir))
		return doCacheDir(dir);
	if (dir.empty())
		return false;
	bool res = resolveFile(dir);
	if (!res)
		return false;
	return checkDirCache(dir);
}

/* Cache a dir. Return true for success.
 */
bool doCacheDir(path &dir) {
//	std::cerr << "cacheing: " << dir.string() << std::endl;
	directory_iterator it(dir);
	directory_iterator fsend;
	while (it != fsend) {
		path cur = *it++;
		string key, val;
		key = cur.string();
		val = lcleaf(cur).string();
//		std::cerr << "Cache put: " << val << " -> " << key << std::endl;
		cache[val] = key;
	}
	dircache.insert(dir.string());
	return true;
}

static x::sregex constructSearchPattern(const std::string &wild) {
	x::sregex scanner;
	scanner = *(x::alnum | x::blank) | (x::s1= x::_);
	std::ostringstream matchstr;
	x::sregex_iterator cur(wild.begin(), wild.end(), scanner);
	x::sregex_iterator end;
	matchstr << "^";
	for (; cur != end; cur++) {
		x::smatch const &what_m = *cur;
		const std::string what = what_m[0];
		if (what == "*")
			matchstr << ".*";
		else if (what == "?")
			matchstr << ".";
		else if (what_m[1] != "")
			matchstr << "[" << what_m[1] << "]";
		else
			matchstr << what_m[0];
	}
	matchstr << "$";
	std::cout << "matchstr " << matchstr.str() << std::endl;
	return x::sregex::compile(matchstr.str());
}

static bool checkSearchPattern(const std::string &match, const x::sregex &l) {
	return x::regex_match(match, l);
}

std::vector<std::string> findByWildcard(std::string dir, std::string wild) {
	cache.clear();
	dircache.clear();

	wild = toLowerCase(wild);

	path dirp(dir, native);
	dirp.normalize();
	if (!resolveFile(dirp))
		return std::vector<std::string>();
	dir = dirp.string();

	if (!doCacheDir(dirp))
		return std::vector<std::string>();
	std::vector<std::string> results;
	x::sregex l = constructSearchPattern(wild);

	std::string lcdir = toLowerCase(dir);
	std::map<string, string>::iterator skey = cache.lower_bound(dir);
	for (; skey != cache.end(); skey++) {
		if (skey->first.length() < lcdir.length())
			break;
		std::string dirpart, filepart; // XXX: we should use boost fops, maybe
		dirpart = skey->first.substr(0, lcdir.length());
		if (dirpart != dir)
			break;
		if (skey->first.length() < lcdir.length() + 2)
			continue;
		filepart = toLowerCase(skey->first.substr(lcdir.length() + 1));
		if (!x::regex_match(filepart, l))
			continue;
		results.push_back(skey->second);
	}
	return results;
}

/* vim: set noet: */
<commit_msg>Add 2 casts to fix Windows build<commit_after>/*
 *  PathResolver.cpp
 *  openc2e
 *
 *  Created by Bryan Donlan
 *  Copyright (c) 2005 Bryan Donlan. All rights reserved.
 *
 *  This library is free software; you can redistribute it and/or
 *  modify it under the terms of the GNU Lesser General Public
 *  License as published by the Free Software Foundation; either
 *  version 2 of the License, or (at your option) any later version.
 *
 *  This library is distributed in the hope that it will be useful,
 *  but WITHOUT ANY WARRANTY; without even the implied warranty of
 *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
 *  Lesser General Public License for more details.
 *
 */

#include "PathResolver.h"

#include <boost/filesystem/path.hpp>
#include <boost/filesystem/operations.hpp>
#include <boost/xpressive/xpressive.hpp>
#include <set>
#include <map>
#include <cctype>
#include <string>
#include <algorithm>
#include <iostream>
#include <sstream>

using std::map;
using std::set;
using std::string;
using namespace boost::filesystem;
namespace x = boost::xpressive;

static set<string> dircache;
static map<string, string> cache;

static bool checkDirCache(path &dir);
static bool doCacheDir(path &dir);

/* C++ is far too verbose for its own good */
static string toLowerCase(string in) {
	transform(in.begin(), in.end(), in.begin(), (int(*)(int))tolower);
	return in;
}

static path lcpath(path &orig) {
	return path(toLowerCase(orig.string()), native);
}

static path lcleaf(path &orig) {
	path br, leaf;
	br = orig.branch_path();
	leaf = path(toLowerCase(orig.leaf()), native);
	return br / leaf;
}

bool resolveFile(path &p) {
	string s = p.string();
	if (!resolveFile(s)) {
		// Maybe something changed underneath us; reset the cache and try again
		cache.clear();
		dircache.clear();
		if (!resolveFile(s))
			return false;
	}
	p = path(s, native);
	return true;
}

bool resolveFile_(string &srcPath) {
	path orig(srcPath, native);
	if (exists(orig))
		return true;

	orig.normalize();
	path dir = orig.branch_path();
	path leaf = path(orig.leaf(), native);

	if (!checkDirCache(dir))
		return false;

	orig = dir / lcpath(leaf);
	string fn = orig.string();

	if (exists(orig)) {
		srcPath = fn;
		return true;
	}

	map<string, string>::iterator i = cache.find(fn);
	if (i == cache.end()) {
		assert(!exists(orig));
		return false;
	}
	srcPath = cache[fn];
	return true;
}

bool resolveFile(std::string &path) {
	std::string orig = path;
	bool res = resolveFile_(path);
#if 0
	std::cerr << orig << " -> ";
	if (!res)
		std::cerr << "(nil)";
	else
		std::cerr << path;
	std::cerr << std::endl;
#endif
	return res;
}

/* If dir is cached, do nothing.
 * If dir exists, cache it.
 * If dir does not exist, see if there's one with different capitalization.
 *
 * If we find a dir, return true. Else, false.
 */
bool checkDirCache(path &dir) {
	if (dir == path())
		dir = path(".");
//	std::cerr << "checkDirCache: " << dir.string() << std::endl;
	if (dircache.end() != dircache.find(dir.string())) {
		return true;
	}
	if (exists(dir))
		return doCacheDir(dir);
	if (dir.empty())
		return false;
	bool res = resolveFile(dir);
	if (!res)
		return false;
	return checkDirCache(dir);
}

/* Cache a dir. Return true for success.
 */
bool doCacheDir(path &dir) {
//	std::cerr << "cacheing: " << dir.string() << std::endl;
	directory_iterator it(dir);
	directory_iterator fsend;
	while (it != fsend) {
		path cur = *it++;
		string key, val;
		key = cur.string();
		val = lcleaf(cur).string();
//		std::cerr << "Cache put: " << val << " -> " << key << std::endl;
		cache[val] = key;
	}
	dircache.insert(dir.string());
	return true;
}

static x::sregex constructSearchPattern(const std::string &wild) {
	x::sregex scanner;
	scanner = *(x::alnum | x::blank) | (x::s1= x::_);
	std::ostringstream matchstr;
	x::sregex_iterator cur(wild.begin(), wild.end(), scanner);
	x::sregex_iterator end;
	matchstr << "^";
	for (; cur != end; cur++) {
		x::smatch const &what_m = *cur;
		const std::string what = what_m[0];
		if (what == "*")
			matchstr << ".*";
		else if (what == "?")
			matchstr << ".";
		else if (what_m[1] != "")
			matchstr << "[" << (std::string)what_m[1] << "]";
		else
			matchstr << (std::string)what_m[0];
	}
	matchstr << "$";
	std::cout << "matchstr " << matchstr.str() << std::endl;
	return x::sregex::compile(matchstr.str());
}

static bool checkSearchPattern(const std::string &match, const x::sregex &l) {
	return x::regex_match(match, l);
}

std::vector<std::string> findByWildcard(std::string dir, std::string wild) {
	cache.clear();
	dircache.clear();

	wild = toLowerCase(wild);

	path dirp(dir, native);
	dirp.normalize();
	if (!resolveFile(dirp))
		return std::vector<std::string>();
	dir = dirp.string();

	if (!doCacheDir(dirp))
		return std::vector<std::string>();
	std::vector<std::string> results;
	x::sregex l = constructSearchPattern(wild);

	std::string lcdir = toLowerCase(dir);
	std::map<string, string>::iterator skey = cache.lower_bound(dir);
	for (; skey != cache.end(); skey++) {
		if (skey->first.length() < lcdir.length())
			break;
		std::string dirpart, filepart; // XXX: we should use boost fops, maybe
		dirpart = skey->first.substr(0, lcdir.length());
		if (dirpart != dir)
			break;
		if (skey->first.length() < lcdir.length() + 2)
			continue;
		filepart = toLowerCase(skey->first.substr(lcdir.length() + 1));
		if (!x::regex_match(filepart, l))
			continue;
		results.push_back(skey->second);
	}
	return results;
}

/* vim: set noet: */
<|endoftext|>