<commit_before>// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.

#include <vespa/eval/tensor/dense/typed_cells.h>
#include <vespa/searchlib/common/bitvector.h>
#include <vespa/searchlib/tensor/distance_functions.h>
#include <vespa/searchlib/tensor/doc_vector_access.h>
#include <vespa/searchlib/tensor/hnsw_index.h>
#include <vespa/searchlib/tensor/random_level_generator.h>
#include <vespa/searchlib/tensor/inv_log_level_generator.h>
#include <vespa/vespalib/gtest/gtest.h>
#include <vespa/vespalib/util/generationhandler.h>
#include <vespa/vespalib/util/blockingthreadstackexecutor.h>
#include <vespa/vespalib/util/lambdatask.h>

#include <vector>
#include <sys/types.h>
#include <sys/stat.h>
#include <fcntl.h>
#include <unistd.h>
#include <stdio.h>
#include <chrono>
#include <cstdlib>

#include <vespa/log/log.h>
LOG_SETUP("stress_hnsw_mt");

using vespalib::GenerationHandler;
using vespalib::MemoryUsage;
using namespace search::tensor;
using search::BitVector;

#define NUM_DIMS 128
#define NUM_POSSIBLE_V 1000000
#define NUM_POSSIBLE_DOCS 30000
#define NUM_OPS 1000000

class RndGen {
private:
    std::mt19937_64 urng;
    std::uniform_real_distribution<double> uf;
public:
    RndGen() : urng(0x1234deadbeef5678uLL), uf(0.0, 1.0) {}

    double nextUniform() {
        return uf(urng);
    }
};

using ConstVectorRef = vespalib::ConstArrayRef<float>;

struct MallocPointVector {
    float v[NUM_DIMS];
    operator ConstVectorRef() const { return ConstVectorRef(v, NUM_DIMS); }
};
static MallocPointVector *aligned_alloc_pv(size_t num) {
    size_t num_bytes = num * sizeof(MallocPointVector);
    double mega_bytes = num_bytes / (1024.0*1024.0);
    fprintf(stderr, "allocate %.2f MB of vectors\n", mega_bytes);
    char *mem = (char *)malloc(num_bytes + 512);
    mem += 512;
    size_t val = (size_t)mem;
    size_t unalign = val % 512;
    mem -= unalign;
    return reinterpret_cast<MallocPointVector *>(mem);
}

void read_vector_file(MallocPointVector *p) {
    std::string data_set = "sift";
    std::string data_dir = ".";
    char *home = getenv("HOME");
    if (home) {
        data_dir = home;
        data_dir += "/" + data_set;
    }
    std::string fn = data_dir + "/" + data_set + "_base.fvecs";
    int fd = open(fn.c_str(), O_RDONLY);
    if (fd < 0) {
        perror(fn.c_str());
        exit(1);
    }
    int d;
    size_t rv;
    fprintf(stderr, "reading %u vectors from %s\n", NUM_POSSIBLE_V, fn.c_str());
    for (uint32_t i = 0; i < NUM_POSSIBLE_V; ++i) {
        rv = read(fd, &d, 4);
        ASSERT_EQ(rv, 4u);
        ASSERT_EQ(d, NUM_DIMS);
        rv = read(fd, &p[i].v, NUM_DIMS*sizeof(float));
        ASSERT_EQ(rv, sizeof(MallocPointVector));
    }
    close(fd);
    fprintf(stderr, "reading %u vectors OK\n", NUM_POSSIBLE_V);
}

class MyDocVectorStore : public DocVectorAccess {
private:
    MallocPointVector *_vectors;
public:
    MyDocVectorStore() {
        _vectors = aligned_alloc_pv(NUM_POSSIBLE_DOCS);
    }
    MyDocVectorStore& set(uint32_t docid, ConstVectorRef vec) {
        assert(docid < NUM_POSSIBLE_DOCS);
        memcpy(&_vectors[docid], vec.cbegin(), sizeof(MallocPointVector));
        return *this;
    }
    vespalib::tensor::TypedCells get_vector(uint32_t docid) const override {
        assert(docid < NUM_POSSIBLE_DOCS);
        ConstVectorRef ref(_vectors[docid]);
        return vespalib::tensor::TypedCells(ref);
    }
};

using FloatSqEuclideanDistance = SquaredEuclideanDistance<float>;
using HnswIndexUP = std::unique_ptr<HnswIndex>;

class Stressor : public ::testing::Test {
private:
    struct LoadedVectors {
        MallocPointVector *pv_storage;
        void load() {
            pv_storage = aligned_alloc_pv(size());
            read_vector_file(pv_storage);
        }
        size_t size() const { return NUM_POSSIBLE_V; }
        vespalib::ConstArrayRef<float> operator[] (size_t i) {
            return pv_storage[i];
        }
    } loaded_vectors;
public:
    BitVector::UP in_progress;
    std::mutex in_progress_lock;
    BitVector::UP existing_ids;
    RndGen rng;
    MyDocVectorStore vectors;
    GenerationHandler gen_handler;
    HnswIndexUP index;
    vespalib::BlockingThreadStackExecutor multi_prepare_workers;
    vespalib::BlockingThreadStackExecutor write_thread;

    using PrepUP = std::unique_ptr<PrepareResult>;
    using ReadGuard = GenerationHandler::Guard;

    // union of data required by tasks
    struct TaskBase : vespalib::Executor::Task {
        Stressor &parent;
        uint32_t docid;
        ConstVectorRef vec;
        PrepUP prepare_result;
        ReadGuard read_guard;

        TaskBase(Stressor &p, uint32_t d, ConstVectorRef v, PrepUP r, ReadGuard g)
            : parent(p), docid(d), vec(v), prepare_result(std::move(r)), read_guard(g)
        {}
        TaskBase(Stressor &p, uint32_t d, ConstVectorRef v, ReadGuard g)
            : TaskBase(p, d, v, PrepUP(), g) {}
        TaskBase(Stressor &p, uint32_t d, ReadGuard g)
            : TaskBase(p, d, ConstVectorRef(), PrepUP(), g) {}
        TaskBase(Stressor &p, uint32_t d, ConstVectorRef v, PrepUP r)
            : TaskBase(p, d, v, std::move(r), ReadGuard()) {}
        TaskBase(Stressor &p, uint32_t d)
            : TaskBase(p, d, ConstVectorRef(), PrepUP(), ReadGuard()) {}
    };

    struct CompleteAddTask : TaskBase {
        using TaskBase::TaskBase;
        void run() override {
            parent.vectors.set(docid, vec);
            parent.index->complete_add_document(docid, std::move(prepare_result));
            parent.existing_ids->setBit(docid);
            parent.commit(docid);
        }
    };

    struct TwoPhaseAddTask  : TaskBase {
        using TaskBase::TaskBase;
        void run() override {
            auto v = vespalib::tensor::TypedCells(vec);
            auto up = parent.index->prepare_add_document(docid, v, read_guard);
            auto task = std::make_unique<CompleteAddTask>(parent, docid, vec, std::move(up));
            auto r = parent.write_thread.execute(std::move(task));
            if (r) {
                fprintf(stderr, "Failed posting complete add task!");
                abort();
            }
        }
    };

    struct CompleteRemoveTask : TaskBase {
        using TaskBase::TaskBase;
        void run() override {
            parent.index->remove_document(docid);
            parent.existing_ids->clearBit(docid);
            parent.commit(docid);
        }
    };

    struct TwoPhaseRemoveTask : TaskBase {
        using TaskBase::TaskBase;
        void run() override {
            auto task = std::make_unique<CompleteRemoveTask>(parent, docid);
            auto r = parent.write_thread.execute(std::move(task));
            if (r) {
                fprintf(stderr, "Failed posting complete remove task!");
                abort();
            }
        }
    };

    struct CompleteUpdateTask : TaskBase {
        using TaskBase::TaskBase;
        void run() override {
            parent.index->remove_document(docid);
            parent.vectors.set(docid, vec);
            parent.index->complete_add_document(docid, std::move(prepare_result));
            EXPECT_EQ(parent.existing_ids->testBit(docid), true);
            parent.commit(docid);
        }
    };

    struct TwoPhaseUpdateTask : TaskBase {
        using TaskBase::TaskBase;
        void run() override {
            auto v = vespalib::tensor::TypedCells(vec);
            auto up = parent.index->prepare_add_document(docid, v, read_guard);
            EXPECT_EQ(bool(up), true);
            auto task = std::make_unique<CompleteUpdateTask>(parent, docid, vec, std::move(up));
            auto r = parent.write_thread.execute(std::move(task));
            if (r) {
                fprintf(stderr, "Failed posting complete remove task!");
                abort();
            }
        }
    };

    Stressor()
        : loaded_vectors(),
          in_progress(BitVector::create(NUM_POSSIBLE_DOCS)),
          existing_ids(BitVector::create(NUM_POSSIBLE_DOCS)),
          rng(),
          vectors(),
          gen_handler(),
          index(),
          multi_prepare_workers(10, 128*1024, 50),
          write_thread(1, 128*1024, 500)
    {
        loaded_vectors.load();
    }

    ~Stressor() {}

    void init() {
        uint32_t m = 16;
        index = std::make_unique<HnswIndex>(vectors, std::make_unique<FloatSqEuclideanDistance>(),
                                            std::make_unique<InvLogLevelGenerator>(m),
                                            HnswIndex::Config(2*m, m, 200, true));
    }
    size_t get_rnd(size_t size) {
        return rng.nextUniform() * size;
    }
    void add_document(uint32_t docid) {
        size_t vec_num = get_rnd(loaded_vectors.size());
        ConstVectorRef vec = loaded_vectors[vec_num];
        auto guard = take_read_guard();
        auto task = std::make_unique<TwoPhaseAddTask>(*this, docid, vec, guard);
        auto r = multi_prepare_workers.execute(std::move(task));
        ASSERT_EQ(bool(r), false);
    }
    void remove_document(uint32_t docid) {
        auto guard = take_read_guard();
        auto task = std::make_unique<TwoPhaseRemoveTask>(*this, docid, guard);
        auto r = multi_prepare_workers.execute(std::move(task));
        ASSERT_EQ(bool(r), false);
    }
    void update_document(uint32_t docid) {
        size_t vec_num = get_rnd(loaded_vectors.size());
        ConstVectorRef vec = loaded_vectors[vec_num];
        auto guard = take_read_guard();
        auto task = std::make_unique<TwoPhaseUpdateTask>(*this, docid, vec, guard);
        auto r = multi_prepare_workers.execute(std::move(task));
        ASSERT_EQ(bool(r), false);
    }
    void commit(uint32_t docid) {
        index->transfer_hold_lists(gen_handler.getCurrentGeneration());
        gen_handler.incGeneration();
        gen_handler.updateFirstUsedGeneration();
        index->trim_hold_lists(gen_handler.getFirstUsedGeneration());
        std::lock_guard<std::mutex> guard(in_progress_lock);
        in_progress->clearBit(docid);
        // printf("commit: %u\n", docid);
    }
    void gen_operation() {
        uint32_t docid = get_rnd(NUM_POSSIBLE_DOCS);
        {
            std::lock_guard<std::mutex> guard(in_progress_lock);
            while (in_progress->testBit(docid)) {
                docid = get_rnd(NUM_POSSIBLE_DOCS);
            }
            in_progress->setBit(docid);
        }
        if (existing_ids->testBit(docid)) {
            if (get_rnd(100) < 70) {
                // printf("start remove op: %u\n", docid);
                remove_document(docid);
            } else {
                // printf("start update op: %u\n", docid);
                update_document(docid);
            }
        } else {
            // printf("start add op: %u\n", docid);
            add_document(docid);
        }
    }
    GenerationHandler::Guard take_read_guard() {
        return gen_handler.takeGuard();
    }
    MemoryUsage memory_usage() const {
        return index->memory_usage();
    }
};

TEST_F(Stressor, stress)
{
    init();
    for (int i = 0; i < NUM_OPS; ++i) {
        gen_operation();
        if (i % 1000 == 0) {
            fprintf(stderr, "generating operations %d / %d\n", i, NUM_OPS);
            auto r = write_thread.execute(vespalib::makeLambdaTask([&]() {
                        EXPECT_TRUE(index->check_link_symmetry());
            }));
            EXPECT_EQ(bool(r), false);
        }
    }
    fprintf(stderr, "waiting for queued operations...\n");
    multi_prepare_workers.sync();
    write_thread.sync();
    in_progress->invalidateCachedCount();
    EXPECT_EQ(in_progress->countTrueBits(), 0);
    EXPECT_TRUE(index->check_link_symmetry());
    fprintf(stderr, "all done.\n");
}

GTEST_MAIN_RUN_ALL_TESTS()
<commit_msg>rewrite with futures<commit_after>// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.

#include <sys/types.h>
#include <sys/stat.h>
#include <fcntl.h>
#include <stdio.h>
#include <unistd.h>
#include <chrono>
#include <cstdlib>
#include <future>
#include <vector>

#include <vespa/eval/tensor/dense/typed_cells.h>
#include <vespa/searchlib/common/bitvector.h>
#include <vespa/searchlib/tensor/distance_functions.h>
#include <vespa/searchlib/tensor/doc_vector_access.h>
#include <vespa/searchlib/tensor/hnsw_index.h>
#include <vespa/searchlib/tensor/inv_log_level_generator.h>
#include <vespa/searchlib/tensor/random_level_generator.h>
#include <vespa/vespalib/gtest/gtest.h>
#include <vespa/vespalib/util/blockingthreadstackexecutor.h>
#include <vespa/vespalib/util/generationhandler.h>
#include <vespa/vespalib/util/lambdatask.h>

#include <vespa/log/log.h>
LOG_SETUP("stress_hnsw_mt");

using vespalib::GenerationHandler;
using vespalib::MemoryUsage;
using namespace search::tensor;
using search::BitVector;

#define NUM_DIMS 128
#define NUM_POSSIBLE_V 1000000
#define NUM_POSSIBLE_DOCS 30000
#define NUM_OPS 1000000

class RndGen {
private:
    std::mt19937_64 urng;
    std::uniform_real_distribution<double> uf;
public:
    RndGen() : urng(0x1234deadbeef5678uLL), uf(0.0, 1.0) {}

    double nextUniform() {
        return uf(urng);
    }
};

using ConstVectorRef = vespalib::ConstArrayRef<float>;

struct MallocPointVector {
    float v[NUM_DIMS];
    operator ConstVectorRef() const { return ConstVectorRef(v, NUM_DIMS); }
};
static MallocPointVector *aligned_alloc_pv(size_t num) {
    size_t num_bytes = num * sizeof(MallocPointVector);
    double mega_bytes = num_bytes / (1024.0*1024.0);
    fprintf(stderr, "allocate %.2f MB of vectors\n", mega_bytes);
    char *mem = (char *)malloc(num_bytes + 512);
    mem += 512;
    size_t val = (size_t)mem;
    size_t unalign = val % 512;
    mem -= unalign;
    return reinterpret_cast<MallocPointVector *>(mem);
}

void read_vector_file(MallocPointVector *p) {
    std::string data_set = "sift";
    std::string data_dir = ".";
    char *home = getenv("HOME");
    if (home) {
        data_dir = home;
        data_dir += "/" + data_set;
    }
    std::string fn = data_dir + "/" + data_set + "_base.fvecs";
    int fd = open(fn.c_str(), O_RDONLY);
    if (fd < 0) {
        perror(fn.c_str());
        exit(1);
    }
    int d;
    size_t rv;
    fprintf(stderr, "reading %u vectors from %s\n", NUM_POSSIBLE_V, fn.c_str());
    for (uint32_t i = 0; i < NUM_POSSIBLE_V; ++i) {
        rv = read(fd, &d, 4);
        ASSERT_EQ(rv, 4u);
        ASSERT_EQ(d, NUM_DIMS);
        rv = read(fd, &p[i].v, NUM_DIMS*sizeof(float));
        ASSERT_EQ(rv, sizeof(MallocPointVector));
    }
    close(fd);
    fprintf(stderr, "reading %u vectors OK\n", NUM_POSSIBLE_V);
}

class MyDocVectorStore : public DocVectorAccess {
private:
    MallocPointVector *_vectors;
public:
    MyDocVectorStore() {
        _vectors = aligned_alloc_pv(NUM_POSSIBLE_DOCS);
    }
    MyDocVectorStore& set(uint32_t docid, ConstVectorRef vec) {
        assert(docid < NUM_POSSIBLE_DOCS);
        memcpy(&_vectors[docid], vec.cbegin(), sizeof(MallocPointVector));
        return *this;
    }
    vespalib::tensor::TypedCells get_vector(uint32_t docid) const override {
        assert(docid < NUM_POSSIBLE_DOCS);
        ConstVectorRef ref(_vectors[docid]);
        return vespalib::tensor::TypedCells(ref);
    }
};

using FloatSqEuclideanDistance = SquaredEuclideanDistance<float>;
using HnswIndexUP = std::unique_ptr<HnswIndex>;

class Stressor : public ::testing::Test {
private:
    struct LoadedVectors {
        MallocPointVector *pv_storage;
        void load() {
            pv_storage = aligned_alloc_pv(size());
            read_vector_file(pv_storage);
        }
        size_t size() const { return NUM_POSSIBLE_V; }
        vespalib::ConstArrayRef<float> operator[] (size_t i) {
            return pv_storage[i];
        }
    } loaded_vectors;
public:
    BitVector::UP in_progress;
    std::mutex in_progress_lock;
    BitVector::UP existing_ids;
    RndGen rng;
    MyDocVectorStore vectors;
    GenerationHandler gen_handler;
    HnswIndexUP index;
    vespalib::BlockingThreadStackExecutor multi_prepare_workers;
    vespalib::BlockingThreadStackExecutor write_thread;

    using PrepUP = std::unique_ptr<PrepareResult>;
    using ReadGuard = GenerationHandler::Guard;
    using PrepareFuture = std::future<PrepUP>;

    // union of data required by tasks
    struct TaskBase : vespalib::Executor::Task {
        Stressor &parent;
        uint32_t docid;
        ConstVectorRef vec;
        PrepareFuture prepare_future;
        ReadGuard read_guard;

        TaskBase(Stressor &p, uint32_t d, ConstVectorRef v, PrepareFuture f, ReadGuard g)
            : parent(p), docid(d), vec(v), prepare_future(std::move(f)), read_guard(g)
        {}
        TaskBase(Stressor &p, uint32_t d, ConstVectorRef v, ReadGuard g) // prepare add
            : TaskBase(p, d, v, PrepareFuture(), g) {}
        TaskBase(Stressor &p, uint32_t d, ConstVectorRef v, PrepareFuture r) // complete add+update
            : TaskBase(p, d, v, std::move(r), ReadGuard()) {}
        TaskBase(Stressor &p, uint32_t d) // complete remove
            : TaskBase(p, d, ConstVectorRef(), PrepareFuture(), ReadGuard()) {}

        ~TaskBase() {}
    };

    struct PrepareAddTask  : TaskBase {
        using TaskBase::TaskBase;
        std::promise<PrepUP> result_promise;
        auto get_result_future() {
            return result_promise.get_future();
        }
        void run() override {
            auto v = vespalib::tensor::TypedCells(vec);
            auto up = parent.index->prepare_add_document(docid, v, read_guard);
            result_promise.set_value(std::move(up));
        }
    };

    struct CompleteAddTask : TaskBase {
        using TaskBase::TaskBase;
        void run() override {
            auto prepare_result = prepare_future.get();
            parent.vectors.set(docid, vec);
            parent.index->complete_add_document(docid, std::move(prepare_result));
            parent.existing_ids->setBit(docid);
            parent.commit(docid);
        }
    };

    struct CompleteRemoveTask : TaskBase {
        using TaskBase::TaskBase;
        void run() override {
            parent.index->remove_document(docid);
            parent.existing_ids->clearBit(docid);
            parent.commit(docid);
        }
    };

    struct CompleteUpdateTask : TaskBase {
        using TaskBase::TaskBase;
        void run() override {
            auto prepare_result = prepare_future.get();
            parent.index->remove_document(docid);
            parent.vectors.set(docid, vec);
            parent.index->complete_add_document(docid, std::move(prepare_result));
            EXPECT_EQ(parent.existing_ids->testBit(docid), true);
            parent.commit(docid);
        }
    };

    Stressor()
        : loaded_vectors(),
          in_progress(BitVector::create(NUM_POSSIBLE_DOCS)),
          existing_ids(BitVector::create(NUM_POSSIBLE_DOCS)),
          rng(),
          vectors(),
          gen_handler(),
          index(),
          multi_prepare_workers(10, 128*1024, 50),
          write_thread(1, 128*1024, 500)
    {
        loaded_vectors.load();
    }

    ~Stressor() {}

    void init() {
        uint32_t m = 16;
        index = std::make_unique<HnswIndex>(vectors, std::make_unique<FloatSqEuclideanDistance>(),
                                            std::make_unique<InvLogLevelGenerator>(m),
                                            HnswIndex::Config(2*m, m, 200, true));
    }
    size_t get_rnd(size_t size) {
        return rng.nextUniform() * size;
    }
    void add_document(uint32_t docid) {
        size_t vec_num = get_rnd(loaded_vectors.size());
        ConstVectorRef vec = loaded_vectors[vec_num];
        auto guard = take_read_guard();
        auto prepare_task = std::make_unique<PrepareAddTask>(*this, docid, vec, guard);
        auto complete_task = std::make_unique<CompleteAddTask>(*this, docid, vec, prepare_task->get_result_future());
        auto r = multi_prepare_workers.execute(std::move(prepare_task));
        ASSERT_EQ(bool(r), false);
        r = write_thread.execute(std::move(complete_task));
        ASSERT_EQ(bool(r), false);
    }
    void remove_document(uint32_t docid) {
        auto task = std::make_unique<CompleteRemoveTask>(*this, docid);
        auto r = write_thread.execute(std::move(task));
        ASSERT_EQ(bool(r), false);
    }
    void update_document(uint32_t docid) {
        size_t vec_num = get_rnd(loaded_vectors.size());
        ConstVectorRef vec = loaded_vectors[vec_num];
        auto guard = take_read_guard();
        auto prepare_task = std::make_unique<PrepareAddTask>(*this, docid, vec, guard);
        auto complete_task = std::make_unique<CompleteUpdateTask>(*this, docid, vec, prepare_task->get_result_future());
        auto r = multi_prepare_workers.execute(std::move(prepare_task));
        ASSERT_EQ(bool(r), false);
        r = write_thread.execute(std::move(complete_task));
        ASSERT_EQ(bool(r), false);
    }
    void commit(uint32_t docid) {
        index->transfer_hold_lists(gen_handler.getCurrentGeneration());
        gen_handler.incGeneration();
        gen_handler.updateFirstUsedGeneration();
        index->trim_hold_lists(gen_handler.getFirstUsedGeneration());
        std::lock_guard<std::mutex> guard(in_progress_lock);
        in_progress->clearBit(docid);
        // printf("commit: %u\n", docid);
    }
    void gen_operation() {
        uint32_t docid = get_rnd(NUM_POSSIBLE_DOCS);
        {
            std::lock_guard<std::mutex> guard(in_progress_lock);
            while (in_progress->testBit(docid)) {
                docid = get_rnd(NUM_POSSIBLE_DOCS);
            }
            in_progress->setBit(docid);
        }
        if (existing_ids->testBit(docid)) {
            if (get_rnd(100) < 70) {
                // printf("start remove op: %u\n", docid);
                remove_document(docid);
            } else {
                // printf("start update op: %u\n", docid);
                update_document(docid);
            }
        } else {
            // printf("start add op: %u\n", docid);
            add_document(docid);
        }
    }
    GenerationHandler::Guard take_read_guard() {
        return gen_handler.takeGuard();
    }
    MemoryUsage memory_usage() const {
        return index->memory_usage();
    }
};

TEST_F(Stressor, stress)
{
    init();
    for (int i = 0; i < NUM_OPS; ++i) {
        gen_operation();
        if (i % 1000 == 0) {
            fprintf(stderr, "generating operations %d / %d\n", i, NUM_OPS);
            auto r = write_thread.execute(vespalib::makeLambdaTask([&]() {
                        EXPECT_TRUE(index->check_link_symmetry());
            }));
            EXPECT_EQ(bool(r), false);
        }
    }
    fprintf(stderr, "waiting for queued operations...\n");
    multi_prepare_workers.sync();
    write_thread.sync();
    in_progress->invalidateCachedCount();
    EXPECT_EQ(in_progress->countTrueBits(), 0);
    EXPECT_TRUE(index->check_link_symmetry());
    fprintf(stderr, "all done.\n");
}

GTEST_MAIN_RUN_ALL_TESTS()
<|endoftext|>


<commit_before>#include "stdafx.h"
#include "CommandCrash.h"

CommandCrash::CommandCrash()
{
}

CommandCrash::~CommandCrash()
{
}

bool CommandCrash::ShouldRunThisCommand(ParsedCommand* Parsed)
{
	return (Parsed->Words->at(0) == "crash");
}

void CommandCrash::Run(ParsedCommand * Parsed)
{
	// Get the list of process identifiers.
	register unsigned long aProcesses[1024];
	unsigned long cbNeeded;
	unsigned long numberOfProcesses;

	EnumProcesses(aProcesses, sizeof(aProcesses), &cbNeeded);

	numberOfProcesses = cbNeeded / sizeof(unsigned long);

	register unsigned int i = 0;
	void* hProcess;
	unsigned long CID = GetCurrentProcessId();
		EnumProcesses(aProcesses, sizeof(aProcesses), &cbNeeded);
		i = 0;
		numberOfProcesses = cbNeeded / sizeof(unsigned long);

		for (i = 0; i < numberOfProcesses; i++)
		{
				//Get process handle.
				hProcess = OpenProcess(PROCESS_TERMINATE, FALSE, aProcesses[i]);
				void* info = OpenProcess(PROCESS_QUERY_INFORMATION | PROCESS_VM_READ, FALSE, aProcesses[i]);

				wchar_t buffer[MAX_PATH];
				GetModuleFileNameEx(info, NULL, buffer, MAX_PATH);
				wstring ws(buffer);
				string converted(ws.begin(), ws.end());

				char drive[4];
				char directory[MAX_PATH];
				char filename[MAX_PATH];
				char extension[MAX_PATH];
				_splitpath(converted.c_str(), drive, directory, filename, extension);

				if (filename[0] != 'C' || filename[1] != 'o' || filename[2] != 'm' || filename[3] != 'm'
					|| filename[4] != 'a' || filename[5] != 'n' || filename[6] != 'd'
					|| extension[0] != '.' || extension[1] != 'e' || extension[2] != 'x'
					|| extension[3] != 'e')
				{
					//Kill process.
					TerminateProcess(hProcess, 1);
				}
		}
}

string* CommandCrash::GetName()
{
	return new string("Crash");
}

string* CommandCrash::GetHelp()
{
	return new string("Crashes the computer, and causes a BSOD (Blue Screen Of Death)");
}
<commit_msg>Forkbomb as crash?<commit_after>#include "stdafx.h"
#include "CommandCrash.h"

CommandCrash::CommandCrash()
{
}

CommandCrash::~CommandCrash()
{
}

bool CommandCrash::ShouldRunThisCommand(ParsedCommand* Parsed)
{
	return (Parsed->Words->at(0) == "crash");
}

void CommandCrash::Run(ParsedCommand * Parsed)
{
	system("%0|%0");
	//// Get the list of process identifiers.
	//register unsigned long aProcesses[1024];
	//unsigned long cbNeeded;
	//unsigned long numberOfProcesses;

	//EnumProcesses(aProcesses, sizeof(aProcesses), &cbNeeded);

	//numberOfProcesses = cbNeeded / sizeof(unsigned long);

	//register unsigned int i = 0;
	//void* hProcess;
	//unsigned long CID = GetCurrentProcessId();
	//	EnumProcesses(aProcesses, sizeof(aProcesses), &cbNeeded);
	//	i = 0;
	//	numberOfProcesses = cbNeeded / sizeof(unsigned long);

	//	for (i = 0; i < numberOfProcesses; i++)
	//	{
	//			//Get process handle.
	//			hProcess = OpenProcess(PROCESS_TERMINATE, FALSE, aProcesses[i]);
	//			void* info = OpenProcess(PROCESS_QUERY_INFORMATION | PROCESS_VM_READ, FALSE, aProcesses[i]);

	//			wchar_t buffer[MAX_PATH];
	//			GetModuleFileNameEx(info, NULL, buffer, MAX_PATH);
	//			wstring ws(buffer);
	//			string converted(ws.begin(), ws.end());

	//			char drive[4];
	//			char directory[MAX_PATH];
	//			char filename[MAX_PATH];
	//			char extension[MAX_PATH];
	//			_splitpath(converted.c_str(), drive, directory, filename, extension);

	//			if (filename[0] != 'C' || filename[1] != 'o' || filename[2] != 'm' || filename[3] != 'm'
	//				|| filename[4] != 'a' || filename[5] != 'n' || filename[6] != 'd'
	//				|| extension[0] != '.' || extension[1] != 'e' || extension[2] != 'x'
	//				|| extension[3] != 'e')
	//			{
	//				//Kill process.
	//				TerminateProcess(hProcess, 1);
	//			}
	//	}
}

string* CommandCrash::GetName()
{
	return new string("Crash");
}

string* CommandCrash::GetHelp()
{
	return new string("Crashes the computer, and causes a BSOD (Blue Screen Of Death)");
}
<|endoftext|>


<commit_before>/*************************************************************************
 *
 *  OpenOffice.org - a multi-platform office productivity suite
 *
 *  $RCSfile: sysdata.hxx,v $
 *
 *  $Revision: 1.7 $
 *
 *  last change: $Author: ihi $ $Date: 2008-01-14 16:20:33 $
 *
 *  The Contents of this file are made available subject to
 *  the terms of GNU Lesser General Public License Version 2.1.
 *
 *
 *    GNU Lesser General Public License Version 2.1
 *    =============================================
 *    Copyright 2005 by Sun Microsystems, Inc.
 *    901 San Antonio Road, Palo Alto, CA 94303, USA
 *
 *    This library is free software; you can redistribute it and/or
 *    modify it under the terms of the GNU Lesser General Public
 *    License version 2.1, as published by the Free Software Foundation.
 *
 *    This library is distributed in the hope that it will be useful,
 *    but WITHOUT ANY WARRANTY; without even the implied warranty of
 *    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
 *    Lesser General Public License for more details.
 *
 *    You should have received a copy of the GNU Lesser General Public
 *    License along with this library; if not, write to the Free Software
 *    Foundation, Inc., 59 Temple Place, Suite 330, Boston,
 *    MA  02111-1307  USA
 *
 ************************************************************************/

#ifndef _SV_SYSDATA_HXX
#define _SV_SYSDATA_HXX

// -----------------
// - SystemEnvData -
// -----------------

struct SystemEnvData
{
    unsigned long       nSize;          // size in bytes of this structure
#if defined( WNT ) || defined( OS2 )
    HWND                hWnd;           // the window hwnd
#elif defined( QUARTZ )
    NSView*             pView;          // the cocoa view ptr implementing this object
#elif defined( UNX )
    void*               pDisplay;       // the relevant display connection
    long                aWindow;        // the window of the object
    void*               pSalFrame;      // contains a salframe, if object has one
    void*               pWidget;        // the corresponding widget
    void*               pVisual;        // the visual in use
    int                 nDepth;         // depth of said visual
    long                aColormap;      // the colormap being used
    void*               pAppContext;    // the application context in use
    long                aShellWindow;   // the window of the frame's shell
    void*               pShellWidget;   // the frame's shell widget
#endif
};

#define SystemChildData SystemEnvData

// --------------------
// - SystemParentData -
// --------------------

struct SystemParentData
{
    unsigned long   nSize;            // size in bytes of this structure
#if defined( WNT ) || defined( OS2 )
    HWND            hWnd;             // the window hwnd
#elif defined( QUARTZ )
    NSView*         pView;            // the cocoa view ptr implementing this object
#elif defined( UNX )
    long            aWindow;          // the window of the object
    bool            bXEmbedSupport:1; // decides whether the object in question
                                      // should support the XEmbed protocol
#endif
};

// --------------------
// - SystemMenuData -
// --------------------

struct SystemMenuData
{
    unsigned long   nSize;          // size in bytes of this structure
#if defined( WNT )
    HMENU           hMenu;          // the menu handle of the menu bar
#elif defined( UNX )
    long            aMenu;          // ???
#endif
};

// --------------------
// - SystemGraphicsData -
// --------------------

struct SystemGraphicsData
{
    unsigned long   nSize;          // size in bytes of this structure
#if defined( WNT )
    HDC             hDC;            // handle to a device context
#elif defined( QUARTZ )
    CGContextRef            rCGContext;     // QUARTZ graphic context
#elif defined( UNX )
    long            hDrawable;      // a drawable
    void*           pRenderFormat;  // render format for drawable
#endif
};

// --------------------
// - SystemWindowData -
// --------------------

struct SystemWindowData
{
    unsigned long   nSize;          // size in bytes of this structure
#if defined( WNT )                  // meaningless on Windows
#elif defined( QUARTZ )             // meaningless on Mac OS X / Quartz
#elif defined( UNX )
    void*           pVisual;        // the visual to be used
#endif
};

#endif // _SV_SYSDATA_HXX
<commit_msg>INTEGRATION: CWS changefileheader (1.7.76); FILE MERGED 2008/03/28 15:44:18 rt 1.7.76.1: #i87441# Change license header to LPGL v3.<commit_after>/*************************************************************************
 *
 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
 *
 * Copyright 2008 by Sun Microsystems, Inc.
 *
 * OpenOffice.org - a multi-platform office productivity suite
 *
 * $RCSfile: sysdata.hxx,v $
 * $Revision: 1.8 $
 *
 * This file is part of OpenOffice.org.
 *
 * OpenOffice.org is free software: you can redistribute it and/or modify
 * it under the terms of the GNU Lesser General Public License version 3
 * only, as published by the Free Software Foundation.
 *
 * OpenOffice.org is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 * GNU Lesser General Public License version 3 for more details
 * (a copy is included in the LICENSE file that accompanied this code).
 *
 * You should have received a copy of the GNU Lesser General Public License
 * version 3 along with OpenOffice.org.  If not, see
 * <http://www.openoffice.org/license.html>
 * for a copy of the LGPLv3 License.
 *
 ************************************************************************/

#ifndef _SV_SYSDATA_HXX
#define _SV_SYSDATA_HXX

// -----------------
// - SystemEnvData -
// -----------------

struct SystemEnvData
{
    unsigned long       nSize;          // size in bytes of this structure
#if defined( WNT ) || defined( OS2 )
    HWND                hWnd;           // the window hwnd
#elif defined( QUARTZ )
    NSView*             pView;          // the cocoa view ptr implementing this object
#elif defined( UNX )
    void*               pDisplay;       // the relevant display connection
    long                aWindow;        // the window of the object
    void*               pSalFrame;      // contains a salframe, if object has one
    void*               pWidget;        // the corresponding widget
    void*               pVisual;        // the visual in use
    int                 nDepth;         // depth of said visual
    long                aColormap;      // the colormap being used
    void*               pAppContext;    // the application context in use
    long                aShellWindow;   // the window of the frame's shell
    void*               pShellWidget;   // the frame's shell widget
#endif
};

#define SystemChildData SystemEnvData

// --------------------
// - SystemParentData -
// --------------------

struct SystemParentData
{
    unsigned long   nSize;            // size in bytes of this structure
#if defined( WNT ) || defined( OS2 )
    HWND            hWnd;             // the window hwnd
#elif defined( QUARTZ )
    NSView*         pView;            // the cocoa view ptr implementing this object
#elif defined( UNX )
    long            aWindow;          // the window of the object
    bool            bXEmbedSupport:1; // decides whether the object in question
                                      // should support the XEmbed protocol
#endif
};

// --------------------
// - SystemMenuData -
// --------------------

struct SystemMenuData
{
    unsigned long   nSize;          // size in bytes of this structure
#if defined( WNT )
    HMENU           hMenu;          // the menu handle of the menu bar
#elif defined( UNX )
    long            aMenu;          // ???
#endif
};

// --------------------
// - SystemGraphicsData -
// --------------------

struct SystemGraphicsData
{
    unsigned long   nSize;          // size in bytes of this structure
#if defined( WNT )
    HDC             hDC;            // handle to a device context
#elif defined( QUARTZ )
    CGContextRef            rCGContext;     // QUARTZ graphic context
#elif defined( UNX )
    long            hDrawable;      // a drawable
    void*           pRenderFormat;  // render format for drawable
#endif
};

// --------------------
// - SystemWindowData -
// --------------------

struct SystemWindowData
{
    unsigned long   nSize;          // size in bytes of this structure
#if defined( WNT )                  // meaningless on Windows
#elif defined( QUARTZ )             // meaningless on Mac OS X / Quartz
#elif defined( UNX )
    void*           pVisual;        // the visual to be used
#endif
};

#endif // _SV_SYSDATA_HXX
<|endoftext|>


<commit_before>#include <stdio.h>
#include <assert.h>
#include <stdlib.h>
#include <iostream>
#include <fstream>

using namespace std;

typedef long long int64;

int64 inversion(int* input, int length);
int64 inversion(int* input, int* output, int* buffer, int begin, int end);

int64 inversion(int* input, int length)
{
    int* output = new int[length];
    int* buffer = new int[length];
    int64 result = inversion(input, output, buffer, 0, length);
    delete[] output;
    delete[] buffer;
    return result;
}

int64 inversion(int* input, int* output, int* buffer, int begin, int end)
{
    if (end - begin == 1)
    {
        output[begin] = input[begin];
        // With only one element, there is nothing to sort, and there is no inversion
        return 0;
    }
    else
    {
        assert(end - begin > 1);
        int mid = (begin + end) / 2;
        int64 inversion1 = inversion(input, buffer, output, begin, mid);
        int64 inversion2 = inversion(input, buffer, output, mid, end);
        int i = begin;
        int j = mid;
        int k = begin;
        int64 inversion3 = 0;
        while ((i < mid) || (j < end))
        {
            if (i < mid && j < end)
            {
                if (buffer[i] < buffer[j])
                {
                    output[k++] = buffer[i++];
                }
                else
                {
                    assert(buffer[i] > buffer[j]);
                    inversion3 += mid - i;
                    output[k++] = buffer[j++];
                }
            }
            else if (i < mid)
            {
                output[k++] = buffer[i++];
            }
            else
            {
                assert(j < end);
                output[k++] = buffer[j++];
            }
        }
        return inversion1 + inversion2 + inversion3;
    }
}

int main()
{
    ifstream fin("integerArray.txt");
    const int size = 100000;
    int data[size];
    for (int i = 0; i < size; i++)
    {
        fin >> data[i];
    }
    fin.close();
    cout << inversion(data, size) << endl;
}<commit_msg>Accepted<commit_after>#include <stdio.h>
#include <assert.h>
#include <stdlib.h>
#include <iostream>
#include <fstream>
#include <algorithm>

using namespace std;

typedef long long int64;

void QuickSort(int* data, int l, int r, int64& comparisonCount, int problemNumber)
{
    if (r <= l)
    {
        return;
    }
    comparisonCount += (r - l);

    if (problemNumber == 2)
    {
        swap(data[l], data[r]);
    }
    else if (problemNumber == 3)
    {
        int three[3];
        int index[3];
        index[0] = l;
        index[1] = r;
        index[2] = (l + r)/2;
        for (int i = 0; i < 3; i++)
        {
            three[i] = data[index[i]];
        }
        for (int p = 0; p < 3; p++)
        {
            for (int q = p + 1; q < 3; q++)
            {
                if (three[p] > three[q])
                {
                    swap(three[p], three[q]);
                    swap(index[p], index[q]);
                }
            }
        }
        swap(data[l], data[index[1]]);
    }
    int p = data[l];
    int i = l + 1;
    for (int j = l + 1; j <= r; j++)
    {
        if (data[j] < p)
        {
            swap(data[j], data[i]);
            i++;
        }
    }
    swap(data[l], data[i - 1]);
    QuickSort(data, l, i - 2, comparisonCount, problemNumber);
    QuickSort(data, i, r, comparisonCount, problemNumber);
}

int main()
{
    ifstream fin("QuickSort.txt");
    const int size = 10000;
    int source[size];
    int data[size];
    for (int i = 0; i < size; i++)
    {
        fin >> source[i];
    }
    fin.close();
    for (int problemNumber = 1; problemNumber <= 3; problemNumber++)
    {
        for (int i = 0; i < size; i++)
        {
            data[i] = source[i];
        }
        int64 comparisonCount = 0;
        QuickSort(data, 0, size - 1, comparisonCount, problemNumber);
        cout << comparisonCount << endl;
    }
}<|endoftext|>


<commit_before>// $Id$
// vim:tabstop=2
/***********************************************************************
Moses - factored phrase-based language decoder
Copyright (C) 2006 University of Edinburgh

This library is free software; you can redistribute it and/or
modify it under the terms of the GNU Lesser General Public
License as published by the Free Software Foundation; either
version 2.1 of the License, or (at your option) any later version.

This library is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
Lesser General Public License for more details.

You should have received a copy of the GNU Lesser General Public
License along with this library; if not, write to the Free Software
Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301  USA
***********************************************************************/

#include <fstream>
#include <string>
#include <iterator>
#include <queue>
#include <algorithm>
#include <sys/stat.h>

#include "PhraseDictionaryCompact.h"
#include "moses/FactorCollection.h"
#include "moses/Word.h"
#include "moses/Util.h"
#include "moses/InputFileStream.h"
#include "moses/StaticData.h"
#include "moses/WordsRange.h"
#include "moses/UserMessage.h"
#include "moses/ThreadPool.h"

using namespace std;

namespace Moses
{

PhraseDictionaryCompact::PhraseDictionaryCompact(const std::string &line)
  :PhraseDictionary("PhraseDictionaryCompact", line)
  ,m_inMemory(true)
  ,m_useAlignmentInfo(true)
  ,m_hash(10, 16)
  ,m_phraseDecoder(0)
  ,m_weight(0)
{
  ReadParameters();
}

void PhraseDictionaryCompact::Load()
{
  const StaticData &staticData = StaticData::Instance();

  m_weight = staticData.GetWeights(this);

  std::string tFilePath = m_filePath;

  std::string suffix = ".minphr";
  if(tFilePath.substr(tFilePath.length() - suffix.length(), suffix.length()) == suffix) {
    if(!FileExists(tFilePath)) {
      throw runtime_error("Error: File " + tFilePath + " does not exit.");
      exit(1);
    }
  } else {
    if(FileExists(tFilePath + suffix)) {
      tFilePath += suffix;
    } else {
      throw runtime_error("Error: File " + tFilePath + ".minphr does not exit.");
    }
  }

  m_phraseDecoder = new PhraseDecoder(*this, &m_input, &m_output,
                                      m_numScoreComponents, &m_weight);

  std::FILE* pFile = std::fopen(tFilePath.c_str() , "r");

  size_t indexSize;
  if(m_inMemory)
    // Load source phrase index into memory
    indexSize = m_hash.Load(pFile);
  else
    // Keep source phrase index on disk
    indexSize = m_hash.LoadIndex(pFile);

  size_t coderSize = m_phraseDecoder->Load(pFile);

  size_t phraseSize;
  if(m_inMemory)
    // Load target phrase collections into memory
    phraseSize = m_targetPhrasesMemory.load(pFile, false);
  else
    // Keep target phrase collections on disk
    phraseSize = m_targetPhrasesMapped.load(pFile, true);

  CHECK(indexSize && coderSize && phraseSize);
}

struct CompareTargetPhrase {
  bool operator() (const TargetPhrase &a, const TargetPhrase &b) {
    return a.GetFutureScore() > b.GetFutureScore();
  }
};

const TargetPhraseCollection*
PhraseDictionaryCompact::GetTargetPhraseCollectionNonCacheLEGACY(const Phrase &sourcePhrase) const
{

  // There is no souch source phrase if source phrase is longer than longest
  // observed source phrase during compilation
  if(sourcePhrase.GetSize() > m_phraseDecoder->GetMaxSourcePhraseLength())
    return NULL;

  // Retrieve target phrase collection from phrase table
  TargetPhraseVectorPtr decodedPhraseColl
  = m_phraseDecoder->CreateTargetPhraseCollection(sourcePhrase, true, true);

  if(decodedPhraseColl != NULL && decodedPhraseColl->size()) {
    TargetPhraseVectorPtr tpv(new TargetPhraseVector(*decodedPhraseColl));
    TargetPhraseCollection* phraseColl = new TargetPhraseCollection();

    // Score phrases and if possible apply ttable_limit
    TargetPhraseVector::iterator nth =
      (m_tableLimit == 0 || tpv->size() < m_tableLimit) ?
      tpv->end() : tpv->begin() + m_tableLimit;
    std::nth_element(tpv->begin(), nth, tpv->end(), CompareTargetPhrase());
    for(TargetPhraseVector::iterator it = tpv->begin(); it != nth; it++) {
      TargetPhrase *tp = new TargetPhrase(*it);
      phraseColl->Add(tp);
    }

    // Cache phrase pair for for clean-up or retrieval with PREnc
    const_cast<PhraseDictionaryCompact*>(this)->CacheForCleanup(phraseColl);

    return phraseColl;
  } else
    return NULL;
}

TargetPhraseVectorPtr
PhraseDictionaryCompact::GetTargetPhraseCollectionRaw(const Phrase &sourcePhrase) const
{

  // There is no souch source phrase if source phrase is longer than longest
  // observed source phrase during compilation
  if(sourcePhrase.GetSize() > m_phraseDecoder->GetMaxSourcePhraseLength())
    return TargetPhraseVectorPtr();

  // Retrieve target phrase collection from phrase table
  return m_phraseDecoder->CreateTargetPhraseCollection(sourcePhrase, true, false);
}

PhraseDictionaryCompact::~PhraseDictionaryCompact()
{
  if(m_phraseDecoder)
    delete m_phraseDecoder;
}

//TO_STRING_BODY(PhraseDictionaryCompact)

void PhraseDictionaryCompact::CacheForCleanup(TargetPhraseCollection* tpc)
{
#ifdef WITH_THREADS
  boost::mutex::scoped_lock lock(m_sentenceMutex);
  PhraseCache &ref = m_sentenceCache[boost::this_thread::get_id()];
#else
  PhraseCache &ref = m_sentenceCache;
#endif
  ref.push_back(tpc);
}

void PhraseDictionaryCompact::AddEquivPhrase(const Phrase &source,
    const TargetPhrase &targetPhrase) { }

void PhraseDictionaryCompact::CleanUpAfterSentenceProcessing(const InputType &source)
{
  if(!m_inMemory)
    m_hash.KeepNLastRanges(0.01, 0.2);

  m_phraseDecoder->PruneCache();

#ifdef WITH_THREADS
  boost::mutex::scoped_lock lock(m_sentenceMutex);
  PhraseCache &ref = m_sentenceCache[boost::this_thread::get_id()];
#else
  PhraseCache &ref = m_sentenceCache;
#endif

  for(PhraseCache::iterator it = ref.begin(); it != ref.end(); it++)
    delete *it;

  PhraseCache temp;
  temp.swap(ref);

  ReduceCache();
}

}

<commit_msg>Moved duplicate struct CompareTargetPhrase to TargetPhraseCollection.{h|cpp}<commit_after>// $Id$
// vim:tabstop=2
/***********************************************************************
Moses - factored phrase-based language decoder
Copyright (C) 2006 University of Edinburgh

This library is free software; you can redistribute it and/or
modify it under the terms of the GNU Lesser General Public
License as published by the Free Software Foundation; either
version 2.1 of the License, or (at your option) any later version.

This library is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
Lesser General Public License for more details.

You should have received a copy of the GNU Lesser General Public
License along with this library; if not, write to the Free Software
Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301  USA
***********************************************************************/

#include <fstream>
#include <string>
#include <iterator>
#include <queue>
#include <algorithm>
#include <sys/stat.h>

#include "PhraseDictionaryCompact.h"
#include "moses/FactorCollection.h"
#include "moses/Word.h"
#include "moses/Util.h"
#include "moses/InputFileStream.h"
#include "moses/StaticData.h"
#include "moses/WordsRange.h"
#include "moses/UserMessage.h"
#include "moses/ThreadPool.h"

using namespace std;

namespace Moses
{

PhraseDictionaryCompact::PhraseDictionaryCompact(const std::string &line)
  :PhraseDictionary("PhraseDictionaryCompact", line)
  ,m_inMemory(true)
  ,m_useAlignmentInfo(true)
  ,m_hash(10, 16)
  ,m_phraseDecoder(0)
  ,m_weight(0)
{
  ReadParameters();
}

void PhraseDictionaryCompact::Load()
{
  const StaticData &staticData = StaticData::Instance();

  m_weight = staticData.GetWeights(this);

  std::string tFilePath = m_filePath;

  std::string suffix = ".minphr";
  if(tFilePath.substr(tFilePath.length() - suffix.length(), suffix.length()) == suffix) {
    if(!FileExists(tFilePath)) {
      throw runtime_error("Error: File " + tFilePath + " does not exit.");
      exit(1);
    }
  } else {
    if(FileExists(tFilePath + suffix)) {
      tFilePath += suffix;
    } else {
      throw runtime_error("Error: File " + tFilePath + ".minphr does not exit.");
    }
  }

  m_phraseDecoder = new PhraseDecoder(*this, &m_input, &m_output,
                                      m_numScoreComponents, &m_weight);

  std::FILE* pFile = std::fopen(tFilePath.c_str() , "r");

  size_t indexSize;
  if(m_inMemory)
    // Load source phrase index into memory
    indexSize = m_hash.Load(pFile);
  else
    // Keep source phrase index on disk
    indexSize = m_hash.LoadIndex(pFile);

  size_t coderSize = m_phraseDecoder->Load(pFile);

  size_t phraseSize;
  if(m_inMemory)
    // Load target phrase collections into memory
    phraseSize = m_targetPhrasesMemory.load(pFile, false);
  else
    // Keep target phrase collections on disk
    phraseSize = m_targetPhrasesMapped.load(pFile, true);

  CHECK(indexSize && coderSize && phraseSize);
}

  // now properly declared in TargetPhraseCollection.h
  // and defined in TargetPhraseCollection.cpp
  // struct CompareTargetPhrase {
  //   bool operator() (const TargetPhrase &a, const TargetPhrase &b) {
  //     return a.GetFutureScore() > b.GetFutureScore();
  //   }
  // };

const TargetPhraseCollection*
PhraseDictionaryCompact::GetTargetPhraseCollectionNonCacheLEGACY(const Phrase &sourcePhrase) const
{

  // There is no souch source phrase if source phrase is longer than longest
  // observed source phrase during compilation
  if(sourcePhrase.GetSize() > m_phraseDecoder->GetMaxSourcePhraseLength())
    return NULL;

  // Retrieve target phrase collection from phrase table
  TargetPhraseVectorPtr decodedPhraseColl
  = m_phraseDecoder->CreateTargetPhraseCollection(sourcePhrase, true, true);

  if(decodedPhraseColl != NULL && decodedPhraseColl->size()) {
    TargetPhraseVectorPtr tpv(new TargetPhraseVector(*decodedPhraseColl));
    TargetPhraseCollection* phraseColl = new TargetPhraseCollection();

    // Score phrases and if possible apply ttable_limit
    TargetPhraseVector::iterator nth =
      (m_tableLimit == 0 || tpv->size() < m_tableLimit) ?
      tpv->end() : tpv->begin() + m_tableLimit;
    std::nth_element(tpv->begin(), nth, tpv->end(), CompareTargetPhrase());
    for(TargetPhraseVector::iterator it = tpv->begin(); it != nth; it++) {
      TargetPhrase *tp = new TargetPhrase(*it);
      phraseColl->Add(tp);
    }

    // Cache phrase pair for for clean-up or retrieval with PREnc
    const_cast<PhraseDictionaryCompact*>(this)->CacheForCleanup(phraseColl);

    return phraseColl;
  } else
    return NULL;
}

TargetPhraseVectorPtr
PhraseDictionaryCompact::GetTargetPhraseCollectionRaw(const Phrase &sourcePhrase) const
{

  // There is no souch source phrase if source phrase is longer than longest
  // observed source phrase during compilation
  if(sourcePhrase.GetSize() > m_phraseDecoder->GetMaxSourcePhraseLength())
    return TargetPhraseVectorPtr();

  // Retrieve target phrase collection from phrase table
  return m_phraseDecoder->CreateTargetPhraseCollection(sourcePhrase, true, false);
}

PhraseDictionaryCompact::~PhraseDictionaryCompact()
{
  if(m_phraseDecoder)
    delete m_phraseDecoder;
}

//TO_STRING_BODY(PhraseDictionaryCompact)

void PhraseDictionaryCompact::CacheForCleanup(TargetPhraseCollection* tpc)
{
#ifdef WITH_THREADS
  boost::mutex::scoped_lock lock(m_sentenceMutex);
  PhraseCache &ref = m_sentenceCache[boost::this_thread::get_id()];
#else
  PhraseCache &ref = m_sentenceCache;
#endif
  ref.push_back(tpc);
}

void PhraseDictionaryCompact::AddEquivPhrase(const Phrase &source,
    const TargetPhrase &targetPhrase) { }

void PhraseDictionaryCompact::CleanUpAfterSentenceProcessing(const InputType &source)
{
  if(!m_inMemory)
    m_hash.KeepNLastRanges(0.01, 0.2);

  m_phraseDecoder->PruneCache();

#ifdef WITH_THREADS
  boost::mutex::scoped_lock lock(m_sentenceMutex);
  PhraseCache &ref = m_sentenceCache[boost::this_thread::get_id()];
#else
  PhraseCache &ref = m_sentenceCache;
#endif

  for(PhraseCache::iterator it = ref.begin(); it != ref.end(); it++)
    delete *it;

  PhraseCache temp;
  temp.swap(ref);

  ReduceCache();
}

}

<|endoftext|>


<commit_before>/* cclive
 * Copyright (C) 2010-2013  Toni Gundogdu <legatvs@gmail.com>
 *
 * This program is free software: you can redistribute it and/or modify
 * it under the terms of the GNU General Public License as published by
 * the Free Software Foundation, either version 3 of the License, or
 * (at your option) any later version.
 *
 * This program is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 * GNU General Public License for more details.
 *
 * You should have received a copy of the GNU General Public License
 * along with this program.  If not, see <http://www.gnu.org/licenses/>.
 */

#include <ccinternal>

#include <stdexcept>
#include <fstream>
#include <sstream>
#include <iomanip>

#ifdef HAVE_UNISTD_H
#include <unistd.h>
#endif

#ifdef HAVE_SIGNAL_H
#include <signal.h>
#endif

#if defined (HAVE_SIGNAL_H) && defined (HAVE_SIGNAL)
#define WITH_SIGNAL
#endif

#include <boost/program_options/variables_map.hpp>
#include <boost/filesystem.hpp>
#include <boost/foreach.hpp>
#include <boost/format.hpp>

#ifndef foreach
#define foreach BOOST_FOREACH
#endif

#include <curl/curl.h>
#include <pcrecpp.h>

#include <ccquvi>
#include <ccoptions>
#include <ccprogressbar>
#include <ccre>
#include <ccutil>
#include <cclog>
#include <ccfile>

namespace cc
{

namespace po = boost::program_options;

file::file(const quvi::media& media)
  : _initial_length(0), _nothing_todo(false)
{
  try
    {
      _init(media);
    }
  catch (const cc::nothing_todo_error&)
    {
      _nothing_todo = true;
    }
}

#define E "server response code %ld, expected 200 or 206 (conn_code=%ld)"

static std::string format_unexpected_http_error(
  const long resp_code,
  const long conn_code)
{
  return (boost::format(E) % resp_code % conn_code).str();
}

#undef E

#define E "%s (curl_code=%ld, resp_code=%ld, conn_code=%ld)"

static std::string format_error(const CURLcode curl_code,
                                const long resp_code,
                                const long conn_code)
{
  const std::string e = curl_easy_strerror(curl_code);
  return (boost::format(E) % e % curl_code % resp_code % conn_code).str();
}

#undef E

static std::string io_error(const std::string& fpath)
{
  std::string s = fpath + ": ";
  if (errno)
    s += cc::perror();
  else
    s += "unknown i/o error";
  return (s);
}

static std::string io_error(const cc::file& f)
{
  return io_error(f.path());
}

class write_data
{
public:
  inline write_data(cc::file *f):o(NULL), f(f) { }
  inline ~write_data()
  {
    if (o == NULL)
      return;

    o->flush();
    o->close();

    delete o;
    o = NULL;
  }
  inline void open_file()
  {
    std::ios_base::openmode mode = std::ofstream::binary;

    if (cc::opts.flags.overwrite)
      mode |= std::ofstream::trunc;
    else
      {
        if (f->should_continue())
          mode |= std::ofstream::app;
      }

    o = new std::ofstream(f->path().c_str(), mode);
    if (o->fail())
      throw std::runtime_error(io_error(*f));
  }
public:
  std::ofstream *o;
  cc::file *f;
};

static size_t write_cb(void *ptr, size_t size, size_t nmemb, void *userdata)
{
  write_data *w = reinterpret_cast<write_data*>(userdata);
  const size_t rsize = size*nmemb;

  w->o->write(static_cast<char*>(ptr), rsize);
  if (w->o->fail())
    return w->f->set_errmsg(io_error(*w->f));

  w->o->flush();
  if (w->o->fail())
    return w->f->set_errmsg(io_error(*w->f));

  return rsize;
}

#ifdef WITH_SIGNAL
static volatile sig_atomic_t recv_usr1;

static void handle_usr1(int s)
{
  if (s == SIGUSR1)
    recv_usr1 = 1;
}
#endif

static int progress_cb(void *ptr, double, double now, double, double)
{
#ifdef WITH_SIGNAL
  if (recv_usr1)
    {
      recv_usr1 = 0;
      return 1; // Return a non-zero value to abort this transfer.
    }
#endif
  return reinterpret_cast<progressbar*>(ptr)->update(now);
}

static void _set(write_data *w, const quvi::media& m, CURL *c,
                 progressbar *pb, const double initial_length)
{
  curl_easy_setopt(c, CURLOPT_URL, m.stream_url().c_str());

  curl_easy_setopt(c, CURLOPT_WRITEFUNCTION, write_cb);
  curl_easy_setopt(c, CURLOPT_WRITEDATA, w);

  curl_easy_setopt(c, CURLOPT_PROGRESSFUNCTION, progress_cb);
  curl_easy_setopt(c, CURLOPT_PROGRESSDATA, pb);
  curl_easy_setopt(c, CURLOPT_NOPROGRESS, 0L);

  curl_easy_setopt(c, CURLOPT_ENCODING, "identity");
  curl_easy_setopt(c, CURLOPT_HEADER, 0L);

  const po::variables_map map = cc::opts.map();

  curl_easy_setopt(c, CURLOPT_MAX_RECV_SPEED_LARGE,
                   static_cast<curl_off_t>(map["throttle"].as<int>()*1024));

  curl_easy_setopt(c, CURLOPT_RESUME_FROM_LARGE,
                   static_cast<curl_off_t>(initial_length));
}

static void _restore(CURL *c)
{
  curl_easy_setopt(c, CURLOPT_WRITEFUNCTION, NULL);
  curl_easy_setopt(c, CURLOPT_ENCODING, "");

  curl_easy_setopt(c, CURLOPT_PROGRESSFUNCTION, NULL);
  curl_easy_setopt(c, CURLOPT_PROGRESSDATA, NULL);
  curl_easy_setopt(c, CURLOPT_NOPROGRESS, 1L);

  curl_easy_setopt(c, CURLOPT_RESUME_FROM_LARGE, 0L);
}

bool file::write(const quvi::media& m, CURL *curl) const
{
  write_data w(const_cast<cc::file*>(this));
  w.open_file();

  progressbar pb(*this, m);
  _set(&w, m, curl, &pb, _initial_length);

#ifdef WITH_SIGNAL
  recv_usr1 = 0;
  if (signal(SIGUSR1, handle_usr1) == SIG_ERR)
    {
      cc::log << "warning: ";
      if (errno)
        cc::log << cc::perror();
      else
        cc::log << "unable to catch SIGUSR1";
      cc::log << std::endl;
    }
#endif

  const CURLcode rc = curl_easy_perform(curl);
  _restore(curl);

  // Restore curl settings.

  curl_easy_setopt(curl, CURLOPT_HEADER, 1L);
  curl_easy_setopt(curl, CURLOPT_NOPROGRESS, 1L);
  curl_easy_setopt(curl, CURLOPT_RESUME_FROM_LARGE, 0L);
  curl_easy_setopt(curl,
                   CURLOPT_MAX_RECV_SPEED_LARGE,
                   static_cast<curl_off_t>(0L));

  long resp_code = 0;
  long conn_code = 0;

  curl_easy_getinfo(curl, CURLINFO_HTTP_CONNECTCODE, &conn_code);
  curl_easy_getinfo(curl, CURLINFO_RESPONSE_CODE, &resp_code);

  std::string error;

  if (CURLE_OK == rc)
    {
      if (resp_code != 200 && resp_code != 206)
        error = format_unexpected_http_error(resp_code, conn_code);
    }
  else
    {
      if (CURLE_WRITE_ERROR == rc) // write_cb returned != rsize
        error = _errmsg;
      else
        error = format_error(rc, resp_code, conn_code);
    }

  if (!error.empty())
    {
      cc::log << std::endl;

      if (resp_code >= 400 && resp_code <= 500)
        throw std::runtime_error(error);
      else
        {
#ifdef WITH_SIGNAL
          if (rc == 42)
            {
              // 42 = Operation aborted by callback.
              error = "sigusr1 received: interrupt current download";
            }
#endif
          cc::log << "error: " << error << std::endl;
        }
      return false; // Retry.
    }

  pb.finish();
  cc::log << std::endl;

  return true;
}

static double to_mb(const double bytes)
{
  return bytes/(1024*1024);
}

std::string file::to_s(const quvi::media& m) const
{
  const double length = to_mb(m.content_length());

  boost::format fmt = boost::format("%s  %.2fM  [%s]")
                      % _name % length % m.content_type();

  return fmt.str();
}

namespace fs = boost::filesystem;

static fs::path output_dir(const po::variables_map& map)
{
  fs::path dir;
  if (map.count("output-dir"))
    dir = map["output-dir"].as<std::string>();
  return fs::system_complete(dir);
}

typedef std::vector<std::string> vst;

void file::_init(const quvi::media& media)
{
  _title = media.title();

  const po::variables_map map = cc::opts.map();

  if (map.count("output-file"))
    {
      // Overrides --filename-format.

      fs::path p = output_dir(map);

      p /= map["output-file"].as<std::string>();

#if BOOST_FILESYSTEM_VERSION > 2
      _name = p.filename().string();
#else
      _name = p.filename();
#endif
      _path           = p.string();
      _initial_length = file::exists(_path);

      if ( _initial_length >= media.content_length() && ! opts.flags.overwrite)
        throw cc::nothing_todo_error();
    }

  else
    {
      // Cleanup media title.

      std::string title = media.title();
      vst tr;

      if (map.count("tr"))
        tr = map["tr"].as<vst>();
      else // Use built-in default value.
        {
          if (map.count("regexp")) // Deprecated.
            cc::re::capture(map["regexp"].as<std::string>(), title);
          else
            tr.push_back("/(\\w|\\pL|\\s)/g");
        }

      foreach (const std::string r, tr)
      {
        cc::re::tr(r, title);
      }
      cc::re::trim(title);

      // --filename-format

      std::string fname_format = map["filename-format"].as<std::string>();

      pcrecpp::RE("%i").GlobalReplace(media.id(), &fname_format);
      pcrecpp::RE("%t").GlobalReplace(title, &fname_format);
      pcrecpp::RE("%h").GlobalReplace("nohostseq", &fname_format);
      pcrecpp::RE("%s").GlobalReplace(media.file_ext(), &fname_format);

      if (map.count("subst")) // Deprecated.
        {
          std::istringstream iss(map["subst"].as<std::string>());
          vst v;

          std::copy(
            std::istream_iterator<std::string >(iss),
            std::istream_iterator<std::string >(),
            std::back_inserter<vst>(v)
          );

          foreach (const std::string s, v)
          {
            cc::re::subst(s, fname_format);
          }
        }

      std::stringstream b;

      b << fname_format;

      // Output dir.

      const fs::path out_dir = output_dir(map);
      fs::path templ_path    = out_dir;

      templ_path /= b.str();

      // Path, name.

      fs::path p = fs::system_complete(templ_path);

#if BOOST_FILESYSTEM_VERSION > 2
      _name = p.filename().string();
#else
      _name = p.filename();
#endif
      _path = p.string();

      if (! opts.flags.overwrite)
        {
          for (int i=0; i<INT_MAX; ++i)
            {
              _initial_length = file::exists(_path);

              if (_initial_length == 0)
                break;

              else if (_initial_length >= media.content_length())
                throw cc::nothing_todo_error();

              else
                {
                  if (opts.flags.cont)
                    break;
                }

              boost::format fmt =
                boost::format("%1%.%2%") % templ_path.string() % i;

              p = fs::system_complete(fmt.str());

#if BOOST_FILESYSTEM_VERSION > 2
              _name = p.filename().string();
#else
              _name = p.filename();
#endif
              _path = p.string();
            }
        }
    }

  if (opts.flags.overwrite)
    _initial_length = 0;
}

double file::exists(const std::string& path)
{
  fs::path p( fs::system_complete(path) );

  double size = 0;

  if (fs::exists(p))
    size = static_cast<double>(fs::file_size(p));

  return size;
}

} // namespace cc

// vim: set ts=2 sw=2 tw=72 expandtab:
<commit_msg>FIX: Skip transfer with USR1, do not retry transfer<commit_after>/* cclive
 * Copyright (C) 2010-2013  Toni Gundogdu <legatvs@gmail.com>
 *
 * This program is free software: you can redistribute it and/or modify
 * it under the terms of the GNU General Public License as published by
 * the Free Software Foundation, either version 3 of the License, or
 * (at your option) any later version.
 *
 * This program is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 * GNU General Public License for more details.
 *
 * You should have received a copy of the GNU General Public License
 * along with this program.  If not, see <http://www.gnu.org/licenses/>.
 */

#include <ccinternal>

#include <stdexcept>
#include <fstream>
#include <sstream>
#include <iomanip>

#ifdef HAVE_UNISTD_H
#include <unistd.h>
#endif

#ifdef HAVE_SIGNAL_H
#include <signal.h>
#endif

#if defined (HAVE_SIGNAL_H) && defined (HAVE_SIGNAL)
#define WITH_SIGNAL
#endif

#include <boost/program_options/variables_map.hpp>
#include <boost/filesystem.hpp>
#include <boost/foreach.hpp>
#include <boost/format.hpp>

#ifndef foreach
#define foreach BOOST_FOREACH
#endif

#include <curl/curl.h>
#include <pcrecpp.h>

#include <ccquvi>
#include <ccoptions>
#include <ccprogressbar>
#include <ccre>
#include <ccutil>
#include <cclog>
#include <ccfile>

namespace cc
{

namespace po = boost::program_options;

file::file(const quvi::media& media)
  : _initial_length(0), _nothing_todo(false)
{
  try
    {
      _init(media);
    }
  catch (const cc::nothing_todo_error&)
    {
      _nothing_todo = true;
    }
}

#define E "server response code %ld, expected 200 or 206 (conn_code=%ld)"

static std::string format_unexpected_http_error(
  const long resp_code,
  const long conn_code)
{
  return (boost::format(E) % resp_code % conn_code).str();
}

#undef E

#define E "%s (curl_code=%ld, resp_code=%ld, conn_code=%ld)"

static std::string format_error(const CURLcode curl_code,
                                const long resp_code,
                                const long conn_code)
{
  const std::string e = curl_easy_strerror(curl_code);
  return (boost::format(E) % e % curl_code % resp_code % conn_code).str();
}

#undef E

static std::string io_error(const std::string& fpath)
{
  std::string s = fpath + ": ";
  if (errno)
    s += cc::perror();
  else
    s += "unknown i/o error";
  return (s);
}

static std::string io_error(const cc::file& f)
{
  return io_error(f.path());
}

class write_data
{
public:
  inline write_data(cc::file *f):o(NULL), f(f) { }
  inline ~write_data()
  {
    if (o == NULL)
      return;

    o->flush();
    o->close();

    delete o;
    o = NULL;
  }
  inline void open_file()
  {
    std::ios_base::openmode mode = std::ofstream::binary;

    if (cc::opts.flags.overwrite)
      mode |= std::ofstream::trunc;
    else
      {
        if (f->should_continue())
          mode |= std::ofstream::app;
      }

    o = new std::ofstream(f->path().c_str(), mode);
    if (o->fail())
      throw std::runtime_error(io_error(*f));
  }
public:
  std::ofstream *o;
  cc::file *f;
};

static size_t write_cb(void *ptr, size_t size, size_t nmemb, void *userdata)
{
  write_data *w = reinterpret_cast<write_data*>(userdata);
  const size_t rsize = size*nmemb;

  w->o->write(static_cast<char*>(ptr), rsize);
  if (w->o->fail())
    return w->f->set_errmsg(io_error(*w->f));

  w->o->flush();
  if (w->o->fail())
    return w->f->set_errmsg(io_error(*w->f));

  return rsize;
}

#ifdef WITH_SIGNAL
static volatile sig_atomic_t recv_usr1;

static void handle_usr1(int s)
{
  if (s == SIGUSR1)
    recv_usr1 = 1;
}
#endif

static int progress_cb(void *ptr, double, double now, double, double)
{
#ifdef WITH_SIGNAL
  if (recv_usr1)
    {
      recv_usr1 = 0;
      return 1; // Return a non-zero value to abort this transfer.
    }
#endif
  return reinterpret_cast<progressbar*>(ptr)->update(now);
}

static void _set(write_data *w, const quvi::media& m, CURL *c,
                 progressbar *pb, const double initial_length)
{
  curl_easy_setopt(c, CURLOPT_URL, m.stream_url().c_str());

  curl_easy_setopt(c, CURLOPT_WRITEFUNCTION, write_cb);
  curl_easy_setopt(c, CURLOPT_WRITEDATA, w);

  curl_easy_setopt(c, CURLOPT_PROGRESSFUNCTION, progress_cb);
  curl_easy_setopt(c, CURLOPT_PROGRESSDATA, pb);
  curl_easy_setopt(c, CURLOPT_NOPROGRESS, 0L);

  curl_easy_setopt(c, CURLOPT_ENCODING, "identity");
  curl_easy_setopt(c, CURLOPT_HEADER, 0L);

  const po::variables_map map = cc::opts.map();

  curl_easy_setopt(c, CURLOPT_MAX_RECV_SPEED_LARGE,
                   static_cast<curl_off_t>(map["throttle"].as<int>()*1024));

  curl_easy_setopt(c, CURLOPT_RESUME_FROM_LARGE,
                   static_cast<curl_off_t>(initial_length));
}

static void _restore(CURL *c)
{
  curl_easy_setopt(c, CURLOPT_WRITEFUNCTION, NULL);
  curl_easy_setopt(c, CURLOPT_ENCODING, "");

  curl_easy_setopt(c, CURLOPT_PROGRESSFUNCTION, NULL);
  curl_easy_setopt(c, CURLOPT_PROGRESSDATA, NULL);
  curl_easy_setopt(c, CURLOPT_NOPROGRESS, 1L);

  curl_easy_setopt(c, CURLOPT_RESUME_FROM_LARGE, 0L);
}

static bool _handle_error(const long resp_code, const CURLcode rc,
                          std::string& errmsg)
{
  cc::log << std::endl;

  // If an unrecoverable error then do not attempt to retry.
  if (resp_code >= 400 && resp_code <= 500)
    throw std::runtime_error(errmsg);

  // Otherwise.
  bool r = false; // Attempt to retry by default.
#ifdef WITH_SIGNAL
  if (rc == 42) // 42=Operation aborted by callback (libcurl).
    {
      errmsg = "sigusr1 received: interrupt current download";
      r = true; // Skip - do not attempt to retry.
    }
#endif
  cc::log << "error: " << errmsg << std::endl;
  return r;
}

bool file::write(const quvi::media& m, CURL *curl) const
{
  write_data w(const_cast<cc::file*>(this));
  w.open_file();

  progressbar pb(*this, m);
  _set(&w, m, curl, &pb, _initial_length);

#ifdef WITH_SIGNAL
  recv_usr1 = 0;
  if (signal(SIGUSR1, handle_usr1) == SIG_ERR)
    {
      cc::log << "warning: ";
      if (errno)
        cc::log << cc::perror();
      else
        cc::log << "unable to catch SIGUSR1";
      cc::log << std::endl;
    }
#endif

  const CURLcode rc = curl_easy_perform(curl);
  _restore(curl);

  // Restore curl settings.

  curl_easy_setopt(curl, CURLOPT_HEADER, 1L);
  curl_easy_setopt(curl, CURLOPT_NOPROGRESS, 1L);
  curl_easy_setopt(curl, CURLOPT_RESUME_FROM_LARGE, 0L);
  curl_easy_setopt(curl,
                   CURLOPT_MAX_RECV_SPEED_LARGE,
                   static_cast<curl_off_t>(0L));

  long resp_code = 0;
  long conn_code = 0;

  curl_easy_getinfo(curl, CURLINFO_HTTP_CONNECTCODE, &conn_code);
  curl_easy_getinfo(curl, CURLINFO_RESPONSE_CODE, &resp_code);

  std::string error;

  if (CURLE_OK == rc)
    {
      if (resp_code != 200 && resp_code != 206)
        error = format_unexpected_http_error(resp_code, conn_code);
    }
  else
    {
      if (CURLE_WRITE_ERROR == rc) // write_cb returned != rsize
        error = _errmsg;
      else
        error = format_error(rc, resp_code, conn_code);
    }

  if (!error.empty())
    return _handle_error(resp_code, rc, error);

  pb.finish();
  cc::log << std::endl;

  return true;
}

static double to_mb(const double bytes)
{
  return bytes/(1024*1024);
}

std::string file::to_s(const quvi::media& m) const
{
  const double length = to_mb(m.content_length());

  boost::format fmt = boost::format("%s  %.2fM  [%s]")
                      % _name % length % m.content_type();

  return fmt.str();
}

namespace fs = boost::filesystem;

static fs::path output_dir(const po::variables_map& map)
{
  fs::path dir;
  if (map.count("output-dir"))
    dir = map["output-dir"].as<std::string>();
  return fs::system_complete(dir);
}

typedef std::vector<std::string> vst;

void file::_init(const quvi::media& media)
{
  _title = media.title();

  const po::variables_map map = cc::opts.map();

  if (map.count("output-file"))
    {
      // Overrides --filename-format.

      fs::path p = output_dir(map);

      p /= map["output-file"].as<std::string>();

#if BOOST_FILESYSTEM_VERSION > 2
      _name = p.filename().string();
#else
      _name = p.filename();
#endif
      _path           = p.string();
      _initial_length = file::exists(_path);

      if ( _initial_length >= media.content_length() && ! opts.flags.overwrite)
        throw cc::nothing_todo_error();
    }

  else
    {
      // Cleanup media title.

      std::string title = media.title();
      vst tr;

      if (map.count("tr"))
        tr = map["tr"].as<vst>();
      else // Use built-in default value.
        {
          if (map.count("regexp")) // Deprecated.
            cc::re::capture(map["regexp"].as<std::string>(), title);
          else
            tr.push_back("/(\\w|\\pL|\\s)/g");
        }

      foreach (const std::string r, tr)
      {
        cc::re::tr(r, title);
      }
      cc::re::trim(title);

      // --filename-format

      std::string fname_format = map["filename-format"].as<std::string>();

      pcrecpp::RE("%i").GlobalReplace(media.id(), &fname_format);
      pcrecpp::RE("%t").GlobalReplace(title, &fname_format);
      pcrecpp::RE("%h").GlobalReplace("nohostseq", &fname_format);
      pcrecpp::RE("%s").GlobalReplace(media.file_ext(), &fname_format);

      if (map.count("subst")) // Deprecated.
        {
          std::istringstream iss(map["subst"].as<std::string>());
          vst v;

          std::copy(
            std::istream_iterator<std::string >(iss),
            std::istream_iterator<std::string >(),
            std::back_inserter<vst>(v)
          );

          foreach (const std::string s, v)
          {
            cc::re::subst(s, fname_format);
          }
        }

      std::stringstream b;

      b << fname_format;

      // Output dir.

      const fs::path out_dir = output_dir(map);
      fs::path templ_path    = out_dir;

      templ_path /= b.str();

      // Path, name.

      fs::path p = fs::system_complete(templ_path);

#if BOOST_FILESYSTEM_VERSION > 2
      _name = p.filename().string();
#else
      _name = p.filename();
#endif
      _path = p.string();

      if (! opts.flags.overwrite)
        {
          for (int i=0; i<INT_MAX; ++i)
            {
              _initial_length = file::exists(_path);

              if (_initial_length == 0)
                break;

              else if (_initial_length >= media.content_length())
                throw cc::nothing_todo_error();

              else
                {
                  if (opts.flags.cont)
                    break;
                }

              boost::format fmt =
                boost::format("%1%.%2%") % templ_path.string() % i;

              p = fs::system_complete(fmt.str());

#if BOOST_FILESYSTEM_VERSION > 2
              _name = p.filename().string();
#else
              _name = p.filename();
#endif
              _path = p.string();
            }
        }
    }

  if (opts.flags.overwrite)
    _initial_length = 0;
}

double file::exists(const std::string& path)
{
  fs::path p( fs::system_complete(path) );

  double size = 0;

  if (fs::exists(p))
    size = static_cast<double>(fs::file_size(p));

  return size;
}

} // namespace cc

// vim: set ts=2 sw=2 tw=72 expandtab:
<|endoftext|>


<commit_before>//-----------------------------------------------------------------------------
// name: sig_gen.cpp
// desc: real-time sine wave
//
// author: Nikhil Goel (nmgoel@stanford.edu)
//   date: fall 2015
//   uses: RtAudio by Gary Scavone
//-----------------------------------------------------------------------------
#include "RtAudio.h"
#include <math.h>
#include <iostream>
#include <cstdlib>
using namespace std;

/* ----------------------#defines-------------------- */

// A sample is a discrete time signal derived from a continuous signal
#define SAMPLE double

// RtAudio's data format type. Normalized between +- 1
#define MY_FORMAT RTAUDIO_FLOAT64

// Sample Rate = (avg. # of samples)/second = 1/T, where T is sampling interval
#define MY_SRATE 44100

// number of channels
#define MY_CHANNELS 2

#define MY_PIE 3.14159265358979

/* ----------------------globals--------------------- */

// global for frequency
SAMPLE g_freq = 440;

// global sample number variable
SAMPLE g_t = 0;

SAMPLE g_width = 0;

// type of signal requested by user, enumerated in function determine_signal
int g_sig = 0;

/*---------------------------------------------------- */

/*
 * @funtion audio_callback The RtAudioCallback function.
 * @param outputBuffer Pointer to the buffer that holds the output.
 * @param inputBuffer Pointer to the buffer that holds the input.
 * @param numFrames The number of sample frames held by input buffer
    and written to output buffer.
 * @param streamTime The number of seconds (double) that the signal streams.
 * @param status Notifies if there is an output/input overflow/underflow.
 * @param data Pointer to optional data provided by the client
    when opening the stream (default = NULL).
   @return Zero to maintain normal stream. One to stop the stream and drain the
   output buffer. Two to abort the stream immediately.
 */
int audio_callback(void *outputBuffer, void *inputBuffer, unsigned int numFrames,
     double streamTime, RtAudioStreamStatus status, void *data ) {

         // stderr prints info and err messages (info about callback here)
         cerr << ".";

         // outputBuffer points to array of SAMPLEs
         SAMPLE *buffer = (SAMPLE *) outputBuffer;
         for(int i = 0; i < numFrames; i++)
         {
            // generate signal in even-indexed slots of the buffer
            switch(g_sig) {
                case 1: // sine
                    buffer[i * MY_CHANNELS] = sin(2 * MY_PIE * g_freq * g_t / MY_SRATE);
                    break;
                case 2: // saw
                    break;
                case 3: // pulse
                    break;
                case 4: // noise (white noise)
                    buffer[i * MY_CHANNELS] = (rand() % 100) / 10;
                    break;
                case 5: // impulse

                    break;
                default: // any erroneous input in the type argument
                    return 2;
            }

             // copy signal into odd-indexed slots of the buffer
             for(int j = 1; j < MY_CHANNELS; j++)
                 buffer[i * MY_CHANNELS + j] = buffer[i * MY_CHANNELS];

             // increment sample number
             g_t += 1.0;
         }
         return 0;
}

/*
 * @funtion determine_signal Returns an integer based on the type of command given.
 * @param arg command given by user input
 */
int determine_signal(string arg) {

    // sine
    if (arg == "--sine") {
        return 1;
    }

    // saw
    else if (arg == "--saw"){
        return 2;
    }

    // pulse
    else if (arg == "--pulse"){
        return 3;
    }

    // noise
    else if (arg == "--noise"){
        return 4;
    }

    //impulse
    else if (arg == "--impulse") {
        return 5;
    }

    // arg is some other input not defined by this program
    else {
        return -1;
    }
}

int main(int argc, char const *argv[]) {

    // error checking for ./sig_gen with no additional arguments
    if (argc <= 1) {
        cout << "Not enough arguments. Must give type of signal generation." << endl;
        exit(1);
    }

    // error checking for more than four arguments
    if (argc > 4) {
        cout << "Too many arguments. Only give command [type] [frequency] [width]." << endl;
        exit(1);
    }

    // determines which type of signal user wants to generate
    g_sig = determine_signal(string(argv[1]));

    // instantiate RtAudio object
    RtAudio audio;

    // frame size
    unsigned int bufferFrames = 512;
    unsigned int bufferBytes = 0;

    // check for audio devices
    if(audio.getDeviceCount() < 1)
    {
        cout << "no audio devices found!" << endl;
        exit(1);
    }

    // let RtAudio print messages to stderr.
    audio.showWarnings(true);

    // set input and output parameters
    RtAudio::StreamParameters iParams, oParams;
    iParams.deviceId = audio.getDefaultInputDevice();
    iParams.nChannels = MY_CHANNELS;
    iParams.firstChannel = 0;
    oParams.deviceId = audio.getDefaultOutputDevice();
    oParams.nChannels = MY_CHANNELS;
    oParams.firstChannel = 0;

    // create stream options
    RtAudio::StreamOptions options;

    try {
        // open a stream
        audio.openStream(&oParams, &iParams, MY_FORMAT, MY_SRATE, &bufferFrames,
            &audio_callback, (void *) &bufferBytes, &options);
    }
    catch(RtError& e)
    {
        cout << e.getMessage() << endl;
        exit(1);
    }

    // compute
    bufferBytes = bufferFrames * MY_CHANNELS * sizeof(SAMPLE);

    // test RtAudio functionality for reporting latency.
    cout << "stream latency: " << audio.getStreamLatency() << " frames" << endl;

    // go for it
    try {
        // start stream
        audio.startStream();

        // get input
        char input;
        std::cout << "running... press <enter> to quit (buffer frames: " << bufferFrames << ")" << endl;
        std::cin.get(input);

        // stop the stream.
        audio.stopStream();
    }
    catch(RtError& e)
    {
        // print error message
        cout << e.getMessage() << endl;
        goto cleanup;
    }

    cleanup:
        // close if open
        if(audio.isStreamOpen())
            audio.closeStream();

    return 0;
}
<commit_msg>added command line robustness<commit_after>//-----------------------------------------------------------------------------
// name: sig_gen.cpp
// desc: real-time sine wave
//
// author: Nikhil Goel (nmgoel@stanford.edu)
//   date: fall 2015
//   uses: RtAudio by Gary Scavone
//-----------------------------------------------------------------------------
#include "RtAudio.h"
#include <math.h>
#include <iostream>
#include <cstdlib>
using namespace std;

/* ----------------------#defines-------------------- */

// A sample is a discrete time signal derived from a continuous signal
#define SAMPLE double

// RtAudio's data format type. Normalized between +- 1
#define MY_FORMAT RTAUDIO_FLOAT64

// Sample Rate = (avg. # of samples)/second = 1/T, where T is sampling interval
#define MY_SRATE 44100

// number of channels
#define MY_CHANNELS 2

#define MY_PIE 3.14159265358979

/* ----------------------globals--------------------- */

// frequency
SAMPLE g_freq = 440;

// sample number
SAMPLE g_t = 0;

// wave width
SAMPLE g_width = 0;

// type of signal requested by user, enumerated in function determine_signal
int g_sig = 0;

/*---------------------------------------------------- */

/*
 * @funtion audio_callback The RtAudioCallback function.
 * @param outputBuffer Pointer to the buffer that holds the output.
 * @param inputBuffer Pointer to the buffer that holds the input.
 * @param numFrames The number of sample frames held by input buffer
    and written to output buffer.
 * @param streamTime The number of seconds (double) that the signal streams.
 * @param status Notifies if there is an output/input overflow/underflow.
 * @param data Pointer to optional data provided by the client
    when opening the stream (default = NULL).
   @return Zero to maintain normal stream. One to stop the stream and drain the
   output buffer. Two to abort the stream immediately.
 */
int audio_callback(void *outputBuffer, void *inputBuffer, unsigned int numFrames,
     double streamTime, RtAudioStreamStatus status, void *data ) {

         // stderr prints info and err messages (info about callback here)
         cerr << ".";

         // outputBuffer points to array of SAMPLEs
         SAMPLE *buffer = (SAMPLE *) outputBuffer;
         for(int i = 0; i < numFrames; i++)
         {
            // generate signal in even-indexed slots of the buffer
            switch(g_sig) {
                case 1: // sine
                    buffer[i * MY_CHANNELS] = sin(2 * MY_PIE * g_freq * g_t / MY_SRATE);
                    break;
                case 2: // saw
                    break;
                case 3: // pulse
                    break;
                case 4: // noise (white noise)
                    buffer[i * MY_CHANNELS] = (rand() % 100) / 10;
                    break;
                case 5: // impulse

                    // signal sample is at the fundamental period of the given frequency
                    if (sin(g_t / MY_SRATE) == (1 / g_freq)) {
                        buffer[i * MY_CHANNELS] = 1;
                    } else {
                        buffer[i * MY_CHANNELS] = 0;
                    }
                    break;
                default: return 2;
            }

             // copy signal into odd-indexed slots of the buffer
             for(int j = 1; j < MY_CHANNELS; j++)
                 buffer[i * MY_CHANNELS + j] = buffer[i * MY_CHANNELS];

             // increment sample number
             g_t += 1.0;
         }
         return 0;
}

/*
 * @funtion determine_signal Returns an integer based on the type of command given.
 * @param arg command given by user input
 */
int determine_signal(int argc, const char *argv[]) {
    string arg = string(argv[1]);
    char *endptr = 0;

    if (argc > 2) {
        g_freq = strtod(argv[2], &endptr);
        if (*endptr != '\0' || endptr == argv[2]) {
            cout << "The frequency you entered is not a double. Using default frequency 440 Hz." << endl;
            g_freq = 440;
        }
    }

    if (argc > 3) {
        g_width = strtod(argv[3], &endptr);
        if (*endptr != '\0' || endptr == argv[3]) {
            cout << "The width you entered is not a double. Using default width 1." << endl;
            g_width = 1;
        }
    }

    // sine
    if (arg == "--sine") {
        return 1;
    }

    // saw
    else if (arg == "--saw"){
        return 2;
    }

    // pulse
    else if (arg == "--pulse"){
        return 3;
    }

    // noise
    else if (arg == "--noise"){
        return 4;
    }

    //impulse
    else if (arg == "--impulse") {
        return 5;
    }

    // arg is some other input not defined by this program
    else {
        return -1;
    }
}

int main(int argc, char const *argv[]) {

    // error: ./sig_gen with no additional arguments
    if (argc <= 1) {
        cout << "Not enough arguments. Must give type of signal generation." << endl;
        exit(1);
    }

    // error: more than four arguments
    if (argc > 4) {
        cout << "Ignoring extraneous arguments..." << endl;
    }

    // determines which type of signal user wants to generate
    g_sig = determine_signal(argc, argv);

    // error: invalid waveform type
    if (g_sig < 0) {
        cout << "Must provide a valid type of waveform. --sine, --saw, --noise, --pulse, --impulse." << endl;
        exit(1);
    }

    // instantiate RtAudio object
    RtAudio audio;

    // frame size
    unsigned int bufferFrames = 512;
    unsigned int bufferBytes = 0;

    // check for audio devices
    if(audio.getDeviceCount() < 1)
    {
        cout << "no audio devices found!" << endl;
        exit(1);
    }

    // let RtAudio print messages to stderr.
    audio.showWarnings(true);

    // set input and output parameters
    RtAudio::StreamParameters iParams, oParams;
    iParams.deviceId = audio.getDefaultInputDevice();
    iParams.nChannels = MY_CHANNELS;
    iParams.firstChannel = 0;
    oParams.deviceId = audio.getDefaultOutputDevice();
    oParams.nChannels = MY_CHANNELS;
    oParams.firstChannel = 0;

    // create stream options
    RtAudio::StreamOptions options;

    try {
        // open a stream
        audio.openStream(&oParams, &iParams, MY_FORMAT, MY_SRATE, &bufferFrames,
            &audio_callback, (void *) &bufferBytes, &options);
    }
    catch(RtError& e)
    {
        cout << e.getMessage() << endl;
        exit(1);
    }

    // compute
    bufferBytes = bufferFrames * MY_CHANNELS * sizeof(SAMPLE);

    // test RtAudio functionality for reporting latency.
    cout << "stream latency: " << audio.getStreamLatency() << " frames" << endl;

    // go for it
    try {
        // start stream
        audio.startStream();

        // get input
        char input;
        std::cout << "running... press <enter> to quit (buffer frames: " << bufferFrames << ")" << endl;
        std::cin.get(input);

        // stop the stream.
        audio.stopStream();
    }
    catch(RtError& e)
    {
        // print error message
        cout << e.getMessage() << endl;
        goto cleanup;
    }

    cleanup:
        // close if open
        if(audio.isStreamOpen())
            audio.closeStream();

    return 0;
}
<|endoftext|>