<commit_before>#include "serialized_fast_value_attribute.h"
#include "streamed_value_saver.h"
#include <vespa/eval/eval/value.h>
#include <vespa/eval/eval/fast_value.hpp>
#include <vespa/eval/streamed/streamed_value_utils.h>
#include <vespa/fastlib/io/bufferedfile.h>
#include <vespa/searchlib/attribute/readerbase.h>
#include <vespa/searchlib/util/fileutil.h>
#include <vespa/vespalib/util/rcuvector.hpp>
#include <vespa/log/log.h>

LOG_SETUP(".searchlib.tensor.serialized_fast_value_attribute");

#include "blob_sequence_reader.h"
#include "tensor_attribute.hpp"

using namespace vespalib;
using namespace vespalib::eval;

namespace search::tensor {

namespace {

constexpr uint32_t TENSOR_ATTRIBUTE_VERSION = 0;

struct ValueBlock : LabelBlock {
    TypedCells cells;
};

class ValueBlockStream {
private:
    const StreamedValueStore::DataFromType &_from_type;
    LabelBlockStream _label_block_stream;
    const char *_cells_ptr;

    size_t dsss() const { return _from_type.dense_subspace_size; }
    auto cell_type() const { return _from_type.cell_type; }
public:
    ValueBlock next_block() {
        auto labels = _label_block_stream.next_block();
        if (labels) {
            TypedCells subspace_cells(_cells_ptr, cell_type(), dsss());
            _cells_ptr += CellTypeUtils::mem_size(cell_type(), dsss());
            return ValueBlock{labels, subspace_cells};
        } else {
            TypedCells none(nullptr, cell_type(), 0);
            return ValueBlock{labels, none};
        }
    }

    ValueBlockStream(const StreamedValueStore::DataFromType &from_type,
                     const StreamedValueStore::StreamedValueData &from_store)
      : _from_type(from_type),
        _label_block_stream(from_store.num_subspaces,
                            from_store.labels_buffer,
                            from_type.num_mapped_dimensions),
        _cells_ptr((const char *)from_store.cells_ref.data)
    {
        _label_block_stream.reset();
    }

    ~ValueBlockStream();
};

ValueBlockStream::~ValueBlockStream() = default;

class OnlyFastValueIndex : public Value {
private:
    const ValueType &_type;
    TypedCells _cells;
    FastValueIndex my_index;
public:
    OnlyFastValueIndex(const ValueType &type,
                       const StreamedValueStore::DataFromType &from_type,
                       const StreamedValueStore::StreamedValueData &from_store)
      : _type(type),
        _cells(from_store.cells_ref),
        my_index(from_type.num_mapped_dimensions,
                 from_store.num_subspaces)
    {
        assert(_type.cell_type() == _cells.type);
        std::vector<vespalib::stringref> address(from_type.num_mapped_dimensions);
        auto block_stream = ValueBlockStream(from_type, from_store);
        size_t ss = 0;
        while (auto block = block_stream.next_block()) {
            size_t idx = my_index.map.add_mapping(block.address);
            if (idx != ss) {
                LOG(error, "add_mapping returned idx=%zu for subspace %zu", idx, ss);
            }
            ++ss;
        }
        if (ss != from_store.num_subspaces) {
            LOG(error, "expected %zu subspaces but got %zu", from_store.num_subspaces, ss);
            abort();
        }
    }

    ~OnlyFastValueIndex();

    const ValueType &type() const final override { return _type; }
    TypedCells cells() const final override { return _cells; }
    const Index &index() const final override { return my_index; }
    vespalib::MemoryUsage get_memory_usage() const final override {
        auto usage = self_memory_usage<OnlyFastValueIndex>();
        usage.merge(my_index.map.estimate_extra_memory_usage());
        return usage;
    }
};

OnlyFastValueIndex::~OnlyFastValueIndex() = default;

}

SerializedFastValueAttribute::SerializedFastValueAttribute(stringref name, const Config &cfg)
  : TensorAttribute(name, cfg, _streamedValueStore),
    _tensor_type(cfg.tensorType()),
    _streamedValueStore(_tensor_type),
    _data_from_type(_tensor_type)
{
}

SerializedFastValueAttribute::~SerializedFastValueAttribute()
{
    getGenerationHolder().clearHoldLists();
    _tensorStore.clearHoldLists();
}

void
SerializedFastValueAttribute::setTensor(DocId docId, const vespalib::eval::Value &tensor)
{
    EntryRef ref = _streamedValueStore.store_tensor(tensor);
    setTensorRef(docId, ref);
    if (!ref.valid()) {
        checkTensorType(tensor);
    }
}

std::unique_ptr<Value>
SerializedFastValueAttribute::getTensor(DocId docId) const
{
    EntryRef ref;
    if (docId < getCommittedDocIdLimit()) {
        ref = _refVector[docId];
    }
    if (!ref.valid()) {
        return {};
    }
    if (auto data_from_store = _streamedValueStore.get_tensor_data(ref)) {
        return std::make_unique<OnlyFastValueIndex>(_tensor_type,
                                                    _data_from_type,
                                                    data_from_store);
    }
    return {};
}

bool
SerializedFastValueAttribute::onLoad()
{
    BlobSequenceReader tensorReader(*this);
    if (!tensorReader.hasData()) {
        return false;
    }
    setCreateSerialNum(tensorReader.getCreateSerialNum());
    assert(tensorReader.getVersion() == TENSOR_ATTRIBUTE_VERSION);
    uint32_t numDocs(tensorReader.getDocIdLimit());
    _refVector.reset();
    _refVector.unsafe_reserve(numDocs);
    vespalib::Array<char> buffer(1024);
    for (uint32_t lid = 0; lid < numDocs; ++lid) {
        uint32_t tensorSize = tensorReader.getNextSize();
        if (tensorSize != 0) {
            if (tensorSize > buffer.size()) {
                buffer.resize(tensorSize + 1024);
            }
            tensorReader.readBlob(&buffer[0], tensorSize);
            vespalib::nbostream source(&buffer[0], tensorSize);
            EntryRef ref = _streamedValueStore.store_encoded_tensor(source);
            _refVector.push_back(ref);
        } else {
            EntryRef invalid;
            _refVector.push_back(invalid);
        }
    }
    setNumDocs(numDocs);
    setCommittedDocIdLimit(numDocs);
    return true;
}

std::unique_ptr<AttributeSaver>
SerializedFastValueAttribute::onInitSave(vespalib::stringref fileName)
{
    vespalib::GenerationHandler::Guard guard(getGenerationHandler().
                                             takeGuard());
    return std::make_unique<StreamedValueSaver>
        (std::move(guard),
         this->createAttributeHeader(fileName),
         getRefCopy(),
         _streamedValueStore);
}

void
SerializedFastValueAttribute::compactWorst()
{
    doCompactWorst<StreamedValueStore::RefType>();
}

}
<commit_msg>more consistent version checking<commit_after>#include "serialized_fast_value_attribute.h"
#include "streamed_value_saver.h"
#include <vespa/eval/eval/value.h>
#include <vespa/eval/eval/fast_value.hpp>
#include <vespa/eval/streamed/streamed_value_utils.h>
#include <vespa/fastlib/io/bufferedfile.h>
#include <vespa/searchlib/attribute/readerbase.h>
#include <vespa/searchlib/util/fileutil.h>
#include <vespa/vespalib/util/rcuvector.hpp>
#include <vespa/log/log.h>

LOG_SETUP(".searchlib.tensor.serialized_fast_value_attribute");

#include "blob_sequence_reader.h"
#include "tensor_attribute.hpp"

using namespace vespalib;
using namespace vespalib::eval;

namespace search::tensor {

namespace {

struct ValueBlock : LabelBlock {
    TypedCells cells;
};

class ValueBlockStream {
private:
    const StreamedValueStore::DataFromType &_from_type;
    LabelBlockStream _label_block_stream;
    const char *_cells_ptr;

    size_t dsss() const { return _from_type.dense_subspace_size; }
    auto cell_type() const { return _from_type.cell_type; }
public:
    ValueBlock next_block() {
        auto labels = _label_block_stream.next_block();
        if (labels) {
            TypedCells subspace_cells(_cells_ptr, cell_type(), dsss());
            _cells_ptr += CellTypeUtils::mem_size(cell_type(), dsss());
            return ValueBlock{labels, subspace_cells};
        } else {
            TypedCells none(nullptr, cell_type(), 0);
            return ValueBlock{labels, none};
        }
    }

    ValueBlockStream(const StreamedValueStore::DataFromType &from_type,
                     const StreamedValueStore::StreamedValueData &from_store)
      : _from_type(from_type),
        _label_block_stream(from_store.num_subspaces,
                            from_store.labels_buffer,
                            from_type.num_mapped_dimensions),
        _cells_ptr((const char *)from_store.cells_ref.data)
    {
        _label_block_stream.reset();
    }

    ~ValueBlockStream();
};

ValueBlockStream::~ValueBlockStream() = default;

class OnlyFastValueIndex : public Value {
private:
    const ValueType &_type;
    TypedCells _cells;
    FastValueIndex my_index;
public:
    OnlyFastValueIndex(const ValueType &type,
                       const StreamedValueStore::DataFromType &from_type,
                       const StreamedValueStore::StreamedValueData &from_store)
      : _type(type),
        _cells(from_store.cells_ref),
        my_index(from_type.num_mapped_dimensions,
                 from_store.num_subspaces)
    {
        assert(_type.cell_type() == _cells.type);
        std::vector<vespalib::stringref> address(from_type.num_mapped_dimensions);
        auto block_stream = ValueBlockStream(from_type, from_store);
        size_t ss = 0;
        while (auto block = block_stream.next_block()) {
            size_t idx = my_index.map.add_mapping(block.address);
            if (idx != ss) {
                LOG(error, "add_mapping returned idx=%zu for subspace %zu", idx, ss);
            }
            ++ss;
        }
        if (ss != from_store.num_subspaces) {
            LOG(error, "expected %zu subspaces but got %zu", from_store.num_subspaces, ss);
            abort();
        }
    }

    ~OnlyFastValueIndex();

    const ValueType &type() const final override { return _type; }
    TypedCells cells() const final override { return _cells; }
    const Index &index() const final override { return my_index; }
    vespalib::MemoryUsage get_memory_usage() const final override {
        auto usage = self_memory_usage<OnlyFastValueIndex>();
        usage.merge(my_index.map.estimate_extra_memory_usage());
        return usage;
    }
};

OnlyFastValueIndex::~OnlyFastValueIndex() = default;

}

SerializedFastValueAttribute::SerializedFastValueAttribute(stringref name, const Config &cfg)
  : TensorAttribute(name, cfg, _streamedValueStore),
    _tensor_type(cfg.tensorType()),
    _streamedValueStore(_tensor_type),
    _data_from_type(_tensor_type)
{
}

SerializedFastValueAttribute::~SerializedFastValueAttribute()
{
    getGenerationHolder().clearHoldLists();
    _tensorStore.clearHoldLists();
}

void
SerializedFastValueAttribute::setTensor(DocId docId, const vespalib::eval::Value &tensor)
{
    EntryRef ref = _streamedValueStore.store_tensor(tensor);
    setTensorRef(docId, ref);
    if (!ref.valid()) {
        checkTensorType(tensor);
    }
}

std::unique_ptr<Value>
SerializedFastValueAttribute::getTensor(DocId docId) const
{
    EntryRef ref;
    if (docId < getCommittedDocIdLimit()) {
        ref = _refVector[docId];
    }
    if (!ref.valid()) {
        return {};
    }
    if (auto data_from_store = _streamedValueStore.get_tensor_data(ref)) {
        return std::make_unique<OnlyFastValueIndex>(_tensor_type,
                                                    _data_from_type,
                                                    data_from_store);
    }
    return {};
}

bool
SerializedFastValueAttribute::onLoad()
{
    BlobSequenceReader tensorReader(*this);
    if (!tensorReader.hasData()) {
        return false;
    }
    setCreateSerialNum(tensorReader.getCreateSerialNum());
    assert(tensorReader.getVersion() == getVersion());
    uint32_t numDocs(tensorReader.getDocIdLimit());
    _refVector.reset();
    _refVector.unsafe_reserve(numDocs);
    vespalib::Array<char> buffer(1024);
    for (uint32_t lid = 0; lid < numDocs; ++lid) {
        uint32_t tensorSize = tensorReader.getNextSize();
        if (tensorSize != 0) {
            if (tensorSize > buffer.size()) {
                buffer.resize(tensorSize + 1024);
            }
            tensorReader.readBlob(&buffer[0], tensorSize);
            vespalib::nbostream source(&buffer[0], tensorSize);
            EntryRef ref = _streamedValueStore.store_encoded_tensor(source);
            _refVector.push_back(ref);
        } else {
            EntryRef invalid;
            _refVector.push_back(invalid);
        }
    }
    setNumDocs(numDocs);
    setCommittedDocIdLimit(numDocs);
    return true;
}

std::unique_ptr<AttributeSaver>
SerializedFastValueAttribute::onInitSave(vespalib::stringref fileName)
{
    vespalib::GenerationHandler::Guard guard(getGenerationHandler().
                                             takeGuard());
    return std::make_unique<StreamedValueSaver>
        (std::move(guard),
         this->createAttributeHeader(fileName),
         getRefCopy(),
         _streamedValueStore);
}

void
SerializedFastValueAttribute::compactWorst()
{
    doCompactWorst<StreamedValueStore::RefType>();
}

}
<|endoftext|>


<commit_before>#include <string>
#include <stdio.h>

#include "stdlib.h"
#include "dag.h"
#include "engine.h"
#include "failure.h"

void diamond_dag() {
    DAG dag("test/diamond.dag");
    Engine engine(dag);

    if (!engine.has_ready_task()) {
        myfailure("Did not queue root tasks");
    }

    Task *a = engine.next_ready_task();
    if (a->name.compare("A") != 0) {
        myfailure("Queued non root task %s", a->name.c_str());
    }

    if (engine.has_ready_task()) {
        myfailure("Queued non-root tasks");
    }

    engine.mark_task_finished(a, 0);

    if (!engine.has_ready_task()) {
        myfailure("Marking did not release tasks");
    }

    Task *bc = engine.next_ready_task();
    if (!engine.has_ready_task()) {
        myfailure("Marking did not release tasks");
    }

    Task *cb = engine.next_ready_task();
    if (engine.has_ready_task()) {
        myfailure("Marking released too many tasks");
    }

    if (bc->name.compare("B") != 0 && bc->name.compare("C") != 0) {
        myfailure("Wrong task released: %s", bc->name.c_str());
    }

    if (cb->name.compare("B") != 0 && cb->name.compare("C") != 0) {
        myfailure("Wrong task released: %s", cb->name.c_str());
    }

    engine.mark_task_finished(bc, 0);
    if (engine.has_ready_task()) {
        myfailure("Marking released a task when it shouldn't");
    }

    engine.mark_task_finished(cb, 0);
    if (!engine.has_ready_task()) {
        myfailure("Marking all parents did not release task D");
    }

    Task *d = engine.next_ready_task();
    if (d->name.compare("D") != 0) {
        myfailure("Not task D");
    }

    if (engine.has_ready_task()) {
        myfailure("No more tasks are available");
    }

    if (engine.is_finished()) {
        myfailure("DAG is not finished");
    }

    engine.mark_task_finished(d, 0);

    if (!engine.is_finished()) {
        myfailure("DAG is finished");
    }
}

void diamond_dag_rescue() {
    DAG dag("test/diamond.dag","test/diamond.rescue");
    Engine engine(dag);

    if (!engine.has_ready_task()) {
        myfailure("Should have ready D task");
    }

    Task *d = engine.next_ready_task();
    if (d->name.compare("D") != 0) {
        myfailure("Ready task is not D");
    }

    engine.mark_task_finished(d, 1);

    if (engine.has_ready_task()) {
        myfailure("Ready tasks even though D failed");
    }

    if (!engine.is_finished()) {
        myfailure("DAG should have been finished after D failed");
    }

    if (!engine.is_failed()) {
        myfailure("DAG should be failed");
    }
}

void diamond_dag_failure() {
    DAG dag("test/diamond.dag");
    Engine engine(dag);

    if (!engine.has_ready_task()) {
        myfailure("Did not queue root tasks");
    }

    Task *a = engine.next_ready_task();
    if (a->name.compare("A") != 0) {
        myfailure("Queued non root task %s", a->name.c_str());
    }

    if (engine.has_ready_task()) {
        myfailure("Queued non-root tasks");
    }

    engine.mark_task_finished(a, 1);

    if (engine.has_ready_task()) {
        myfailure("Released tasks even though parent failed");
    }

    if (!engine.is_finished()) {
        myfailure("DAG should have been finished after A failed");
    }
}

void read_file(char *fname, char *buf) {
    FILE *f = fopen(fname, "r");
    int read = fread(buf, 1, 1024, f);
    buf[read] = '\0';
    fclose(f);
}

void diamond_dag_newrescue() {
    char temp[1024];
    sprintf(temp,"file_XXXXXX");
    mkstemp(temp);

    DAG dag("test/diamond.dag");
    Engine engine(dag, temp);

    Task *a = engine.next_ready_task();
    engine.mark_task_finished(a, 0);

    Task *bc = engine.next_ready_task();
    engine.mark_task_finished(bc, 0);

    Task *cb = engine.next_ready_task();
    engine.mark_task_finished(cb, 0);

    Task *d = engine.next_ready_task();
    engine.mark_task_finished(d, 1);

    if (!engine.is_finished()) {
        myfailure("DAG should be finished");
    }

    if (!engine.is_failed()) {
        myfailure("DAG should be failed");
    }

    char buf[1024];
    read_file(temp, buf);
    if (strcmp(buf, "\nDONE A\nDONE B\nDONE C") != 0) {
        myfailure("Rescue file not updated properly: %s", temp);
    } else {
        unlink(temp);
    }
}

void diamond_dag_oldrescue() {
    char temp[1024];
    sprintf(temp, "file_XXXXXX");
    mkstemp(temp);

    DAG dag("test/diamond.dag", "test/diamond.rescue");
    Engine engine(dag, temp);

    if (!engine.has_ready_task()) {
        myfailure("Should have ready D task");
    }

    Task *d = engine.next_ready_task();
    if (d->name.compare("D") != 0) {
        myfailure("Ready task is not D");
    }

    engine.mark_task_finished(d, 0);

    if (engine.has_ready_task()) {
        myfailure("Ready tasks even though D finished");
    }

    if (!engine.is_finished()) {
        myfailure("DAG should have been finished after D finished");
    }

    if (engine.is_failed()) {
        myfailure("DAG should not be failed");
    }

    char buf[1024];
    read_file(temp, buf);
    if (strcmp(buf, "\nDONE A\nDONE B\nDONE C\nDONE D") != 0) {
        myfailure("Rescue file not updated properly: %s: %s", temp, buf);
    } else {
        unlink(temp);
    }
}

void diamond_dag_max_failures() {
    DAG dag("test/diamond.dag");
    Engine engine(dag, "", 1);

    if (!engine.has_ready_task()) {
        myfailure("Did not queue root tasks");
    }

    Task *a = engine.next_ready_task();
    if (a->name.compare("A") != 0) {
        myfailure("Queued non root task %s", a->name.c_str());
    }

    if (engine.has_ready_task()) {
        myfailure("Queued non-root tasks");
    }

    engine.mark_task_finished(a, 0);

    Task *bc = engine.next_ready_task();

    engine.mark_task_finished(bc, 1);

    if (engine.has_ready_task()) {
        myfailure("DAG should not have a ready task because %s failed", bc->name.c_str());
    }
}

void diamond_dag_retries() {
    int tries = 3;

    DAG dag("test/diamond.dag");
    Engine engine(dag, "", 0, tries);

    Task *a;

    for (int i=0; i<tries; i++) {
        if (!engine.has_ready_task()) {
            myfailure("A should have been ready");
        }

        a = engine.next_ready_task();
        if (a->name.compare("A") != 0) {
            myfailure("A should have been ready");
        }

        engine.mark_task_finished(a, 1);
    }

    if (engine.has_ready_task()) {
        myfailure("DAG should not have a ready task because A failed");
    }
}

void diamond_dag_retries2() {
    int tries = 3;

    DAG dag("test/diamond.dag");
    Engine engine(dag, "", 0, tries);

    Task *a;

    for (int i=0; i<tries-1; i++) {
        if (!engine.has_ready_task()) {
            myfailure("A should have been ready");
        }

        a = engine.next_ready_task();
        if (a->name.compare("A") != 0) {
            myfailure("A should have been ready");
        }

        engine.mark_task_finished(a, 1);
    }

    if (!engine.has_ready_task()) {
        myfailure("A should have been ready");
    }

    a = engine.next_ready_task();
    if (a->name.compare("A") != 0) {
        myfailure("A should have been ready");
    }

    engine.mark_task_finished(a, 0);

    if (!engine.has_ready_task()) {
        myfailure("DAG should have a ready task because A finally succeeded");
    }

    Task *bc = engine.next_ready_task();
    if (bc->name.compare("B")!=0 && bc->name.compare("C")!=0) {
        myfailure("B or C should have been ready");
    }
}

int main(int argc, char *argv[]) {
    diamond_dag();
    diamond_dag_failure();
    diamond_dag_max_failures();
    diamond_dag_retries();
    diamond_dag_retries2();
    diamond_dag_oldrescue();
    diamond_dag_newrescue();
    diamond_dag_rescue();
    return 0;
}
<commit_msg>PMC: Add unistd.h to test-engine.cpp<commit_after>#include <string>
#include <stdio.h>
#include <unistd.h>

#include "stdlib.h"
#include "dag.h"
#include "engine.h"
#include "failure.h"

void diamond_dag() {
    DAG dag("test/diamond.dag");
    Engine engine(dag);

    if (!engine.has_ready_task()) {
        myfailure("Did not queue root tasks");
    }

    Task *a = engine.next_ready_task();
    if (a->name.compare("A") != 0) {
        myfailure("Queued non root task %s", a->name.c_str());
    }

    if (engine.has_ready_task()) {
        myfailure("Queued non-root tasks");
    }

    engine.mark_task_finished(a, 0);

    if (!engine.has_ready_task()) {
        myfailure("Marking did not release tasks");
    }

    Task *bc = engine.next_ready_task();
    if (!engine.has_ready_task()) {
        myfailure("Marking did not release tasks");
    }

    Task *cb = engine.next_ready_task();
    if (engine.has_ready_task()) {
        myfailure("Marking released too many tasks");
    }

    if (bc->name.compare("B") != 0 && bc->name.compare("C") != 0) {
        myfailure("Wrong task released: %s", bc->name.c_str());
    }

    if (cb->name.compare("B") != 0 && cb->name.compare("C") != 0) {
        myfailure("Wrong task released: %s", cb->name.c_str());
    }

    engine.mark_task_finished(bc, 0);
    if (engine.has_ready_task()) {
        myfailure("Marking released a task when it shouldn't");
    }

    engine.mark_task_finished(cb, 0);
    if (!engine.has_ready_task()) {
        myfailure("Marking all parents did not release task D");
    }

    Task *d = engine.next_ready_task();
    if (d->name.compare("D") != 0) {
        myfailure("Not task D");
    }

    if (engine.has_ready_task()) {
        myfailure("No more tasks are available");
    }

    if (engine.is_finished()) {
        myfailure("DAG is not finished");
    }

    engine.mark_task_finished(d, 0);

    if (!engine.is_finished()) {
        myfailure("DAG is finished");
    }
}

void diamond_dag_rescue() {
    DAG dag("test/diamond.dag","test/diamond.rescue");
    Engine engine(dag);

    if (!engine.has_ready_task()) {
        myfailure("Should have ready D task");
    }

    Task *d = engine.next_ready_task();
    if (d->name.compare("D") != 0) {
        myfailure("Ready task is not D");
    }

    engine.mark_task_finished(d, 1);

    if (engine.has_ready_task()) {
        myfailure("Ready tasks even though D failed");
    }

    if (!engine.is_finished()) {
        myfailure("DAG should have been finished after D failed");
    }

    if (!engine.is_failed()) {
        myfailure("DAG should be failed");
    }
}

void diamond_dag_failure() {
    DAG dag("test/diamond.dag");
    Engine engine(dag);

    if (!engine.has_ready_task()) {
        myfailure("Did not queue root tasks");
    }

    Task *a = engine.next_ready_task();
    if (a->name.compare("A") != 0) {
        myfailure("Queued non root task %s", a->name.c_str());
    }

    if (engine.has_ready_task()) {
        myfailure("Queued non-root tasks");
    }

    engine.mark_task_finished(a, 1);

    if (engine.has_ready_task()) {
        myfailure("Released tasks even though parent failed");
    }

    if (!engine.is_finished()) {
        myfailure("DAG should have been finished after A failed");
    }
}

void read_file(char *fname, char *buf) {
    FILE *f = fopen(fname, "r");
    int read = fread(buf, 1, 1024, f);
    buf[read] = '\0';
    fclose(f);
}

void diamond_dag_newrescue() {
    char temp[1024];
    sprintf(temp,"file_XXXXXX");
    mkstemp(temp);

    DAG dag("test/diamond.dag");
    Engine engine(dag, temp);

    Task *a = engine.next_ready_task();
    engine.mark_task_finished(a, 0);

    Task *bc = engine.next_ready_task();
    engine.mark_task_finished(bc, 0);

    Task *cb = engine.next_ready_task();
    engine.mark_task_finished(cb, 0);

    Task *d = engine.next_ready_task();
    engine.mark_task_finished(d, 1);

    if (!engine.is_finished()) {
        myfailure("DAG should be finished");
    }

    if (!engine.is_failed()) {
        myfailure("DAG should be failed");
    }

    char buf[1024];
    read_file(temp, buf);
    if (strcmp(buf, "\nDONE A\nDONE B\nDONE C") != 0) {
        myfailure("Rescue file not updated properly: %s", temp);
    } else {
        unlink(temp);
    }
}

void diamond_dag_oldrescue() {
    char temp[1024];
    sprintf(temp, "file_XXXXXX");
    mkstemp(temp);

    DAG dag("test/diamond.dag", "test/diamond.rescue");
    Engine engine(dag, temp);

    if (!engine.has_ready_task()) {
        myfailure("Should have ready D task");
    }

    Task *d = engine.next_ready_task();
    if (d->name.compare("D") != 0) {
        myfailure("Ready task is not D");
    }

    engine.mark_task_finished(d, 0);

    if (engine.has_ready_task()) {
        myfailure("Ready tasks even though D finished");
    }

    if (!engine.is_finished()) {
        myfailure("DAG should have been finished after D finished");
    }

    if (engine.is_failed()) {
        myfailure("DAG should not be failed");
    }

    char buf[1024];
    read_file(temp, buf);
    if (strcmp(buf, "\nDONE A\nDONE B\nDONE C\nDONE D") != 0) {
        myfailure("Rescue file not updated properly: %s: %s", temp, buf);
    } else {
        unlink(temp);
    }
}

void diamond_dag_max_failures() {
    DAG dag("test/diamond.dag");
    Engine engine(dag, "", 1);

    if (!engine.has_ready_task()) {
        myfailure("Did not queue root tasks");
    }

    Task *a = engine.next_ready_task();
    if (a->name.compare("A") != 0) {
        myfailure("Queued non root task %s", a->name.c_str());
    }

    if (engine.has_ready_task()) {
        myfailure("Queued non-root tasks");
    }

    engine.mark_task_finished(a, 0);

    Task *bc = engine.next_ready_task();

    engine.mark_task_finished(bc, 1);

    if (engine.has_ready_task()) {
        myfailure("DAG should not have a ready task because %s failed", bc->name.c_str());
    }
}

void diamond_dag_retries() {
    int tries = 3;

    DAG dag("test/diamond.dag");
    Engine engine(dag, "", 0, tries);

    Task *a;

    for (int i=0; i<tries; i++) {
        if (!engine.has_ready_task()) {
            myfailure("A should have been ready");
        }

        a = engine.next_ready_task();
        if (a->name.compare("A") != 0) {
            myfailure("A should have been ready");
        }

        engine.mark_task_finished(a, 1);
    }

    if (engine.has_ready_task()) {
        myfailure("DAG should not have a ready task because A failed");
    }
}

void diamond_dag_retries2() {
    int tries = 3;

    DAG dag("test/diamond.dag");
    Engine engine(dag, "", 0, tries);

    Task *a;

    for (int i=0; i<tries-1; i++) {
        if (!engine.has_ready_task()) {
            myfailure("A should have been ready");
        }

        a = engine.next_ready_task();
        if (a->name.compare("A") != 0) {
            myfailure("A should have been ready");
        }

        engine.mark_task_finished(a, 1);
    }

    if (!engine.has_ready_task()) {
        myfailure("A should have been ready");
    }

    a = engine.next_ready_task();
    if (a->name.compare("A") != 0) {
        myfailure("A should have been ready");
    }

    engine.mark_task_finished(a, 0);

    if (!engine.has_ready_task()) {
        myfailure("DAG should have a ready task because A finally succeeded");
    }

    Task *bc = engine.next_ready_task();
    if (bc->name.compare("B")!=0 && bc->name.compare("C")!=0) {
        myfailure("B or C should have been ready");
    }
}

int main(int argc, char *argv[]) {
    diamond_dag();
    diamond_dag_failure();
    diamond_dag_max_failures();
    diamond_dag_retries();
    diamond_dag_retries2();
    diamond_dag_oldrescue();
    diamond_dag_newrescue();
    diamond_dag_rescue();
    return 0;
}
<|endoftext|>


<commit_before>//===-- GDBRemoteCommunicationReplayServer.cpp ------------------*- C++ -*-===//
//
// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
// See https://llvm.org/LICENSE.txt for license information.
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
//
//===----------------------------------------------------------------------===//

#include <errno.h>

#include "lldb/Host/Config.h"

#include "GDBRemoteCommunicationReplayServer.h"
#include "ProcessGDBRemoteLog.h"

// C Includes
// C++ Includes
#include <cstring>

// Project includes
#include "lldb/Host/ThreadLauncher.h"
#include "lldb/Utility/ConstString.h"
#include "lldb/Utility/Event.h"
#include "lldb/Utility/FileSpec.h"
#include "lldb/Utility/StreamString.h"
#include "lldb/Utility/StringExtractorGDBRemote.h"

using namespace llvm;
using namespace lldb;
using namespace lldb_private;
using namespace lldb_private::process_gdb_remote;

static bool unexpected(llvm::StringRef expected, llvm::StringRef actual) {
  // The 'expected' string contains the raw data, including the leading $ and
  // trailing checksum. The 'actual' string contains only the packet's content.
  if (expected.contains(actual))
    return false;
  if (expected == "+" || actual == "+")
    return false;
  // Contains a PID which might be different.
  if (expected.contains("vAttach"))
    return false;
  // Contains a ascii-hex-path.
  if (expected.contains("QSetSTD"))
    return false;
  // Contains environment values.
  if (expected.contains("QEnvironment"))
    return false;

  return true;
}

GDBRemoteCommunicationReplayServer::GDBRemoteCommunicationReplayServer()
    : GDBRemoteCommunication("gdb-remote.server",
                             "gdb-remote.server.rx_packet"),
      m_async_broadcaster(nullptr, "lldb.gdb-remote.server.async-broadcaster"),
      m_async_listener_sp(
          Listener::MakeListener("lldb.gdb-remote.server.async-listener")),
      m_async_thread_state_mutex(), m_skip_acks(false) {
  m_async_broadcaster.SetEventName(eBroadcastBitAsyncContinue,
                                   "async thread continue");
  m_async_broadcaster.SetEventName(eBroadcastBitAsyncThreadShouldExit,
                                   "async thread should exit");

  const uint32_t async_event_mask =
      eBroadcastBitAsyncContinue | eBroadcastBitAsyncThreadShouldExit;
  m_async_listener_sp->StartListeningForEvents(&m_async_broadcaster,
                                               async_event_mask);
}

GDBRemoteCommunicationReplayServer::~GDBRemoteCommunicationReplayServer() {
  StopAsyncThread();
}

GDBRemoteCommunication::PacketResult
GDBRemoteCommunicationReplayServer::GetPacketAndSendResponse(
    Timeout<std::micro> timeout, Status &error, bool &interrupt, bool &quit) {
  std::lock_guard<std::recursive_mutex> guard(m_async_thread_state_mutex);

  StringExtractorGDBRemote packet;
  PacketResult packet_result = WaitForPacketNoLock(packet, timeout, false);

  if (packet_result != PacketResult::Success) {
    if (!IsConnected()) {
      error.SetErrorString("lost connection");
      quit = true;
    } else {
      error.SetErrorString("timeout");
    }
    return packet_result;
  }

  m_async_broadcaster.BroadcastEvent(eBroadcastBitAsyncContinue);

  if (m_skip_acks) {
    const StringExtractorGDBRemote::ServerPacketType packet_type =
        packet.GetServerPacketType();
    switch (packet_type) {
    case StringExtractorGDBRemote::eServerPacketType_nack:
    case StringExtractorGDBRemote::eServerPacketType_ack:
      return PacketResult::Success;
    default:
      break;
    }
  } else if (packet.GetStringRef() == "QStartNoAckMode") {
    m_skip_acks = true;
    m_send_acks = false;
  }

  // A QEnvironment packet is sent for every environment variable. If the
  // number of environment variables is different during replay, the replies
  // become out of sync.
  if (packet.GetStringRef().find("QEnvironment") == 0) {
    return SendRawPacketNoLock("$OK#9a", true);
  }

  Log *log(ProcessGDBRemoteLog::GetLogIfAllCategoriesSet(GDBR_LOG_PROCESS));
  while (!m_packet_history.empty()) {
    // Pop last packet from the history.
    GDBRemoteCommunicationHistory::Entry entry = m_packet_history.back();
    m_packet_history.pop_back();

    if (entry.type == GDBRemoteCommunicationHistory::ePacketTypeSend) {
      if (unexpected(entry.packet.data, packet.GetStringRef())) {
        LLDB_LOG(log,
                 "GDBRemoteCommunicationReplayServer expected packet: '{0}'",
                 entry.packet.data);
        LLDB_LOG(log,
                 "GDBRemoteCommunicationReplayServer actual packet: '{0}'",
                 packet.GetStringRef());
      }

      // Ignore QEnvironment packets as they're handled earlier.
      if (entry.packet.data.find("QEnvironment") == 1) {
        assert(m_packet_history.back().type ==
               GDBRemoteCommunicationHistory::ePacketTypeRecv);
        m_packet_history.pop_back();
      }

      continue;
    }

    if (entry.type == GDBRemoteCommunicationHistory::ePacketTypeInvalid) {
      LLDB_LOG(
          log,
          "GDBRemoteCommunicationReplayServer skipped invalid packet: '{0}'",
          packet.GetStringRef());
      continue;
    }

    return SendRawPacketNoLock(entry.packet.data, true);
  }

  quit = true;

  return packet_result;
}

LLVM_YAML_IS_DOCUMENT_LIST_VECTOR(
    std::vector<
        lldb_private::process_gdb_remote::GDBRemoteCommunicationHistory::Entry>)

llvm::Error
GDBRemoteCommunicationReplayServer::LoadReplayHistory(const FileSpec &path) {
  auto error_or_file = MemoryBuffer::getFile(path.GetPath());
  if (auto err = error_or_file.getError())
    return errorCodeToError(err);

  yaml::Input yin((*error_or_file)->getBuffer());
  yin >> m_packet_history;

  if (auto err = yin.error())
    return errorCodeToError(err);

  // We want to manipulate the vector like a stack so we need to reverse the
  // order of the packets to have the oldest on at the back.
  std::reverse(m_packet_history.begin(), m_packet_history.end());

  return Error::success();
}

bool GDBRemoteCommunicationReplayServer::StartAsyncThread() {
  std::lock_guard<std::recursive_mutex> guard(m_async_thread_state_mutex);
  if (!m_async_thread.IsJoinable()) {
    // Create a thread that watches our internal state and controls which
    // events make it to clients (into the DCProcess event queue).
    m_async_thread = ThreadLauncher::LaunchThread(
        "<lldb.gdb-remote.server.async>",
        GDBRemoteCommunicationReplayServer::AsyncThread, this, nullptr);
  }

  // Wait for handshake.
  m_async_broadcaster.BroadcastEvent(eBroadcastBitAsyncContinue);

  return m_async_thread.IsJoinable();
}

void GDBRemoteCommunicationReplayServer::StopAsyncThread() {
  std::lock_guard<std::recursive_mutex> guard(m_async_thread_state_mutex);

  if (!m_async_thread.IsJoinable())
    return;

  // Request thread to stop.
  m_async_broadcaster.BroadcastEvent(eBroadcastBitAsyncThreadShouldExit);

  // Disconnect client.
  Disconnect();

  // Stop the thread.
  m_async_thread.Join(nullptr);
  m_async_thread.Reset();
}

void GDBRemoteCommunicationReplayServer::ReceivePacket(
    GDBRemoteCommunicationReplayServer &server, bool &done) {
  Status error;
  bool interrupt;
  auto packet_result = server.GetPacketAndSendResponse(std::chrono::seconds(1),
                                                       error, interrupt, done);
  if (packet_result != GDBRemoteCommunication::PacketResult::Success &&
      packet_result !=
          GDBRemoteCommunication::PacketResult::ErrorReplyTimeout) {
    done = true;
  } else {
    server.m_async_broadcaster.BroadcastEvent(eBroadcastBitAsyncContinue);
  }
}

thread_result_t GDBRemoteCommunicationReplayServer::AsyncThread(void *arg) {
  GDBRemoteCommunicationReplayServer *server =
      (GDBRemoteCommunicationReplayServer *)arg;

  EventSP event_sp;
  bool done = false;

  while (true) {
    if (server->m_async_listener_sp->GetEvent(event_sp, llvm::None)) {
      const uint32_t event_type = event_sp->GetType();
      if (event_sp->BroadcasterIs(&server->m_async_broadcaster)) {
        switch (event_type) {
        case eBroadcastBitAsyncContinue:
          ReceivePacket(*server, done);
          if (done)
            return {};
          break;
        case eBroadcastBitAsyncThreadShouldExit:
        default:
          return {};
        }
      }
    }
  }

  return {};
}
<commit_msg>[Reproducers] Fix GDB remote flakiness during replay<commit_after>//===-- GDBRemoteCommunicationReplayServer.cpp ------------------*- C++ -*-===//
//
// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
// See https://llvm.org/LICENSE.txt for license information.
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
//
//===----------------------------------------------------------------------===//

#include <errno.h>

#include "lldb/Host/Config.h"

#include "GDBRemoteCommunicationReplayServer.h"
#include "ProcessGDBRemoteLog.h"

// C Includes
// C++ Includes
#include <cstring>

// Project includes
#include "lldb/Host/ThreadLauncher.h"
#include "lldb/Utility/ConstString.h"
#include "lldb/Utility/Event.h"
#include "lldb/Utility/FileSpec.h"
#include "lldb/Utility/StreamString.h"
#include "lldb/Utility/StringExtractorGDBRemote.h"

using namespace llvm;
using namespace lldb;
using namespace lldb_private;
using namespace lldb_private::process_gdb_remote;

static bool unexpected(llvm::StringRef expected, llvm::StringRef actual) {
  // The 'expected' string contains the raw data, including the leading $ and
  // trailing checksum. The 'actual' string contains only the packet's content.
  if (expected.contains(actual))
    return false;
  // Contains a PID which might be different.
  if (expected.contains("vAttach"))
    return false;
  // Contains a ascii-hex-path.
  if (expected.contains("QSetSTD"))
    return false;
  // Contains environment values.
  if (expected.contains("QEnvironment"))
    return false;

  return true;
}

GDBRemoteCommunicationReplayServer::GDBRemoteCommunicationReplayServer()
    : GDBRemoteCommunication("gdb-replay", "gdb-replay.rx_packet"),
      m_async_broadcaster(nullptr, "lldb.gdb-replay.async-broadcaster"),
      m_async_listener_sp(
          Listener::MakeListener("lldb.gdb-replay.async-listener")),
      m_async_thread_state_mutex(), m_skip_acks(false) {
  m_async_broadcaster.SetEventName(eBroadcastBitAsyncContinue,
                                   "async thread continue");
  m_async_broadcaster.SetEventName(eBroadcastBitAsyncThreadShouldExit,
                                   "async thread should exit");

  const uint32_t async_event_mask =
      eBroadcastBitAsyncContinue | eBroadcastBitAsyncThreadShouldExit;
  m_async_listener_sp->StartListeningForEvents(&m_async_broadcaster,
                                               async_event_mask);
}

GDBRemoteCommunicationReplayServer::~GDBRemoteCommunicationReplayServer() {
  StopAsyncThread();
}

GDBRemoteCommunication::PacketResult
GDBRemoteCommunicationReplayServer::GetPacketAndSendResponse(
    Timeout<std::micro> timeout, Status &error, bool &interrupt, bool &quit) {
  std::lock_guard<std::recursive_mutex> guard(m_async_thread_state_mutex);

  StringExtractorGDBRemote packet;
  PacketResult packet_result = WaitForPacketNoLock(packet, timeout, false);

  if (packet_result != PacketResult::Success) {
    if (!IsConnected()) {
      error.SetErrorString("lost connection");
      quit = true;
    } else {
      error.SetErrorString("timeout");
    }
    return packet_result;
  }

  m_async_broadcaster.BroadcastEvent(eBroadcastBitAsyncContinue);

  // If m_send_acks is true, we're before the handshake phase. We've already
  // acknowledge the '+' packet so we're done here.
  if (m_send_acks && packet.GetStringRef() == "+")
    return PacketResult::Success;

  // This completes the handshake. Since m_send_acks was true, we can unset it
  // already.
  if (packet.GetStringRef() == "QStartNoAckMode")
    m_send_acks = false;

  // A QEnvironment packet is sent for every environment variable. If the
  // number of environment variables is different during replay, the replies
  // become out of sync.
  if (packet.GetStringRef().find("QEnvironment") == 0) {
    return SendRawPacketNoLock("$OK#9a");
  }

  Log *log(ProcessGDBRemoteLog::GetLogIfAllCategoriesSet(GDBR_LOG_PROCESS));
  while (!m_packet_history.empty()) {
    // Pop last packet from the history.
    GDBRemoteCommunicationHistory::Entry entry = m_packet_history.back();
    m_packet_history.pop_back();

    // We're handled the handshake implicitly before. Skip the packet and move
    // on.
    if (entry.packet.data == "+")
      continue;

    if (entry.type == GDBRemoteCommunicationHistory::ePacketTypeSend) {
      if (unexpected(entry.packet.data, packet.GetStringRef())) {
        LLDB_LOG(log,
                 "GDBRemoteCommunicationReplayServer expected packet: '{0}'",
                 entry.packet.data);
        LLDB_LOG(log, "GDBRemoteCommunicationReplayServer actual packet: '{0}'",
                 packet.GetStringRef());
      }

      // Ignore QEnvironment packets as they're handled earlier.
      if (entry.packet.data.find("QEnvironment") == 1) {
        assert(m_packet_history.back().type ==
               GDBRemoteCommunicationHistory::ePacketTypeRecv);
        m_packet_history.pop_back();
      }

      continue;
    }

    if (entry.type == GDBRemoteCommunicationHistory::ePacketTypeInvalid) {
      LLDB_LOG(
          log,
          "GDBRemoteCommunicationReplayServer skipped invalid packet: '{0}'",
          packet.GetStringRef());
      continue;
    }

    LLDB_LOG(log,
             "GDBRemoteCommunicationReplayServer replied to '{0}' with '{1}'",
             packet.GetStringRef(), entry.packet.data);
    return SendRawPacketNoLock(entry.packet.data);
  }

  quit = true;

  return packet_result;
}

LLVM_YAML_IS_DOCUMENT_LIST_VECTOR(
    std::vector<
        lldb_private::process_gdb_remote::GDBRemoteCommunicationHistory::Entry>)

llvm::Error
GDBRemoteCommunicationReplayServer::LoadReplayHistory(const FileSpec &path) {
  auto error_or_file = MemoryBuffer::getFile(path.GetPath());
  if (auto err = error_or_file.getError())
    return errorCodeToError(err);

  yaml::Input yin((*error_or_file)->getBuffer());
  yin >> m_packet_history;

  if (auto err = yin.error())
    return errorCodeToError(err);

  // We want to manipulate the vector like a stack so we need to reverse the
  // order of the packets to have the oldest on at the back.
  std::reverse(m_packet_history.begin(), m_packet_history.end());

  return Error::success();
}

bool GDBRemoteCommunicationReplayServer::StartAsyncThread() {
  std::lock_guard<std::recursive_mutex> guard(m_async_thread_state_mutex);
  if (!m_async_thread.IsJoinable()) {
    // Create a thread that watches our internal state and controls which
    // events make it to clients (into the DCProcess event queue).
    m_async_thread = ThreadLauncher::LaunchThread(
        "<lldb.gdb-replay.async>",
        GDBRemoteCommunicationReplayServer::AsyncThread, this, nullptr);
  }

  // Wait for handshake.
  m_async_broadcaster.BroadcastEvent(eBroadcastBitAsyncContinue);

  return m_async_thread.IsJoinable();
}

void GDBRemoteCommunicationReplayServer::StopAsyncThread() {
  std::lock_guard<std::recursive_mutex> guard(m_async_thread_state_mutex);

  if (!m_async_thread.IsJoinable())
    return;

  // Request thread to stop.
  m_async_broadcaster.BroadcastEvent(eBroadcastBitAsyncThreadShouldExit);

  // Disconnect client.
  Disconnect();

  // Stop the thread.
  m_async_thread.Join(nullptr);
  m_async_thread.Reset();
}

void GDBRemoteCommunicationReplayServer::ReceivePacket(
    GDBRemoteCommunicationReplayServer &server, bool &done) {
  Status error;
  bool interrupt;
  auto packet_result = server.GetPacketAndSendResponse(std::chrono::seconds(1),
                                                       error, interrupt, done);
  if (packet_result != GDBRemoteCommunication::PacketResult::Success &&
      packet_result !=
          GDBRemoteCommunication::PacketResult::ErrorReplyTimeout) {
    done = true;
  } else {
    server.m_async_broadcaster.BroadcastEvent(eBroadcastBitAsyncContinue);
  }
}

thread_result_t GDBRemoteCommunicationReplayServer::AsyncThread(void *arg) {
  GDBRemoteCommunicationReplayServer *server =
      (GDBRemoteCommunicationReplayServer *)arg;

  EventSP event_sp;
  bool done = false;

  while (true) {
    if (server->m_async_listener_sp->GetEvent(event_sp, llvm::None)) {
      const uint32_t event_type = event_sp->GetType();
      if (event_sp->BroadcasterIs(&server->m_async_broadcaster)) {
        switch (event_type) {
        case eBroadcastBitAsyncContinue:
          ReceivePacket(*server, done);
          if (done)
            return {};
          break;
        case eBroadcastBitAsyncThreadShouldExit:
        default:
          return {};
        }
      }
    }
  }

  return {};
}
<|endoftext|>


<commit_before><commit_msg>algorithms: use std::stable_sort<commit_after><|endoftext|>


<commit_before><commit_msg>Treeview may lose focus when left/right arrow is used<commit_after><|endoftext|>


<commit_before>/*************************************************************************
 *
 *  OpenOffice.org - a multi-platform office productivity suite
 *
 *  $RCSfile: numhead.cxx,v $
 *
 *  $Revision: 1.5 $
 *
 *  last change: $Author: rt $ $Date: 2005-09-08 16:34:04 $
 *
 *  The Contents of this file are made available subject to
 *  the terms of GNU Lesser General Public License Version 2.1.
 *
 *
 *    GNU Lesser General Public License Version 2.1
 *    =============================================
 *    Copyright 2005 by Sun Microsystems, Inc.
 *    901 San Antonio Road, Palo Alto, CA 94303, USA
 *
 *    This library is free software; you can redistribute it and/or
 *    modify it under the terms of the GNU Lesser General Public
 *    License version 2.1, as published by the Free Software Foundation.
 *
 *    This library is distributed in the hope that it will be useful,
 *    but WITHOUT ANY WARRANTY; without even the implied warranty of
 *    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
 *    Lesser General Public License for more details.
 *
 *    You should have received a copy of the GNU Lesser General Public
 *    License along with this library; if not, write to the Free Software
 *    Foundation, Inc., 59 Temple Place, Suite 330, Boston,
 *    MA  02111-1307  USA
 *
 ************************************************************************/
#ifndef GCC
#pragma hdrstop
#endif

#ifndef _DEBUG_HXX //autogen
#include <tools/debug.hxx>
#endif

#include "numhead.hxx"

//      ID's fuer Dateien:
#define SV_NUMID_SIZES                      0x4200

// STATIC DATA -----------------------------------------------------------

//SEG_EOFGLOBALS()

// =======================================================================
/*                              wird fuer SvNumberformatter nicht gebraucht
//#pragma SEG_FUNCDEF(numhead_01)

SvNumReadHeader::SvNumReadHeader(SvStream& rNewStream) :
    rStream( rNewStream )
{
    ULONG nDataSize;
    rStream >> nDataSize;
    nDataEnd = rStream.Tell() + nDataSize;
}

//#pragma SEG_FUNCDEF(numhead_02)

SvNumReadHeader::~SvNumReadHeader()
{
    ULONG nReadEnd = rStream.Tell();
    DBG_ASSERT( nReadEnd <= nDataEnd, "zuviele Bytes gelesen" );
    if ( nReadEnd != nDataEnd )
        rStream.Seek(nDataEnd);                     // Rest ueberspringen
}

//#pragma SEG_FUNCDEF(numhead_03)

ULONG SvNumReadHeader::BytesLeft() const
{
    ULONG nReadEnd = rStream.Tell();
    if (nReadEnd <= nDataEnd)
        return nDataEnd-nReadEnd;

    DBG_ERROR("Fehler bei SvNumReadHeader::BytesLeft");
    return 0;
}

// -----------------------------------------------------------------------

//#pragma SEG_FUNCDEF(numhead_04)

SvNumWriteHeader::SvNumWriteHeader(SvStream& rNewStream, ULONG nDefault) :
    rStream( rNewStream )
{
    nDataSize = nDefault;
    rStream << nDataSize;
    nDataPos = rStream.Tell();
}

//#pragma SEG_FUNCDEF(numhead_05)

SvNumWriteHeader::~SvNumWriteHeader()
{
    ULONG nPos = rStream.Tell();

    if ( nPos - nDataPos != nDataSize )             // Default getroffen?
    {
        nDataSize = nPos - nDataPos;
        rStream.Seek(nDataPos - sizeof(ULONG));
        rStream << nDataSize;                       // Groesse am Anfang eintragen
        rStream.Seek(nPos);
    }
}
*/

// =======================================================================

//#pragma SEG_FUNCDEF(numhead_06)

//! mit Skip() synchron
ImpSvNumMultipleReadHeader::ImpSvNumMultipleReadHeader(SvStream& rNewStream) :
    rStream( rNewStream )
{
    ULONG nDataSize;
    rStream >> nDataSize;
    ULONG nDataPos = rStream.Tell();
    nEntryEnd = nDataPos;

    rStream.SeekRel(nDataSize);
    USHORT nID;
    rStream >> nID;
    if (nID != SV_NUMID_SIZES)
    {
        DBG_ERROR("SV_NUMID_SIZES nicht gefunden");
    }
    ULONG nSizeTableLen;
    rStream >> nSizeTableLen;
    pBuf = new char[nSizeTableLen];
    rStream.Read( pBuf, nSizeTableLen );
    pMemStream = new SvMemoryStream( pBuf, nSizeTableLen, STREAM_READ );

    nEndPos = rStream.Tell();
    rStream.Seek( nDataPos );
}

//#pragma SEG_FUNCDEF(numhead_07)

ImpSvNumMultipleReadHeader::~ImpSvNumMultipleReadHeader()
{
    DBG_ASSERT( pMemStream->Tell() == pMemStream->GetSize(),
                "Sizes nicht vollstaendig gelesen" );
    delete pMemStream;
    delete [] pBuf;

    rStream.Seek(nEndPos);
}

//! mit ctor synchron
// static
void ImpSvNumMultipleReadHeader::Skip( SvStream& rStream )
{
    ULONG nDataSize;
    rStream >> nDataSize;
    rStream.SeekRel( nDataSize );
    USHORT nID;
    rStream >> nID;
    if ( nID != SV_NUMID_SIZES )
    {
        DBG_ERROR("SV_NUMID_SIZES nicht gefunden");
    }
    ULONG nSizeTableLen;
    rStream >> nSizeTableLen;
    rStream.SeekRel( nSizeTableLen );
}

//#pragma SEG_FUNCDEF(numhead_08)

void ImpSvNumMultipleReadHeader::EndEntry()
{
    ULONG nPos = rStream.Tell();
    DBG_ASSERT( nPos <= nEntryEnd, "zuviel gelesen" );
    if ( nPos != nEntryEnd )
        rStream.Seek( nEntryEnd );          // Rest ueberspringen
}

//#pragma SEG_FUNCDEF(numhead_0d)

void ImpSvNumMultipleReadHeader::StartEntry()
{
    ULONG nPos = rStream.Tell();
    ULONG nEntrySize;
    (*pMemStream) >> nEntrySize;

    nEntryEnd = nPos + nEntrySize;
}

//#pragma SEG_FUNCDEF(numhead_09)

ULONG ImpSvNumMultipleReadHeader::BytesLeft() const
{
    ULONG nReadEnd = rStream.Tell();
    if (nReadEnd <= nEntryEnd)
        return nEntryEnd-nReadEnd;

    DBG_ERROR("Fehler bei ImpSvNumMultipleReadHeader::BytesLeft");
    return 0;
}

// -----------------------------------------------------------------------

//#pragma SEG_FUNCDEF(numhead_0a)

ImpSvNumMultipleWriteHeader::ImpSvNumMultipleWriteHeader(SvStream& rNewStream,
                                                   ULONG nDefault) :
    rStream( rNewStream ),
    aMemStream( 4096, 4096 )
{
    nDataSize = nDefault;
    rStream << nDataSize;

    nDataPos = rStream.Tell();
    nEntryStart = nDataPos;
}

//#pragma SEG_FUNCDEF(numhead_0b)

ImpSvNumMultipleWriteHeader::~ImpSvNumMultipleWriteHeader()
{
    ULONG nDataEnd = rStream.Tell();

    rStream << (USHORT) SV_NUMID_SIZES;
    rStream << aMemStream.Tell();
    rStream.Write( aMemStream.GetData(), aMemStream.Tell() );

    if ( nDataEnd - nDataPos != nDataSize )                 // Default getroffen?
    {
        nDataSize = nDataEnd - nDataPos;
        ULONG nPos = rStream.Tell();
        rStream.Seek(nDataPos-sizeof(ULONG));
        rStream << nDataSize;                               // Groesse am Anfang eintragen
        rStream.Seek(nPos);
    }
}

//#pragma SEG_FUNCDEF(numhead_0c)

void ImpSvNumMultipleWriteHeader::EndEntry()
{
    ULONG nPos = rStream.Tell();
    aMemStream << nPos - nEntryStart;
}

//#pragma SEG_FUNCDEF(numhead_0e)

void ImpSvNumMultipleWriteHeader::StartEntry()
{
    ULONG nPos = rStream.Tell();
    nEntryStart = nPos;
}

<commit_msg>INTEGRATION: CWS sixtyfour02 (1.5.178); FILE MERGED 2006/02/22 12:02:10 cmc 1.5.178.1: #i62425# streaming longs to disk<commit_after>/*************************************************************************
 *
 *  OpenOffice.org - a multi-platform office productivity suite
 *
 *  $RCSfile: numhead.cxx,v $
 *
 *  $Revision: 1.6 $
 *
 *  last change: $Author: vg $ $Date: 2006-03-16 13:05:56 $
 *
 *  The Contents of this file are made available subject to
 *  the terms of GNU Lesser General Public License Version 2.1.
 *
 *
 *    GNU Lesser General Public License Version 2.1
 *    =============================================
 *    Copyright 2005 by Sun Microsystems, Inc.
 *    901 San Antonio Road, Palo Alto, CA 94303, USA
 *
 *    This library is free software; you can redistribute it and/or
 *    modify it under the terms of the GNU Lesser General Public
 *    License version 2.1, as published by the Free Software Foundation.
 *
 *    This library is distributed in the hope that it will be useful,
 *    but WITHOUT ANY WARRANTY; without even the implied warranty of
 *    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
 *    Lesser General Public License for more details.
 *
 *    You should have received a copy of the GNU Lesser General Public
 *    License along with this library; if not, write to the Free Software
 *    Foundation, Inc., 59 Temple Place, Suite 330, Boston,
 *    MA  02111-1307  USA
 *
 ************************************************************************/
#ifndef GCC
#pragma hdrstop
#endif

#ifndef _DEBUG_HXX //autogen
#include <tools/debug.hxx>
#endif

#include "numhead.hxx"

//      ID's fuer Dateien:
#define SV_NUMID_SIZES                      0x4200

// STATIC DATA -----------------------------------------------------------

//SEG_EOFGLOBALS()

// =======================================================================
/*                              wird fuer SvNumberformatter nicht gebraucht
//#pragma SEG_FUNCDEF(numhead_01)

SvNumReadHeader::SvNumReadHeader(SvStream& rNewStream) :
    rStream( rNewStream )
{
    ULONG nDataSize;
    rStream >> nDataSize;
    nDataEnd = rStream.Tell() + nDataSize;
}

//#pragma SEG_FUNCDEF(numhead_02)

SvNumReadHeader::~SvNumReadHeader()
{
    ULONG nReadEnd = rStream.Tell();
    DBG_ASSERT( nReadEnd <= nDataEnd, "zuviele Bytes gelesen" );
    if ( nReadEnd != nDataEnd )
        rStream.Seek(nDataEnd);                     // Rest ueberspringen
}

//#pragma SEG_FUNCDEF(numhead_03)

ULONG SvNumReadHeader::BytesLeft() const
{
    ULONG nReadEnd = rStream.Tell();
    if (nReadEnd <= nDataEnd)
        return nDataEnd-nReadEnd;

    DBG_ERROR("Fehler bei SvNumReadHeader::BytesLeft");
    return 0;
}

// -----------------------------------------------------------------------

//#pragma SEG_FUNCDEF(numhead_04)

SvNumWriteHeader::SvNumWriteHeader(SvStream& rNewStream, ULONG nDefault) :
    rStream( rNewStream )
{
    nDataSize = nDefault;
    rStream << nDataSize;
    nDataPos = rStream.Tell();
}

//#pragma SEG_FUNCDEF(numhead_05)

SvNumWriteHeader::~SvNumWriteHeader()
{
    ULONG nPos = rStream.Tell();

    if ( nPos - nDataPos != nDataSize )             // Default getroffen?
    {
        nDataSize = nPos - nDataPos;
        rStream.Seek(nDataPos - sizeof(ULONG));
        rStream << nDataSize;                       // Groesse am Anfang eintragen
        rStream.Seek(nPos);
    }
}
*/

// =======================================================================

//#pragma SEG_FUNCDEF(numhead_06)

//! mit Skip() synchron
ImpSvNumMultipleReadHeader::ImpSvNumMultipleReadHeader(SvStream& rNewStream) :
    rStream( rNewStream )
{
    sal_uInt32 nDataSize;
    rStream >> nDataSize;
    ULONG nDataPos = rStream.Tell();
    nEntryEnd = nDataPos;

    rStream.SeekRel(nDataSize);
    USHORT nID;
    rStream >> nID;
    if (nID != SV_NUMID_SIZES)
    {
        DBG_ERROR("SV_NUMID_SIZES nicht gefunden");
    }
    sal_uInt32 nSizeTableLen;
    rStream >> nSizeTableLen;
    pBuf = new char[nSizeTableLen];
    rStream.Read( pBuf, nSizeTableLen );
    pMemStream = new SvMemoryStream( pBuf, nSizeTableLen, STREAM_READ );

    nEndPos = rStream.Tell();
    rStream.Seek( nDataPos );
}

//#pragma SEG_FUNCDEF(numhead_07)

ImpSvNumMultipleReadHeader::~ImpSvNumMultipleReadHeader()
{
    DBG_ASSERT( pMemStream->Tell() == pMemStream->GetSize(),
                "Sizes nicht vollstaendig gelesen" );
    delete pMemStream;
    delete [] pBuf;

    rStream.Seek(nEndPos);
}

//! mit ctor synchron
// static
void ImpSvNumMultipleReadHeader::Skip( SvStream& rStream )
{
    sal_uInt32 nDataSize;
    rStream >> nDataSize;
    rStream.SeekRel( nDataSize );
    USHORT nID;
    rStream >> nID;
    if ( nID != SV_NUMID_SIZES )
    {
        DBG_ERROR("SV_NUMID_SIZES nicht gefunden");
    }
    sal_uInt32 nSizeTableLen;
    rStream >> nSizeTableLen;
    rStream.SeekRel( nSizeTableLen );
}

//#pragma SEG_FUNCDEF(numhead_08)

void ImpSvNumMultipleReadHeader::EndEntry()
{
    ULONG nPos = rStream.Tell();
    DBG_ASSERT( nPos <= nEntryEnd, "zuviel gelesen" );
    if ( nPos != nEntryEnd )
        rStream.Seek( nEntryEnd );          // Rest ueberspringen
}

//#pragma SEG_FUNCDEF(numhead_0d)

void ImpSvNumMultipleReadHeader::StartEntry()
{
    ULONG nPos = rStream.Tell();
    sal_uInt32 nEntrySize;
    (*pMemStream) >> nEntrySize;

    nEntryEnd = nPos + nEntrySize;
}

//#pragma SEG_FUNCDEF(numhead_09)

ULONG ImpSvNumMultipleReadHeader::BytesLeft() const
{
    ULONG nReadEnd = rStream.Tell();
    if (nReadEnd <= nEntryEnd)
        return nEntryEnd-nReadEnd;

    DBG_ERROR("Fehler bei ImpSvNumMultipleReadHeader::BytesLeft");
    return 0;
}

// -----------------------------------------------------------------------

//#pragma SEG_FUNCDEF(numhead_0a)

ImpSvNumMultipleWriteHeader::ImpSvNumMultipleWriteHeader(SvStream& rNewStream,
                                                   ULONG nDefault) :
    rStream( rNewStream ),
    aMemStream( 4096, 4096 )
{
    nDataSize = nDefault;
    rStream << nDataSize;

    nDataPos = rStream.Tell();
    nEntryStart = nDataPos;
}

//#pragma SEG_FUNCDEF(numhead_0b)

ImpSvNumMultipleWriteHeader::~ImpSvNumMultipleWriteHeader()
{
    ULONG nDataEnd = rStream.Tell();

    rStream << (USHORT) SV_NUMID_SIZES;
    rStream << static_cast<sal_uInt32>(aMemStream.Tell());
    rStream.Write( aMemStream.GetData(), aMemStream.Tell() );

    if ( nDataEnd - nDataPos != nDataSize )                 // Default getroffen?
    {
        nDataSize = nDataEnd - nDataPos;
        ULONG nPos = rStream.Tell();
        rStream.Seek(nDataPos-sizeof(ULONG));
        rStream << nDataSize;                               // Groesse am Anfang eintragen
        rStream.Seek(nPos);
    }
}

//#pragma SEG_FUNCDEF(numhead_0c)

void ImpSvNumMultipleWriteHeader::EndEntry()
{
    ULONG nPos = rStream.Tell();
    aMemStream << static_cast<sal_uInt32>(nPos - nEntryStart);
}

//#pragma SEG_FUNCDEF(numhead_0e)

void ImpSvNumMultipleWriteHeader::StartEntry()
{
    ULONG nPos = rStream.Tell();
    nEntryStart = nPos;
}

<|endoftext|>


<commit_before><commit_msg>Cleaning up commented out code.<commit_after><|endoftext|>


<commit_before>#include "recorder_base.h"
#include "globals.h"
#include "threads.h"
#include "sift_assert.h"
#include "recorder_control.h"
#include "syscall_modeling.h"

#include <iostream>

VOID countInsns(THREADID threadid, INT32 count)
{
   thread_data[threadid].icount += count;

   if (thread_data[threadid].icount >= fast_forward_target && !KnobUseROI.Value() && !KnobMPIImplicitROI.Value())
   {
      if (KnobVerbose.Value())
         std::cerr << "[SIFT_RECORDER:" << app_id << ":" << thread_data[threadid].thread_num << "] Changing to detailed after " << thread_data[threadid].icount << " instructions" << std::endl;
      if (!thread_data[threadid].output)
         openFile(threadid);
      thread_data[threadid].icount = 0;
      any_thread_in_detail = true;
      PIN_RemoveInstrumentation();
   }
}

VOID sendInstruction(THREADID threadid, ADDRINT addr, UINT32 size, UINT32 num_addresses, BOOL is_branch, BOOL taken, BOOL is_predicate, BOOL executing, BOOL isbefore, BOOL ispause)
{
   // We're still called for instructions in the same basic block as ROI end, ignore these
   if (!thread_data[threadid].output)
      return;

   ++thread_data[threadid].icount;
   ++thread_data[threadid].icount_detailed;

   // Reconstruct basic blocks (we could ask Pin, but do it the same way as TraceThread will do it)
   if (thread_data[threadid].bbv_end || thread_data[threadid].bbv_last != addr)
   {
      // We're the start of a new basic block
      thread_data[threadid].bbv->count(thread_data[threadid].bbv_base, thread_data[threadid].bbv_count);
      thread_data[threadid].bbv_base = addr;
      thread_data[threadid].bbv_count = 0;
   }
   thread_data[threadid].bbv_count++;
   thread_data[threadid].bbv_last = addr + size;
   // Force BBV end on non-taken branches
   thread_data[threadid].bbv_end = is_branch;

   uint64_t addresses[Sift::MAX_DYNAMIC_ADDRESSES] = { 0 };
   for(uint8_t i = 0; i < num_addresses; ++i)
   {
      addresses[i] = thread_data[threadid].dyn_address_queue->front();
      sift_assert(!thread_data[threadid].dyn_address_queue->empty());
      thread_data[threadid].dyn_address_queue->pop_front();

      if (isbefore)
      {
         // If the instruction hasn't executed yet, access the address to ensure a page fault if the mapping wasn't set up yet
         static char dummy = 0;
         dummy += *(char *)addresses[i];
      }
   }
   sift_assert(thread_data[threadid].dyn_address_queue->empty());

   thread_data[threadid].output->Instruction(addr, size, num_addresses, addresses, is_branch, taken, is_predicate, executing);

   if (KnobUseResponseFiles.Value() && KnobFlowControl.Value() && (thread_data[threadid].icount > thread_data[threadid].flowcontrol_target || ispause))
   {
      thread_data[threadid].output->Sync();
      thread_data[threadid].flowcontrol_target = thread_data[threadid].icount + KnobFlowControl.Value();
   }

   if (detailed_target != 0 && thread_data[threadid].icount_detailed >= detailed_target)
   {
      closeFile(threadid);
      PIN_Detach();
      return;
   }

   if (blocksize && thread_data[threadid].icount >= blocksize)
   {
      openFile(threadid);
      thread_data[threadid].icount = 0;
   }
}

VOID handleMemory(THREADID threadid, ADDRINT address)
{
   // We're still called for instructions in the same basic block as ROI end, ignore these
   if (!thread_data[threadid].output)
      return;

   thread_data[threadid].dyn_address_queue->push_back(address);
}

UINT32 addMemoryModeling(INS ins)
{
   UINT32 num_addresses = 0;

   if (INS_IsMemoryRead (ins) || INS_IsMemoryWrite (ins))
   {
      for (unsigned int i = 0; i < INS_MemoryOperandCount(ins); i++)
      {
         INS_InsertCall(ins, IPOINT_BEFORE,
               AFUNPTR(handleMemory),
               IARG_THREAD_ID,
               IARG_MEMORYOP_EA, i,
               IARG_END);
         num_addresses++;
      }
   }
   sift_assert(num_addresses <= Sift::MAX_DYNAMIC_ADDRESSES);

   return num_addresses;
}

VOID insertCall(INS ins, IPOINT ipoint, UINT32 num_addresses, BOOL is_branch, BOOL taken)
{
   INS_InsertCall(ins, ipoint,
      AFUNPTR(sendInstruction),
      IARG_THREAD_ID,
      IARG_ADDRINT, INS_Address(ins),
      IARG_UINT32, UINT32(INS_Size(ins)),
      IARG_UINT32, num_addresses,
      IARG_BOOL, is_branch,
      IARG_BOOL, taken,
      IARG_BOOL, INS_IsPredicated(ins),
      IARG_EXECUTING,
      IARG_BOOL, ipoint == IPOINT_BEFORE,
      IARG_BOOL, INS_Opcode(ins) == XED_ICLASS_PAUSE,
      IARG_END);
}

static VOID traceCallback(TRACE trace, void *v)
{
   BBL bbl_head = TRACE_BblHead(trace);

   for (BBL bbl = bbl_head; BBL_Valid(bbl); bbl = BBL_Next(bbl))
   {
      for(INS ins = BBL_InsHead(bbl); ; ins = INS_Next(ins))
      {
         // Simics-style magic instruction: xchg bx, bx
         if (INS_IsXchg(ins) && INS_OperandReg(ins, 0) == REG_BX && INS_OperandReg(ins, 1) == REG_BX)
         {
            INS_InsertPredicatedCall(ins, IPOINT_BEFORE, (AFUNPTR)handleMagic, IARG_RETURN_REGS, REG_GAX, IARG_THREAD_ID, IARG_REG_VALUE, REG_GAX, IARG_REG_VALUE, REG_GBX, IARG_REG_VALUE, REG_GCX, IARG_END);
         }

         // Handle emulated syscalls
         if (INS_IsSyscall(ins))
         {
            INS_InsertPredicatedCall(ins, IPOINT_BEFORE, AFUNPTR(emulateSyscallFunc), IARG_THREAD_ID, IARG_CONST_CONTEXT, IARG_END);
         }

         if (ins == BBL_InsTail(bbl))
            break;
      }

      if (!any_thread_in_detail)
      {
         BBL_InsertCall(bbl, IPOINT_ANYWHERE, (AFUNPTR)countInsns, IARG_THREAD_ID, IARG_UINT32, BBL_NumIns(bbl), IARG_END);
      }
      else
      {
         for(INS ins = BBL_InsHead(bbl); ; ins = INS_Next(ins))
         {
            // For memory instructions, we should populate data items before we send the MicroOp
            UINT32 num_addresses = addMemoryModeling(ins);

            bool is_branch = INS_IsBranch(ins) && INS_HasFallThrough(ins);

            if (is_branch)
            {
               insertCall(ins, IPOINT_AFTER,        num_addresses, true  /* is_branch */, false /* taken */);
               insertCall(ins, IPOINT_TAKEN_BRANCH, num_addresses, true  /* is_branch */, true  /* taken */);
            }
            else
            {
               // Whenever possible, use IPOINT_AFTER as this allows us to process addresses after the application has used them.
               // This ensures that their logical to physical mapping has been set up.
               insertCall(ins, INS_HasFallThrough(ins) ? IPOINT_AFTER : IPOINT_BEFORE, num_addresses, false /* is_branch */, false /* taken */);
            }

            if (ins == BBL_InsTail(bbl))
               break;
         }
      }
   }
}

void initRecorderBase()
{
   TRACE_AddInstrumentFunction(traceCallback, 0);
}
<commit_msg>[sift_recorder] Fix magic instruction support for the 32-bit architecture<commit_after>#include "recorder_base.h"
#include "globals.h"
#include "threads.h"
#include "sift_assert.h"
#include "recorder_control.h"
#include "syscall_modeling.h"

#include <iostream>

VOID countInsns(THREADID threadid, INT32 count)
{
   thread_data[threadid].icount += count;

   if (thread_data[threadid].icount >= fast_forward_target && !KnobUseROI.Value() && !KnobMPIImplicitROI.Value())
   {
      if (KnobVerbose.Value())
         std::cerr << "[SIFT_RECORDER:" << app_id << ":" << thread_data[threadid].thread_num << "] Changing to detailed after " << thread_data[threadid].icount << " instructions" << std::endl;
      if (!thread_data[threadid].output)
         openFile(threadid);
      thread_data[threadid].icount = 0;
      any_thread_in_detail = true;
      PIN_RemoveInstrumentation();
   }
}

VOID sendInstruction(THREADID threadid, ADDRINT addr, UINT32 size, UINT32 num_addresses, BOOL is_branch, BOOL taken, BOOL is_predicate, BOOL executing, BOOL isbefore, BOOL ispause)
{
   // We're still called for instructions in the same basic block as ROI end, ignore these
   if (!thread_data[threadid].output)
      return;

   ++thread_data[threadid].icount;
   ++thread_data[threadid].icount_detailed;

   // Reconstruct basic blocks (we could ask Pin, but do it the same way as TraceThread will do it)
   if (thread_data[threadid].bbv_end || thread_data[threadid].bbv_last != addr)
   {
      // We're the start of a new basic block
      thread_data[threadid].bbv->count(thread_data[threadid].bbv_base, thread_data[threadid].bbv_count);
      thread_data[threadid].bbv_base = addr;
      thread_data[threadid].bbv_count = 0;
   }
   thread_data[threadid].bbv_count++;
   thread_data[threadid].bbv_last = addr + size;
   // Force BBV end on non-taken branches
   thread_data[threadid].bbv_end = is_branch;

   uint64_t addresses[Sift::MAX_DYNAMIC_ADDRESSES] = { 0 };
   for(uint8_t i = 0; i < num_addresses; ++i)
   {
      addresses[i] = thread_data[threadid].dyn_address_queue->front();
      sift_assert(!thread_data[threadid].dyn_address_queue->empty());
      thread_data[threadid].dyn_address_queue->pop_front();

      if (isbefore)
      {
         // If the instruction hasn't executed yet, access the address to ensure a page fault if the mapping wasn't set up yet
         static char dummy = 0;
         dummy += *(char *)addresses[i];
      }
   }
   sift_assert(thread_data[threadid].dyn_address_queue->empty());

   thread_data[threadid].output->Instruction(addr, size, num_addresses, addresses, is_branch, taken, is_predicate, executing);

   if (KnobUseResponseFiles.Value() && KnobFlowControl.Value() && (thread_data[threadid].icount > thread_data[threadid].flowcontrol_target || ispause))
   {
      thread_data[threadid].output->Sync();
      thread_data[threadid].flowcontrol_target = thread_data[threadid].icount + KnobFlowControl.Value();
   }

   if (detailed_target != 0 && thread_data[threadid].icount_detailed >= detailed_target)
   {
      closeFile(threadid);
      PIN_Detach();
      return;
   }

   if (blocksize && thread_data[threadid].icount >= blocksize)
   {
      openFile(threadid);
      thread_data[threadid].icount = 0;
   }
}

VOID handleMemory(THREADID threadid, ADDRINT address)
{
   // We're still called for instructions in the same basic block as ROI end, ignore these
   if (!thread_data[threadid].output)
      return;

   thread_data[threadid].dyn_address_queue->push_back(address);
}

UINT32 addMemoryModeling(INS ins)
{
   UINT32 num_addresses = 0;

   if (INS_IsMemoryRead (ins) || INS_IsMemoryWrite (ins))
   {
      for (unsigned int i = 0; i < INS_MemoryOperandCount(ins); i++)
      {
         INS_InsertCall(ins, IPOINT_BEFORE,
               AFUNPTR(handleMemory),
               IARG_THREAD_ID,
               IARG_MEMORYOP_EA, i,
               IARG_END);
         num_addresses++;
      }
   }
   sift_assert(num_addresses <= Sift::MAX_DYNAMIC_ADDRESSES);

   return num_addresses;
}

VOID insertCall(INS ins, IPOINT ipoint, UINT32 num_addresses, BOOL is_branch, BOOL taken)
{
   INS_InsertCall(ins, ipoint,
      AFUNPTR(sendInstruction),
      IARG_THREAD_ID,
      IARG_ADDRINT, INS_Address(ins),
      IARG_UINT32, UINT32(INS_Size(ins)),
      IARG_UINT32, num_addresses,
      IARG_BOOL, is_branch,
      IARG_BOOL, taken,
      IARG_BOOL, INS_IsPredicated(ins),
      IARG_EXECUTING,
      IARG_BOOL, ipoint == IPOINT_BEFORE,
      IARG_BOOL, INS_Opcode(ins) == XED_ICLASS_PAUSE,
      IARG_END);
}

static VOID traceCallback(TRACE trace, void *v)
{
   BBL bbl_head = TRACE_BblHead(trace);

   for (BBL bbl = bbl_head; BBL_Valid(bbl); bbl = BBL_Next(bbl))
   {
      for(INS ins = BBL_InsHead(bbl); ; ins = INS_Next(ins))
      {
         // Simics-style magic instruction: xchg bx, bx
         if (INS_IsXchg(ins) && INS_OperandReg(ins, 0) == REG_BX && INS_OperandReg(ins, 1) == REG_BX)
         {
            INS_InsertPredicatedCall(ins, IPOINT_BEFORE, (AFUNPTR)handleMagic, IARG_RETURN_REGS, REG_GAX, IARG_THREAD_ID, IARG_REG_VALUE, REG_GAX,
#ifdef TARGET_IA32
                                     IARG_REG_VALUE, REG_GDX,
#else
                                     IARG_REG_VALUE, REG_GBX,
#endif
                                     IARG_REG_VALUE, REG_GCX, IARG_END);
         }

         // Handle emulated syscalls
         if (INS_IsSyscall(ins))
         {
            INS_InsertPredicatedCall(ins, IPOINT_BEFORE, AFUNPTR(emulateSyscallFunc), IARG_THREAD_ID, IARG_CONST_CONTEXT, IARG_END);
         }

         if (ins == BBL_InsTail(bbl))
            break;
      }

      if (!any_thread_in_detail)
      {
         BBL_InsertCall(bbl, IPOINT_ANYWHERE, (AFUNPTR)countInsns, IARG_THREAD_ID, IARG_UINT32, BBL_NumIns(bbl), IARG_END);
      }
      else
      {
         for(INS ins = BBL_InsHead(bbl); ; ins = INS_Next(ins))
         {
            // For memory instructions, we should populate data items before we send the MicroOp
            UINT32 num_addresses = addMemoryModeling(ins);

            bool is_branch = INS_IsBranch(ins) && INS_HasFallThrough(ins);

            if (is_branch)
            {
               insertCall(ins, IPOINT_AFTER,        num_addresses, true  /* is_branch */, false /* taken */);
               insertCall(ins, IPOINT_TAKEN_BRANCH, num_addresses, true  /* is_branch */, true  /* taken */);
            }
            else
            {
               // Whenever possible, use IPOINT_AFTER as this allows us to process addresses after the application has used them.
               // This ensures that their logical to physical mapping has been set up.
               insertCall(ins, INS_HasFallThrough(ins) ? IPOINT_AFTER : IPOINT_BEFORE, num_addresses, false /* is_branch */, false /* taken */);
            }

            if (ins == BBL_InsTail(bbl))
               break;
         }
      }
   }
}

void initRecorderBase()
{
   TRACE_AddInstrumentFunction(traceCallback, 0);
}
<|endoftext|>


<commit_before>/* This is free and unencumbered software released into the public domain. */

#ifdef HAVE_CONFIG_H
#include <config.h>
#endif

#include "rdfjson.h"

#include <cassert> /* for assert() */

namespace {
  struct implementation : public rdf::writer::implementation {
    implementation(FILE* stream,
      const char* content_type,
      const char* charset,
      const char* base_uri);
    virtual ~implementation() noexcept override;
    virtual void configure(const char* key, const char* value) override;
    virtual void begin() override;
    virtual void finish() override;
    virtual void write_triple(const rdf::triple& triple) override;
    virtual void write_quad(const rdf::quad& quad) override;
    virtual void write_comment(const char* comment) override;
    virtual void flush() override;
  };
}

rdf::writer::implementation*
rdf_writer_for_rdfjson(FILE* const stream,
                      const char* const content_type,
                      const char* const charset,
                      const char* const base_uri) {
  (void)stream, (void)content_type, (void)charset, (void)base_uri;
  return nullptr; // TODO
}
<commit_msg>Implemented a simple native RDF/JSON serializer.<commit_after>/* This is free and unencumbered software released into the public domain. */

#ifdef HAVE_CONFIG_H
#include <config.h>
#endif

#include "rdfjson.h"
#include <rfc/json/json_writer.h>

#include <cassert> /* for assert() */
#include <map>     /* for std::map */
#include <memory>  /* for std::unique_ptr */
#include <vector>  /* for std::vector */

using namespace rfc4627;

namespace {
  struct implementation : public rdf::writer::implementation {
    implementation(FILE* stream,
      const char* content_type,
      const char* charset,
      const char* base_uri);
    virtual ~implementation() noexcept override;
    virtual void configure(const char* key, const char* value) override;
    virtual void begin() override;
    virtual void finish() override;
    virtual void write_triple(const rdf::triple& triple) override;
    virtual void write_quad(const rdf::quad& quad) override;
    virtual void write_comment(const char* comment) override;
    virtual void flush() override;

  protected:
    void write_object_term(const rdf::term& term);

  private:
    using object_set    = std::vector<std::unique_ptr<rdf::term>>;
    using predicate_map = std::map<std::string, object_set>;
    using subject_map   = std::map<std::string, predicate_map>;
    json_writer _json;
    subject_map _data;
  };
}

rdf::writer::implementation*
rdf_writer_for_rdfjson(FILE* const stream,
                      const char* const content_type,
                      const char* const charset,
                      const char* const base_uri) {
  return new implementation(stream, content_type, charset, base_uri);
}

implementation::implementation(FILE* const stream,
                               const char* const content_type,
                               const char* const charset,
                               const char* const base_uri)
  : _json{stream} {
  assert(stream != nullptr);
  static_cast<void>(content_type);
  static_cast<void>(charset);
  static_cast<void>(base_uri);
}

implementation::~implementation() noexcept {}

void
implementation::configure(const char* const key,
                          const char* const value) {
  static_cast<void>(key); // TODO
  static_cast<void>(value);
}

void
implementation::begin() {
  /* @see http://www.w3.org/TR/rdf-json/#section-serialization */
  _json.begin_object();
}

void
implementation::finish() {
  /* @see http://www.w3.org/TR/rdf-json/#section-serialization */
  auto& subject_map = _data;
  for (auto& element : subject_map) {
    const auto& subject_string = element.first.c_str();
    auto& predicate_map = element.second;

    _json.write_string(subject_string);
    _json.begin_object();

    for (auto& element : predicate_map) {
      const auto& predicate_uri = element.first.c_str();
      auto& object_set = element.second;

      _json.write_string(predicate_uri);
      _json.begin_array();

      for (const auto& object : object_set) {
        write_object_term(*object);
      }
      object_set.clear();

      _json.finish_array();
    }
    predicate_map.clear();

    _json.finish_object();
  }
  subject_map.clear();

  _json.finish_object();
}

void
implementation::write_triple(const rdf::triple& triple) {
  const auto& subject   = *triple.subject;
  const auto& predicate = *triple.predicate;
  const auto& object    = *triple.object;

  _data[subject.string][predicate.string].emplace_back(object.clone());

  _count++;
}

void
implementation::write_quad(const rdf::quad& quad) {
  const auto& subject   = *quad.subject;
  const auto& predicate = *quad.predicate;
  const auto& object    = *quad.object;

  _data[subject.string][predicate.string].emplace_back(object.clone());

  _count++;
}

void
implementation::write_comment(const char* const comment) {
  static_cast<void>(comment); /* JSON doesn't support comments */
}

void
implementation::flush() {
  _json.flush();
}

void
implementation::write_object_term(const rdf::term& term_) {
  _json.begin_object();

  switch (term_.type) {
    case rdf::term_type::uri_reference: {
      const auto& term = dynamic_cast<const rdf::uri_reference&>(term_);
      _json.write_string("type");
      _json.write_string("uri");
      _json.write_string("value");
      _json.write_string(term.string.c_str());
      break;
    }

    case rdf::term_type::blank_node: {
      const auto& term = dynamic_cast<const rdf::blank_node&>(term_);
      _json.write_string("type");
      _json.write_string("bnode");
      _json.write_string("value");
      std::string buffer{"_:"};
      buffer.append(term.string);
      _json.write_string(buffer.c_str());
      break;
    }

    case rdf::term_type::plain_literal: {
      const auto& term = dynamic_cast<const rdf::plain_literal&>(term_);
      _json.write_string("type");
      _json.write_string("literal");
      _json.write_string("value");
      _json.write_string(term.string.c_str());
      if (!term.language_tag.empty()) {
        _json.write_string("lang");
        _json.write_string(term.language_tag.c_str());
      }
      break;
    }

    case rdf::term_type::typed_literal: {
      const auto& term = dynamic_cast<const rdf::typed_literal&>(term_);
      _json.write_string("type");
      _json.write_string("literal");
      _json.write_string("value");
      _json.write_string(term.string.c_str());
      _json.write_string("datatype");
      _json.write_string(term.datatype_uri.c_str());
      break;
    }

    case rdf::term_type::none:
    default:
      assert(false && "invalid term type for #write_object_term");
  }

  _json.finish_object();
}
<|endoftext|>


<commit_before>/*
 * bacteria-core, core for cellular automaton
 * Copyright (C) 2016 Pavel Dolgov
 *
 * See the LICENSE file for terms of use.
 */

#ifndef CORE_CONSTANTS_HPP_
#define CORE_CONSTANTS_HPP_

/** Mass increase per one 'eat' command */
const int EAT_MASS = 1;

/** Mass increase per one 'go' command */
const int GO_MASS = -1;

/** Mass increase per one 'clon' command */
const int CLON_MASS = -10;

/** Mass increase per one 'str' command */
const int STR_MASS = -1;

/** Max additional damage per one 'str' command */
const int MAX_STR_DAMAGE = -2;

const int MAX_COMMANDS_PER_INSTRUCTION = 100;

const int MIN_COMMANDS_PER_INSTRUCTION = 1;

/** Maximum randomly generated number of actions (per instruction) */
const int RANDOM_MAX_ACTIONS = 5;

/** Maximum number of actions per move */
const int MAX_ACTIONS = 1;

/** Maximum number of pseudo actions per move */
const int MAX_PSEUDO_ACTIONS = 30;

/** Default mass of new bacteria */
const int DEFAULT_MASS = 5;

/** Default mass of cloned bacteria */
const int DEFAULT_CLON_MASS = 2;

/** Minimum width of board */
const int MIN_WIDTH = 5;

/** Maximum width of board */
const int MAX_WIDTH = 500;

/** Minimum height of board */
const int MIN_HEIGHT = 5;

/** Maximum height of board */
const int MAX_HEIGHT = 500;

#endif
<commit_msg>Add constant PSEUDO_ACTIONS_EXCESS_PENALTY<commit_after>/*
 * bacteria-core, core for cellular automaton
 * Copyright (C) 2016 Pavel Dolgov
 *
 * See the LICENSE file for terms of use.
 */

#ifndef CORE_CONSTANTS_HPP_
#define CORE_CONSTANTS_HPP_

/** Mass increase per one 'eat' command */
const int EAT_MASS = 1;

/** Mass increase per one 'go' command */
const int GO_MASS = -1;

/** Mass increase per one 'clon' command */
const int CLON_MASS = -10;

/** Mass increase per one 'str' command */
const int STR_MASS = -1;

/** Max additional damage per one 'str' command */
const int MAX_STR_DAMAGE = -2;

const int MAX_COMMANDS_PER_INSTRUCTION = 100;

const int MIN_COMMANDS_PER_INSTRUCTION = 1;

/** Maximum randomly generated number of actions (per instruction) */
const int RANDOM_MAX_ACTIONS = 5;

/** Maximum number of actions per move */
const int MAX_ACTIONS = 1;

/** Maximum number of pseudo actions per move */
const int MAX_PSEUDO_ACTIONS = 30;

/** Mass penalty (if pseudo actions >= MAX_PSEUDO_ACTIONS) */
const int PSEUDO_ACTIONS_EXCESS_PENALTY = -5;

/** Default mass of new bacteria */
const int DEFAULT_MASS = 5;

/** Default mass of cloned bacteria */
const int DEFAULT_CLON_MASS = 2;

/** Minimum width of board */
const int MIN_WIDTH = 5;

/** Maximum width of board */
const int MAX_WIDTH = 500;

/** Minimum height of board */
const int MIN_HEIGHT = 5;

/** Maximum height of board */
const int MAX_HEIGHT = 500;

#endif
<|endoftext|>


<commit_before>#pragma once

#include <sstream>
#include <iod/utils.hh>
#include <silicon/symbols.hh>

namespace sl
{

  using s::_primary_key;
  using s::_auto_increment;
  using s::_read_only;

  namespace sql_orm_internals
  {
    auto remove_members_with_attribute = [] (const auto& o, const auto& a)
    {
      typedef std::decay_t<decltype(a)> A;
      return foreach2(o) | [&] (auto& m)
      {
        typedef typename std::decay_t<decltype(m)>::attributes_type attrs;
        return ::iod::static_if<!has_symbol<attrs, A>::value>(
          [&] () { return m; },
          [&] () {});
      };
    };

    auto extract_members_with_attribute = [] (const auto& o, const auto& a)
    {
      typedef std::decay_t<decltype(a)> A;
      return foreach2(o) | [&] (auto& m)
      {
        typedef typename std::decay_t<decltype(m)>::attributes_type attrs;
        return ::iod::static_if<has_symbol<attrs, A>::value>(
          [&] () { return m; },
          [&] () {});
      };
    };

    template <typename T>
    using remove_auto_increment_t = decltype(remove_members_with_attribute(std::declval<T>(), _auto_increment));
    template <typename T>
    using remove_read_only_fields_t = decltype(remove_members_with_attribute(std::declval<T>(), _read_only));
    template <typename T>
    using extract_primary_keys_t = decltype(extract_members_with_attribute(std::declval<T>(), _primary_key));
    template <typename T>
    using remove_primary_keys_t = decltype(remove_members_with_attribute(std::declval<T>(), _primary_key));

  }

  template <typename C, typename O>
  struct sql_orm
  {
    typedef O object_type;

    // O without auto increment for create procedure.
    typedef sql_orm_internals::remove_auto_increment_t<O> without_auto_inc_type;

    // Object with only the primary keys for the delete and update procedures.
    typedef sql_orm_internals::extract_primary_keys_t<O> PKS;

    static_assert(!std::is_same<PKS, void>::value, "You must set at least one member of the CRUD object as primary key.");

    sql_orm(const std::string& table, C& con) : table_name_(table), con_(con) {}

    int find_by_id(int id, O& o)
    {
      return con_("SELECT * from " + table_name_ + " where id = ?")(id) >> o;
    }

    // save all fields except auto increment.
    // The db will automatically fill auto increment keys.
    template <typename N>
    int insert(const N& o)
    {
      std::stringstream ss;
      std::stringstream vs;
      ss << "INSERT into " << table_name_ << "(";

      bool first = true;
      auto values = foreach(without_auto_inc_type()) | [&] (auto& m) {
        if (!first) { ss << ","; vs << ","; }
        first = false;
        ss << m.symbol().name();
        vs << "?";
        return m.symbol() = m.symbol().member_access(o);
      };

      ss << ") VALUES (" << vs.str() << ")";
      auto req = con_(ss.str());
      apply(values, req);
      return req.last_insert_id();

    };

    // Iterate on all the rows of the table.
    template <typename F>
    void forall(F f)
    {
      std::stringstream ss;
      ss << "SELECT * from " << table_name_;
      con_(ss.str())() | f;
    }

    // Update N's members except auto increment members.
    // N must have at least one primary key.
    template <typename N>
    int update(const N& o)
    {
      // check if N has at least one member of PKS.
      auto pk = intersect(o, PKS());
      static_assert(decltype(pk)::size() > 0, "You must provide at least one primary key to update an object.");
      std::stringstream ss;
      ss << "UPDATE " << table_name_ << " SET ";

      bool first = true;
      auto values = foreach(o) | [&] (auto& m) {
        if (!first) ss << ",";
        first = false;
        ss << m.symbol().name() << " = ?";
        return m;
      };

      ss << " WHERE ";

      first = true;
      auto pks_values = foreach(pk) | [&] (auto& m) {
        if (!first) ss << " and ";
        first = false;
        ss << m.symbol().name() << " = ? ";
        return m.symbol() = m.value();
      };

      auto stmt = con_(ss.str());
      apply(values, pks_values, stmt);
      return stmt.last_insert_id();
    }

    template <typename T>
    void destroy(const T& o)
    {
      std::stringstream ss;
      ss << "DELETE from " << table_name_ << " WHERE ";

      bool first = true;
      auto values = foreach(PKS()) | [&] (auto& m) {
        if (!first) ss << " and ";
        first = false;
        ss << m.symbol().name() << " = ? ";
        return m.symbol() = o[m.symbol()];
      };

      apply(values, con_(ss.str()));
    }

    std::string table_name_;
    C con_;
  };

  struct sqlite_connection;
  struct mysql_connection;

  template <typename C, typename O>
  struct sql_orm_factory
  {
    typedef O object_type;
    typedef sql_orm<C, O> instance_type;

    sql_orm_factory(const std::string& table_name) : table_name_(table_name) {}

    void initialize(C& c)
    {
      std::stringstream ss;
      ss << "CREATE TABLE if not exists " <<  table_name_ << " (";

      bool first = true;
      foreach(O()) | [&] (auto& m)
      {
        if (!first) ss << ", ";
        ss << m.symbol().name() << " " << c.type_to_string(m.value());

        if (std::is_same<C, sqlite_connection>::value)
        {
          if (m.attributes().has(_auto_increment) || m.attributes().has(_primary_key))
            ss << " PRIMARY KEY ";
        }

        if (std::is_same<C, mysql_connection>::value)
        {
          if (m.attributes().has(_auto_increment))
            ss << " AUTO_INCREMENT NOT NULL";
          if (m.attributes().has(_primary_key))
            ss << " PRIMARY KEY ";
        }

        // To activate when pgsql_connection is implemented.
        // if (std::is_same<C, pgsql_connection>::value and
        //     m.attributes().has(_auto_increment))
        //   ss << " SERIAL ";

        first = false;
      };
      ss << ");";
      try
      {
        c(ss.str())();
      }
      catch (std::exception e)
      {
        std::cout << "Warning: Silicon could not create the " << table_name_ << " sql table." << std::endl
                  << "You can ignore this message if the table already exists."
                  << "The sql error is: " << e.what() << std::endl;
      }
    }

    sql_orm<C, O> instantiate(C& con) {
      return sql_orm<C, O>(table_name_, con);
    };

    std::string table_name_;
  };

}
<commit_msg>Add iostream header<commit_after>#pragma once

#include <iostream>
#include <sstream>
#include <iod/utils.hh>
#include <silicon/symbols.hh>

namespace sl
{

  using s::_primary_key;
  using s::_auto_increment;
  using s::_read_only;

  namespace sql_orm_internals
  {
    auto remove_members_with_attribute = [] (const auto& o, const auto& a)
    {
      typedef std::decay_t<decltype(a)> A;
      return foreach2(o) | [&] (auto& m)
      {
        typedef typename std::decay_t<decltype(m)>::attributes_type attrs;
        return ::iod::static_if<!has_symbol<attrs, A>::value>(
          [&] () { return m; },
          [&] () {});
      };
    };

    auto extract_members_with_attribute = [] (const auto& o, const auto& a)
    {
      typedef std::decay_t<decltype(a)> A;
      return foreach2(o) | [&] (auto& m)
      {
        typedef typename std::decay_t<decltype(m)>::attributes_type attrs;
        return ::iod::static_if<has_symbol<attrs, A>::value>(
          [&] () { return m; },
          [&] () {});
      };
    };

    template <typename T>
    using remove_auto_increment_t = decltype(remove_members_with_attribute(std::declval<T>(), _auto_increment));
    template <typename T>
    using remove_read_only_fields_t = decltype(remove_members_with_attribute(std::declval<T>(), _read_only));
    template <typename T>
    using extract_primary_keys_t = decltype(extract_members_with_attribute(std::declval<T>(), _primary_key));
    template <typename T>
    using remove_primary_keys_t = decltype(remove_members_with_attribute(std::declval<T>(), _primary_key));

  }

  template <typename C, typename O>
  struct sql_orm
  {
    typedef O object_type;

    // O without auto increment for create procedure.
    typedef sql_orm_internals::remove_auto_increment_t<O> without_auto_inc_type;

    // Object with only the primary keys for the delete and update procedures.
    typedef sql_orm_internals::extract_primary_keys_t<O> PKS;

    static_assert(!std::is_same<PKS, void>::value, "You must set at least one member of the CRUD object as primary key.");

    sql_orm(const std::string& table, C& con) : table_name_(table), con_(con) {}

    int find_by_id(int id, O& o)
    {
      return con_("SELECT * from " + table_name_ + " where id = ?")(id) >> o;
    }

    // save all fields except auto increment.
    // The db will automatically fill auto increment keys.
    template <typename N>
    int insert(const N& o)
    {
      std::stringstream ss;
      std::stringstream vs;
      ss << "INSERT into " << table_name_ << "(";

      bool first = true;
      auto values = foreach(without_auto_inc_type()) | [&] (auto& m) {
        if (!first) { ss << ","; vs << ","; }
        first = false;
        ss << m.symbol().name();
        vs << "?";
        return m.symbol() = m.symbol().member_access(o);
      };

      ss << ") VALUES (" << vs.str() << ")";
      auto req = con_(ss.str());
      apply(values, req);
      return req.last_insert_id();

    };

    // Iterate on all the rows of the table.
    template <typename F>
    void forall(F f)
    {
      std::stringstream ss;
      ss << "SELECT * from " << table_name_;
      con_(ss.str())() | f;
    }

    // Update N's members except auto increment members.
    // N must have at least one primary key.
    template <typename N>
    int update(const N& o)
    {
      // check if N has at least one member of PKS.
      auto pk = intersect(o, PKS());
      static_assert(decltype(pk)::size() > 0, "You must provide at least one primary key to update an object.");
      std::stringstream ss;
      ss << "UPDATE " << table_name_ << " SET ";

      bool first = true;
      auto values = foreach(o) | [&] (auto& m) {
        if (!first) ss << ",";
        first = false;
        ss << m.symbol().name() << " = ?";
        return m;
      };

      ss << " WHERE ";

      first = true;
      auto pks_values = foreach(pk) | [&] (auto& m) {
        if (!first) ss << " and ";
        first = false;
        ss << m.symbol().name() << " = ? ";
        return m.symbol() = m.value();
      };

      auto stmt = con_(ss.str());
      apply(values, pks_values, stmt);
      return stmt.last_insert_id();
    }

    template <typename T>
    void destroy(const T& o)
    {
      std::stringstream ss;
      ss << "DELETE from " << table_name_ << " WHERE ";

      bool first = true;
      auto values = foreach(PKS()) | [&] (auto& m) {
        if (!first) ss << " and ";
        first = false;
        ss << m.symbol().name() << " = ? ";
        return m.symbol() = o[m.symbol()];
      };

      apply(values, con_(ss.str()));
    }

    std::string table_name_;
    C con_;
  };

  struct sqlite_connection;
  struct mysql_connection;

  template <typename C, typename O>
  struct sql_orm_factory
  {
    typedef O object_type;
    typedef sql_orm<C, O> instance_type;

    sql_orm_factory(const std::string& table_name) : table_name_(table_name) {}

    void initialize(C& c)
    {
      std::stringstream ss;
      ss << "CREATE TABLE if not exists " <<  table_name_ << " (";

      bool first = true;
      foreach(O()) | [&] (auto& m)
      {
        if (!first) ss << ", ";
        ss << m.symbol().name() << " " << c.type_to_string(m.value());

        if (std::is_same<C, sqlite_connection>::value)
        {
          if (m.attributes().has(_auto_increment) || m.attributes().has(_primary_key))
            ss << " PRIMARY KEY ";
        }

        if (std::is_same<C, mysql_connection>::value)
        {
          if (m.attributes().has(_auto_increment))
            ss << " AUTO_INCREMENT NOT NULL";
          if (m.attributes().has(_primary_key))
            ss << " PRIMARY KEY ";
        }

        // To activate when pgsql_connection is implemented.
        // if (std::is_same<C, pgsql_connection>::value and
        //     m.attributes().has(_auto_increment))
        //   ss << " SERIAL ";

        first = false;
      };
      ss << ");";
      try
      {
        c(ss.str())();
      }
      catch (std::exception e)
      {
        std::cout << "Warning: Silicon could not create the " << table_name_ << " sql table." << std::endl
                  << "You can ignore this message if the table already exists."
                  << "The sql error is: " << e.what() << std::endl;
      }
    }

    sql_orm<C, O> instantiate(C& con) {
      return sql_orm<C, O>(table_name_, con);
    };

    std::string table_name_;
  };

}
<|endoftext|>