<commit_before>/*
 * file: packindex.cpp
 *
 * Purpose:
 *
 * canned Format:
 *
 * The MRF packed format consist of a header of size 16 + 16 * ((49151 + isize) / 49152)
 * followed by the 512 byte blocks of the original MRF index that do hold non-zero values
 * The output index will be 1:3072 (0.03255%) of the original virtual size, rounded up to 16,
 * plus the actual content blocks.
 *
 * This is because the canned file header contains a bitmask that holds presence info for every
 * 512 byte block of the input, stored in 96 block groups, stored as a 96 bit line prefixed
 * by a 32 bit running count of previously existing blocks. Thus, each line will use 16 bytes.
 * The canned file starts with a 16 byte canned index metadata that describes it's structure
 * followed by the bitmap, followed by the data blocks.

 * Since the size of the header can be calculated from the input MRF index size
 * it can be used as a check that the header is correct. Also the total number of set bits
 * in the header has to be equal to the number of blocks in the file
 *
 * The canned index file metadata line contains two 32bit integers and one 64bit int
 * all in big endian
 * | "MRF\0" | size of bitmap in 16 byte units | original index size in 16 byte units |
 *
 * The bitmap structure has 4 32bit unsigned integers, in big endian format
 * |start_count | bits 0 to 32 | bits 33 to 63 | bits 64 to 95 |
 * Where start_count is the total count of the bits set in the previous lines
 *
 */

#include <string>
#include <vector>
#include <fstream>
#include <iostream>
#include <cassert>
#include <cstdio>
#include <cstdlib>

// For memset only
#include <cstring>

#if defined(_WIN32)
#define FSEEK _fseeki64
#define FTELL _ftelli64

// Windows is always little endian, supply functions to swap bytes
// These are defined in <cstdlib>
#define htobe16 _byteswap_ushort
#define be16toh _byteswap_ushort
#define htobe32 _byteswap_ulong
#define be32toh _byteswap_ulong
#define htobe64 _byteswap_uint64
#define be64toh _byteswap_uint64

#else
#include <endian.h>
#define FSEEK fseek
#define FTELL ftell
#endif

using namespace std;

// Block size used, do not modify
const int BSZ = 512;

// 4 byte length signature string
const char *SIG = "IDX";

// Compare a substring of src with cmp, return true if same
// offset can be negative, in which case it is measured from the end of the src, python style
static bool substr_equal(const string &src, const string &cmp, int off = 0, size_t len = 0) {
    if (off < 0)
        off += static_cast<int>(src.size());
    if (off < 0 || off >= src.size())
        return false; // usage Error
    return cmp == (len ? src.substr(off, len) : src.substr(off));
}

// Returns true if the block is filled with zeros
inline bool check(const char *src, size_t len = BSZ) {
    assert(len % 8 == 0); // So we can use uint64_t
    len /= 8;
    const uint64_t *p = reinterpret_cast<const uint64_t *>(src);
    uint64_t accumulator = 0;
    while (len--)
        accumulator |= *p++;
    return 0 == accumulator;
}

// Program options
struct options {
    options() : un(false), quiet(false) {}
    vector<string> file_names;
    string error; // Empty if parsing went fine
    bool un; // reverse
    bool quiet;  // Verbose by default
};

static options parse(int argc, char **argv) {
    options opt;
    bool optend = false;
    for (int i = 1; i < argc; i++) {
        string arg(argv[i]);
        if (!optend && 0 == arg.find_first_of("-")) {
            if (arg == "-q") {
                opt.quiet = true;
                continue;
            }
            else if (arg == "-u") {
                opt.un = true;
                continue;
            }
            else if (arg == "-") { // Could be stdin or stdout file name
                opt.file_names.push_back(arg);
            }
            else if (arg == "--") { // End of options
                optend = true;
            }
            else {
                opt.error = string("Unknown option ") + arg;
                return opt;
            }
        }
        else { // File names, accumulate
            opt.file_names.push_back(arg);
        }
    }

    return opt;
}

enum ERRORS { NO_ERR = 0, USAGE_ERR, IO_ERR, INTERNAL_ERR };

static int Usage(const string &error) {
    cerr << error << endl;
    return USAGE_ERR;
}

// Output has a 16 bit reserved line plus the counted bitmap
static inline uint64_t canned_size(uint64_t in_size) {
    return 16 + 16 * ((96 * BSZ - 1 + in_size) / (96 * BSZ));
}

int can(const options &opt) {
    if (opt.file_names.size() != 2)
        return Usage("Need an input and an output name");

    string in_idx_name(opt.file_names[0]);
    string out_idx_name(opt.file_names[1]);

    if (!substr_equal(in_idx_name, ".idx", -4))
        return Usage("Input file should have an .idx extension");

    if (!substr_equal(out_idx_name, ".ix", -3))
        return Usage("Output file should have an .ix extension");

    FILE *in_idx = fopen(in_idx_name.c_str(), "rb");
    FILE *out_idx = fopen(out_idx_name.c_str(), "wb");

    if (!in_idx || !out_idx) {
        cerr << "Can't open " << (in_idx ? out_idx_name : in_idx_name) << endl;
        return IO_ERR;
    }

    FSEEK(in_idx, 0, SEEK_END);
    uint64_t in_size = static_cast<uint64_t>(FTELL(in_idx));
    FSEEK(in_idx, 0, SEEK_SET);

    // Input has to be an index, which is always a multiple of 16 bytes
    if (in_size % 16) {
        cerr << "Input file is not an index file, size is not a multiple of 16\n";
        return USAGE_ERR;
    }

    uint64_t header_size = canned_size(in_size);
    if (!opt.quiet)
        cout << "Header will be " << header_size << " bytes" << endl;

    // Get space for the header and reserve space on disk,
    // Header will be written at the end
    vector<uint32_t> header(header_size / sizeof(uint32_t));

    // Reserve space for the header
    FSEEK(out_idx, header_size, SEEK_SET);

    // Running count of output blocks
    size_t count = 0;
    size_t in_block_count = (BSZ - 1 + in_size) / BSZ;
    // Block buffer
    char buffer[BSZ];

    // Current line start within header as a 32bit int index
    // always a multiple of 4, since there are 4 ints per line
    // Skip the reserved line
    int line = 4;
    // and current bit position within that line
    int bit_pos = 0;

#define BIT_SET(line, i) header[line + 1 + i / 32] |= 1 << (i % 32)

    // Check all full blocks, transferring them as needed
    while (--in_block_count) {
        if (BSZ != fread(buffer, 1, BSZ, in_idx)) {
            cerr << "Error reading block from input file\n";
            return IO_ERR;
        }

        if (!check(buffer)) {
            if (BSZ != fwrite(buffer, 1, BSZ, out_idx)) {
                cerr << "Error writing to output file\n";
                return IO_ERR;
            }
            BIT_SET(line, bit_pos);
            count++;
        }

        bit_pos++;
        if (96 == bit_pos) {
            // Start a new line, store the running count
            bit_pos = 0;
            line += 4;
            // If there is another line, initialize running count
            if (line < header.size())
                header[line] = static_cast<uint32_t>(count);
        }
    }

    auto extra_bytes = in_size % BSZ;

    // The very last block may be partial
    if (extra_bytes) {
        memset(buffer, 0, BSZ);
        if (extra_bytes != fread(buffer, 1, extra_bytes, in_idx)) {
            cerr << "Error reading block from input file\n";
            return IO_ERR;
        }

        if (!check(buffer)) {
            if (extra_bytes != fwrite(buffer, 1, extra_bytes, out_idx)) {
                cerr << "Error writing to output file\n";
                return IO_ERR;
            }
            BIT_SET(line, bit_pos);
        }
        line += 4; // Points right at the end of the header
    }

#undef BIT_SET

    if (!opt.quiet)
        cout << "Index packed from " << in_size << " to " << FTELL(out_idx) << endl;

    // line should point to the last line or the one after the last
    if (!(header.size() == line))
        cerr << "Something is wrong, line is " << line << " header is " << header.size() << endl;

    // swap all header values to big endian
    for (auto &v : header)
        v = htobe32(v);

    // Write the header line, the signature is not dependent of endianess
    header[0] = *reinterpret_cast<const uint32_t *>(SIG);

    // Last piece of the header line, the size of the header itself, in 16 byte units
    // This imposes a size limit of 64GB for the header, which translates into
    // 192PB for the source index, unlikely to be ever reached
    header[1] = static_cast<uint32_t>(htobe64(header.size() / 4));

    // The initial index size, in 16byte units, big endian, this fills
    // header[2] and header[3]
    *reinterpret_cast<uint64_t *>(&header[2]) = htobe64(in_size / 16);

    // Done, write the header at the begining of the file
    fclose(in_idx);
    FSEEK(out_idx, 0, SEEK_SET);
    if (header.size() != fwrite(header.data(), sizeof(uint32_t), header.size(), out_idx)) {
        cerr << "Error writing output header\n";
        return IO_ERR;
    }
    fclose(out_idx);

    return NO_ERR;
}

int uncan(const options &opt) {
    if (opt.file_names.size() != 2)
        return Usage("Need an input and an output name, use - to use stdin or stdout");

    string in_idx_name(opt.file_names[0]);
    string out_idx_name(opt.file_names[1]);

    // TODO: handle stdin and stdout as in / out
    if (!substr_equal(in_idx_name, ".ix", -3))
        return Usage("Input file should have an .ix extension");

    if (!substr_equal(out_idx_name, ".idx", -4))
        return Usage("Output file should have an .idx extension");

    FILE *in_idx = fopen(in_idx_name.c_str(), "rb");
    FILE *out_idx = fopen(out_idx_name.c_str(), "wb");

    if (!in_idx || !out_idx) {
        cerr << "Can't open " << (in_idx ? out_idx_name : in_idx_name) << endl;
        return IO_ERR;
    }

    vector<uint32_t> header(4);
    if (4 != fread(header.data(), sizeof(uint32_t), 4, in_idx)) {
        cerr << "Error reading from input header\n";
        return IO_ERR;
    }

    // Verify and unpack the header line
    if (header[0] != *reinterpret_cast<const uint32_t *>(SIG))
        return Usage("Input is not a canned MRF index");
    // in 16 byte units, convert it to 4 byte units
    uint32_t header_size = 4 * be32toh(header[1]);

    // in bytes
    uint64_t out_size = 16 * be64toh(*reinterpret_cast<uint64_t *>(&header[2]));

    // Verify that the sizes make sense
    if (header_size != canned_size(out_size))
        return Usage("Input header is corrupt");

    return INTERNAL_ERR;
}

int main(int argc, char **argv)
{
    options opt(parse(argc, argv));
    if (!opt.error.empty())
        return Usage(opt.error);

    if (opt.un)
        return uncan(opt);
    return can(opt);
}
<commit_msg>Wrote uncan<commit_after>/*
 * file: packindex.cpp
 *
 * Purpose:
 *
 * canned Format:
 *
 * The MRF packed format consist of a header of size 16 + 16 * ((49151 + isize) / 49152)
 * followed by the 512 byte blocks of the original MRF index that do hold non-zero values
 * The output index will be 1:3072 (0.03255%) of the original virtual size, rounded up to 16,
 * plus the actual content blocks.
 *
 * This is because the canned file header contains a bitmask that holds presence info for every
 * 512 byte block of the input, stored in 96 block groups, stored as a 96 bit line prefixed
 * by a 32 bit running count of previously existing blocks. Thus, each line will use 16 bytes.
 * The canned file starts with a 16 byte canned index metadata that describes it's structure
 * followed by the bitmap, followed by the data blocks.

 * Since the size of the header can be calculated from the input MRF index size
 * it can be used as a check that the header is correct. Also the total number of set bits
 * in the header has to be equal to the number of blocks in the file
 *
 * The canned index file metadata line contains two 32bit integers and one 64bit int
 * all in big endian
 * | "MRF\0" | size of bitmap in 16 byte units | original index size in 16 byte units |
 *
 * The bitmap structure has 4 32bit unsigned integers, in big endian format
 * |start_count | bits 0 to 32 | bits 33 to 63 | bits 64 to 95 |
 * Where start_count is the total count of the bits set in the previous lines
 *
 */

#if defined(_WIN32)
#include <Windows.h>
#include <io.h>

#define FSEEK _fseeki64
#define FTELL _ftelli64

 // Windows is always little endian, supply functions to swap bytes
 // These are defined in <cstdlib>
#define htobe16 _byteswap_ushort
#define be16toh _byteswap_ushort
#define htobe32 _byteswap_ulong
#define be32toh _byteswap_ulong
#define htobe64 _byteswap_uint64
#define be64toh _byteswap_uint64
#else
#include <unistd.h>
#include <endian.h>
#define FSEEK fseek
#define FTELL ftell
#define SETSPARSE(X) true
#endif

#include <string>
#include <vector>
#include <algorithm>
#include <fstream>
#include <iostream>
#include <cassert>
#include <cstdio>
#include <cstdlib>

 // For memset only
#include <cstring>

using namespace std;

// Block size used, do not modify
const int BSZ = 512;

// 4 byte length signature string
const char *SIG = "IDX";

// Make file end at current offset, return true on success
static bool MARK_END(FILE *f) {
#if defined(_WIN32)
    HANDLE h = (HANDLE) _get_osfhandle(_fileno(f));
    if (h == INVALID_HANDLE_VALUE)
        return false; // Possibly not a seekable file
    return 0 != SetEndOfFile(h);
#else
    return !ftruncate(fileno(f), ftell(f));
#endif
}

#if defined(_WIN32)
// Set file as sparse, returns true if all went fine
static bool SETSPARSE(FILE *f) {
    DWORD dw;
    HANDLE h = (HANDLE)_get_osfhandle(_fileno(f));
    if (h = INVALID_HANDLE_VALUE)
        return false;
    return 0 != DeviceIoControl(h, FSCTL_SET_SPARSE, nullptr, 0, nullptr, 0, &dw, nullptr);
}
#endif

// Compare a substring of src with cmp, return true if same
// offset can be negative, in which case it is measured from the end of the src, python style
static bool substr_equal(const string &src, const string &cmp, int off = 0, size_t len = 0) {
    if (off < 0)
        off += static_cast<int>(src.size());
    if (off < 0 || off >= src.size())
        return false; // usage Error
    return cmp == (len ? src.substr(off, len) : src.substr(off));
}

// Returns true if the block is filled with zeros
inline bool check(const char *src, size_t len = BSZ) {
    assert(len % 8 == 0); // So we can use uint64_t
    len /= 8;
    const uint64_t *p = reinterpret_cast<const uint64_t *>(src);
    uint64_t accumulator = 0;
    while (len--)
        accumulator |= *p++;
    return 0 == accumulator;
}

// Program options
struct options {
    options() : un(false), quiet(false) {}
    vector<string> file_names;
    string error; // Empty if parsing went fine
    bool un; // reverse
    bool quiet;  // Verbose by default
};

static options parse(int argc, char **argv) {
    options opt;
    bool optend = false;
    for (int i = 1; i < argc; i++) {
        string arg(argv[i]);
        if (!optend && 0 == arg.find_first_of("-")) {
            if (arg == "-q") {
                opt.quiet = true;
                continue;
            }
            else if (arg == "-u") {
                opt.un = true;
                continue;
            }
            else if (arg == "-") { // Could be stdin or stdout file name
                opt.file_names.push_back(arg);
            }
            else if (arg == "--") { // End of options
                optend = true;
            }
            else {
                opt.error = string("Unknown option ") + arg;
                return opt;
            }
        }
        else { // File names, accumulate
            opt.file_names.push_back(arg);
        }
    }

    return opt;
}

enum ERRORS { NO_ERR = 0, USAGE_ERR, IO_ERR, INTERNAL_ERR };

static int Usage(const string &error) {
    cerr << error << endl;
    return USAGE_ERR;
}

// Output has a 16 bit reserved line plus the counted bitmap
static inline uint64_t canned_size(uint64_t in_size) {
    return 16 + 16 * ((96 * BSZ - 1 + in_size) / (96 * BSZ));
}

int can(const options &opt) {
    if (opt.file_names.size() != 2)
        return Usage("Need an input and an output name");

    string in_idx_name(opt.file_names[0]);
    string out_idx_name(opt.file_names[1]);

    if (!substr_equal(in_idx_name, ".idx", -4))
        return Usage("Input file should have an .idx extension");

    if (!substr_equal(out_idx_name, ".ix", -3))
        return Usage("Output file should have an .ix extension");

    FILE *in_idx = fopen(in_idx_name.c_str(), "rb");
    FILE *out_idx = fopen(out_idx_name.c_str(), "wb");

    if (!in_idx || !out_idx) {
        cerr << "Can't open " << (in_idx ? out_idx_name : in_idx_name) << endl;
        return IO_ERR;
    }

    FSEEK(in_idx, 0, SEEK_END);
    uint64_t in_size = static_cast<uint64_t>(FTELL(in_idx));
    FSEEK(in_idx, 0, SEEK_SET);

    // Input has to be an index, which is always a multiple of 16 bytes
    if (in_size % 16) {
        cerr << "Input file is not an index file, size is not a multiple of 16\n";
        return USAGE_ERR;
    }

    uint64_t header_size = canned_size(in_size);
    if (!opt.quiet)
        cout << "Header will be " << header_size << " bytes" << endl;

    // Get space for the header and reserve space on disk,
    // Header will be written at the end
    vector<uint32_t> header(header_size / sizeof(uint32_t));

    // Reserve space for the header
    FSEEK(out_idx, header_size, SEEK_SET);

    // Running count of output blocks
    size_t count = 0;
    size_t in_block_count = (BSZ - 1 + in_size) / BSZ;
    // Block buffer
    char buffer[BSZ];

    // Current line start within header as a 32bit int index
    // always a multiple of 4, since there are 4 ints per line
    // Skip the reserved line
    int line = 4;
    // and current bit position within that line
    int bit_pos = 0;

#define BIT_SET(line, i) header[line + 1 + i / 32] |= 1 << (i % 32)

    // Check all full blocks, transferring them as needed
    while (--in_block_count) {
        if (BSZ != fread(buffer, 1, BSZ, in_idx)) {
            cerr << "Error reading block from input file\n";
            return IO_ERR;
        }

        if (!check(buffer)) {
            if (BSZ != fwrite(buffer, 1, BSZ, out_idx)) {
                cerr << "Error writing to output file\n";
                return IO_ERR;
            }
            BIT_SET(line, bit_pos);
            count++;
        }

        bit_pos++;
        if (96 == bit_pos) {
            // Start a new line, store the running count
            bit_pos = 0;
            line += 4;
            // If there is another line, initialize running count
            if (line < header.size())
                header[line] = static_cast<uint32_t>(count);
        }
    }

    auto extra_bytes = in_size % BSZ;

    // The very last block may be partial
    if (extra_bytes) {
        memset(buffer, 0, BSZ);
        if (extra_bytes != fread(buffer, 1, extra_bytes, in_idx)) {
            cerr << "Error reading block from input file\n";
            return IO_ERR;
        }

        if (!check(buffer)) {
            if (extra_bytes != fwrite(buffer, 1, extra_bytes, out_idx)) {
                cerr << "Error writing to output file\n";
                return IO_ERR;
            }
            BIT_SET(line, bit_pos);
        }
        line += 4; // Points right at the end of the header
    }

#undef BIT_SET

    if (!opt.quiet)
        cout << "Index packed from " << in_size << " to " << FTELL(out_idx) << endl;

    // line should point to the last line or the one after the last
    if (!(header.size() == line))
        cerr << "Something is wrong, line is " << line << " header is " << header.size() << endl;

    // swap all header values to big endian
    for (auto &v : header)
        v = htobe32(v);

    // Write the header line, the signature is not dependent of endianess
    header[0] = *reinterpret_cast<const uint32_t *>(SIG);

    // Last piece of the header line, the size of the header itself, in 16 byte units
    // This imposes a size limit of 64GB for the header, which translates into
    // 192PB for the source index, unlikely to be ever reached
    header[1] = static_cast<uint32_t>(htobe64(header.size() / 4));

    // The initial index size, in 16byte units, big endian, this fills
    // header[2] and header[3]
    *reinterpret_cast<uint64_t *>(&header[2]) = htobe64(in_size / 16);

    // Done, write the header at the begining of the file
    fclose(in_idx);
    FSEEK(out_idx, 0, SEEK_SET);
    if (header.size() != fwrite(header.data(), sizeof(uint32_t), header.size(), out_idx)) {
        cerr << "Error writing output header\n";
        return IO_ERR;
    }
    fclose(out_idx);

    return NO_ERR;
}

// The bit state at a given position in a line, assumes native endianess
inline bool is_on(uint32_t *values, int bit) {
    return 0 != (values[1 + bit / 32] & (static_cast<uint32_t>(1) << bit % 32));
}

// transfer len bytes from in to out, prints an error and returns false if it erred
// len has to be under or equal to BSZ
inline int transfer(FILE *in_file, FILE *out_file, size_t len = BSZ) {
    assert(len <= BSZ);
    static char buffer[BSZ];

    if (len != fread(buffer, 1, len, in_file)) {
        cerr << "Read error\n";
        return false;
    }

    if (len != fwrite(buffer, 1, len, out_file)) {
        cerr << "Write error\n";
        return false;
    }

    return true;
}

int uncan(const options &opt) {
    if (opt.file_names.size() != 2)
        return Usage("Need an input and an output name, use - to use stdin or stdout");

    string in_idx_name(opt.file_names[0]);
    string out_idx_name(opt.file_names[1]);

    // TODO: handle stdin and stdout as in / out
    if (!substr_equal(in_idx_name, ".ix", -3))
        return Usage("Input file should have an .ix extension");

    if (!substr_equal(out_idx_name, ".idx", -4))
        return Usage("Output file should have an .idx extension");

    FILE *in_idx = fopen(in_idx_name.c_str(), "rb");
    FILE *out_idx = fopen(out_idx_name.c_str(), "wb");
    SETSPARSE(out_idx);

    if (!in_idx || !out_idx) {
        cerr << "Can't open " << (in_idx ? out_idx_name : in_idx_name) << endl;
        return IO_ERR;
    }

    vector<uint32_t> header(4);
    if (4 != fread(header.data(), sizeof(uint32_t), 4, in_idx)) {
        cerr << "Error reading from input header\n";
        return IO_ERR;
    }

    // Verify and unpack the header line
    if (header[0] != *reinterpret_cast<const uint32_t *>(SIG))
        return Usage("Input is not a canned MRF index, wrong magick");
    // in 16 byte units, convert it to 4 byte units
    uint32_t header_size = 4 * be32toh(header[1]);

    // in bytes
    uint64_t out_size = 16 * be64toh(*reinterpret_cast<uint64_t *>(&header[2]));

    // Verify that the sizes make sense
    if (header_size != canned_size(out_size))
        return Usage("Input header is corrupt");

    if (!opt.quiet)
        cout << "Output size will be " << out_size << endl;

    header_size -= 4;

    // The bitmap part of the header
    vector<uint32_t> bitmap(header_size);

    if (header_size != fread(bitmap.data(), sizeof(uint32_t), header_size, in_idx)) {
        cerr << "Error reading input bitmap\n";
        return IO_ERR;
    }

    // Swap bitmap to host
    for (auto &it : bitmap)
        it = be32toh(it);

    // Full output blocks, there might be one more, partial
    uint64_t num_blocks = out_size / BSZ;

    uint32_t count = 0; // The running count of output blocks with data
    uint32_t line = 0; // increments by 4

    // How many output blocks are empty
    uint64_t empties = 0;

    // Loop over input lines
    while (num_blocks) {
        int bits = 96;
        if (num_blocks < 96)
            bits = static_cast<int>(num_blocks);

        // Check that the running count for the line agrees
        if (count != bitmap[line])
            return Usage("Input bitmap is corrupt\n");

        for (int bit= 0; bit < bits; bit++, num_blocks--) {

            if (!is_on(&bitmap[line], bit)) {
                empties++;
                continue;
            }

            // Flush
            if (empties)
                FSEEK(out_idx, empties * BSZ, SEEK_CUR);
            empties = 0;

            // Transfer the block from in to out
            if (!transfer(in_idx, out_idx))
                return IO_ERR;

            count++; // Transferred one more block
        }

        line += 4;
    }

    // Flush
    if (empties)
        FSEEK(out_idx, empties * BSZ, SEEK_CUR);

    // Might have a partial block at the end
    int extra_bytes = out_size % BSZ;
    if (extra_bytes) {
        // Which line we're on
        line = static_cast<uint32_t>((out_size / BSZ / 96) * 4);
        int bit = (out_size / BSZ) % 96;
        // Check the running count if the partial block is bit 0
        if ((bit == 0) && (count != bitmap[line]))
            return Usage("Input bitmap is corrupt\n");

        if (is_on(&bitmap[line], bit)) {
            // Transfer the partial block, we're done
            if (!transfer(in_idx, out_idx, extra_bytes))
                return IO_ERR;
        }
        else {
            FSEEK(out_idx, extra_bytes, SEEK_CUR);
        }
    }

    fclose(in_idx);
    // Need to end the file at the current position, to get the right size
    MARK_END(out_idx);
    fclose(out_idx);
    return INTERNAL_ERR;
}

int main(int argc, char **argv)
{
    options opt(parse(argc, argv));
    if (!opt.error.empty())
        return Usage(opt.error);

    if (opt.un)
        return uncan(opt);
    return can(opt);
}
<|endoftext|>


<commit_before>/****************************************************************************
** Meta object code from reading C++ file 'treemodel.h'
**
** Created by: The Qt Meta Object Compiler version 63 (Qt 4.8.5)
**
** WARNING! All changes made in this file will be lost!
*****************************************************************************/

#include "treemodel.h"
#if !defined(Q_MOC_OUTPUT_REVISION)
#error "The header file 'treemodel.h' doesn't include <QObject>."
#elif Q_MOC_OUTPUT_REVISION != 63
#error "This file was generated using the moc from 4.8.5. It"
#error "cannot be used with the include files from this version of Qt."
#error "(The moc has changed too much.)"
#endif

QT_BEGIN_MOC_NAMESPACE
static const uint qt_meta_data_TreeModel[] = {

 // content:
       6,       // revision
       0,       // classname
       0,    0, // classinfo
       0,    0, // methods
       0,    0, // properties
       0,    0, // enums/sets
       0,    0, // constructors
       0,       // flags
       0,       // signalCount

       0        // eod
};

static const char qt_meta_stringdata_TreeModel[] = {
    "TreeModel\0"
};

void TreeModel::qt_static_metacall(QObject *_o, QMetaObject::Call _c, int _id, void **_a)
{
    Q_UNUSED(_o);
    Q_UNUSED(_id);
    Q_UNUSED(_c);
    Q_UNUSED(_a);
}

const QMetaObjectExtraData TreeModel::staticMetaObjectExtraData = {
    0,  qt_static_metacall
};

const QMetaObject TreeModel::staticMetaObject = {
    { &QAbstractItemModel::staticMetaObject, qt_meta_stringdata_TreeModel,
      qt_meta_data_TreeModel, &staticMetaObjectExtraData }
};

#ifdef Q_NO_DATA_RELOCATION
const QMetaObject &TreeModel::getStaticMetaObject() { return staticMetaObject; }
#endif //Q_NO_DATA_RELOCATION

const QMetaObject *TreeModel::metaObject() const
{
    return QObject::d_ptr->metaObject ? QObject::d_ptr->metaObject : &staticMetaObject;
}

void *TreeModel::qt_metacast(const char *_clname)
{
    if (!_clname) return 0;
    if (!strcmp(_clname, qt_meta_stringdata_TreeModel))
        return static_cast<void*>(const_cast< TreeModel*>(this));
    return QAbstractItemModel::qt_metacast(_clname);
}

int TreeModel::qt_metacall(QMetaObject::Call _c, int _id, void **_a)
{
    _id = QAbstractItemModel::qt_metacall(_c, _id, _a);
    if (_id < 0)
        return _id;
    return _id;
}
QT_END_MOC_NAMESPACE
<commit_msg>Delete moc_treemodel.cpp<commit_after><|endoftext|>


<commit_before>#include "StorageMemoChunk.h"
#include "StorageChunkSerializer.h"
#include "StorageChunkWriter.h"
#include "StorageBulkCursor.h"
#include "StorageEnvironment.h"
#include "StorageAsyncGet.h"

static inline int KeyCmp(const ReadBuffer& a, const ReadBuffer& b)
{
    return ReadBuffer::Cmp(a, b);
}

static inline const ReadBuffer Key(const StorageMemoKeyValue* kv)
{
    return kv->GetKey();
}

static inline const ReadBuffer Key(const StorageFileKeyValue* kv)
{
    return kv->GetKey();
}

uint32_t StorageMemoKeyValueAllocator::GetFreeSize()
{
    return size - pos;
}

StorageMemoChunk::StorageMemoChunk(uint64_t chunkID_, bool useBloomFilter_)
{
    chunkID = chunkID_;
    useBloomFilter = useBloomFilter_;

    serialized = false;
    minLogSegmentID = 0;
    maxLogSegmentID = 0;
    maxLogCommandID = 0;
    size = 0;
    avgSize = 0.0;
    fileChunk = NULL;
    deleted = false;
}

StorageMemoChunk::~StorageMemoChunk()
{
    StorageMemoKeyValueAllocator*   allocator;

    keyValueBlocks.DeleteQueue();

    FOREACH_FIRST (allocator, allocators)
    {
        allocators.Remove(allocator);
        free((void*) allocator);
    }

    if (fileChunk != NULL)
        delete fileChunk;
}

StorageChunk::ChunkState StorageMemoChunk::GetChunkState()
{
    if (serialized)
        return StorageChunk::Serialized;
    else
        return StorageChunk::Tree;
}

void StorageMemoChunk::NextBunch(StorageBulkCursor& cursor, StorageShard* shard)
{
    bool                    first;
    int                     cmpres;
    ReadBuffer              nextKey, key, value;
    StorageMemoKeyValue*    it;
    uint32_t                total;

    if (keyValues.GetCount() == 0)
    {
        cursor.SetLast(true);
        return;
    }

    nextKey = cursor.GetNextKey();
    it = keyValues.Locate(nextKey, cmpres);

    total = 0;
    first = true;
    while (it)
    {
        if (!shard->RangeContains(it->GetKey()))
        {
            it = keyValues.Next(it);
            continue;
        }

        if (!first && total + it->GetKey().GetLength() + it->GetValue().GetLength() > STORAGE_MEMO_BUNCH_GRAN)
            break;

        total += it->GetKey().GetLength() + it->GetValue().GetLength();
        cursor.AppendKeyValue(it);

        it = keyValues.Next(it);
        first = false;
    }
    cursor.FinalizeKeyValues();

    if (!it)
    {
        cursor.SetLast(true);
    }
    else
    {
        cursor.SetNextKey(it->GetKey());
        cursor.SetLast(false);
    }
}

//void StorageMemoChunk::SetChunkID(uint64_t chunkID_)
//{
//    chunkID = chunkID_;
//}
//
//void StorageMemoChunk::SetUseBloomFilter(bool useBloomFilter_)
//{
//    useBloomFilter = useBloomFilter_;
//}

uint64_t StorageMemoChunk::GetChunkID()
{
    return chunkID;
}

bool StorageMemoChunk::UseBloomFilter()
{
    return useBloomFilter;
}

StorageKeyValue* StorageMemoChunk::Get(ReadBuffer& key)
{
    if (keyValues.GetCount() == 0)
        return NULL;

    return keyValues.Get(key);
}

void StorageMemoChunk::AsyncGet(StorageAsyncGet* asyncGet)
{
    StorageKeyValue*    kv;

    Log_Debug("StorageMemoChunk::AsyncGet");

    kv = Get(asyncGet->key);
    if (kv == NULL || kv->GetType() == STORAGE_KEYVALUE_TYPE_DELETE)
    {
        asyncGet->ret = false;
        asyncGet->completed = true;
        return;
    }

    asyncGet->value = kv->GetValue();
    asyncGet->ret = true;
    asyncGet->completed = true;
}

bool StorageMemoChunk::Set(ReadBuffer key, ReadBuffer value)
{
    int                     cmpres;
    StorageMemoKeyValue*    it;

    ASSERT(key.GetLength() > 0);

    if (keyValues.GetCount() != 0)
    {
        it = keyValues.Locate<const ReadBuffer&>(key, cmpres);
        if (cmpres == 0)
        {
            it->Set(key, value, this);
            return true;
        }
    }

    it = NewStorageMemoKeyValue();
    it->Set(key, value, this);
    keyValues.Insert<const ReadBuffer>(it);

    return true;
}

bool StorageMemoChunk::Delete(ReadBuffer key)
{
    int                     cmpres;
    StorageMemoKeyValue*    it;

    ASSERT(key.GetLength() > 0);

    if (keyValues.GetCount() != 0)
    {
        it = keyValues.Locate<const ReadBuffer&>(key, cmpres);
        if (cmpres == 0)
        {
            it->Delete(key, this);
            return true;
        }
    }

    it = NewStorageMemoKeyValue();
    it->Delete(key, this);
    keyValues.Insert<const ReadBuffer>(it);

    return true;
}

void StorageMemoChunk::RegisterLogCommand(uint64_t logSegmentID_, uint32_t logCommandID_)
{
    if (minLogSegmentID == 0)
        minLogSegmentID = logSegmentID_;

    if (logSegmentID_ > maxLogSegmentID)
    {
        maxLogSegmentID = logSegmentID_;
        maxLogCommandID = logCommandID_;
    }
    else if (logSegmentID_ == maxLogSegmentID)
    {
        if (logCommandID_ > maxLogCommandID)
            maxLogCommandID = logCommandID_;
    }
}

uint64_t StorageMemoChunk::GetMinLogSegmentID()
{
    return minLogSegmentID;
}

uint64_t StorageMemoChunk::GetMaxLogSegmentID()
{
    return maxLogSegmentID;
}

uint32_t StorageMemoChunk::GetMaxLogCommandID()
{
    return maxLogCommandID;
}

ReadBuffer StorageMemoChunk::GetFirstKey()
{
    if (keyValues.First())
        return keyValues.First()->GetKey();
    else
        return ReadBuffer();
}

ReadBuffer StorageMemoChunk::GetLastKey()
{
    if (keyValues.Last())
        return keyValues.Last()->GetKey();
    else
        return ReadBuffer();
}

uint64_t StorageMemoChunk::GetSize()
{
    return size;
}

ReadBuffer StorageMemoChunk::GetMidpoint()
{
    if (keyValues.Mid())
        return keyValues.Mid()->GetKey();
    else
        return ReadBuffer();
}

bool StorageMemoChunk::IsEmpty()
{
    return (size == 0);
}

StorageFileChunk* StorageMemoChunk::RemoveFileChunk()
{
    StorageFileChunk*   ret;

    ret = fileChunk;
    fileChunk = NULL;
    return ret;
}

void StorageMemoChunk::RemoveFirst()
{
    StorageMemoKeyValueBlock*   block;
    StorageMemoKeyValue*        first;

    first = keyValues.First();
    keyValues.Remove(first);

    first->Free(this);

    block = keyValueBlocks.First();
    ASSERT(first >= block->keyValues && first < (block->keyValues + STORAGE_BLOCK_NUM_KEY_VALUE));
    block->last++;
    if (block->last == STORAGE_BLOCK_NUM_KEY_VALUE)
    {
        keyValueBlocks.Dequeue();
        delete block;
        size -= sizeof(StorageMemoKeyValueBlock);
    }
}

StorageMemoKeyValue* StorageMemoChunk::NewStorageMemoKeyValue()
{
    StorageMemoKeyValueBlock*   block;
    StorageMemoKeyValue*        keyValue;
    bool                        isNewBlockNeeded;

    isNewBlockNeeded = false;

    if (keyValueBlocks.GetLength() == 0)
        isNewBlockNeeded = true;
    else
    {
        block = keyValueBlocks.Last();
        if (block->first == STORAGE_BLOCK_NUM_KEY_VALUE)
            isNewBlockNeeded = true;
    }

    if (isNewBlockNeeded)
    {
        block = new StorageMemoKeyValueBlock;
        block->next = block;
        block->first = 0;
        block->last = 0;
        keyValueBlocks.Enqueue(block);
        size += sizeof(StorageMemoKeyValueBlock);
    }

    keyValue = &block->keyValues[block->first];
    block->first++;

    return keyValue;
}

char* StorageMemoChunk::Alloc(size_t size_)
{
    StorageMemoKeyValueAllocator*   allocator;
    StorageMemoKeyValueAllocator*   prevAllocator;
    char*                           memory;
    uint32_t                        allocatedSize;
    size_t                          requiredSize;

    // Each allocated piece of memory is prefixed with an uint32_t offset
    // from the start of the buffer of the allocator. Therefore it is
    // possible to find the allocator object of a given pointer.
    requiredSize = size_ + sizeof(uint32_t);
    avgSize = (avgSize * keyValues.GetCount() + requiredSize) / (keyValues.GetCount() + 1);

    // save the last allocator to avoid fragmentation
    prevAllocator = allocator = allocators.Last();

    if (allocator == NULL || allocator->GetFreeSize() < requiredSize)
    {
        allocatedSize = MAX(requiredSize, STORAGE_MEMO_ALLOCATOR_DEFAULT_SIZE);
        // avoid calling malloc twice by allocating memory and setting the pointers
        memory = (char*) malloc(sizeof(StorageMemoKeyValueAllocator) + allocatedSize);

        allocator = (StorageMemoKeyValueAllocator*) memory;
        allocator->buffer = memory + sizeof(StorageMemoKeyValueAllocator);
        allocator->size = allocatedSize;
        allocator->pos = 0;
        allocator->num = 0;
        allocator->next = allocator;
        allocator->prev = allocator;

        // TODO: better strategy to avoid fragmentation
        if (prevAllocator != NULL &&
         prevAllocator->GetFreeSize() > (uint32_t)(avgSize + 0.5))
        {
            allocators.Remove(prevAllocator);
            allocators.Append(allocator);
            allocators.Append(prevAllocator);
        }
        else
            allocators.Append(allocator);

        size += sizeof(StorageMemoKeyValueAllocator) + allocator->size;
    }

    // The memory layout is the following:
    //
    // +---------------------
    // | pos (4 bytes)
    // +---------------------
    // | memory (size_ bytes)
    // +---------------------
    //
    memory = allocator->buffer + allocator->pos;
    allocator->pos += sizeof(uint32_t);
    *(uint32_t*) memory = (uint32_t) allocator->pos;
    memory += sizeof(uint32_t);
    allocator->pos += size_;
    allocator->num++;

    return memory;
}

void StorageMemoChunk::Free(char* buffer)
{
    ReadBuffer                      key;
    uint32_t                        pos;
    StorageMemoKeyValueAllocator*   allocator;

    pos = *(uint32_t*)(buffer - sizeof(uint32_t));
    allocator = (StorageMemoKeyValueAllocator*)(buffer - pos - sizeof(StorageMemoKeyValueAllocator));
    allocator->num--;

    if (allocator->num == 0)
    {
        size -= sizeof(StorageMemoKeyValueAllocator) + allocator->size;
        allocators.Remove(allocator);
        free(allocator);

        // adjust the average size
        avgSize = (double) size / keyValues.GetCount();
    }
}
<commit_msg>Refactored code.<commit_after>#include "StorageMemoChunk.h"
#include "StorageChunkSerializer.h"
#include "StorageChunkWriter.h"
#include "StorageBulkCursor.h"
#include "StorageEnvironment.h"
#include "StorageAsyncGet.h"

static inline int KeyCmp(const ReadBuffer& a, const ReadBuffer& b)
{
    return ReadBuffer::Cmp(a, b);
}

static inline const ReadBuffer Key(const StorageMemoKeyValue* kv)
{
    return kv->GetKey();
}

static inline const ReadBuffer Key(const StorageFileKeyValue* kv)
{
    return kv->GetKey();
}

uint32_t StorageMemoKeyValueAllocator::GetFreeSize()
{
    return size - pos;
}

StorageMemoChunk::StorageMemoChunk(uint64_t chunkID_, bool useBloomFilter_)
{
    chunkID = chunkID_;
    useBloomFilter = useBloomFilter_;

    serialized = false;
    minLogSegmentID = 0;
    maxLogSegmentID = 0;
    maxLogCommandID = 0;
    size = 0;
    avgSize = 0.0;
    fileChunk = NULL;
    deleted = false;
}

StorageMemoChunk::~StorageMemoChunk()
{
    StorageMemoKeyValueAllocator*   allocator;

    keyValueBlocks.DeleteQueue();

    FOREACH_FIRST (allocator, allocators)
    {
        allocators.Remove(allocator);
        free((void*) allocator);
    }

    if (fileChunk != NULL)
        delete fileChunk;
}

StorageChunk::ChunkState StorageMemoChunk::GetChunkState()
{
    if (serialized)
        return StorageChunk::Serialized;
    else
        return StorageChunk::Tree;
}

void StorageMemoChunk::NextBunch(StorageBulkCursor& cursor, StorageShard* shard)
{
    bool                    first;
    int                     cmpres;
    ReadBuffer              nextKey, key, value;
    StorageMemoKeyValue*    it;
    uint32_t                total;

    if (keyValues.GetCount() == 0)
    {
        cursor.SetLast(true);
        return;
    }

    nextKey = cursor.GetNextKey();
    it = keyValues.Locate(nextKey, cmpres);

    total = 0;
    first = true;
    while (it)
    {
        if (!shard->RangeContains(it->GetKey()))
        {
            it = keyValues.Next(it);
            continue;
        }

        if (!first && total + it->GetKey().GetLength() + it->GetValue().GetLength() > STORAGE_MEMO_BUNCH_GRAN)
            break;

        total += it->GetKey().GetLength() + it->GetValue().GetLength();
        cursor.AppendKeyValue(it);

        it = keyValues.Next(it);
        first = false;
    }
    cursor.FinalizeKeyValues();

    if (!it)
    {
        cursor.SetLast(true);
    }
    else
    {
        cursor.SetNextKey(it->GetKey());
        cursor.SetLast(false);
    }
}

//void StorageMemoChunk::SetChunkID(uint64_t chunkID_)
//{
//    chunkID = chunkID_;
//}
//
//void StorageMemoChunk::SetUseBloomFilter(bool useBloomFilter_)
//{
//    useBloomFilter = useBloomFilter_;
//}

uint64_t StorageMemoChunk::GetChunkID()
{
    return chunkID;
}

bool StorageMemoChunk::UseBloomFilter()
{
    return useBloomFilter;
}

StorageKeyValue* StorageMemoChunk::Get(ReadBuffer& key)
{
    if (keyValues.GetCount() == 0)
        return NULL;

    return keyValues.Get(key);
}

void StorageMemoChunk::AsyncGet(StorageAsyncGet* asyncGet)
{
    StorageKeyValue*    kv;

    Log_Debug("StorageMemoChunk::AsyncGet");

    kv = Get(asyncGet->key);
    if (kv == NULL || kv->GetType() == STORAGE_KEYVALUE_TYPE_DELETE)
    {
        asyncGet->ret = false;
        asyncGet->completed = true;
        return;
    }

    asyncGet->value = kv->GetValue();
    asyncGet->ret = true;
    asyncGet->completed = true;
}

bool StorageMemoChunk::Set(ReadBuffer key, ReadBuffer value)
{
    StorageMemoKeyValue* kv;

    ASSERT(key.GetLength() > 0);

    kv = keyValues.Get(key);
    if (kv)
    {
        kv->Set(key, value, this);
        return true;
    }

    kv = NewStorageMemoKeyValue();
    kv->Set(key, value, this);
    keyValues.Insert<const ReadBuffer>(kv);

    return true;
}

bool StorageMemoChunk::Delete(ReadBuffer key)
{
    StorageMemoKeyValue* kv;

    ASSERT(key.GetLength() > 0);

    kv = keyValues.Get(key);
    if (kv)
    {
        kv->Delete(key, this);
        return true;
    }

    kv = NewStorageMemoKeyValue();
    kv->Delete(key, this);
    keyValues.Insert<const ReadBuffer>(kv);

    return true;
}

void StorageMemoChunk::RegisterLogCommand(uint64_t logSegmentID_, uint32_t logCommandID_)
{
    if (minLogSegmentID == 0)
        minLogSegmentID = logSegmentID_;

    if (logSegmentID_ > maxLogSegmentID)
    {
        maxLogSegmentID = logSegmentID_;
        maxLogCommandID = logCommandID_;
    }
    else if (logSegmentID_ == maxLogSegmentID)
    {
        if (logCommandID_ > maxLogCommandID)
            maxLogCommandID = logCommandID_;
    }
}

uint64_t StorageMemoChunk::GetMinLogSegmentID()
{
    return minLogSegmentID;
}

uint64_t StorageMemoChunk::GetMaxLogSegmentID()
{
    return maxLogSegmentID;
}

uint32_t StorageMemoChunk::GetMaxLogCommandID()
{
    return maxLogCommandID;
}

ReadBuffer StorageMemoChunk::GetFirstKey()
{
    if (keyValues.First())
        return keyValues.First()->GetKey();
    else
        return ReadBuffer();
}

ReadBuffer StorageMemoChunk::GetLastKey()
{
    if (keyValues.Last())
        return keyValues.Last()->GetKey();
    else
        return ReadBuffer();
}

uint64_t StorageMemoChunk::GetSize()
{
    return size;
}

ReadBuffer StorageMemoChunk::GetMidpoint()
{
    if (keyValues.Mid())
        return keyValues.Mid()->GetKey();
    else
        return ReadBuffer();
}

bool StorageMemoChunk::IsEmpty()
{
    return (size == 0);
}

StorageFileChunk* StorageMemoChunk::RemoveFileChunk()
{
    StorageFileChunk*   ret;

    ret = fileChunk;
    fileChunk = NULL;
    return ret;
}

void StorageMemoChunk::RemoveFirst()
{
    StorageMemoKeyValueBlock*   block;
    StorageMemoKeyValue*        first;

    first = keyValues.First();
    keyValues.Remove(first);

    first->Free(this);

    block = keyValueBlocks.First();
    ASSERT(first >= block->keyValues && first < (block->keyValues + STORAGE_BLOCK_NUM_KEY_VALUE));
    block->last++;
    if (block->last == STORAGE_BLOCK_NUM_KEY_VALUE)
    {
        keyValueBlocks.Dequeue();
        delete block;
        size -= sizeof(StorageMemoKeyValueBlock);
    }
}

StorageMemoKeyValue* StorageMemoChunk::NewStorageMemoKeyValue()
{
    StorageMemoKeyValueBlock*   block;
    StorageMemoKeyValue*        keyValue;
    bool                        isNewBlockNeeded;

    isNewBlockNeeded = false;

    if (keyValueBlocks.GetLength() == 0)
        isNewBlockNeeded = true;
    else
    {
        block = keyValueBlocks.Last();
        if (block->first == STORAGE_BLOCK_NUM_KEY_VALUE)
            isNewBlockNeeded = true;
    }

    if (isNewBlockNeeded)
    {
        block = new StorageMemoKeyValueBlock;
        block->next = block;
        block->first = 0;
        block->last = 0;
        keyValueBlocks.Enqueue(block);
        size += sizeof(StorageMemoKeyValueBlock);
    }

    keyValue = &block->keyValues[block->first];
    block->first++;

    return keyValue;
}

char* StorageMemoChunk::Alloc(size_t size_)
{
    StorageMemoKeyValueAllocator*   allocator;
    StorageMemoKeyValueAllocator*   prevAllocator;
    char*                           memory;
    uint32_t                        allocatedSize;
    size_t                          requiredSize;

    // Each allocated piece of memory is prefixed with an uint32_t offset
    // from the start of the buffer of the allocator. Therefore it is
    // possible to find the allocator object of a given pointer.
    requiredSize = size_ + sizeof(uint32_t);
    avgSize = (avgSize * keyValues.GetCount() + requiredSize) / (keyValues.GetCount() + 1);

    // save the last allocator to avoid fragmentation
    prevAllocator = allocator = allocators.Last();

    if (allocator == NULL || allocator->GetFreeSize() < requiredSize)
    {
        allocatedSize = MAX(requiredSize, STORAGE_MEMO_ALLOCATOR_DEFAULT_SIZE);
        // avoid calling malloc twice by allocating memory and setting the pointers
        memory = (char*) malloc(sizeof(StorageMemoKeyValueAllocator) + allocatedSize);

        allocator = (StorageMemoKeyValueAllocator*) memory;
        allocator->buffer = memory + sizeof(StorageMemoKeyValueAllocator);
        allocator->size = allocatedSize;
        allocator->pos = 0;
        allocator->num = 0;
        allocator->next = allocator;
        allocator->prev = allocator;

        // TODO: better strategy to avoid fragmentation
        if (prevAllocator != NULL &&
         prevAllocator->GetFreeSize() > (uint32_t)(avgSize + 0.5))
        {
            allocators.Remove(prevAllocator);
            allocators.Append(allocator);
            allocators.Append(prevAllocator);
        }
        else
            allocators.Append(allocator);

        size += sizeof(StorageMemoKeyValueAllocator) + allocator->size;
    }

    // The memory layout is the following:
    //
    // +---------------------
    // | pos (4 bytes)
    // +---------------------
    // | memory (size_ bytes)
    // +---------------------
    //
    memory = allocator->buffer + allocator->pos;
    allocator->pos += sizeof(uint32_t);
    *(uint32_t*) memory = (uint32_t) allocator->pos;
    memory += sizeof(uint32_t);
    allocator->pos += size_;
    allocator->num++;

    return memory;
}

void StorageMemoChunk::Free(char* buffer)
{
    ReadBuffer                      key;
    uint32_t                        pos;
    StorageMemoKeyValueAllocator*   allocator;

    pos = *(uint32_t*)(buffer - sizeof(uint32_t));
    allocator = (StorageMemoKeyValueAllocator*)(buffer - pos - sizeof(StorageMemoKeyValueAllocator));
    allocator->num--;

    if (allocator->num == 0)
    {
        size -= sizeof(StorageMemoKeyValueAllocator) + allocator->size;
        allocators.Remove(allocator);
        free(allocator);

        // adjust the average size
        avgSize = (double) size / keyValues.GetCount();
    }
}
<|endoftext|>


<commit_before>#include "StorageMemoChunk.h"
#include "StorageChunkSerializer.h"
#include "StorageChunkWriter.h"
#include "StorageBulkCursor.h"
#include "StorageEnvironment.h"
#include "StorageAsyncGet.h"

static inline int KeyCmp(const ReadBuffer& a, const ReadBuffer& b)
{
    return ReadBuffer::Cmp(a, b);
}

static inline const ReadBuffer Key(const StorageMemoKeyValue* kv)
{
    return kv->GetKey();
}

static inline const ReadBuffer Key(const StorageFileKeyValue* kv)
{
    return kv->GetKey();
}

StorageMemoChunk::StorageMemoChunk(uint64_t chunkID_, bool useBloomFilter_)
{
    chunkID = chunkID;
    useBloomFilter = useBloomFilter_;

    serialized = false;
    minLogSegmentID = 0;
    maxLogSegmentID = 0;
    maxLogCommandID = 0;
    size = 0;
    fileChunk = NULL;
    deleted = false;
    haveUncommitedWrites = false;
}

StorageMemoChunk::~StorageMemoChunk()
{
    keyValues.DeleteTree();
    if (fileChunk != NULL)
        delete fileChunk;
}

StorageChunk::ChunkState StorageMemoChunk::GetChunkState()
{
    if (serialized)
        return StorageChunk::Serialized;
    else
        return StorageChunk::Tree;
}

void StorageMemoChunk::NextBunch(StorageBulkCursor& cursor, StorageShard* shard)
{
    bool                    first;
    int                     cmpres;
    ReadBuffer              nextKey, key, value;
    StorageMemoKeyValue*    it;
    uint32_t                total;

    if (keyValues.GetCount() == 0)
    {
        cursor.SetLast(true);
        return;
    }

    nextKey = cursor.GetNextKey();
    it = keyValues.Locate(nextKey, cmpres);

    total = 0;
    first = true;
    while (it)
    {
        if (!shard->RangeContains(it->GetKey()))
        {
            it = keyValues.Next(it);
            continue;
        }

        if (!first && total + it->GetKey().GetLength() + it->GetValue().GetLength() > STORAGE_MEMO_BUNCH_GRAN)
            break;

        total += it->GetKey().GetLength() + it->GetValue().GetLength();
        cursor.AppendKeyValue(it);

        it = keyValues.Next(it);
        first = false;
    }
    cursor.FinalizeKeyValues();

    if (!it)
    {
        cursor.SetLast(true);
    }
    else
    {
        cursor.SetNextKey(it->GetKey());
        cursor.SetLast(false);
    }
}

//void StorageMemoChunk::SetChunkID(uint64_t chunkID_)
//{
//    chunkID = chunkID_;
//}
//
//void StorageMemoChunk::SetUseBloomFilter(bool useBloomFilter_)
//{
//    useBloomFilter = useBloomFilter_;
//}

uint64_t StorageMemoChunk::GetChunkID()
{
    return chunkID;
}

bool StorageMemoChunk::UseBloomFilter()
{
    return useBloomFilter;
}

StorageKeyValue* StorageMemoChunk::Get(ReadBuffer& key)
{
    int                     cmpres;
    StorageMemoKeyValue*    it;

    if (keyValues.GetCount() == 0)
        return NULL;

    it = keyValues.Locate<const ReadBuffer&>(key, cmpres);
    if (cmpres != 0)
        return NULL;

    return it;
}

void StorageMemoChunk::AsyncGet(StorageAsyncGet* asyncGet)
{
    StorageKeyValue*    kv;

    Log_Debug("StorageMemoChunk::AsyncGet");

    kv = Get(asyncGet->key);
    if (kv == NULL || kv->GetType() == STORAGE_KEYVALUE_TYPE_DELETE)
    {
        asyncGet->ret = false;
        asyncGet->completed = true;
        return;
    }

    asyncGet->value = kv->GetValue();
    asyncGet->ret = true;
    asyncGet->completed = true;
}

bool StorageMemoChunk::Set(ReadBuffer key, ReadBuffer value)
{
    int                     cmpres;
    StorageMemoKeyValue*    it;

    if (keyValues.GetCount() != 0)
    {
        it = keyValues.Locate<ReadBuffer&>(key, cmpres);
        if (cmpres == 0)
        {
            size -= it->GetLength();
            it->Set(key, value);
            size += it->GetLength();
            return true;
        }
    }

    it = new StorageMemoKeyValue;
    it->Set(key, value);
    keyValues.Insert(it);
    size += it->GetLength();

    return true;
}

bool StorageMemoChunk::Delete(ReadBuffer key)
{
    int                     cmpres;
    StorageMemoKeyValue*    it;

    if (keyValues.GetCount() != 0)
    {
        it = keyValues.Locate<ReadBuffer&>(key, cmpres);
        if (cmpres == 0)
        {
            size -= it->GetLength();
            it->Delete(key);
            size += it->GetLength();
            return true;
        }
    }

    it = new StorageMemoKeyValue;
    it->Delete(key);
    keyValues.Insert(it);
    size += it->GetLength();

    return true;
}

void StorageMemoChunk::RegisterLogCommand(uint64_t logSegmentID_, uint32_t logCommandID_)
{
    haveUncommitedWrites = true;

    if (minLogSegmentID == 0)
        minLogSegmentID = logSegmentID_;

    if (logSegmentID_ > maxLogSegmentID)
    {
        maxLogSegmentID = logSegmentID_;
        maxLogCommandID = logCommandID_;
    }
    else if (logSegmentID_ == maxLogSegmentID)
    {
        if (logCommandID_ > maxLogCommandID)
            maxLogCommandID = logCommandID_;
    }
}

uint64_t StorageMemoChunk::GetMinLogSegmentID()
{
    return minLogSegmentID;
}

uint64_t StorageMemoChunk::GetMaxLogSegmentID()
{
    return maxLogSegmentID;
}

uint32_t StorageMemoChunk::GetMaxLogCommandID()
{
    return maxLogCommandID;
}

ReadBuffer StorageMemoChunk::GetFirstKey()
{
    if (keyValues.First())
        return keyValues.First()->GetKey();
    else
        return ReadBuffer();
}

ReadBuffer StorageMemoChunk::GetLastKey()
{
    if (keyValues.Last())
        return keyValues.Last()->GetKey();
    else
        return ReadBuffer();
}

uint64_t StorageMemoChunk::GetSize()
{
    return size;
}

ReadBuffer StorageMemoChunk::GetMidpoint()
{
    if (keyValues.Mid())
        return keyValues.Mid()->GetKey();
    else
        return ReadBuffer();
}

StorageFileChunk* StorageMemoChunk::RemoveFileChunk()
{
    StorageFileChunk*   ret;

    ret = fileChunk;
    fileChunk = NULL;
    return ret;
}

void StorageMemoChunk::RemoveFirst()
{
    StorageMemoKeyValue* first;

    first = keyValues.First();
    size -= first->GetLength();
    keyValues.Remove(first);
    delete first;
}

<commit_msg>Refactored StorageEnvironment::TrySerializeChunks().<commit_after>#include "StorageMemoChunk.h"
#include "StorageChunkSerializer.h"
#include "StorageChunkWriter.h"
#include "StorageBulkCursor.h"
#include "StorageEnvironment.h"
#include "StorageAsyncGet.h"

static inline int KeyCmp(const ReadBuffer& a, const ReadBuffer& b)
{
    return ReadBuffer::Cmp(a, b);
}

static inline const ReadBuffer Key(const StorageMemoKeyValue* kv)
{
    return kv->GetKey();
}

static inline const ReadBuffer Key(const StorageFileKeyValue* kv)
{
    return kv->GetKey();
}

StorageMemoChunk::StorageMemoChunk(uint64_t chunkID_, bool useBloomFilter_)
{
    chunkID = chunkID_;
    useBloomFilter = useBloomFilter_;

    serialized = false;
    minLogSegmentID = 0;
    maxLogSegmentID = 0;
    maxLogCommandID = 0;
    size = 0;
    fileChunk = NULL;
    deleted = false;
    haveUncommitedWrites = false;
}

StorageMemoChunk::~StorageMemoChunk()
{
    keyValues.DeleteTree();
    if (fileChunk != NULL)
        delete fileChunk;
}

StorageChunk::ChunkState StorageMemoChunk::GetChunkState()
{
    if (serialized)
        return StorageChunk::Serialized;
    else
        return StorageChunk::Tree;
}

void StorageMemoChunk::NextBunch(StorageBulkCursor& cursor, StorageShard* shard)
{
    bool                    first;
    int                     cmpres;
    ReadBuffer              nextKey, key, value;
    StorageMemoKeyValue*    it;
    uint32_t                total;

    if (keyValues.GetCount() == 0)
    {
        cursor.SetLast(true);
        return;
    }

    nextKey = cursor.GetNextKey();
    it = keyValues.Locate(nextKey, cmpres);

    total = 0;
    first = true;
    while (it)
    {
        if (!shard->RangeContains(it->GetKey()))
        {
            it = keyValues.Next(it);
            continue;
        }

        if (!first && total + it->GetKey().GetLength() + it->GetValue().GetLength() > STORAGE_MEMO_BUNCH_GRAN)
            break;

        total += it->GetKey().GetLength() + it->GetValue().GetLength();
        cursor.AppendKeyValue(it);

        it = keyValues.Next(it);
        first = false;
    }
    cursor.FinalizeKeyValues();

    if (!it)
    {
        cursor.SetLast(true);
    }
    else
    {
        cursor.SetNextKey(it->GetKey());
        cursor.SetLast(false);
    }
}

//void StorageMemoChunk::SetChunkID(uint64_t chunkID_)
//{
//    chunkID = chunkID_;
//}
//
//void StorageMemoChunk::SetUseBloomFilter(bool useBloomFilter_)
//{
//    useBloomFilter = useBloomFilter_;
//}

uint64_t StorageMemoChunk::GetChunkID()
{
    return chunkID;
}

bool StorageMemoChunk::UseBloomFilter()
{
    return useBloomFilter;
}

StorageKeyValue* StorageMemoChunk::Get(ReadBuffer& key)
{
    int                     cmpres;
    StorageMemoKeyValue*    it;

    if (keyValues.GetCount() == 0)
        return NULL;

    it = keyValues.Locate<const ReadBuffer&>(key, cmpres);
    if (cmpres != 0)
        return NULL;

    return it;
}

void StorageMemoChunk::AsyncGet(StorageAsyncGet* asyncGet)
{
    StorageKeyValue*    kv;

    Log_Debug("StorageMemoChunk::AsyncGet");

    kv = Get(asyncGet->key);
    if (kv == NULL || kv->GetType() == STORAGE_KEYVALUE_TYPE_DELETE)
    {
        asyncGet->ret = false;
        asyncGet->completed = true;
        return;
    }

    asyncGet->value = kv->GetValue();
    asyncGet->ret = true;
    asyncGet->completed = true;
}

bool StorageMemoChunk::Set(ReadBuffer key, ReadBuffer value)
{
    int                     cmpres;
    StorageMemoKeyValue*    it;

    if (keyValues.GetCount() != 0)
    {
        it = keyValues.Locate<ReadBuffer&>(key, cmpres);
        if (cmpres == 0)
        {
            size -= it->GetLength();
            it->Set(key, value);
            size += it->GetLength();
            return true;
        }
    }

    it = new StorageMemoKeyValue;
    it->Set(key, value);
    keyValues.Insert(it);
    size += it->GetLength();

    return true;
}

bool StorageMemoChunk::Delete(ReadBuffer key)
{
    int                     cmpres;
    StorageMemoKeyValue*    it;

    if (keyValues.GetCount() != 0)
    {
        it = keyValues.Locate<ReadBuffer&>(key, cmpres);
        if (cmpres == 0)
        {
            size -= it->GetLength();
            it->Delete(key);
            size += it->GetLength();
            return true;
        }
    }

    it = new StorageMemoKeyValue;
    it->Delete(key);
    keyValues.Insert(it);
    size += it->GetLength();

    return true;
}

void StorageMemoChunk::RegisterLogCommand(uint64_t logSegmentID_, uint32_t logCommandID_)
{
    haveUncommitedWrites = true;

    if (minLogSegmentID == 0)
        minLogSegmentID = logSegmentID_;

    if (logSegmentID_ > maxLogSegmentID)
    {
        maxLogSegmentID = logSegmentID_;
        maxLogCommandID = logCommandID_;
    }
    else if (logSegmentID_ == maxLogSegmentID)
    {
        if (logCommandID_ > maxLogCommandID)
            maxLogCommandID = logCommandID_;
    }
}

uint64_t StorageMemoChunk::GetMinLogSegmentID()
{
    return minLogSegmentID;
}

uint64_t StorageMemoChunk::GetMaxLogSegmentID()
{
    return maxLogSegmentID;
}

uint32_t StorageMemoChunk::GetMaxLogCommandID()
{
    return maxLogCommandID;
}

ReadBuffer StorageMemoChunk::GetFirstKey()
{
    if (keyValues.First())
        return keyValues.First()->GetKey();
    else
        return ReadBuffer();
}

ReadBuffer StorageMemoChunk::GetLastKey()
{
    if (keyValues.Last())
        return keyValues.Last()->GetKey();
    else
        return ReadBuffer();
}

uint64_t StorageMemoChunk::GetSize()
{
    return size;
}

ReadBuffer StorageMemoChunk::GetMidpoint()
{
    if (keyValues.Mid())
        return keyValues.Mid()->GetKey();
    else
        return ReadBuffer();
}

StorageFileChunk* StorageMemoChunk::RemoveFileChunk()
{
    StorageFileChunk*   ret;

    ret = fileChunk;
    fileChunk = NULL;
    return ret;
}

void StorageMemoChunk::RemoveFirst()
{
    StorageMemoKeyValue* first;

    first = keyValues.First();
    size -= first->GetLength();
    keyValues.Remove(first);
    delete first;
}

<|endoftext|>


<commit_before>
#include <boost/test/unit_test.hpp>

#include "SoftXMT.hpp"
#include "Delegate.hpp"

BOOST_AUTO_TEST_SUITE( Stealing_tests );

int num_tasks = 10000;
int64_t num_finished=0;
GlobalAddress<int64_t> nf_addr;

struct task1_arg {
    int num;
    Thread * parent;
};

void wake_f( task1_arg * args, size_t size, void * payload, size_t payload_size ) {
    SoftXMT_wake( args->parent );
}

void task1_f( void * arg ) {
    task1_arg* targ = (task1_arg*) arg;
    int mynum = targ->num;
    Thread * parent = targ->parent;

    if (mynum % 500 == 0) {
        BOOST_MESSAGE( CURRENT_THREAD << " with task " << mynum << " about to yield" );
    }
    SoftXMT_yield( );

    int64_t result = SoftXMT_delegate_fetch_and_add_word( nf_addr, 1 );
    if ( result == num_tasks-1 ) {
        BOOST_MESSAGE( CURRENT_THREAD << " with task " << mynum << " result=" << result );
        if ( SoftXMT_mynode() == 0 ) {
            SoftXMT_wake( parent );
        } else {
            SoftXMT_call_on( 0, wake_f, targ );
        }
    }
}

void user_main( Thread * me, void * args )
{
  nf_addr = GlobalAddress<int64_t>::TwoDimensional( &num_finished );

  task1_arg argss[num_tasks];
  for (int ta = 0; ta<num_tasks; ta++) {
    argss[ta] = { ta, me };
    SoftXMT_publicTask( &task1_f, &argss[ta] );
  }

  SoftXMT_suspend( );  // currently here just not to starve workers

  BOOST_MESSAGE( "user main is exiting" );
  SoftXMT_signal_done();
}

BOOST_AUTO_TEST_CASE( test1 ) {

  SoftXMT_init( &(boost::unit_test::framework::master_test_suite().argc),
                &(boost::unit_test::framework::master_test_suite().argv) );

  SoftXMT_activate();

  // force certain run parameters
  //BOOST_CHECK_EQUAL( FLAGS_chunk_size, 4 );
  //BOOST_CHECK_EQUAL( FLAGS_num_starting_workers, 4 );

  DVLOG(1) << "Spawning user main Thread....";
  SoftXMT_run_user_main( &user_main, NULL );
  BOOST_CHECK( SoftXMT_done() == true );

  SoftXMT_finish( 0 );
}

BOOST_AUTO_TEST_SUITE_END();

<commit_msg>make Stealing_test more controlled by having paired barriers on local and stolen tasks<commit_after>
#include <boost/test/unit_test.hpp>

#include "SoftXMT.hpp"
#include "Delegate.hpp"
#include "Tasking.hpp"

/// 8 tasks are spawned; first 4 meant to be stolen, then 4 meant to stay put
/// chunksize is 4 to ensure half the 8 will be released and stolen
/// these 4 stolen tasks are meant to wake the unstolen tasks
BOOST_AUTO_TEST_SUITE( Stealing_tests );

int num_tasks = 8;
int64_t num_finished=0;
int64_t vals[4] = { 0 };
Thread * threads[4];

struct task1_arg {
    int num;
    Thread * parent;
};

struct wake_arg {
    Thread * wakee;
};

void wake_f( wake_arg * args, size_t size, void * payload, size_t payload_size ) {
    SoftXMT_wake( args->wakee );
}

struct wakeindex_args {
    int index;
};

void wakeindex_f( wakeindex_args * args, size_t size, void * payload, size_t payload_size ) {
    SoftXMT_wake( threads[args->index] );
}

void pairsBarrier( int index ) {
    // increment the val on Node 0
    GlobalAddress<int64_t> vals_addr = GlobalAddress<int64_t>::TwoDimensional( &vals[index], 0 );
    int64_t result = SoftXMT_delegate_fetch_and_add_word( vals_addr, 1 );
    if ( 0 == result ) {
        // I am first so suspend
        SoftXMT_suspend( );
    } else if ( 1 == result ) {
        // I am second so wake other
        wakeindex_args warg = { index };
        SoftXMT_call_on( 1-SoftXMT_mynode(), &wakeindex_f, &warg );
    } else {
        BOOST_CHECK( false );
    }
}

void task_local( task1_arg * arg ) {
    int mynum = arg->num;
    Thread * parent = arg->parent;

    // this task should not have been stolen and running on 0
    BOOST_CHECK_EQUAL( 0, SoftXMT_mynode() );

    BOOST_MESSAGE( CURRENT_THREAD << " with task(local) " << mynum << " about to enter pair barrier" );
    threads[mynum] = CURRENT_THREAD; // store my Thread ptr in local global array
    pairsBarrier( mynum );

    // increment the global counter
    GlobalAddress<int64_t> nf_addr = GlobalAddress<int64_t>::TwoDimensional( &num_finished, 0 );
    int64_t result = SoftXMT_delegate_fetch_and_add_word( nf_addr, 1 );
    if ( result == num_tasks-1 ) {
        BOOST_MESSAGE( CURRENT_THREAD << " with task(local) " << mynum << " result=" << result );
        SoftXMT_wake( parent );
    }
}

void task_stolen( task1_arg * arg ) {
    int mynum = arg->num;
    Thread * parent = arg->parent;

    // this task should have been stolen and running on 1
    BOOST_CHECK_EQUAL( 1, SoftXMT_mynode() );

    // wake the corresponding task on Node 0
    BOOST_MESSAGE( CURRENT_THREAD << " with task(stolen) " << mynum << " about to enter pair barrier" );
    threads[mynum] = CURRENT_THREAD; // store my Thread ptr in local global array
    pairsBarrier( mynum );

    // increment the global counter
    GlobalAddress<int64_t> nf_addr = GlobalAddress<int64_t>::TwoDimensional( &num_finished, 0 );
    int64_t result = SoftXMT_delegate_fetch_and_add_word( nf_addr, 1 );
    if ( result == num_tasks-1 ) {
        BOOST_MESSAGE( CURRENT_THREAD << " with task(stolen) " << mynum << " result=" << result );
        wake_arg wwarg = { parent };
        SoftXMT_call_on( 0, &wake_f, &wwarg );
    }

}

void user_main( Thread * me, void * args )
{
  task1_arg argss[num_tasks];
  for (int ta = 0; ta<(num_tasks/2); ta++) {
    argss[ta] = { ta, me };
    SoftXMT_publicTask( &task_stolen, &argss[ta] );
  }
  for (int ta = 0; ta<(num_tasks/2); ta++) {
    argss[ta] = { ta, me };
    SoftXMT_publicTask( &task_local, &argss[ta] );
  }

  SoftXMT_suspend( );  // currently here just not to starve workers

  BOOST_MESSAGE( "user main is exiting" );
  SoftXMT_signal_done();
}

BOOST_AUTO_TEST_CASE( test1 ) {

  SoftXMT_init( &(boost::unit_test::framework::master_test_suite().argc),
                &(boost::unit_test::framework::master_test_suite().argv) );

  SoftXMT_activate();

  // force certain run parameters
  //BOOST_CHECK_EQUAL( FLAGS_chunk_size, 4 );
  //BOOST_CHECK_EQUAL( FLAGS_num_starting_workers, 4 );

  DVLOG(1) << "Spawning user main Thread....";
  SoftXMT_run_user_main( &user_main, NULL );
  BOOST_CHECK( SoftXMT_done() == true );

  SoftXMT_finish( 0 );
}

BOOST_AUTO_TEST_SUITE_END();

<|endoftext|>


<commit_before>//=====================================================================//
/*! @file
    @brief  net_core @n
			Copyright 2017 Kunihito Hiramatsu
    @author 平松邦仁 (hira@rvf-rc45.net)
*/
//=====================================================================//
#include "common/renesas.hpp"
#include "net/dhcp_client.hpp"

namespace net {

	//+++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++//
	/*!
		@brief  net_core テンプレート・クラス
		@param[in]	ETHER_IO	インサーネット・ドライバー
	*/
	//+++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++//
	template <class ETHER_IO>
	class net_core {
	public:

	private:

		enum class task {
			wait_link,	// リンクアップを待つ
			wait_dhcp,	// DHCP IP アドレスの取得を待つ
			setup_ip,	// IP アドレスを直接設定
			main_loop,	// メイン・ループ
		};

		ETHER_IO&	io_;

		typedef dhcp_client<ETHER_IO> DHCP;
		DHCP		dhcp_;
///		DHCP::dhcp_t dhcp_status_;

		task		task_;

	public:
		//-----------------------------------------------------------------//
		/*!
			@brief  コンストラクター
			@param[in]	io	インサーネット・ドライバー・クラス
		*/
		//-----------------------------------------------------------------//
		net_core(ETHER_IO& io) : io_(io), dhcp_(io),
			task_(task::wait_link)
			{ }

		//-----------------------------------------------------------------//
		/*!
			@brief  開始
			@param[in]	mac	インサーネット・コントローラー MAC アドレス
		*/
		//-----------------------------------------------------------------//
		bool start(const uint8_t* mac)
		{
			bool ret = io_.open(mac);
			if(ret) {
				utils::format("Ether open OK\n");
			} else {
				utils::format("Ether open NG\n");
			}

			task_ = task::wait_link;

			return ret;
		}

		//-----------------------------------------------------------------//
		/*!
			@brief  サービス（10ミリ秒毎に呼び出す）
		*/
		//-----------------------------------------------------------------//
		void service()
		{
			switch(task_) {

			case task::wait_link:
				io_.polling_link_status();
				if(io_.link_process()) {
					utils::format("Ether Link UP: OK\n");
					task_ = task::wait_dhcp;
				}
				break;

			case task::wait_dhcp:
				if(dhcp_.start()) {
					utils::format("DHCP OK\n");
					const DHCP_INFO& info = dhcp_.get_info();
					info.list();
					task_ = task::main_loop;
				} else {
					utils::format("DHCP NG\n");
					task_ = task::setup_ip;
				}
				break;

			case task::setup_ip:
				break;

			case task::main_loop:
				break;

			default:
				break;
			}
		}
	};

}
<commit_msg>update linkup interval<commit_after>//=====================================================================//
/*! @file
    @brief  net_core @n
			Copyright 2017 Kunihito Hiramatsu
    @author 平松邦仁 (hira@rvf-rc45.net)
*/
//=====================================================================//
#include "common/renesas.hpp"
#include "net/dhcp_client.hpp"

namespace net {

	//+++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++//
	/*!
		@brief  net_core テンプレート・クラス
		@param[in]	ETHER_IO	インサーネット・ドライバー
	*/
	//+++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++//
	template <class ETHER_IO>
	class net_core {
	public:

	private:

		enum class task {
			wait_link,	// リンクアップを待つ
			wait_dhcp,	// DHCP IP アドレスの取得を待つ
			setup_ip,	// IP アドレスを直接設定
			main_loop,	// メイン・ループ
		};

		ETHER_IO&	io_;

		typedef dhcp_client<ETHER_IO> DHCP;
		DHCP		dhcp_;

		task		task_;

		uint8_t	link_interval_;
		bool	link_up_;

	public:
		//-----------------------------------------------------------------//
		/*!
			@brief  コンストラクター
			@param[in]	io	インサーネット・ドライバー・クラス
		*/
		//-----------------------------------------------------------------//
		net_core(ETHER_IO& io) : io_(io), dhcp_(io),
			task_(task::wait_link), link_interval_(0), link_up_(false)
			{ }

		//-----------------------------------------------------------------//
		/*!
			@brief  開始
			@param[in]	mac	インサーネット・コントローラー MAC アドレス
		*/
		//-----------------------------------------------------------------//
		bool start(const uint8_t* mac)
		{
			bool ret = io_.open(mac);
			if(ret) {
				utils::format("Ether open OK\n");
			} else {
				utils::format("Ether open NG\n");
			}

			task_ = task::wait_link;

			return ret;
		}

		//-----------------------------------------------------------------//
		/*!
			@brief  サービス（10ミリ秒毎に呼び出す）
		*/
		//-----------------------------------------------------------------//
		void service()
		{
			if(link_interval_ >= 100) {
				io_.polling_link_status();
				link_interval_ = 0;
			}
			++link_interval_;

			bool link = io_.link_process();

			switch(task_) {

			case task::wait_link:
				if(!link_up_ && link) {
					utils::format("Ether Link UP: OK\n");
					task_ = task::wait_dhcp;
				}
				break;

			case task::wait_dhcp:
				if(dhcp_.start()) {
					utils::format("DHCP OK\n");
					const DHCP_INFO& info = dhcp_.get_info();
					info.list();
					task_ = task::main_loop;
				} else {
					utils::format("DHCP NG\n");
					task_ = task::setup_ip;
				}
				break;

			case task::setup_ip:
				break;

			case task::main_loop:
				break;

			default:
				break;
			}

			if(link_up_ && !link) {  // link is down
				task_ = task::wait_link;
			}

			link_up_ = link;
		}
	};

}
<|endoftext|>


<commit_before>//------------------------------------------------------------------------------
// File: mmodule.hpp
//
// Desc: Module for Data Mining.
//
// Copyright (c) 2014-2018. veyesys.com All rights reserved.
//------------------------------------------------------------------------------
#ifndef __M_MODULE_HPP__
#define __M_MODULE_HPP__
#include "utility.hpp"
#include "debug.hpp"
#include "videotype.hpp"
#include "miningtype.hpp"
#include "minterface.hpp"
#include "minterfacemgr.hpp"
#include "factory.hpp"
#include "cppkit/ck_string.h"
#include "cppkit/ck_memory.h"
#include "cppkit/ck_command_queue.h"
#include "cppkit/ck_dynamic_library.h"
#include "cppkit/ck_byte_ptr.h"

using namespace std;
using namespace cppkit;

typedef std::map<u32, MiningInterfaceMgr *> MInterfaceMgrMap;
class MiningModule
{
public:
	inline MiningModule(astring strPath, Factory &pFactory);
	inline ~MiningModule();
public:
	inline BOOL Valid();
	inline BOOL Init();
	inline BOOL InitOneDevice(DeviceParam & Param);

private:
	MiningInterface * m_module;
	ck_dynamic_library m_dynLib;
	MiningCreateDeviceFunc m_CreateDevice;
	astring m_strPath;
	BOOL m_Init;
	Factory &m_pFactory;
	MInterfaceMgrMap m_mgrMap;
};

#include "mmoduleimpl.hpp"

#endif /* __M_MODULE_HPP__ */
<commit_msg>add device callback<commit_after>//------------------------------------------------------------------------------
// File: mmodule.hpp
//
// Desc: Module for Data Mining.
//
// Copyright (c) 2014-2018. veyesys.com All rights reserved.
//------------------------------------------------------------------------------
#ifndef __M_MODULE_HPP__
#define __M_MODULE_HPP__
#include "utility.hpp"
#include "debug.hpp"
#include "videotype.hpp"
#include "miningtype.hpp"
#include "minterface.hpp"
#include "minterfacemgr.hpp"
#include "factory.hpp"
#include "cppkit/ck_string.h"
#include "cppkit/ck_memory.h"
#include "cppkit/ck_command_queue.h"
#include "cppkit/ck_dynamic_library.h"
#include "cppkit/ck_byte_ptr.h"

using namespace std;
using namespace cppkit;

typedef std::map<u32, MiningInterfaceMgr *> MInterfaceMgrMap;
class MiningModule
{
public:
	inline MiningModule(astring strPath, Factory &pFactory);
	inline ~MiningModule();
public:
	inline BOOL Valid();
	inline BOOL Init();
	inline BOOL InitOneDevice(DeviceParam & Param);
	inline BOOL DeleteOneDevice(int id);
    	inline static BOOL DeviceChangeCallbackFunc(void* pData,
						FactoryDeviceChangeData change);
    	inline BOOL DeviceChangeCallbackFunc1(FactoryDeviceChangeData change);

private:
	MiningInterface * m_module;
	ck_dynamic_library m_dynLib;
	MiningCreateDeviceFunc m_CreateDevice;
	astring m_strPath;
	BOOL m_Init;
	Factory &m_pFactory;
	MInterfaceMgrMap m_mgrMap;
};

#include "mmoduleimpl.hpp"

#endif /* __M_MODULE_HPP__ */
<|endoftext|>


<commit_before>//////////////////////////////////////////////////////////////////////////
//
//  Copyright (c) 2007-2010, Image Engine Design Inc. All rights reserved.
//
//  Redistribution and use in source and binary forms, with or without
//  modification, are permitted provided that the following conditions are
//  met:
//
//     * Redistributions of source code must retain the above copyright
//       notice, this list of conditions and the following disclaimer.
//
//     * Redistributions in binary form must reproduce the above copyright
//       notice, this list of conditions and the following disclaimer in the
//       documentation and/or other materials provided with the distribution.
//
//     * Neither the name of Image Engine Design nor the names of any
//       other contributors to this software may be used to endorse or
//       promote products derived from this software without specific prior
//       written permission.
//
//  THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS
//  IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO,
//  THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
//  PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR
//  CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
//  EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
//  PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
//  PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
//  LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
//  NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
//  SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
//
//////////////////////////////////////////////////////////////////////////

#include "boost/python.hpp"

#include <sstream>

#include "IECore/CompoundObject.h"
#include "IECorePython/RunTimeTypedBinding.h"

#include <exception>

using std::string;
using namespace boost;
using namespace boost::python;
using namespace IECore;

namespace IECorePython
{

static std::string repr( CompoundObject &o )
{
	std::stringstream s;

	s << "IECore." << o.typeName() << "(";

	bool added = false;
	for (
		CompoundObject::ObjectMap::const_iterator it = o.members().begin(); it != o.members().end(); ++it )
	{
		const std::string &key = it->first;

		object item( it->second );

		if ( item.attr( "__repr__" ) != object() )
		{
			std::string v = call_method< std::string >( item.ptr(), "__repr__" );

			if ( !added )
			{
				added = true;
				s << "{";
			}
			else
			{
				s << ",";
			}

			s << "'";
			s << key;
			s << "':";
			s << v;
		}
	}

	if ( added )
	{
		s << "}";
	}

	s << ")";

	return s.str();
}

static unsigned int len( const CompoundObject &o )
{
	return o.members().size();
}

static ObjectPtr getItem( const CompoundObject &o, const char *n )
{
	CompoundObject::ObjectMap::const_iterator it = o.members().find( n );
	if( it==o.members().end() )
	{
		PyErr_SetString( PyExc_KeyError, n );
		throw_error_already_set();
	}
	return it->second;
}

static void setItem( CompoundObject &o, const char *n, Object &v )
{
	o.members()[n] = &v;
}

static void delItem( CompoundObject &o, const char *n )
{
	CompoundObject::ObjectMap::iterator it = o.members().find( n );
	if( it==o.members().end() )
	{
		PyErr_SetString( PyExc_KeyError, n );
		throw_error_already_set();
	}
	o.members().erase( it );
}

static bool contains( const CompoundObject &o, const char *n )
{
	CompoundObject::ObjectMap::const_iterator it = o.members().find( n );
	if( it==o.members().end() )
	{
		return false;
	}
	return true;
}

static bool has_key( const CompoundObject &o, const char *n )
{
	CompoundObject::ObjectMap::const_iterator it = o.members().find( n );
	return ( it != o.members().end() );
}

static boost::python::list keys( const CompoundObject &o )
{
	boost::python::list result;
	CompoundObject::ObjectMap::const_iterator it;
	for( it = o.members().begin(); it!=o.members().end(); it++ )
	{
		result.append( it->first.value() );
	}
	return result;
}

static boost::python::list values( const CompoundObject &o )
{
	boost::python::list result;
	CompoundObject::ObjectMap::const_iterator it;
	for( it = o.members().begin(); it!=o.members().end(); it++ )
	{
		result.append( it->second );
	}
	return result;
}

static boost::python::list items( const CompoundObject &o )
{
	boost::python::list result;
	CompoundObject::ObjectMap::const_iterator it;
	for( it = o.members().begin(); it!=o.members().end(); it++ )
	{
		result.append( boost::python::make_tuple( it->first.value(), it->second ) );
	}
	return result;
}

class CompoundObjectFromPythonDict
{
	public :

		CompoundObjectFromPythonDict()
		{
			converter::registry::push_back(
				&convertible,
				&construct,
				type_id<CompoundObjectPtr> ()
			);
		}

	private :

		static void *convertible( PyObject *obj_ptr )
		{
			if ( !PyDict_Check( obj_ptr ) )
			{
				return 0;
			}
			return obj_ptr;
		}

		static void construct(
	        	PyObject *obj_ptr,
	        	converter::rvalue_from_python_stage1_data *data )
		{
			assert( obj_ptr );
			assert( PyDict_Check( obj_ptr ) );

			handle<> h( borrowed(obj_ptr) );
			dict d( h );

			void* storage = (( converter::rvalue_from_python_storage<CompoundObjectPtr>* ) data )->storage.bytes;
			new( storage ) CompoundObjectPtr( compoundObjectFromDict( d ) );
			data->convertible = storage;
		}

		static CompoundObjectPtr compoundObjectFromDict( const dict &v )
		{
			CompoundObjectPtr x = new CompoundObject();
			list values = v.values();
			list keys = v.keys();

			for (int i = 0; i < keys.attr("__len__")(); i++)
			{
				object key(keys[i]);
				object value(values[i]);
				extract<const char *> keyElem(key);
				if (!keyElem.check())
				{
					PyErr_SetString(PyExc_TypeError, "Incompatible key type. Only strings accepted.");
					throw_error_already_set();
				}

				extract< Object& > valueElem(value);
				if (valueElem.check())
				{
					setItem( *x, keyElem(), valueElem() );
					continue;
				}
				extract<dict> dictValueE( value );
				if( dictValueE.check() )
				{
					CompoundObjectPtr sub = compoundObjectFromDict( dictValueE() );
					setItem( *x, keyElem(), *sub );
					continue;
				}
				else
				{
					PyErr_SetString(PyExc_TypeError, "Incompatible value type - must be Object or dict.");
					throw_error_already_set();
				}
			}

			return x;
		}
};

/// binding for update method
static void update( CompoundObject &x, ConstCompoundObjectPtr y )
{
	assert( y );
	CompoundObject::ObjectMap::const_iterator it = y->members().begin();

	for (; it != y->members().end(); it++)
	{
		setItem( x, it->first.value().c_str(), *it->second );
	}
}

/// copy constructor
static CompoundObjectPtr copyConstructor( ConstCompoundObjectPtr other )
{
	assert( other );
	CompoundObjectPtr r = new CompoundObject();
	update( *r, other );
	return r;
}

/// binding for get method
static ObjectPtr get( const CompoundObject &o, const char *key, ObjectPtr defaultValue )
{
	CompoundObject::ObjectMap::const_iterator it = o.members().find( key );
	if ( it == o.members().end() )
	{
		return defaultValue;
	}
	// return the value from the CompoundObject
	return it->second;
}

static CompoundObjectPtr defaultInstance()
{
	return CompoundObject::defaultInstance();
}

void bindCompoundObject()
{

	RunTimeTypedClass<CompoundObject>()
		.def( init<>() )
		.def("__init__", make_constructor(&copyConstructor), "Copy constructor.")
		.def( "__repr__", &repr )
		.def( "__len__", &len )
		.def( "__getitem__", &getItem )
		.def( "__setitem__", &setItem )
		.def( "__delitem__", &delItem )
		.def( "__contains__", &contains )
		.def( "has_key", &has_key )
		.def( "keys", &keys )
		.def( "values", &values )
		.def( "items", &items )
		.def( "update", &update )
		.def( "get", &get, "m.get(k [, v])\nReturns m[k] if found; otherwise, returns v.",
			(
				boost::python::arg( "self" ),
				boost::python::arg( "key" ),
				boost::python::arg( "defaultValue" ) = object()
			)
		)
		.def( "defaultInstance", &defaultInstance ).staticmethod( "defaultInstance" )
	;

	CompoundObjectFromPythonDict();

}

} // namespace IECorePython
<commit_msg>Optimizing conversion from Python dict to CompoundObject.<commit_after>//////////////////////////////////////////////////////////////////////////
//
//  Copyright (c) 2007-2010, Image Engine Design Inc. All rights reserved.
//
//  Redistribution and use in source and binary forms, with or without
//  modification, are permitted provided that the following conditions are
//  met:
//
//     * Redistributions of source code must retain the above copyright
//       notice, this list of conditions and the following disclaimer.
//
//     * Redistributions in binary form must reproduce the above copyright
//       notice, this list of conditions and the following disclaimer in the
//       documentation and/or other materials provided with the distribution.
//
//     * Neither the name of Image Engine Design nor the names of any
//       other contributors to this software may be used to endorse or
//       promote products derived from this software without specific prior
//       written permission.
//
//  THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS
//  IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO,
//  THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
//  PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR
//  CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
//  EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
//  PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
//  PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
//  LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
//  NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
//  SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
//
//////////////////////////////////////////////////////////////////////////

#include "boost/python.hpp"

#include <sstream>

#include "IECore/CompoundObject.h"
#include "IECorePython/RunTimeTypedBinding.h"

#include <exception>

using std::string;
using namespace boost;
using namespace boost::python;
using namespace IECore;

namespace IECorePython
{

static std::string repr( CompoundObject &o )
{
	std::stringstream s;

	s << "IECore." << o.typeName() << "(";

	bool added = false;
	for (
		CompoundObject::ObjectMap::const_iterator it = o.members().begin(); it != o.members().end(); ++it )
	{
		const std::string &key = it->first;

		object item( it->second );

		if ( item.attr( "__repr__" ) != object() )
		{
			std::string v = call_method< std::string >( item.ptr(), "__repr__" );

			if ( !added )
			{
				added = true;
				s << "{";
			}
			else
			{
				s << ",";
			}

			s << "'";
			s << key;
			s << "':";
			s << v;
		}
	}

	if ( added )
	{
		s << "}";
	}

	s << ")";

	return s.str();
}

static unsigned int len( const CompoundObject &o )
{
	return o.members().size();
}

static ObjectPtr getItem( const CompoundObject &o, const char *n )
{
	CompoundObject::ObjectMap::const_iterator it = o.members().find( n );
	if( it==o.members().end() )
	{
		PyErr_SetString( PyExc_KeyError, n );
		throw_error_already_set();
	}
	return it->second;
}

static void setItem( CompoundObject &o, const char *n, Object &v )
{
	o.members()[n] = &v;
}

static void delItem( CompoundObject &o, const char *n )
{
	CompoundObject::ObjectMap::iterator it = o.members().find( n );
	if( it==o.members().end() )
	{
		PyErr_SetString( PyExc_KeyError, n );
		throw_error_already_set();
	}
	o.members().erase( it );
}

static bool contains( const CompoundObject &o, const char *n )
{
	CompoundObject::ObjectMap::const_iterator it = o.members().find( n );
	if( it==o.members().end() )
	{
		return false;
	}
	return true;
}

static bool has_key( const CompoundObject &o, const char *n )
{
	CompoundObject::ObjectMap::const_iterator it = o.members().find( n );
	return ( it != o.members().end() );
}

static boost::python::list keys( const CompoundObject &o )
{
	boost::python::list result;
	CompoundObject::ObjectMap::const_iterator it;
	for( it = o.members().begin(); it!=o.members().end(); it++ )
	{
		result.append( it->first.value() );
	}
	return result;
}

static boost::python::list values( const CompoundObject &o )
{
	boost::python::list result;
	CompoundObject::ObjectMap::const_iterator it;
	for( it = o.members().begin(); it!=o.members().end(); it++ )
	{
		result.append( it->second );
	}
	return result;
}

static boost::python::list items( const CompoundObject &o )
{
	boost::python::list result;
	CompoundObject::ObjectMap::const_iterator it;
	for( it = o.members().begin(); it!=o.members().end(); it++ )
	{
		result.append( boost::python::make_tuple( it->first.value(), it->second ) );
	}
	return result;
}

class CompoundObjectFromPythonDict
{
	public :

		CompoundObjectFromPythonDict()
		{
			converter::registry::push_back(
				&convertible,
				&construct,
				type_id<CompoundObjectPtr> ()
			);
		}

	private :

		static void *convertible( PyObject *obj_ptr )
		{
			if ( !PyDict_Check( obj_ptr ) )
			{
				return 0;
			}
			return obj_ptr;
		}

		static void construct(
	        	PyObject *obj_ptr,
	        	converter::rvalue_from_python_stage1_data *data )
		{
			assert( obj_ptr );
			assert( PyDict_Check( obj_ptr ) );

			handle<> h( borrowed(obj_ptr) );
			dict d( h );

			void* storage = (( converter::rvalue_from_python_storage<CompoundObjectPtr>* ) data )->storage.bytes;
			new( storage ) CompoundObjectPtr( compoundObjectFromDict( d ) );
			data->convertible = storage;
		}

		static CompoundObjectPtr compoundObjectFromDict( const dict &v )
		{
			CompoundObjectPtr x = new CompoundObject();
			list keys = v.keys();
			int len = boost::python::len(v);
			object key,value;
			for (int i = 0; i < len; i++)
			{
				key = keys[i];
				value = v[key];
				extract<const char *> keyElem(key);
				if (!keyElem.check())
				{
					PyErr_SetString(PyExc_TypeError, "Incompatible key type. Only strings accepted.");
					throw_error_already_set();
				}

				extract< Object& > valueElem(value);
				if (valueElem.check())
				{
					setItem( *x, keyElem(), valueElem() );
					continue;
				}
				extract<dict> dictValueE( value );
				if( dictValueE.check() )
				{
					CompoundObjectPtr sub = compoundObjectFromDict( dictValueE() );
					setItem( *x, keyElem(), *sub );
					continue;
				}
				else
				{
					PyErr_SetString(PyExc_TypeError, "Incompatible value type - must be Object or dict.");
					throw_error_already_set();
				}
			}

			return x;
		}
};

/// binding for update method
static void update( CompoundObject &x, ConstCompoundObjectPtr y )
{
	assert( y );
	CompoundObject::ObjectMap::const_iterator it = y->members().begin();

	for (; it != y->members().end(); it++)
	{
		setItem( x, it->first.value().c_str(), *it->second );
	}
}

/// copy constructor
static CompoundObjectPtr copyConstructor( ConstCompoundObjectPtr other )
{
	assert( other );
	CompoundObjectPtr r = new CompoundObject();
	update( *r, other );
	return r;
}

/// binding for get method
static ObjectPtr get( const CompoundObject &o, const char *key, ObjectPtr defaultValue )
{
	CompoundObject::ObjectMap::const_iterator it = o.members().find( key );
	if ( it == o.members().end() )
	{
		return defaultValue;
	}
	// return the value from the CompoundObject
	return it->second;
}

static CompoundObjectPtr defaultInstance()
{
	return CompoundObject::defaultInstance();
}

void bindCompoundObject()
{

	RunTimeTypedClass<CompoundObject>()
		.def( init<>() )
		.def("__init__", make_constructor(&copyConstructor), "Copy constructor.")
		.def( "__repr__", &repr )
		.def( "__len__", &len )
		.def( "__getitem__", &getItem )
		.def( "__setitem__", &setItem )
		.def( "__delitem__", &delItem )
		.def( "__contains__", &contains )
		.def( "has_key", &has_key )
		.def( "keys", &keys )
		.def( "values", &values )
		.def( "items", &items )
		.def( "update", &update )
		.def( "get", &get, "m.get(k [, v])\nReturns m[k] if found; otherwise, returns v.",
			(
				boost::python::arg( "self" ),
				boost::python::arg( "key" ),
				boost::python::arg( "defaultValue" ) = object()
			)
		)
		.def( "defaultInstance", &defaultInstance ).staticmethod( "defaultInstance" )
	;

	CompoundObjectFromPythonDict();

}

} // namespace IECorePython
<|endoftext|>


<commit_before>#include <cstdint>
#include <cstddef>
#include <unistd.h>

#include "ThoughtProcess.h"
#include "ThoughtProcess_StraightLine.h"
#include "SensorRTIMU.h"
#include "PiWars.h"
#include "Powertrain.h"
#include <iostream>

namespace PiWars {

ThoughtProcess_StraightLine::ThoughtProcess_StraightLine(PiWars *robot) : ThoughtProcess(robot), _rtimu(new SensorRTIMU()) {
}

ThoughtProcess_StraightLine::~ThoughtProcess_StraightLine() {
  if(_rtimu) {
    _rtimu->disable();
    delete _rtimu;
    _rtimu = nullptr;
  }
}

bool ThoughtProcess_StraightLine::available() {
  return _rtimu->exists();
}

bool ThoughtProcess_StraightLine::prepare() {
  return _rtimu->enable();
}

void ThoughtProcess_StraightLine::run() {
  bool running = true;
  float lastPowerLeft = -1.0, lastPowerRight = -1.0;
  float pitch, roll, yaw;

  float heading;

  // Let the sensor settle down
  std::this_thread::sleep_for (std::chrono::seconds(2));

  // Get the heading that we want to maintain
  _rtimu->fusion(pitch, roll, yaw);
  heading = yaw + 180;

  while(true) {
    float currentHeading, offset, powerLeft, powerRight;

    // Get the heading that we want to maintain
    _rtimu->fusion(pitch, roll, yaw);

    // Work out our offset versus the heading
    currentHeading = yaw + 180;

    // What's the difference?
    offset = currentHeading - heading;

    // Very simple checks
    if(offset < 1.0) {
      // We want to move left slightly
      powerLeft = 0.35;
      powerRight = 0.50;
    }
    else if(offset > 1.0) {
      // Move right
      powerLeft = 0.50;
      powerRight = 0.35;
    }
    else {
      // straight on!
      powerLeft = 0.50;
      powerRight = 0.50;
    }

    // Set the motors
    if(lastPowerLeft != powerLeft || lastPowerRight != powerRight) {
      robot()->powertrain()->setPower(powerLeft, powerRight);
      std::cerr << "setting power to " << powerLeft << ":" << powerRight << std::endl;
      lastPowerLeft = powerLeft;
      lastPowerRight = powerRight;
    }

    // Let the robot actually move
    std::this_thread::sleep_for (std::chrono::microseconds(200));
  }

  robot()->powertrain()->stop();
}

void ThoughtProcess_StraightLine::stop() {
}

}
<commit_msg>Updated straightline code to stop after 5 second, to run at a slightly higher speed, revers the left/right choice depending on the offset.<commit_after>#include <cstdint>
#include <cstddef>
#include <unistd.h>

#include "ThoughtProcess.h"
#include "ThoughtProcess_StraightLine.h"
#include "SensorRTIMU.h"
#include "PiWars.h"
#include "Powertrain.h"
#include <iostream>

namespace PiWars {

ThoughtProcess_StraightLine::ThoughtProcess_StraightLine(PiWars *robot) : ThoughtProcess(robot), _rtimu(new SensorRTIMU()) {
}

ThoughtProcess_StraightLine::~ThoughtProcess_StraightLine() {
  if(_rtimu) {
    _rtimu->disable();
    delete _rtimu;
    _rtimu = nullptr;
  }
}

bool ThoughtProcess_StraightLine::available() {
  return _rtimu->exists();
}

bool ThoughtProcess_StraightLine::prepare() {
  return _rtimu->enable();
}

void ThoughtProcess_StraightLine::run() {
  bool running = true;
  float lastPowerLeft = -1.0, lastPowerRight = -1.0;
  float pitch, roll, yaw;

  std::chrono::time_point<std::chrono::system_clock> start, end;
  float heading;

  // Let the sensor settle down
  std::this_thread::sleep_for (std::chrono::seconds(2));

  // Get the heading that we want to maintain
  _rtimu->fusion(pitch, roll, yaw);
  heading = yaw + 180;

  std::cerr << "Selected heading " << heading << std::endl;

  // Start the motors off at 50% so we don't pull too much current when we jump to 66%
  robot()->powertrain()->setPower(0.50, 0.50);

  // Note the start time
  start = std::chrono::system_clock::now();

  while(true)
  {
    float currentHeading, offset, powerLeft, powerRight;

    // Get the heading that we want to maintain
    _rtimu->fusion(pitch, roll, yaw);

    // Work out our offset versus the heading
    currentHeading = yaw + 180;

    // What's the difference?
    offset = currentHeading - heading;

    std::cerr << "Current heading " << currentHeading << " offset " << offset << "\r" << std::flush;
    // Very simple checks
    if(offset > 0) {
      // We want to move left slightly
      powerLeft = 0.55;
      powerRight = 0.66;
    }
    else if(offset < 0) {
      // Move right
      powerLeft = 0.66;
      powerRight = 0.55;
    }
    else {
      // straight on!
      powerLeft = 0.66;
      powerRight = 0.66;
    }

    // Set the motors
    if(lastPowerLeft != powerLeft || lastPowerRight != powerRight) {
      robot()->powertrain()->setPower(powerLeft, powerRight);
      //std::cerr << "setting power to " << powerLeft << ":" << powerRight << std::endl;
      lastPowerLeft = powerLeft;
      lastPowerRight = powerRight;
    }

    // Let the robot actually move
    std::this_thread::sleep_for (std::chrono::microseconds(100));

    end = std::chrono::system_clock::now();
    std::chrono::duration<float> duration = end - start;

    if(duration.count() >= 5.0){
      break;
    }
  }

  robot()->powertrain()->stop();

  std::cerr << std::endl << "Done!" << std::endl;
}

void ThoughtProcess_StraightLine::stop() {
}

}
<|endoftext|>