<commit_before>#include <string.h>
#include <stdlib.h>

#include "lexer.h"

namespace glsl {

// Lookup table of keywords
#undef KEYWORD
#define KEYWORD(X) { #X, kKeyword_##X },
static const keywordInfo kKeywords[] = {
    #include "lexemes.h"
};
#undef KEYWORD
#define KEYWORD(...)

// Lookup table of operators
#undef OPERATOR
#define OPERATOR(X, S, PREC) { #X, S, PREC },
static const operatorInfo kOperators[] = {
    #include "lexemes.h"
};
#undef OPERATOR
#define OPERATOR(...)

int token::precedence() const {
    return kOperators[m_operator].precedence;
}

/// location
location::location()
    : column(1)
    , line(1)
    , position(0)
{
}

void location::advanceColumn(size_t count) {
    column += count;
    position += count;
}

void location::advanceLine() {
    line++;
    position++;
    column = 1;
}

static inline bool isDigit(int ch) {
    return unsigned(ch) - '0' < 10;
}

static inline bool isChar(int ch) {
    return (unsigned(ch) | 32) - 'a' < 26;
}

static inline bool isOctal(int ch) {
    return unsigned(ch) - '0' < 8;
}

static inline bool isHex(int ch) {
    return (ch >= 'a' && ch <= 'f') || (ch >= 'A' && ch >= 'F') || isDigit(ch);
}

static inline bool isSpace(int ch) {
    return (ch >= '\t' && ch <= '\r') || ch == ' ';
}

lexer::lexer(const std::string &string)
    : m_data(string)
{
}

int lexer::at(int offset) const {
    if (position() + offset < m_data.length())
        return m_data[position() + offset];
    return 0;
}

void lexer::read(token &out) {
    // TODO: Line continuation (backslash `\'.)
    if (position() == m_data.length()) {
        out.m_type = kType_eof;
        return;
    }

    int ch1 = at(1);
    int ch2 = at(2);

    // Lex numerics
    if (isDigit(at()) || (at() == '.' && isDigit(ch1)))
    {
        bool isFloat = false;
        bool isDouble = false;
        bool isUnsigned = false;
        bool isOctalish = false;
        bool isHexish = false;

        if (at() == '0') {
            if (ch1 && (ch1 == 'x' || ch1 == 'X')) {
                isHexish = true;
                m_location.advanceColumn(2);
            }
            // TODO: Octal
        }

        std::string numeric = readNumeric(isOctalish, isHexish);
        if (position() != m_data.length() && at() == '.') {
            isFloat = true;
            numeric.push_back('.');
            m_location.advanceColumn();
            numeric += readNumeric(isOctalish, isHexish);
        }

        if (position() != m_data.length() && (at() == 'e' || at() == 'E')) {
            ch1 = at(1);
            ch2 = at(2);
            if ((ch1 == '+' || ch1 == '-') && (ch2 >= '0' && ch2 <= '9')) {
                numeric.push_back(ch1);
                numeric.push_back(ch2);
                m_location.advanceColumn(2);
                numeric += readNumeric(isOctalish, isHexish);
                isFloat = true;
            } else {
                m_error = "invalid numeric literal";
                return;
            }
        }

        if (position() != m_data.length() && isChar(at())) {
            ch1 = at(1);
            if (at() == 'f' || at() == 'F') {
                isFloat = true;
            } else if ((at() == 'l' && ch1 == 'f') || (at() == 'L' && ch1 == 'F')) {
                isFloat = false;
                isDouble = true;
            } else if (at() == 'u' || at() == 'U') {
                if (isFloat) {
                    m_error = "invalid use of suffix on literal";
                    return;
                }
                isUnsigned = true;
            } else {
                m_error = "invalid numeric literal";
                return;
            }
            m_location.advanceColumn();
        }

        if (isHexish && (isFloat || isDouble)) {
            m_error = "invalid numeric literal";
            return;
        }

        int base = isHexish ? 16 : (isOctalish ? 8 : 10);
        if (isFloat) {
            out.m_type = kType_constant_float;
            out.asFloat = strtof(numeric.c_str(), 0);
        } else if (isDouble) {
            out.m_type = kType_constant_double;
            out.asDouble = strtof(numeric.c_str(), 0);
        } else if (isUnsigned) {
            out.m_type = kType_constant_uint;
            out.asUnsigned = strtoul(numeric.c_str(), 0, base);
        } else {
            out.m_type = kType_constant_int;
            out.asInt = strtol(numeric.c_str(), 0, base);
        }
    } else if (isChar(at()) || at() == '_') {
        // Identifiers
        out.m_type = kType_identifier;
        out.m_identifier.clear();
        while (position() != m_data.length() && (isChar(at()) || isDigit(at()) || at() == '_')) {
            out.m_identifier.push_back(at());
            m_location.advanceColumn();
        }

        for (size_t i = 0; i < sizeof(kKeywords)/sizeof(kKeywords[0]); i++) {
            if (strcmp(kKeywords[i].name, out.m_identifier.c_str()))
                continue;
            out.m_type = kType_keyword;
            out.m_keyword = int(i);
        }
    } else {
        switch (at()) {
        // Non operators
        case '\n':
        case '\t':
        case '\f':
        case '\v':
        case '\r':
        case ' ':
            while (position() < m_data.length() && isSpace(at())) {
                if (at() == '\n')
                    m_location.advanceLine();
                else
                    m_location.advanceColumn();
            }
            out.m_type = kType_whitespace; // Whitespace already skippped
            break;
        case ';':
            out.m_type = kType_semicolon;
            m_location.advanceColumn();
            break;
        case '{':
            out.m_type = kType_scope_begin;
            m_location.advanceColumn();
            break;
        case '}':
            out.m_type = kType_scope_end;
            m_location.advanceColumn();
            break;

        // Operators
        case '.':
            out.m_type = kType_operator;
            out.m_operator = kOperator_dot;
            break;
        case '+':
            out.m_type = kType_operator;
            if (ch1 == '+')
                out.m_operator = kOperator_increment;
            else if (ch1 == '=')
                out.m_operator = kOperator_add_assign;
            else
                out.m_operator = kOperator_plus;
            break;
        case '-':
            out.m_type = kType_operator;
            if (ch1 == '-')
                out.m_operator = kOperator_decrement;
            else if (ch1 == '=')
                out.m_operator = kOperator_sub_assign;
            else
                out.m_operator = kOperator_minus;
            break;
        case '/':
            if (ch1 == '/') {
                // Skip line comments
                while (position() != m_data.length()) {
                    if (at() == '\n') {
                        m_location.advanceLine();
                        break;
                    }
                    m_location.advanceColumn();
                }
                out.m_type = kType_comment;
            } else if (ch1 == '*') {
                // Skip block comments
                while (position() != m_data.length()) {
                    if (at() == '\n') {
                        m_location.advanceLine();
                        continue;
                    }
                    if (at() == '*' && position() + 1 < m_data.length() && m_data[position() + 1] == '/') {
                        m_location.advanceColumn(2);
                        break;
                    }
                    m_location.advanceColumn();
                }
                out.m_type = kType_comment;
            } else if (ch1 == '=') {
                out.m_type = kType_operator;
                out.m_operator = kOperator_divide_assign;
            } else {
                out.m_type = kType_operator;
                out.m_operator = kOperator_divide;
            }
            break;
        case '*':
            out.m_type = kType_operator;
            if (ch1 == '=')
                out.m_operator = kOperator_multiply_assign;
            else
                out.m_operator = kOperator_multiply;
            break;
        case '%':
            out.m_type = kType_operator;
            if (ch1 == '=')
                out.m_operator = kOperator_modulus_assign;
            else
                out.m_operator = kOperator_modulus;
            break;
        case '<':
            out.m_type = kType_operator;
            if (ch1 == '<' && ch2 == '=')
                out.m_operator = kOperator_shift_left_assign;
            else if (ch1 == '<')
                out.m_operator = kOperator_shift_left;
            else if (ch1 == '=')
                out.m_operator = kOperator_less_equal;
            else
                out.m_operator = kOperator_less;
            break;
        case '>':
            out.m_type = kType_operator;
            if (ch1 == '>' && ch2 == '=')
                out.m_operator = kOperator_shift_right_assign;
            else if (ch1 == '>')
                out.m_operator = kOperator_shift_right;
            else if (ch1 == '=')
                out.m_operator = kOperator_greater_equal;
            else
                out.m_operator = kOperator_greater;
            break;
        case '[':
            out.m_type = kType_operator;
            out.m_operator = kOperator_bracket_begin;
            break;
        case ']':
            out.m_type = kType_operator;
            out.m_operator = kOperator_bracket_end;
            break;
        case '(':
            out.m_type = kType_operator;
            out.m_operator = kOperator_paranthesis_begin;
            break;
        case ')':
            out.m_type = kType_operator;
            out.m_operator = kOperator_paranthesis_end;
            break;
        case '^':
            out.m_type = kType_operator;
            if (ch1 == '^')
                out.m_operator = kOperator_logical_xor;
            else if (ch1 == '=')
                out.m_operator = kOperator_bit_xor_assign;
            else
                out.m_operator = kOperator_bit_xor;
            break;
        case '|':
            out.m_type = kType_operator;
            if (ch1 == '|')
                out.m_operator = kOperator_logical_or;
            else if (ch1 == '=')
                out.m_operator = kOperator_bit_or_assign;
            else
                out.m_operator = kOperator_bit_or;
            break;
        case '&':
            out.m_type = kType_operator;
            if (ch1 == '&')
                out.m_operator = kOperator_logical_and;
            else if (ch1 == '=')
                out.m_operator = kOperator_bit_and_assign;
            else
                out.m_operator = kOperator_bit_and;
            break;
        case '~':
            out.m_type = kType_operator;
            out.m_operator = kOperator_bit_not;
            break;
        case '=':
            out.m_type = kType_operator;
            if (ch1 == '=')
                out.m_operator = kOperator_equal;
            else
                out.m_operator = kOperator_assign;
            break;
        case '!':
            out.m_type = kType_operator;
            if (ch1 == '=')
                out.m_operator = kOperator_not_equal;
            else
                out.m_operator = kOperator_logical_not;
            break;
        case ':':
            out.m_type = kType_operator;
            out.m_operator = kOperator_colon;
            break;
        case ',':
            out.m_type = kType_operator;
            out.m_operator = kOperator_comma;
            break;
        case '?':
            out.m_type = kType_operator;
            out.m_operator = kOperator_questionmark;
            break;
        default:
            m_error = "invalid character encountered";
            return;
        }
        // Skip whitespace for operator
        if (out.m_type == kType_operator)
            m_location.advanceColumn(strlen(kOperators[out.m_operator].string));
    }
}

std::string lexer::readNumeric(bool isOctalish, bool isHexish) {
    std::string digits;
    if (isOctalish) {
        while (position() < m_data.length() && isOctal(at())) {
            digits.push_back(at());
            m_location.advanceColumn();
        }
    } else if (isHexish) {
        while (position() < m_data.length() && isHex(at())) {
            digits.push_back(at());
            m_location.advanceColumn();
        }
    } else {
        while (position() < m_data.length() && isDigit(at())) {
            digits.push_back(at());
            m_location.advanceColumn();
        }
    }
    return digits;
}

token lexer::peek() {
    token out;
    backup();
    read(out, true);
    restore();
    return out;
}

void lexer::read(token &out, bool) {
    do {
        read(out);
    } while (out.m_type == kType_whitespace || out.m_type == kType_comment);
}

const char *lexer::error() const {
    return m_error.size() ? m_error.c_str() : NULL;
}

void lexer::backup() {
    m_backup = m_location;
}

void lexer::restore() {
    m_location = m_backup;
}

}
<commit_msg>Fix hexadecimal lexing<commit_after>#include <string.h>
#include <stdlib.h>

#include "lexer.h"

namespace glsl {

// Lookup table of keywords
#undef KEYWORD
#define KEYWORD(X) { #X, kKeyword_##X },
static const keywordInfo kKeywords[] = {
    #include "lexemes.h"
};
#undef KEYWORD
#define KEYWORD(...)

// Lookup table of operators
#undef OPERATOR
#define OPERATOR(X, S, PREC) { #X, S, PREC },
static const operatorInfo kOperators[] = {
    #include "lexemes.h"
};
#undef OPERATOR
#define OPERATOR(...)

int token::precedence() const {
    return kOperators[m_operator].precedence;
}

/// location
location::location()
    : column(1)
    , line(1)
    , position(0)
{
}

void location::advanceColumn(size_t count) {
    column += count;
    position += count;
}

void location::advanceLine() {
    line++;
    position++;
    column = 1;
}

static inline bool isDigit(int ch) {
    return unsigned(ch) - '0' < 10;
}

static inline bool isChar(int ch) {
    return (unsigned(ch) | 32) - 'a' < 26;
}

static inline bool isOctal(int ch) {
    return unsigned(ch) - '0' < 8;
}

static inline bool isHex(int ch) {
    return (ch >= 'a' && ch <= 'f') || (ch >= 'A' && ch <= 'F') || isDigit(ch);
}

static inline bool isSpace(int ch) {
    return (ch >= '\t' && ch <= '\r') || ch == ' ';
}

lexer::lexer(const std::string &string)
    : m_data(string)
{
}

int lexer::at(int offset) const {
    if (position() + offset < m_data.length())
        return m_data[position() + offset];
    return 0;
}

void lexer::read(token &out) {
    // TODO: Line continuation (backslash `\'.)
    if (position() == m_data.length()) {
        out.m_type = kType_eof;
        return;
    }

    int ch1 = at(1);
    int ch2 = at(2);

    // Lex numerics
    if (isDigit(at()) || (at() == '.' && isDigit(ch1)))
    {
        bool isFloat = false;
        bool isDouble = false;
        bool isUnsigned = false;
        bool isOctalish = false;
        bool isHexish = false;

        if (at() == '0') {
            if (ch1 && (ch1 == 'x' || ch1 == 'X')) {
                isHexish = true;
                m_location.advanceColumn(2);
            }
            // TODO: Octal
        }

        std::string numeric = readNumeric(isOctalish, isHexish);
        if (position() != m_data.length() && at() == '.') {
            isFloat = true;
            numeric.push_back('.');
            m_location.advanceColumn();
            numeric += readNumeric(isOctalish, isHexish);
        }

        if (position() != m_data.length() && (at() == 'e' || at() == 'E')) {
            ch1 = at(1);
            ch2 = at(2);
            if ((ch1 == '+' || ch1 == '-') && (ch2 >= '0' && ch2 <= '9')) {
                numeric.push_back(ch1);
                numeric.push_back(ch2);
                m_location.advanceColumn(2);
                numeric += readNumeric(isOctalish, isHexish);
                isFloat = true;
            } else {
                m_error = "invalid numeric literal";
                return;
            }
        }

        if (position() != m_data.length() && isChar(at())) {
            ch1 = at(1);
            if (at() == 'f' || at() == 'F') {
                isFloat = true;
            } else if ((at() == 'l' && ch1 == 'f') || (at() == 'L' && ch1 == 'F')) {
                isFloat = false;
                isDouble = true;
            } else if (at() == 'u' || at() == 'U') {
                if (isFloat) {
                    m_error = "invalid use of suffix on literal";
                    return;
                }
                isUnsigned = true;
            } else {
                m_error = "invalid numeric literal";
                return;
            }
            m_location.advanceColumn();
        }

        if (isHexish && (isFloat || isDouble)) {
            m_error = "invalid numeric literal";
            return;
        }

        int base = isHexish ? 16 : (isOctalish ? 8 : 10);
        if (isFloat) {
            out.m_type = kType_constant_float;
            out.asFloat = strtof(numeric.c_str(), 0);
        } else if (isDouble) {
            out.m_type = kType_constant_double;
            out.asDouble = strtof(numeric.c_str(), 0);
        } else if (isUnsigned) {
            out.m_type = kType_constant_uint;
            out.asUnsigned = strtoul(numeric.c_str(), 0, base);
        } else {
            out.m_type = kType_constant_int;
            out.asInt = strtol(numeric.c_str(), 0, base);
        }
    } else if (isChar(at()) || at() == '_') {
        // Identifiers
        out.m_type = kType_identifier;
        out.m_identifier.clear();
        while (position() != m_data.length() && (isChar(at()) || isDigit(at()) || at() == '_')) {
            out.m_identifier.push_back(at());
            m_location.advanceColumn();
        }

        for (size_t i = 0; i < sizeof(kKeywords)/sizeof(kKeywords[0]); i++) {
            if (strcmp(kKeywords[i].name, out.m_identifier.c_str()))
                continue;
            out.m_type = kType_keyword;
            out.m_keyword = int(i);
        }
    } else {
        switch (at()) {
        // Non operators
        case '\n':
        case '\t':
        case '\f':
        case '\v':
        case '\r':
        case ' ':
            while (position() < m_data.length() && isSpace(at())) {
                if (at() == '\n')
                    m_location.advanceLine();
                else
                    m_location.advanceColumn();
            }
            out.m_type = kType_whitespace; // Whitespace already skippped
            break;
        case ';':
            out.m_type = kType_semicolon;
            m_location.advanceColumn();
            break;
        case '{':
            out.m_type = kType_scope_begin;
            m_location.advanceColumn();
            break;
        case '}':
            out.m_type = kType_scope_end;
            m_location.advanceColumn();
            break;

        // Operators
        case '.':
            out.m_type = kType_operator;
            out.m_operator = kOperator_dot;
            break;
        case '+':
            out.m_type = kType_operator;
            if (ch1 == '+')
                out.m_operator = kOperator_increment;
            else if (ch1 == '=')
                out.m_operator = kOperator_add_assign;
            else
                out.m_operator = kOperator_plus;
            break;
        case '-':
            out.m_type = kType_operator;
            if (ch1 == '-')
                out.m_operator = kOperator_decrement;
            else if (ch1 == '=')
                out.m_operator = kOperator_sub_assign;
            else
                out.m_operator = kOperator_minus;
            break;
        case '/':
            if (ch1 == '/') {
                // Skip line comments
                while (position() != m_data.length()) {
                    if (at() == '\n') {
                        m_location.advanceLine();
                        break;
                    }
                    m_location.advanceColumn();
                }
                out.m_type = kType_comment;
            } else if (ch1 == '*') {
                // Skip block comments
                while (position() != m_data.length()) {
                    if (at() == '\n') {
                        m_location.advanceLine();
                        continue;
                    }
                    if (at() == '*' && position() + 1 < m_data.length() && m_data[position() + 1] == '/') {
                        m_location.advanceColumn(2);
                        break;
                    }
                    m_location.advanceColumn();
                }
                out.m_type = kType_comment;
            } else if (ch1 == '=') {
                out.m_type = kType_operator;
                out.m_operator = kOperator_divide_assign;
            } else {
                out.m_type = kType_operator;
                out.m_operator = kOperator_divide;
            }
            break;
        case '*':
            out.m_type = kType_operator;
            if (ch1 == '=')
                out.m_operator = kOperator_multiply_assign;
            else
                out.m_operator = kOperator_multiply;
            break;
        case '%':
            out.m_type = kType_operator;
            if (ch1 == '=')
                out.m_operator = kOperator_modulus_assign;
            else
                out.m_operator = kOperator_modulus;
            break;
        case '<':
            out.m_type = kType_operator;
            if (ch1 == '<' && ch2 == '=')
                out.m_operator = kOperator_shift_left_assign;
            else if (ch1 == '<')
                out.m_operator = kOperator_shift_left;
            else if (ch1 == '=')
                out.m_operator = kOperator_less_equal;
            else
                out.m_operator = kOperator_less;
            break;
        case '>':
            out.m_type = kType_operator;
            if (ch1 == '>' && ch2 == '=')
                out.m_operator = kOperator_shift_right_assign;
            else if (ch1 == '>')
                out.m_operator = kOperator_shift_right;
            else if (ch1 == '=')
                out.m_operator = kOperator_greater_equal;
            else
                out.m_operator = kOperator_greater;
            break;
        case '[':
            out.m_type = kType_operator;
            out.m_operator = kOperator_bracket_begin;
            break;
        case ']':
            out.m_type = kType_operator;
            out.m_operator = kOperator_bracket_end;
            break;
        case '(':
            out.m_type = kType_operator;
            out.m_operator = kOperator_paranthesis_begin;
            break;
        case ')':
            out.m_type = kType_operator;
            out.m_operator = kOperator_paranthesis_end;
            break;
        case '^':
            out.m_type = kType_operator;
            if (ch1 == '^')
                out.m_operator = kOperator_logical_xor;
            else if (ch1 == '=')
                out.m_operator = kOperator_bit_xor_assign;
            else
                out.m_operator = kOperator_bit_xor;
            break;
        case '|':
            out.m_type = kType_operator;
            if (ch1 == '|')
                out.m_operator = kOperator_logical_or;
            else if (ch1 == '=')
                out.m_operator = kOperator_bit_or_assign;
            else
                out.m_operator = kOperator_bit_or;
            break;
        case '&':
            out.m_type = kType_operator;
            if (ch1 == '&')
                out.m_operator = kOperator_logical_and;
            else if (ch1 == '=')
                out.m_operator = kOperator_bit_and_assign;
            else
                out.m_operator = kOperator_bit_and;
            break;
        case '~':
            out.m_type = kType_operator;
            out.m_operator = kOperator_bit_not;
            break;
        case '=':
            out.m_type = kType_operator;
            if (ch1 == '=')
                out.m_operator = kOperator_equal;
            else
                out.m_operator = kOperator_assign;
            break;
        case '!':
            out.m_type = kType_operator;
            if (ch1 == '=')
                out.m_operator = kOperator_not_equal;
            else
                out.m_operator = kOperator_logical_not;
            break;
        case ':':
            out.m_type = kType_operator;
            out.m_operator = kOperator_colon;
            break;
        case ',':
            out.m_type = kType_operator;
            out.m_operator = kOperator_comma;
            break;
        case '?':
            out.m_type = kType_operator;
            out.m_operator = kOperator_questionmark;
            break;
        default:
            m_error = "invalid character encountered";
            return;
        }
        // Skip whitespace for operator
        if (out.m_type == kType_operator)
            m_location.advanceColumn(strlen(kOperators[out.m_operator].string));
    }
}

std::string lexer::readNumeric(bool isOctalish, bool isHexish) {
    std::string digits;
    if (isOctalish) {
        while (position() < m_data.length() && isOctal(at())) {
            digits.push_back(at());
            m_location.advanceColumn();
        }
    } else if (isHexish) {
        while (position() < m_data.length() && isHex(at())) {
            digits.push_back(at());
            m_location.advanceColumn();
        }
    } else {
        while (position() < m_data.length() && isDigit(at())) {
            digits.push_back(at());
            m_location.advanceColumn();
        }
    }
    return digits;
}

token lexer::peek() {
    token out;
    backup();
    read(out, true);
    restore();
    return out;
}

void lexer::read(token &out, bool) {
    do {
        read(out);
    } while (out.m_type == kType_whitespace || out.m_type == kType_comment);
}

const char *lexer::error() const {
    return m_error.size() ? m_error.c_str() : NULL;
}

void lexer::backup() {
    m_backup = m_location;
}

void lexer::restore() {
    m_location = m_backup;
}

}
<|endoftext|>


<commit_before>#include <iostream>
#include <string>

using namespace std;

ALPHABET_SIZE = 26;

int index_of(char c)
{
    return tolower(c) - 'a';
}

struct TrieNode
{
    TrieNode(char key, bool present = false);

    char key;
    bool present;

    TrieNode* children[ALPHABET_SIZE];
};

TrieNode::TrieNode(char key, bool present = false)
{
    this->key = key;
    this->present = present;

    for(int i = 0; i < ALPHABET_SIZE; i++) {
        children[i] = NULL;
    }
}

typedef TrieNode* NodePtr;

class StringSet
{
    public:
        StringSet();

        ~StringSet();

        // Returns true if and only if 'str' is in the set.
        bool contains(string str);

        // Adds str to the set. If 'str' is present in the set, this operation has no effect.
        void insert(string str);

        // Removes 'str' from the set. If 'str' is not present in the se,t this operation has no effect.
        void remove(string str);

    private:
        NodePtr root;
};
<commit_msg>Minor errors in Trie<commit_after>#include <iostream>
#include <string>

using namespace std;

const int ALPHABET_SIZE = 26;

int index_of(char c) {
    return tolower(c) - 'a';
}

struct TrieNode
{
    TrieNode(char key, bool present = false);

    char key;
    bool present;

    TrieNode* children[ALPHABET_SIZE];
};

TrieNode::TrieNode(char key, bool present)
{
    this->key = key;
    this->present = present;

    for(int i = 0; i < ALPHABET_SIZE; i++) {
        children[i] = NULL;
    }
}

typedef TrieNode* NodePtr;

class StringSet
{
    public:
        StringSet();

        ~StringSet();

        // Returns true if and only if 'str' is in the set.
        bool contains(string str);

        // Adds str to the set. If 'str' is present in the set, this operation has no effect.
        void insert(string str);

        // Removes 'str' from the set. If 'str' is not present in the se,t this operation has no effect.
        void remove(string str);

    private:
        NodePtr root;
};
<|endoftext|>


<commit_before>#include <iostream>
#include <fstream>
#include <boost/program_options.hpp>
#include <boost/algorithm/string.hpp>
#include <boost/filesystem.hpp>
#include "sentence.hpp"
#include "modality.hpp"
#include "eval.hpp"

bool mkdir(const boost::filesystem::path &dir_path) {
	std::cerr << dir_path.string() << std::endl;
	if (!boost::filesystem::exists(dir_path)) {
		std::cerr << "mkdir " << dir_path.string() << std::endl;
		if (!boost::filesystem::create_directories(dir_path)) {
			std::cerr << "ERROR: mkdir " << dir_path.string() << " failed" << std::endl;
			return false;
		}
	}
	return true;
}

int main(int argc, char *argv[]) {

	unsigned int split_num = 5;

	boost::program_options::options_description opt("Usage", 200);
	opt.add_options()
		("path,p", boost::program_options::value< std::vector<std::string> >()->multitoken(), "input data to learn (required)")
		("list,l", boost::program_options::value< std::vector<std::string> >()->multitoken(), "list files of input data")
		("input,i", boost::program_options::value<int>(), "input format\n 0 - BCCWJ-style XML tagged format re-parsed by CaboCha [default]\n 1 - BCCWJ-style XML tagged format re-parsed by KNP\n 2 - depndency parsed format by CaboCha/J.DepP\n 3 - dependency parsed format by KNP\n 4 - predicate-argument structure analyzed format by SynCha/ChaPAS\n 5 - predicate-argument structure analyzed format by KNP")
		("pos", boost::program_options::value<int>(), "POS tag for CaboCha/J.DepP (optional)\n 0 - IPA/Naist-jdic [default]\n 1 - JumanDic\n 2 - UniDic")
		("posset", boost::program_options::value<std::string>(), "POS set to parse (optional")
		("with-fsem", "functional expressions are labeled with semantic labels")
		("cross,x", "enable cross validation (optional): default off")
		("split,g", boost::program_options::value<unsigned int>(), "number of groups for cross validation (optional): default 5")
		("outdir,o", boost::program_options::value<std::string>(), "directory to store output files (optional)\n simple training -  stores model file and feature file to \"model (default)\"\n cross validation - stores model file, feature file and result file to \"output (default)\"")
		("dic,d", boost::program_options::value<std::string>(), "dictionary directory (optional)")
		("target,t", boost::program_options::value<unsigned int>(), "method of detecting token to be learned\n 0 - by part of speech [default]\n 1 - predicate detected by a predicate-argument structure analyzer (only SynCha format is supported)\n 2 - by machine learning (has not been implemented)\n 3 - by gold data")
		("help,h", "Show help messages")
		("version,v", "Show version informaion");

	boost::program_options::variables_map argmap;
	boost::program_options::store(parse_command_line(argc, argv, opt), argmap);
	boost::program_options::notify(argmap);

	if (argmap.count("help")) {
		std::cout << opt << std::endl;
		return 1;
	}

	if (argmap.count("version")) {
		std::cout << PACKAGE_VERSION << std::endl;
		return 1;
	}

	std::string outdir;
	if (argmap.count("outdir")) {
		outdir = argmap["outdir"].as<std::string>();
	}
	else {
		if (argmap.count("cross")) {
			outdir = "output";
		}
		else {
			outdir = "model";
		}
	}
	boost::filesystem::path outdir_path(outdir);
	mkdir(outdir_path);

	std::string dic_dir = DICDIR;
	if (argmap.count("dic")) {
		dic_dir = argmap["dic"].as<std::string>();
	}
	boost::filesystem::path dic_dir_path(dic_dir);

	int input_layer;
	if (argmap.count("input")) {
		input_layer = argmap["input"].as<int>();
	}

	int pos_tag = modality::POS_IPA;
	if (argmap.count("pos")) {
		pos_tag = argmap["pos"].as<int>();
	}

	std::string pos_set;
	if (argmap.count("posset")) {
		pos_set = argmap["posset"].as<std::string>();
	}

	if (!argmap.count("path") && !argmap.count("list")) {
		std::cerr << "ERROR: no input data" << std::endl;
		return false;
	}
	std::vector<std::string> files;
	if (argmap.count("path")) {
		BOOST_FOREACH (std::string file, argmap["path"].as< std::vector<std::string> >() ) {
			files.push_back(file);
		}
	}
	if (argmap.count("list")) {
		BOOST_FOREACH (std::string list_file, argmap["list"].as< std::vector<std::string> >() ) {
			std::ifstream ifs(list_file.c_str());
			std::string buf;
			while (getline(ifs, buf)) {
				boost::algorithm::trim_right(buf);
				files.push_back(buf);
			}
		}
	}

	sort(files.begin(), files.end());
	files.erase(unique(files.begin(), files.end()), files.end());
	std::cout << "found " << files.size() << " files" << std::endl;

	if (files.size() == 0) {
		std::cerr << "ERROR: no learn data" << std::endl;
		exit(-1);
	}

	modality::parser mod_parser(outdir_path.string(), dic_dir);
	mod_parser.set_pos_tag(pos_tag, pos_set);

	boost::filesystem::path kt_path("keyterms.txt");
	kt_path = dic_dir_path / kt_path;
	mod_parser.fgen.load_keyterms(kt_path);

	if (argmap.count("target")) {
		mod_parser.target_detection = argmap["target"].as<unsigned int>();
	}

	if (argmap.count("with-fsem")) {
		mod_parser.has_fsem = true;
	}

	switch (input_layer) {
		case modality::IN_DEP_CAB:
		case modality::IN_DEP_KNP:
		case modality::IN_PAS_SYN:
		case modality::IN_PAS_KNP:
			mod_parser.load_deppasmods(files, input_layer);
			break;
		case modality::IN_XML_CAB:
		case modality::IN_XML_KNP:
			mod_parser.load_xmls(files, input_layer);
			break;
		default:
			std::cerr << "ERROR: invalid input format" << std::endl;
			exit(-1);
	}

	unsigned int cnt_inst = 0;
	BOOST_FOREACH (nlp::sentence sent, mod_parser.learning_data) {
		BOOST_FOREACH (nlp::chunk chk, sent.chunks) {
			BOOST_FOREACH (nlp::token tok, chk.tokens) {
				if (tok.has_mod) {
					cnt_inst++;
				}
			}
		}
	}
	std::cout << "load done" << std::endl;
 	std::cout << "   " << mod_parser.learning_data.size() << " sents" << std::endl;
 	std::cout << "   " << cnt_inst << " instances" << std::endl;
	std::cout << std::endl;

	if (argmap.count("cross")) {
		evaluator evals[LABEL_NUM];

		if (argmap.count("split")) {
			split_num = argmap["split"].as<unsigned int>();
		}
		unsigned int grp_size = mod_parser.learning_data.size() / split_num;
		std::vector< std::vector< nlp::sentence > > split_data;

		for (unsigned int i=0 ; i<split_num ; ++i) {
			std::vector< nlp::sentence > grp;
			// last group = rest of learning data
			if (i+1 == split_num) {
				std::copy(mod_parser.learning_data.begin() + i*grp_size, mod_parser.learning_data.end(), std::back_inserter(grp));
			}
			// divided data by split_num
			else {
				std::copy(mod_parser.learning_data.begin() + i*grp_size, mod_parser.learning_data.begin() + (i+1)*grp_size, std::back_inserter(grp));
			}
			split_data.push_back(grp);
		}
		// split_data contains split_num data sets

		unsigned int cnt = 0;
		BOOST_FOREACH (std::vector<nlp::sentence> grp, split_data) {
			std::cout << " group " << cnt << ": " << grp.size() << std::endl;
			cnt++;
		}
		std::cout << std::endl;

		for (unsigned int step=0 ; step<split_num ; ++step) {
			std::cout << "* step " << step << std::endl;
			mod_parser.learning_data.clear();

			boost::filesystem::path model_path[LABEL_NUM];
			boost::filesystem::path feat_path[LABEL_NUM];
			boost::filesystem::path result_path[LABEL_NUM];

			BOOST_FOREACH (unsigned int i, mod_parser.analyze_tags) {
				std::stringstream suffix_ss;
				suffix_ss << std::setw(3) << std::setfill('0') << step;
				boost::filesystem::path mp("model_" + mod_parser.id2tag(i) + suffix_ss.str());
				model_path[i] = outdir_path / mp;
				boost::filesystem::path fp("feat_" + mod_parser.id2tag(i) + suffix_ss.str());
				feat_path[i] = outdir_path / fp;
				boost::filesystem::path rp("result_" + mod_parser.id2tag(i) + suffix_ss.str());
				result_path[i] = outdir_path / rp;
			}

			for (unsigned int i=0 ; i<split_num ; ++i) {
				if (i != step) {
					// set learning data. used No.step data set as test data
					mod_parser.learning_data.insert(mod_parser.learning_data.end(), split_data[i].begin(), split_data[i].end());
				}
			}
			std::cout << " learning data size: " << mod_parser.learning_data.size() << std::endl;

			mod_parser.learn(model_path, feat_path);

			mod_parser.load_models(model_path);

			std::ofstream *os = new std::ofstream[LABEL_NUM];
			BOOST_FOREACH (unsigned int i, mod_parser.analyze_tags) {
				os[i].open(result_path[i].string().c_str());
			}

			std::vector< nlp::sentence > test_data;
			BOOST_FOREACH (nlp::sentence sent, split_data[step]) {
				test_data.push_back(sent);
			}

			for (unsigned int test_cnt=0 ; test_cnt<test_data.size() ; ++test_cnt) {
#ifdef _MODEBUG
				std::cout << "before " << test_data[test_cnt].sent_id << ": ";
				BOOST_FOREACH (nlp::chunk chk, test_data[test_cnt].chunks) {
					BOOST_FOREACH (nlp::token tok, chk.tokens) {
						if (tok.has_mod) {
							std::string mod_str;
							tok.mod.str(mod_str);
							std::cout << "  " << mod_str << std::endl;
						}
					}
				}
#endif

				// when target detection is DETECT_BY_GOLD, only bool value of having modality is set to test data
				if (mod_parser.target_detection == modality::DETECT_BY_GOLD) {
					BOOST_FOREACH (nlp::chunk chk, test_data[test_cnt].chunks) {
						BOOST_FOREACH (nlp::token tok, chk.tokens) {
							if (tok.has_mod) {
								tok.mod.tag.clear();
							}
						}
					}
				}
				else {
					test_data[test_cnt].clear_mod();
				}

				// tokens to be analyzed are detected by specified method in analyze() and gold data validation
				mod_parser.analyze(test_data[test_cnt]);

#ifdef _MODEBUG
				std::cout << "after  " << test_data[test_cnt].sent_id << ": ";
				BOOST_FOREACH (nlp::chunk chk, test_data[test_cnt].chunks) {
					BOOST_FOREACH (nlp::token tok, chk.tokens) {
						if (tok.has_mod) {
							std::string mod_str;
							tok.mod.str(mod_str);
							std::cout << "  " << mod_str << std::endl;
						}
					}
				}
#endif

				for (unsigned int chk_cnt=0 ; chk_cnt<test_data[test_cnt].chunks.size() ; ++chk_cnt) {
					for (unsigned int tok_cnt=0 ; tok_cnt<test_data[test_cnt].chunks[chk_cnt].tokens.size() ; ++tok_cnt) {
						nlp::token tok_gold = split_data[step][test_cnt].chunks[chk_cnt].tokens[tok_cnt];
						nlp::token tok_sys = test_data[test_cnt].chunks[chk_cnt].tokens[tok_cnt];
						if (tok_gold.has_mod && tok_sys.has_mod) {
							std::stringstream id_ss;
							id_ss << test_data[test_cnt].sent_id << "_" << tok_sys.id;

							BOOST_FOREACH (unsigned int i, mod_parser.analyze_tags) {
								evals[i].add( id_ss.str() , tok_gold.mod.tag[mod_parser.id2tag(i)], tok_sys.mod.tag[mod_parser.id2tag(i)] );
								os[i] << id_ss.str() << "," << tok_gold.mod.tag[mod_parser.id2tag(i)] << "," << tok_sys.mod.tag[mod_parser.id2tag(i)] << std::endl;
							}
						}
						else if (tok_sys.has_mod && !tok_gold.has_mod) {
							std::cerr << "ERROR: modality tag does not exist in token " << tok_sys.id << " in " << test_data[test_cnt].sent_id << std::endl;
						}
					}
				}
			}

			BOOST_FOREACH (unsigned int i, mod_parser.analyze_tags) {
				os[i].close();
			}
		}

		BOOST_FOREACH (unsigned int i, mod_parser.analyze_tags) {
			std::cout << "* " << mod_parser.id2tag(i) << std::endl;
			evals[i].eval();
			double acc = evals[i].accuracy();
			std::cout << acc << std::endl;
			evals[i].print_confusion_matrix();
			evals[i].print_prec_rec();
		}

	}
	else {
		mod_parser.learn();
	}

	mod_parser.save_f2i();
	mod_parser.save_l2i();
	mod_parser.save_i2l();

	return 1;
}

<commit_msg>fixed the bug of input layer for training<commit_after>#include <iostream>
#include <fstream>
#include <boost/program_options.hpp>
#include <boost/algorithm/string.hpp>
#include <boost/filesystem.hpp>
#include "sentence.hpp"
#include "modality.hpp"
#include "eval.hpp"

bool mkdir(const boost::filesystem::path &dir_path) {
	std::cerr << dir_path.string() << std::endl;
	if (!boost::filesystem::exists(dir_path)) {
		std::cerr << "mkdir " << dir_path.string() << std::endl;
		if (!boost::filesystem::create_directories(dir_path)) {
			std::cerr << "ERROR: mkdir " << dir_path.string() << " failed" << std::endl;
			return false;
		}
	}
	return true;
}

int main(int argc, char *argv[]) {

	unsigned int split_num = 5;

	boost::program_options::options_description opt("Usage", 200);
	opt.add_options()
		("path,p", boost::program_options::value< std::vector<std::string> >()->multitoken(), "input data to learn (required)")
		("list,l", boost::program_options::value< std::vector<std::string> >()->multitoken(), "list files of input data")
		("input,i", boost::program_options::value<int>(), "input layer (optional)\n 1 - dependency parsed layer by CaboCha/J.DepP\n 2 - dependency parsed layer by KNP\n 3 - predicate-argument structure analyzed layer by SynCha/ChaPAS\n 4 - predicate-argument structure analyzed layer by KNP\n 5 - BCCWJ-style XML format (inputs are analyzed by MeCab and CaboCha with IPA PoS tag)\n 6 - BCCWJ XML format (inputs are analyzed by Juman and KNP)")
		("pos", boost::program_options::value<int>(), "POS tag for CaboCha/J.DepP (optional)\n 0 - IPA/Naist-jdic [default]\n 1 - JumanDic\n 2 - UniDic")
		("posset", boost::program_options::value<std::string>(), "POS set to parse (optional")
		("with-fsem", "functional expressions are labeled with semantic labels")
		("cross,x", "enable cross validation (optional): default off")
		("split,g", boost::program_options::value<unsigned int>(), "number of groups for cross validation (optional): default 5")
		("outdir,o", boost::program_options::value<std::string>(), "directory to store output files (optional)\n simple training -  stores model file and feature file to \"model (default)\"\n cross validation - stores model file, feature file and result file to \"output (default)\"")
		("dic,d", boost::program_options::value<std::string>(), "dictionary directory (optional)")
		("target,t", boost::program_options::value<unsigned int>(), "method of detecting token to be learned\n 0 - by part of speech [default]\n 1 - predicate detected by a predicate-argument structure analyzer (only SynCha format is supported)\n 2 - by machine learning (has not been implemented)\n 3 - by gold data")
		("help,h", "Show help messages")
		("version,v", "Show version informaion");

	boost::program_options::variables_map argmap;
	boost::program_options::store(parse_command_line(argc, argv, opt), argmap);
	boost::program_options::notify(argmap);

	if (argmap.count("help")) {
		std::cout << opt << std::endl;
		return 1;
	}

	if (argmap.count("version")) {
		std::cout << PACKAGE_VERSION << std::endl;
		return 1;
	}

	std::string outdir;
	if (argmap.count("outdir")) {
		outdir = argmap["outdir"].as<std::string>();
	}
	else {
		if (argmap.count("cross")) {
			outdir = "output";
		}
		else {
			outdir = "model";
		}
	}
	boost::filesystem::path outdir_path(outdir);
	mkdir(outdir_path);

	std::string dic_dir = DICDIR;
	if (argmap.count("dic")) {
		dic_dir = argmap["dic"].as<std::string>();
	}
	boost::filesystem::path dic_dir_path(dic_dir);

	int input_layer;
	if (argmap.count("input")) {
		input_layer = argmap["input"].as<int>();
	}

	int pos_tag = modality::POS_IPA;
	if (argmap.count("pos")) {
		pos_tag = argmap["pos"].as<int>();
	}

	std::string pos_set;
	if (argmap.count("posset")) {
		pos_set = argmap["posset"].as<std::string>();
	}

	if (!argmap.count("path") && !argmap.count("list")) {
		std::cerr << "ERROR: no input data" << std::endl;
		return false;
	}
	std::vector<std::string> files;
	if (argmap.count("path")) {
		BOOST_FOREACH (std::string file, argmap["path"].as< std::vector<std::string> >() ) {
			files.push_back(file);
		}
	}
	if (argmap.count("list")) {
		BOOST_FOREACH (std::string list_file, argmap["list"].as< std::vector<std::string> >() ) {
			std::ifstream ifs(list_file.c_str());
			std::string buf;
			while (getline(ifs, buf)) {
				boost::algorithm::trim_right(buf);
				files.push_back(buf);
			}
		}
	}

	sort(files.begin(), files.end());
	files.erase(unique(files.begin(), files.end()), files.end());
	std::cout << "found " << files.size() << " files" << std::endl;

	if (files.size() == 0) {
		std::cerr << "ERROR: no learn data" << std::endl;
		exit(-1);
	}

	modality::parser mod_parser(outdir_path.string(), dic_dir);
	mod_parser.set_pos_tag(pos_tag, pos_set);

	boost::filesystem::path kt_path("keyterms.txt");
	kt_path = dic_dir_path / kt_path;
	mod_parser.fgen.load_keyterms(kt_path);

	if (argmap.count("target")) {
		mod_parser.target_detection = argmap["target"].as<unsigned int>();
	}

	if (argmap.count("with-fsem")) {
		mod_parser.has_fsem = true;
	}

	switch (input_layer) {
		case modality::IN_DEP_CAB:
		case modality::IN_DEP_KNP:
		case modality::IN_PAS_SYN:
		case modality::IN_PAS_KNP:
			mod_parser.load_deppasmods(files, input_layer);
			break;
		case modality::IN_XML_CAB:
		case modality::IN_XML_KNP:
			mod_parser.load_xmls(files, input_layer);
			break;
		default:
			std::cerr << "ERROR: invalid input format" << std::endl;
			exit(-1);
	}

	unsigned int cnt_inst = 0;
	BOOST_FOREACH (nlp::sentence sent, mod_parser.learning_data) {
		BOOST_FOREACH (nlp::chunk chk, sent.chunks) {
			BOOST_FOREACH (nlp::token tok, chk.tokens) {
				if (tok.has_mod) {
					cnt_inst++;
				}
			}
		}
	}
	std::cout << "load done" << std::endl;
 	std::cout << "   " << mod_parser.learning_data.size() << " sents" << std::endl;
 	std::cout << "   " << cnt_inst << " instances" << std::endl;
	std::cout << std::endl;

	if (argmap.count("cross")) {
		evaluator evals[LABEL_NUM];

		if (argmap.count("split")) {
			split_num = argmap["split"].as<unsigned int>();
		}
		unsigned int grp_size = mod_parser.learning_data.size() / split_num;
		std::vector< std::vector< nlp::sentence > > split_data;

		for (unsigned int i=0 ; i<split_num ; ++i) {
			std::vector< nlp::sentence > grp;
			// last group = rest of learning data
			if (i+1 == split_num) {
				std::copy(mod_parser.learning_data.begin() + i*grp_size, mod_parser.learning_data.end(), std::back_inserter(grp));
			}
			// divided data by split_num
			else {
				std::copy(mod_parser.learning_data.begin() + i*grp_size, mod_parser.learning_data.begin() + (i+1)*grp_size, std::back_inserter(grp));
			}
			split_data.push_back(grp);
		}
		// split_data contains split_num data sets

		unsigned int cnt = 0;
		BOOST_FOREACH (std::vector<nlp::sentence> grp, split_data) {
			std::cout << " group " << cnt << ": " << grp.size() << std::endl;
			cnt++;
		}
		std::cout << std::endl;

		for (unsigned int step=0 ; step<split_num ; ++step) {
			std::cout << "* step " << step << std::endl;
			mod_parser.learning_data.clear();

			boost::filesystem::path model_path[LABEL_NUM];
			boost::filesystem::path feat_path[LABEL_NUM];
			boost::filesystem::path result_path[LABEL_NUM];

			BOOST_FOREACH (unsigned int i, mod_parser.analyze_tags) {
				std::stringstream suffix_ss;
				suffix_ss << std::setw(3) << std::setfill('0') << step;
				boost::filesystem::path mp("model_" + mod_parser.id2tag(i) + suffix_ss.str());
				model_path[i] = outdir_path / mp;
				boost::filesystem::path fp("feat_" + mod_parser.id2tag(i) + suffix_ss.str());
				feat_path[i] = outdir_path / fp;
				boost::filesystem::path rp("result_" + mod_parser.id2tag(i) + suffix_ss.str());
				result_path[i] = outdir_path / rp;
			}

			for (unsigned int i=0 ; i<split_num ; ++i) {
				if (i != step) {
					// set learning data. used No.step data set as test data
					mod_parser.learning_data.insert(mod_parser.learning_data.end(), split_data[i].begin(), split_data[i].end());
				}
			}
			std::cout << " learning data size: " << mod_parser.learning_data.size() << std::endl;

			mod_parser.learn(model_path, feat_path);

			mod_parser.load_models(model_path);

			std::ofstream *os = new std::ofstream[LABEL_NUM];
			BOOST_FOREACH (unsigned int i, mod_parser.analyze_tags) {
				os[i].open(result_path[i].string().c_str());
			}

			std::vector< nlp::sentence > test_data;
			BOOST_FOREACH (nlp::sentence sent, split_data[step]) {
				test_data.push_back(sent);
			}

			for (unsigned int test_cnt=0 ; test_cnt<test_data.size() ; ++test_cnt) {
#ifdef _MODEBUG
				std::cout << "before " << test_data[test_cnt].sent_id << ": ";
				BOOST_FOREACH (nlp::chunk chk, test_data[test_cnt].chunks) {
					BOOST_FOREACH (nlp::token tok, chk.tokens) {
						if (tok.has_mod) {
							std::string mod_str;
							tok.mod.str(mod_str);
							std::cout << "  " << mod_str << std::endl;
						}
					}
				}
#endif

				// when target detection is DETECT_BY_GOLD, only bool value of having modality is set to test data
				if (mod_parser.target_detection == modality::DETECT_BY_GOLD) {
					BOOST_FOREACH (nlp::chunk chk, test_data[test_cnt].chunks) {
						BOOST_FOREACH (nlp::token tok, chk.tokens) {
							if (tok.has_mod) {
								tok.mod.tag.clear();
							}
						}
					}
				}
				else {
					test_data[test_cnt].clear_mod();
				}

				// tokens to be analyzed are detected by specified method in analyze() and gold data validation
				mod_parser.analyze(test_data[test_cnt]);

#ifdef _MODEBUG
				std::cout << "after  " << test_data[test_cnt].sent_id << ": ";
				BOOST_FOREACH (nlp::chunk chk, test_data[test_cnt].chunks) {
					BOOST_FOREACH (nlp::token tok, chk.tokens) {
						if (tok.has_mod) {
							std::string mod_str;
							tok.mod.str(mod_str);
							std::cout << "  " << mod_str << std::endl;
						}
					}
				}
#endif

				for (unsigned int chk_cnt=0 ; chk_cnt<test_data[test_cnt].chunks.size() ; ++chk_cnt) {
					for (unsigned int tok_cnt=0 ; tok_cnt<test_data[test_cnt].chunks[chk_cnt].tokens.size() ; ++tok_cnt) {
						nlp::token tok_gold = split_data[step][test_cnt].chunks[chk_cnt].tokens[tok_cnt];
						nlp::token tok_sys = test_data[test_cnt].chunks[chk_cnt].tokens[tok_cnt];
						if (tok_gold.has_mod && tok_sys.has_mod) {
							std::stringstream id_ss;
							id_ss << test_data[test_cnt].sent_id << "_" << tok_sys.id;

							BOOST_FOREACH (unsigned int i, mod_parser.analyze_tags) {
								evals[i].add( id_ss.str() , tok_gold.mod.tag[mod_parser.id2tag(i)], tok_sys.mod.tag[mod_parser.id2tag(i)] );
								os[i] << id_ss.str() << "," << tok_gold.mod.tag[mod_parser.id2tag(i)] << "," << tok_sys.mod.tag[mod_parser.id2tag(i)] << std::endl;
							}
						}
						else if (tok_sys.has_mod && !tok_gold.has_mod) {
							std::cerr << "ERROR: modality tag does not exist in token " << tok_sys.id << " in " << test_data[test_cnt].sent_id << std::endl;
						}
					}
				}
			}

			BOOST_FOREACH (unsigned int i, mod_parser.analyze_tags) {
				os[i].close();
			}
		}

		BOOST_FOREACH (unsigned int i, mod_parser.analyze_tags) {
			std::cout << "* " << mod_parser.id2tag(i) << std::endl;
			evals[i].eval();
			double acc = evals[i].accuracy();
			std::cout << acc << std::endl;
			evals[i].print_confusion_matrix();
			evals[i].print_prec_rec();
		}

	}
	else {
		mod_parser.learn();
	}

	mod_parser.save_f2i();
	mod_parser.save_l2i();
	mod_parser.save_i2l();

	return 1;
}

<|endoftext|>


<commit_before>#include "platform.h"
#include <iostream>
#include <numeric>

#include "naive.h"
<<<<<<< HEAD
#include "NTRAI_Leo.h"
=======
#include "april.h"
#include "trevor.h"
#include "senorsen.h"
>>>>>>> ed275f1d7742debf11ce0487c35f0583cbd98db6
using namespace std;

Platform::Platform() {
	//TODO: 初始化各个玩家
	initLevels();

	vector<int> t;
	Player *p = new Naive(level);
	string name = p->yourName();
	players.insert(pair <string, Player*> (name, p));
	scoreCards.insert(pair <string, vector<int>> (name, t));

	p = new NTRAI_Leo(level);
	name = p->yourName();
	players.insert(pair <string, Player*> (name, p));
	scoreCards.insert(pair <string, vector<int>> (name, t));

<<<<<<< HEAD
=======
	p = new Trevor(level);
	name = p->yourName();
	players.insert(pair <string, Player*> (name, p));
	scoreCards.insert(pair <string, vector<int>> (name, t));

	p = new Senorsen(level);
	name = p->yourName();
	players.insert(pair <string, Player*> (name, p));
	scoreCards.insert(pair <string, vector<int>> (name, t));

>>>>>>> ed275f1d7742debf11ce0487c35f0583cbd98db6
	for (auto iter = players.begin(); iter != players.end(); ++iter) {
		names.push_back(iter->first);
	}

	initCards();
	init();
}

int Platform::getRoundTime() {
	return (MAX_CARD - extraCards.size() - 4) / players.size();
}

void Platform::initLevels() {

	// http://zh.wikipedia.org/wiki/%E8%B0%81%E6%98%AF%E7%89%9B%E5%A4%B4%E7%8E%8B
	for (int i = 1; i <= MAX_CARD; ++i) {
		if ( (i - 5) % 5 == 0) {
			level.insert(pair <int, int>(i, 2));
			continue;
		}
		if (i % 10 == 0) {
			level.insert(pair <int, int>(i, 3));
			continue;
		}
		if (i % 11 == 0) {
			level.insert(pair <int, int>(i, 5));
			continue;
		}
		if (i == 55) {
			level.insert(pair <int, int>(i, 7));
			continue;
		}
		level.insert(pair <int, int>(i, 1));
	}
	// for (auto iter = level.begin(); iter != level.end(); ++iter) {
	// 	cerr << iter->first << ' ' << iter->second<<endl;
	// }
}

void Platform::initCards() {
	cerr << "Initialing Cards\n";
	for (int i = 1; i <= MAX_CARD; ++i)
		cards.push_back(i);
	random_shuffle(cards.begin(), cards.end());

	auto iter = cards.begin();

	int avaliableCount = MAX_CARD - 4;
	extraCards.resize(avaliableCount % player_count);
	copy(iter, iter + (avaliableCount % player_count), extraCards.begin());
	sort(extraCards.begin(), extraCards.end());
	iter += avaliableCount % player_count;

	for (int i = 0; i != player_count; ++i) {
		vector<int> tempCards(avaliableCount / player_count);
		copy(iter, iter + avaliableCount / player_count, tempCards.begin());
		sort(tempCards.begin(), tempCards.end());
		userCards.insert(pair <string, vector<int>> (names[i], tempCards));
		iter = iter + avaliableCount / player_count;
	}

	while (iter != cards.end()) {
		vector<int> heap;
		heap.push_back(*iter);
		heapCards.push_back(heap);
		++iter;
	}
	cerr << "Initial Card complete\n";

}

void Platform::printVector(vector<int> vec) {
	for (auto ai = vec.begin(); ai != vec.end(); ++ai)
		cerr << *ai << ' ';
	cerr << endl;
}

void Platform::singleRound() {
	map<int, string> operation;
	map<string, int> operationReversed;

	for (auto iter = players.begin(); iter != players.end(); ++iter) {

		cerr << "\tUser " << iter->first << " Start." << endl;
		int retCard = iter->second->run(heapCards, scoreCards);

		vector<int> &tempUserCard = userCards[iter->first];
		// 判断是否存在这张牌
		if (find(tempUserCard.begin(), tempUserCard.end(), retCard) == tempUserCard.end()) {
			cerr << "User: " + iter->first + " returns error AT run:" << retCard << endl;
			cerr << "Heap top dumped:" << endl;
			for (auto heapIter = heapCards.begin(); heapIter != heapCards.end(); ++heapIter) {
				printVector(*heapIter);
			}
			exit(1);
		}
		tempUserCard.erase(find(tempUserCard.begin(), tempUserCard.end(), retCard));

		operation.insert(pair <int, string> (retCard, iter->first));
		operationReversed.insert(pair <string, int> (iter->first, retCard));
		// 将出牌序列保存起来
	}

	for (auto iter = operation.begin(); iter != operation.end(); ++iter) {
		Player *player = players[iter->second];
		int opt = iter->first;

		// 找出一个最大的，符合要求的牌堆
		int cardMax = -1;
		auto maxIter = heapCards.begin();
		for (auto hiter = heapCards.begin(); hiter != heapCards.end(); ++hiter) {
			int tmpMax = *max_element(hiter->begin(), hiter->end());
			if (tmpMax > cardMax && tmpMax < opt) {
				cardMax = tmpMax;
				maxIter = hiter;
			}
		}

		if (cardMax == -1) {
			//没有找到，需要询问他删除那个牌堆了
			int heapToGet = player->getHeap(heapCards, operationReversed);
			if (heapToGet < 0 || heapToGet > 3) {
				cerr << "User: " + iter->first << " returns error AT getHeap: " << heapToGet <<endl;
				exit(1);
			}

			// 将即将积分的堆临时保存
			auto heapIter = &heapCards[heapToGet];
			sort(heapIter->begin(), heapIter->end());
			player->notifyGetScore(*heapIter);
			heapIter->clear();
			heapIter->push_back(iter->first);
			scoreCards[iter->second].insert(scoreCards[iter->second].end(), heapIter->begin(), heapIter->end());

		} else {
			// 找到了一个最大的，并且应该将它放上去
			maxIter->push_back(iter->first);
			sort(maxIter->begin(), maxIter->end());
			if (maxIter->size() > 5) {
				//超过5个，需要放进去了
				player->notifyGetScore(*maxIter);
				scoreCards[iter->second].insert(scoreCards[iter->second].end(), maxIter->begin(), maxIter->end());
				maxIter->clear();
				maxIter->push_back(iter->first);
			} else {
				// 没发生什么事情
			}
		}
	}

	for (auto iter = operation.begin(); iter != operation.end(); ++iter) {
		Player *player = players[iter->second];
		player->notifyPostRun(operationReversed);
	}
}

void Platform::init() {
	for (auto iter = names.begin(); iter != names.end(); ++iter) {
		players[*iter]->init(names, userCards[*iter], extraCards);
	}
}

void Platform::notifyFinish() {
	cerr << endl << endl << "User Score List:" << endl;
	for (auto iter = scoreCards.begin(); iter != scoreCards.end(); ++iter) {
		int sum = accumulate(iter->second.begin(), iter->second.end(), 0);
		cerr << iter->first << '\t' << sum << endl;
	}
	cerr << endl;
}

map<string, int> Platform::getScore() {
	map <string, int> scoreMap;
	for (auto iter = scoreCards.begin(); iter != scoreCards.end(); ++iter) {
		int sum = accumulate(iter->second.begin(), iter->second.end(), 0);
		scoreMap.insert(pair <string, int> (iter->first, sum));
	}
<<<<<<< HEAD
	cout << endl;
}
=======
	return scoreMap;
}
>>>>>>> ed275f1d7742debf11ce0487c35f0583cbd98db6
<commit_msg>fix merge<commit_after>#include "platform.h"
#include <iostream>
#include <numeric>

#include "naive.h"
#include "NTRAI_Leo.h"
#include "april.h"
#include "trevor.h"
#include "senorsen.h"
using namespace std;

Platform::Platform() {
	//TODO: 初始化各个玩家
	initLevels();

	vector<int> t;
	Player *p = new Naive(level);
	string name = p->yourName();
	players.insert(pair <string, Player*> (name, p));
	scoreCards.insert(pair <string, vector<int>> (name, t));

	p = new NTRAI_Leo(level);
	name = p->yourName();
	players.insert(pair <string, Player*> (name, p));
	scoreCards.insert(pair <string, vector<int>> (name, t));

	p = new Trevor(level);
	name = p->yourName();
	players.insert(pair <string, Player*> (name, p));
	scoreCards.insert(pair <string, vector<int>> (name, t));

	p = new Senorsen(level);
	name = p->yourName();
	players.insert(pair <string, Player*> (name, p));
	scoreCards.insert(pair <string, vector<int>> (name, t));

	for (auto iter = players.begin(); iter != players.end(); ++iter) {
		names.push_back(iter->first);
	}

	initCards();
	init();
}

int Platform::getRoundTime() {
	return (MAX_CARD - extraCards.size() - 4) / players.size();
}

void Platform::initLevels() {

	// http://zh.wikipedia.org/wiki/%E8%B0%81%E6%98%AF%E7%89%9B%E5%A4%B4%E7%8E%8B
	for (int i = 1; i <= MAX_CARD; ++i) {
		if ( (i - 5) % 5 == 0) {
			level.insert(pair <int, int>(i, 2));
			continue;
		}
		if (i % 10 == 0) {
			level.insert(pair <int, int>(i, 3));
			continue;
		}
		if (i % 11 == 0) {
			level.insert(pair <int, int>(i, 5));
			continue;
		}
		if (i == 55) {
			level.insert(pair <int, int>(i, 7));
			continue;
		}
		level.insert(pair <int, int>(i, 1));
	}
	// for (auto iter = level.begin(); iter != level.end(); ++iter) {
	// 	cerr << iter->first << ' ' << iter->second<<endl;
	// }
}

void Platform::initCards() {
	cerr << "Initialing Cards\n";
	for (int i = 1; i <= MAX_CARD; ++i)
		cards.push_back(i);
	random_shuffle(cards.begin(), cards.end());

	auto iter = cards.begin();

	int avaliableCount = MAX_CARD - 4;
	extraCards.resize(avaliableCount % player_count);
	copy(iter, iter + (avaliableCount % player_count), extraCards.begin());
	sort(extraCards.begin(), extraCards.end());
	iter += avaliableCount % player_count;

	for (int i = 0; i != player_count; ++i) {
		vector<int> tempCards(avaliableCount / player_count);
		copy(iter, iter + avaliableCount / player_count, tempCards.begin());
		sort(tempCards.begin(), tempCards.end());
		userCards.insert(pair <string, vector<int>> (names[i], tempCards));
		iter = iter + avaliableCount / player_count;
	}

	while (iter != cards.end()) {
		vector<int> heap;
		heap.push_back(*iter);
		heapCards.push_back(heap);
		++iter;
	}
	cerr << "Initial Card complete\n";

}

void Platform::printVector(vector<int> vec) {
	for (auto ai = vec.begin(); ai != vec.end(); ++ai)
		cerr << *ai << ' ';
	cerr << endl;
}

void Platform::singleRound() {
	map<int, string> operation;
	map<string, int> operationReversed;

	for (auto iter = players.begin(); iter != players.end(); ++iter) {

		cerr << "\tUser " << iter->first << " Start." << endl;
		int retCard = iter->second->run(heapCards, scoreCards);

		vector<int> &tempUserCard = userCards[iter->first];
		// 判断是否存在这张牌
		if (find(tempUserCard.begin(), tempUserCard.end(), retCard) == tempUserCard.end()) {
			cerr << "User: " + iter->first + " returns error AT run:" << retCard << endl;
			cerr << "Heap top dumped:" << endl;
			for (auto heapIter = heapCards.begin(); heapIter != heapCards.end(); ++heapIter) {
				printVector(*heapIter);
			}
			exit(1);
		}
		tempUserCard.erase(find(tempUserCard.begin(), tempUserCard.end(), retCard));

		operation.insert(pair <int, string> (retCard, iter->first));
		operationReversed.insert(pair <string, int> (iter->first, retCard));
		// 将出牌序列保存起来
	}

	for (auto iter = operation.begin(); iter != operation.end(); ++iter) {
		Player *player = players[iter->second];
		int opt = iter->first;

		// 找出一个最大的，符合要求的牌堆
		int cardMax = -1;
		auto maxIter = heapCards.begin();
		for (auto hiter = heapCards.begin(); hiter != heapCards.end(); ++hiter) {
			int tmpMax = *max_element(hiter->begin(), hiter->end());
			if (tmpMax > cardMax && tmpMax < opt) {
				cardMax = tmpMax;
				maxIter = hiter;
			}
		}

		if (cardMax == -1) {
			//没有找到，需要询问他删除那个牌堆了
			int heapToGet = player->getHeap(heapCards, operationReversed);
			if (heapToGet < 0 || heapToGet > 3) {
				cerr << "User: " + iter->first << " returns error AT getHeap: " << heapToGet <<endl;
				exit(1);
			}

			// 将即将积分的堆临时保存
			auto heapIter = &heapCards[heapToGet];
			sort(heapIter->begin(), heapIter->end());
			player->notifyGetScore(*heapIter);
			heapIter->clear();
			heapIter->push_back(iter->first);
			scoreCards[iter->second].insert(scoreCards[iter->second].end(), heapIter->begin(), heapIter->end());

		} else {
			// 找到了一个最大的，并且应该将它放上去
			maxIter->push_back(iter->first);
			sort(maxIter->begin(), maxIter->end());
			if (maxIter->size() > 5) {
				//超过5个，需要放进去了
				player->notifyGetScore(*maxIter);
				scoreCards[iter->second].insert(scoreCards[iter->second].end(), maxIter->begin(), maxIter->end());
				maxIter->clear();
				maxIter->push_back(iter->first);
			} else {
				// 没发生什么事情
			}
		}
	}

	for (auto iter = operation.begin(); iter != operation.end(); ++iter) {
		Player *player = players[iter->second];
		player->notifyPostRun(operationReversed);
	}
}

void Platform::init() {
	for (auto iter = names.begin(); iter != names.end(); ++iter) {
		players[*iter]->init(names, userCards[*iter], extraCards);
	}
}

void Platform::notifyFinish() {
	cerr << endl << endl << "User Score List:" << endl;
	for (auto iter = scoreCards.begin(); iter != scoreCards.end(); ++iter) {
		int sum = accumulate(iter->second.begin(), iter->second.end(), 0);
		cerr << iter->first << '\t' << sum << endl;
	}
	cerr << endl;
}

map<string, int> Platform::getScore() {
	map <string, int> scoreMap;
	for (auto iter = scoreCards.begin(); iter != scoreCards.end(); ++iter) {
		int sum = accumulate(iter->second.begin(), iter->second.end(), 0);
		scoreMap.insert(pair <string, int> (iter->first, sum));
	}
	return scoreMap;
}
<|endoftext|>


<commit_before>
#include <libclientserver.h>

int retval = 0;

class Handler : public IServerHandler
{
private:
	bool m_Quit;

public:
	Handler()
	{
		m_Quit = false;
	}

	void OnPreNewConnection()
	{
		//printf("PreNewConnection\n");
	}

	void OnPostNewConnection(IServerConnection *Connection)
	{
		//printf("PostNewConnection\n");
	}

	void OnDisconnect(IServerConnection *Connection)
	{
		//printf("Disconnect\n");
	}

	int OnRequest(IServerConnection *Connection, Json::Value &request, Json::Value &response)
	{
		std::stringstream ss;
		std::string Command = request["action"].asString();

		//ss << "OnRequest: " << Command << " ";
		//printf("%s\n", ss.str().c_str());

		if (Command == "PING")
			return 0;

		if (Command == "QUIT")
		{
			m_Quit = true;
			return 0;
		}

		return -ENOSYS;
	}

	int OnCommand(IServerConnection *Connection, Json::Value &request)
	{
		printf("OnCommand: %s\n", request["action"].asString().c_str());
		return -ENOSYS;
	}

	int OnJsonRequest(IServerConnection *Connection, Json::Value &req, Json::Value &res)
	{
		printf("OnJsonRequest\n");
		return -ENOSYS;
	}

	int OnJsonCommand(IServerConnection *Connection, Json::Value &req)
	{
		printf("OnJsonCommand\n");
		return -ENOSYS;
	}

	void OnBadLine(IServerConnection *Connection, const std::string *line)
	{
		printf("Bad Line: %s\n", line->c_str());
	}

	void Wait()
	{
		while(m_Quit == false)
		{
			printf("Running\n");
			sleep(1);
		}
	}

};

class TestServer : Thread
{
	public:
		TestServer()
		{
			m_quit = false;
			Start();
		}

		~TestServer()
		{
			m_quit = true;
			Stop();
		}

		virtual void Run()
		{
			Handler SrvHandler;
			ServerManager Manager(&SrvHandler);
	        Manager.SetThreads(4);
			ServerUnix Unix("/tmp/TestServer");
			Manager.ServerAdd(&Unix);
			while(m_quit == false)
			{	Json::Value event;
				event["action"] = "RUNNING";
				Manager.SendEvent(event);
				sleep(1);
			}

			Manager.ServerRemove(&Unix);
		}
	private:
		bool m_quit;
};

class TestClient : Thread {
    public:
        TestClient(ClientBase *cli) {
            m_cli = cli;
            m_quit = false;
            Start();
        }

        ~TestClient() {
            m_quit = true;
			Stop();
        }

        int Ping(ClientBase *Client) {
            PerfCounter PC("PING");
            Json::Value request;
            Json::Value response;

            request["action"] = "PING";
            return Client->SendRequest(request, response);
        }

        virtual void Run() {
            while(m_quit == false) {
                if (Ping(m_cli) != 0)
                    retval = 1;
            }
        }

    private:
        ClientBase *m_cli;
        bool m_quit;
};

int main(int argc, char **argv)
{
	TestServer TS;
    ClientBase *Client = Client::Create("unix:/tmp/TestServer");
    do {
        Client->Connect();
        Client->WaitForConnect();

        std::list<std::shared_ptr<TestClient> > lst;
        lst.push_back(std::make_shared<TestClient>(Client));
        lst.push_back(std::make_shared<TestClient>(Client));
        lst.push_back(std::make_shared<TestClient>(Client));
        lst.push_back(std::make_shared<TestClient>(Client));
        for(int i =0;i<10000;i++)
        {
            Client->Disconnect();
            Client->Connect();
        }

        lst.clear();
        Client->Disconnect();
    } while(0);

	delete Client;

	return retval;
}

<commit_msg>Fix bad exit code on test. This test should normally show a crash when it fails<commit_after>
#include <libclientserver.h>

int retval = 0;

class Handler : public IServerHandler
{
private:
	bool m_Quit;

public:
	Handler()
	{
		m_Quit = false;
	}

	void OnPreNewConnection()
	{
		//printf("PreNewConnection\n");
	}

	void OnPostNewConnection(IServerConnection *Connection)
	{
		//printf("PostNewConnection\n");
	}

	void OnDisconnect(IServerConnection *Connection)
	{
		//printf("Disconnect\n");
	}

	int OnRequest(IServerConnection *Connection, Json::Value &request, Json::Value &response)
	{
		std::stringstream ss;
		std::string Command = request["action"].asString();

		//ss << "OnRequest: " << Command << " ";
		//printf("%s\n", ss.str().c_str());

		if (Command == "PING")
			return 0;

		if (Command == "QUIT")
		{
			m_Quit = true;
			return 0;
		}

		return -ENOSYS;
	}

	int OnCommand(IServerConnection *Connection, Json::Value &request)
	{
		printf("OnCommand: %s\n", request["action"].asString().c_str());
		return -ENOSYS;
	}

	int OnJsonRequest(IServerConnection *Connection, Json::Value &req, Json::Value &res)
	{
		printf("OnJsonRequest\n");
		return -ENOSYS;
	}

	int OnJsonCommand(IServerConnection *Connection, Json::Value &req)
	{
		printf("OnJsonCommand\n");
		return -ENOSYS;
	}

	void OnBadLine(IServerConnection *Connection, const std::string *line)
	{
		printf("Bad Line: %s\n", line->c_str());
	}

	void Wait()
	{
		while(m_Quit == false)
		{
			printf("Running\n");
			sleep(1);
		}
	}

};

class TestServer : Thread
{
	public:
		TestServer()
		{
			m_quit = false;
			Start();
		}

		~TestServer()
		{
			m_quit = true;
			Stop();
		}

		virtual void Run()
		{
			Handler SrvHandler;
			ServerManager Manager(&SrvHandler);
	        Manager.SetThreads(4);
			ServerUnix Unix("/tmp/TestServer");
			Manager.ServerAdd(&Unix);
			while(m_quit == false)
			{	Json::Value event;
				event["action"] = "RUNNING";
				Manager.SendEvent(event);
				sleep(1);
			}

			Manager.ServerRemove(&Unix);
		}
	private:
		bool m_quit;
};

class TestClient : Thread {
    public:
        TestClient(ClientBase *cli) {
            m_cli = cli;
            m_quit = false;
            Start();
        }

        ~TestClient() {
            m_quit = true;
			Stop();
        }

        int Ping(ClientBase *Client) {
            PerfCounter PC("PING");
            Json::Value request;
            Json::Value response;

            request["action"] = "PING";
            return Client->SendRequest(request, response);
        }

        virtual void Run() {
            while(m_quit == false) {
                Ping(m_cli);
            }
        }

    private:
        ClientBase *m_cli;
        bool m_quit;
};

int main(int argc, char **argv)
{
	TestServer TS;
    ClientBase *Client = Client::Create("unix:/tmp/TestServer");
    do {
        Client->Connect();
        Client->WaitForConnect();

        std::list<std::shared_ptr<TestClient> > lst;
        lst.push_back(std::make_shared<TestClient>(Client));
        lst.push_back(std::make_shared<TestClient>(Client));
        lst.push_back(std::make_shared<TestClient>(Client));
        lst.push_back(std::make_shared<TestClient>(Client));
        for(int i =0;i<10000;i++)
        {
            Client->Disconnect();
            Client->Connect();
        }

        lst.clear();
        Client->Disconnect();
    } while(0);

	delete Client;

	return retval;
}

<|endoftext|>


<commit_before>// Copyright 2015 The Crashpad Authors. All rights reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

#include "util/thread/thread_log_messages.h"

#include <string.h>
#include <sys/types.h>

#include "base/logging.h"
#include "base/stl_util.h"
#include "base/strings/stringprintf.h"
#include "gtest/gtest.h"
#include "util/thread/thread.h"

namespace crashpad {
namespace test {
namespace {

TEST(ThreadLogMessages, Empty) {
  ThreadLogMessages thread_log_messages;

  const std::vector<std::string>& log_messages =
      thread_log_messages.log_messages();

  EXPECT_TRUE(log_messages.empty());
}

// For a message formatted like "[preamble] message\n", returns just "message".
// If the message is not formatted as expected, a Google Test expectation
// failure will be recorded and this function will return an empty string.
std::string MessageString(const std::string& log_message) {
  if (log_message.size() < 1) {
    EXPECT_GE(log_message.size(), 1u);
    return std::string();
  }

  constexpr char kStartChar = '[';
  if (log_message[0] != kStartChar) {
    EXPECT_EQ(log_message[0], kStartChar);
    return std::string();
  }

  static constexpr char kFindString[] = "] ";
  size_t pos = log_message.find(kFindString);
  if (pos == std::string::npos) {
    EXPECT_NE(pos, std::string::npos);
    return std::string();
  }

  std::string message_string = log_message.substr(pos + strlen(kFindString));
  if (message_string.size() < 1) {
    EXPECT_GE(message_string.size(), 1u);
    return std::string();
  }

  constexpr char kEndChar = '\n';
  if (message_string[message_string.size() - 1] != kEndChar) {
    EXPECT_NE(message_string[message_string.size() - 1], kEndChar);
    return std::string();
  }

  message_string.resize(message_string.size() - 1);
  return message_string;
}

TEST(ThreadLogMessages, Basic) {
  // Logging must be enabled at least at this level for this test to work.
  ASSERT_TRUE(LOG_IS_ON(INFO));

  {
    static constexpr const char* kMessages[] = {
      "An info message",
      "A warning message",
      "An error message",
    };

    ThreadLogMessages thread_log_messages;

    LOG(INFO) << kMessages[0];
    LOG(WARNING) << kMessages[1];
    LOG(ERROR) << kMessages[2];

    const std::vector<std::string>& log_messages =
        thread_log_messages.log_messages();

    EXPECT_EQ(log_messages.size(), base::size(kMessages));
    for (size_t index = 0; index < base::size(kMessages); ++index) {
      EXPECT_EQ(MessageString(log_messages[index]), kMessages[index])
          << "index " << index;
    }
  }

  {
    static constexpr char kMessage[] = "Sample error message";

    ThreadLogMessages thread_log_messages;

    LOG(ERROR) << kMessage;

    const std::vector<std::string>& log_messages =
        thread_log_messages.log_messages();

    EXPECT_EQ(log_messages.size(), 1u);
    EXPECT_EQ(MessageString(log_messages[0]), kMessage);
  }

  {
    ThreadLogMessages thread_log_messages;

    LOG(INFO) << "I can't believe I " << "streamed" << " the whole thing.";

    const std::vector<std::string>& log_messages =
        thread_log_messages.log_messages();

    EXPECT_EQ(log_messages.size(), 1u);
    EXPECT_EQ(MessageString(log_messages[0]),
              "I can't believe I streamed the whole thing.");
  }
}

class LoggingTestThread : public Thread {
 public:
  LoggingTestThread() : thread_number_(0), start_(0), count_(0) {}
  ~LoggingTestThread() override {}

  void Initialize(size_t thread_number, int start, int count) {
    thread_number_ = thread_number;
    start_ = start;
    count_ = count;
  }

 private:
  void ThreadMain() override {
    ThreadLogMessages thread_log_messages;

    std::vector<std::string> expected_messages;
    for (int index = start_; index < start_ + count_; ++index) {
      std::string message = base::StringPrintf("message %d", index);
      expected_messages.push_back(message);
      LOG(WARNING) << message;
    }

    const std::vector<std::string>& log_messages =
        thread_log_messages.log_messages();

    ASSERT_EQ(log_messages.size(), static_cast<size_t>(count_));
    for (size_t index = 0; index < log_messages.size(); ++index) {
      EXPECT_EQ(MessageString(log_messages[index]), expected_messages[index])
          << "thread_number_ " << thread_number_ << ", index " << index;
    }
  }

  size_t thread_number_;
  int start_;
  int count_;

  DISALLOW_COPY_AND_ASSIGN(LoggingTestThread);
};

TEST(ThreadLogMessages, Multithreaded) {
  // Logging must be enabled at least at this level for this test to work.
  ASSERT_TRUE(LOG_IS_ON(WARNING));

  LoggingTestThread threads[20];
  int start = 0;
  for (size_t index = 0; index < base::size(threads); ++index) {
    threads[index].Initialize(
        index, static_cast<int>(start), static_cast<int>(index));
    start += static_cast<int>(index);

    ASSERT_NO_FATAL_FAILURE(threads[index].Start());
  }

  for (LoggingTestThread& thread : threads) {
    thread.Join();
  }
}

}  // namespace
}  // namespace test
}  // namespace crashpad
<commit_msg>Fix ThreadLogMessagesTest on Chrome OS<commit_after>// Copyright 2015 The Crashpad Authors. All rights reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

#include "util/thread/thread_log_messages.h"

#include <string.h>
#include <sys/types.h>

#include "base/logging.h"
#include "base/stl_util.h"
#include "base/strings/stringprintf.h"
#include "gtest/gtest.h"
#include "util/thread/thread.h"

namespace crashpad {
namespace test {
namespace {

TEST(ThreadLogMessages, Empty) {
  ThreadLogMessages thread_log_messages;

  const std::vector<std::string>& log_messages =
      thread_log_messages.log_messages();

  EXPECT_TRUE(log_messages.empty());
}

void ExpectLogMessage(const std::string& message, const std::string& expected) {
  ASSERT_GT(message.size(), expected.size());
  EXPECT_EQ(message.back(), '\n');
  EXPECT_STREQ(
      message.substr(message.size() - expected.size() - 1, expected.size())
          .c_str(),
      expected.c_str());
}

TEST(ThreadLogMessages, Basic) {
  // Logging must be enabled at least at this level for this test to work.
  ASSERT_TRUE(LOG_IS_ON(INFO));

  {
    static constexpr const char* kMessages[] = {
      "An info message",
      "A warning message",
      "An error message",
    };

    ThreadLogMessages thread_log_messages;

    LOG(INFO) << kMessages[0];
    LOG(WARNING) << kMessages[1];
    LOG(ERROR) << kMessages[2];

    const std::vector<std::string>& log_messages =
        thread_log_messages.log_messages();

    EXPECT_EQ(log_messages.size(), base::size(kMessages));
    for (size_t index = 0; index < base::size(kMessages); ++index) {
      ASSERT_NO_FATAL_FAILURE(
          ExpectLogMessage(log_messages[index], kMessages[index]))
          << "index " << index;
    }
  }

  {
    static constexpr char kMessage[] = "Sample error message";

    ThreadLogMessages thread_log_messages;

    LOG(ERROR) << kMessage;

    const std::vector<std::string>& log_messages =
        thread_log_messages.log_messages();

    EXPECT_EQ(log_messages.size(), 1u);
    ExpectLogMessage(log_messages[0], kMessage);
  }

  {
    ThreadLogMessages thread_log_messages;

    LOG(INFO) << "I can't believe I " << "streamed" << " the whole thing.";

    const std::vector<std::string>& log_messages =
        thread_log_messages.log_messages();

    EXPECT_EQ(log_messages.size(), 1u);
    ExpectLogMessage(log_messages[0],
                     "I can't believe I streamed the whole thing.");
  }
}

class LoggingTestThread : public Thread {
 public:
  LoggingTestThread() : thread_number_(0), start_(0), count_(0) {}
  ~LoggingTestThread() override {}

  void Initialize(size_t thread_number, int start, int count) {
    thread_number_ = thread_number;
    start_ = start;
    count_ = count;
  }

 private:
  void ThreadMain() override {
    ThreadLogMessages thread_log_messages;

    std::vector<std::string> expected_messages;
    for (int index = start_; index < start_ + count_; ++index) {
      std::string message = base::StringPrintf("message %d", index);
      expected_messages.push_back(message);
      LOG(WARNING) << message;
    }

    const std::vector<std::string>& log_messages =
        thread_log_messages.log_messages();

    ASSERT_EQ(log_messages.size(), static_cast<size_t>(count_));
    for (size_t index = 0; index < log_messages.size(); ++index) {
      ASSERT_NO_FATAL_FAILURE(
          ExpectLogMessage(log_messages[index], expected_messages[index]))
          << "thread_number_ " << thread_number_ << ", index " << index;
    }
  }

  size_t thread_number_;
  int start_;
  int count_;

  DISALLOW_COPY_AND_ASSIGN(LoggingTestThread);
};

TEST(ThreadLogMessages, Multithreaded) {
  // Logging must be enabled at least at this level for this test to work.
  ASSERT_TRUE(LOG_IS_ON(WARNING));

  LoggingTestThread threads[20];
  int start = 0;
  for (size_t index = 0; index < base::size(threads); ++index) {
    threads[index].Initialize(
        index, static_cast<int>(start), static_cast<int>(index));
    start += static_cast<int>(index);

    ASSERT_NO_FATAL_FAILURE(threads[index].Start());
  }

  for (LoggingTestThread& thread : threads) {
    thread.Join();
  }
}

}  // namespace
}  // namespace test
}  // namespace crashpad
<|endoftext|>


<commit_before>/**
* Copyright 2016 BitTorrent Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
*    http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
#include "scraps/SHA256.h"
#include "scraps/apple/SHA256.h"
#include "scraps/sodium/SHA256.h"
#include "scraps/random.h"
#include "complexity.h"

#include <benchmark/benchmark.h>
#include <iostream>
#include <random>

using namespace scraps;

namespace {
auto GetRandomTestBytes(size_t numBytes) {
    static std::random_device rd;
    static auto rng = [&]{
        std::mt19937 rng;
        rng.seed(rd());
        return rng;
    }();

    return RandomBytes(numBytes, rng);
}
} // anonymous namespace

template <typename SHA256Type>
void SHA256Complexity(benchmark::State& state) {
    SHA256Type sha256;
    std::array<unsigned char, SHA256Type::kHashSize> result;
    while (state.KeepRunning()) {
        state.PauseTiming();
        auto bytes = GetRandomTestBytes(state.range(0));
        memset(result.data(), 0, result.size());
        sha256.reset();
        state.ResumeTiming();
        sha256.update(bytes.data(), bytes.size());
        sha256.finish(result.data());
        benchmark::DoNotOptimize(&result);
        benchmark::ClobberMemory();
    }
    state.SetComplexityN(state.range(0));
}

void SHA256ComplexityAllInOne(benchmark::State& state) {
    while (state.KeepRunning()) {
        state.PauseTiming();
        auto bytes = GetRandomTestBytes(state.range(0));
        state.ResumeTiming();
        auto result = GetSHA256(gsl::as_span(bytes));
        benchmark::DoNotOptimize(&result);
        benchmark::ClobberMemory();
    }
    state.SetComplexityN(state.range(0));
}

void SHA256ComplexityApple(benchmark::State& state) {
    SHA256Complexity<scraps::apple::SHA256>(state);
}

void SHA256ComplexitySodium(benchmark::State& state) {
    SHA256Complexity<scraps::sodium::SHA256>(state);
}

BENCHMARK(SHA256ComplexityAllInOne)->RangeMultiplier(4)->Range(256, 1<<15)->Complexity();
BENCHMARK(SHA256ComplexitySodium)->RangeMultiplier(4)->Range(256, 1<<15)->Complexity();
BENCHMARK(SHA256ComplexityApple)->RangeMultiplier(4)->Range(256, 1<<15)->Complexity();
EXPECT_COMPLEXITY_LE(SHA256ComplexityAllInOne, benchmark::oN);
EXPECT_COMPLEXITY_LE(SHA256ComplexitySodium, benchmark::oN);
EXPECT_COMPLEXITY_LE(SHA256ComplexityApple, benchmark::oN);
<commit_msg>Fix benchmark builds on Linux<commit_after>/**
* Copyright 2016 BitTorrent Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
*    http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
#include "scraps/SHA256.h"

#if SCRAPS_APPLE
#include "scraps/apple/SHA256.h"
#endif

#include "scraps/sodium/SHA256.h"
#include "scraps/random.h"
#include "complexity.h"

#include <benchmark/benchmark.h>
#include <iostream>
#include <random>

using namespace scraps;

namespace {
auto GetRandomTestBytes(size_t numBytes) {
    static std::random_device rd;
    static auto rng = [&]{
        std::mt19937 rng;
        rng.seed(rd());
        return rng;
    }();

    return RandomBytes(numBytes, rng);
}
} // anonymous namespace

template <typename SHA256Type>
void SHA256Complexity(benchmark::State& state) {
    SHA256Type sha256;
    std::array<unsigned char, SHA256Type::kHashSize> result;
    while (state.KeepRunning()) {
        state.PauseTiming();
        auto bytes = GetRandomTestBytes(state.range(0));
        memset(result.data(), 0, result.size());
        sha256.reset();
        state.ResumeTiming();
        sha256.update(bytes.data(), bytes.size());
        sha256.finish(result.data());
        benchmark::DoNotOptimize(&result);
        benchmark::ClobberMemory();
    }
    state.SetComplexityN(state.range(0));
}

void SHA256ComplexityAllInOne(benchmark::State& state) {
    while (state.KeepRunning()) {
        state.PauseTiming();
        auto bytes = GetRandomTestBytes(state.range(0));
        state.ResumeTiming();
        auto result = GetSHA256(gsl::as_span(bytes));
        benchmark::DoNotOptimize(&result);
        benchmark::ClobberMemory();
    }
    state.SetComplexityN(state.range(0));
}

#if SCRAPS_APPLE

void SHA256ComplexityApple(benchmark::State& state) {
    SHA256Complexity<scraps::apple::SHA256>(state);
}
BENCHMARK(SHA256ComplexityApple)->RangeMultiplier(4)->Range(256, 1<<15)->Complexity();
EXPECT_COMPLEXITY_LE(SHA256ComplexityApple, benchmark::oN);

#endif // SCRAPS_APPLE

void SHA256ComplexitySodium(benchmark::State& state) {
    SHA256Complexity<scraps::sodium::SHA256>(state);
}

BENCHMARK(SHA256ComplexityAllInOne)->RangeMultiplier(4)->Range(256, 1<<15)->Complexity();
BENCHMARK(SHA256ComplexitySodium)->RangeMultiplier(4)->Range(256, 1<<15)->Complexity();

EXPECT_COMPLEXITY_LE(SHA256ComplexityAllInOne, benchmark::oN);
EXPECT_COMPLEXITY_LE(SHA256ComplexitySodium, benchmark::oN);
<|endoftext|>