<commit_before>// Copyright 2008 Paul Hodge

#include "common_headers.h"

#include "testing.h"
#include "circa.h"

namespace circa {
namespace parser_tests {

void literal_float()
{
    token_stream::TokenStream tokens("1.0");
    ast::Expression *expr = parser::atom(tokens);
    ast::LiteralFloat *literal = dynamic_cast<ast::LiteralFloat*>(expr);
    test_assert(literal != NULL);
    test_assert(literal->text == "1.0");
    test_assert(literal->toString() == "1.0");
    test_assert(literal->hasQuestionMark == false);

    delete expr;

    tokens.reset("5.0?");
    expr = parser::atom(tokens);
    literal = dynamic_cast<ast::LiteralFloat*>(expr);
    test_assert(literal != NULL);
    test_assert(literal->text == "5.0");
    test_assert(literal->toString() == "5.0");
    test_assert(literal->hasQuestionMark == true);

    delete expr;
}

void literal_string()
{
    token_stream::TokenStream tokens("\"quoted string\"");
    ast::Expression *expr = parser::atom(tokens);
    ast::LiteralString *literal = dynamic_cast<ast::LiteralString*>(expr);
    test_assert(literal != NULL);
    test_assert(literal->text == "quoted string");
    test_assert(literal->toString() == "\"quoted string\"");

    delete expr;
}

void function_call()
{
    token_stream::TokenStream tokens("add(1,2)");
    ast::FunctionCall *functionCall = parser::functionCall(tokens);
    test_assert(functionCall != NULL);
    test_assert(functionCall->functionName == "add");

    ast::LiteralInteger *arg1 =
        dynamic_cast<ast::LiteralInteger*>(functionCall->arguments[0]->expression);
    test_assert(arg1 != NULL);
    test_assert(arg1->text == "1");
    ast::LiteralInteger *arg2 =
        dynamic_cast<ast::LiteralInteger*>(functionCall->arguments[1]->expression);
    test_assert(arg2 != NULL);
    test_assert(arg2->text == "2");
    test_assert(functionCall->toString() == "add(1,2)");

    delete functionCall;
}

void name_binding_expression()
{
    token_stream::TokenStream tokens("name = hi(2,u)");
    ast::ExpressionStatement *statement = parser::expressionStatement(tokens);

    test_assert(statement->nameBinding == "name");

    ast::FunctionCall *functionCall =
        dynamic_cast<ast::FunctionCall*>(statement->expression);

    test_assert(functionCall->functionName == "hi");
    test_assert(functionCall->toString() == "hi(2,u)");
    test_assert(statement->toString() == "name = hi(2,u)");

    delete statement;
}

void test_to_string()
{
    // Build an AST programatically so that the toString function
    // can't cheat.
    ast::FunctionCall *functionCall = new ast::FunctionCall("something");
    functionCall->addArgument(new ast::LiteralString("input0"), " ", "");
    functionCall->addArgument(new ast::LiteralInteger("4"), "", " ");
    test_assert(functionCall->toString() == "something( \"input0\",4 )");
    delete functionCall;
}

void create_literals()
{
    Branch branch;

    ast::LiteralInteger *lint = new ast::LiteralInteger("13");
    Term *int_t = lint->createTerm(branch);
    test_assert(int_t->type == INT_TYPE);
    test_assert(as_int(int_t) == 13);
    delete lint;

    ast::LiteralFloat *lfloat = new ast::LiteralFloat("1.432");
    Term *float_t = lfloat->createTerm(branch);
    test_assert(float_t->type == FLOAT_TYPE);
    test_assert(as_float(float_t) > 1.431 && as_float(float_t) < 1.433);
    delete lfloat;

    ast::LiteralString *lstr = new ast::LiteralString("hello");
    Term *str_t = lstr->createTerm(branch);
    test_assert(str_t->type == STRING_TYPE);
    test_assert(as_string(str_t) == "hello");
    delete lstr;
}

void create_function_call()
{
    Branch branch;

    ast::FunctionCall *functionCall = new ast::FunctionCall("add");
    functionCall->addArgument(new ast::LiteralFloat("2"), "", "");
    functionCall->addArgument(new ast::LiteralFloat("3"), "", "");

    Term *term = functionCall->createTerm(branch);
    test_assert(as_function(term->function).name == "add");

    // make sure this term is not evaluated yet
    test_assert(term->needsUpdate);

    evaluate_term(term);

    test_assert(!term->needsUpdate);
    test_assert(as_float(term) == 5);

    delete functionCall;
}

void test_apply_statement()
{
    Branch branch;

    Term* result = apply_statement(branch, "something = add(5.0,3.0)");

    test_assert(result != NULL);
    test_assert(result->name == "something");
    test_assert(as_function(result->function).name == "add");
    test_assert(result->needsUpdate);
    evaluate_term(result);
    test_assert(as_float(result) == 8);
    test_assert(!result->needsUpdate);
}

void function_decl_ast()
{
    // Create a FunctionDecl from scratch
    ast::FunctionDecl decl;

    decl.header = new ast::FunctionHeader();
    decl.header->functionName = "add";
    decl.header->outputType = "float";
    decl.header->addArgument("float","arg1");
    decl.header->addArgument("float","arg2");

    decl.statements = new ast::StatementList();

    token_stream::TokenStream tokens("x = add(arg1,arg2)");
    decl.statements->push(parser::statement(tokens));

    test_assert(decl.toString() == "function add(float arg1, float arg2)\nx = add(arg1,arg2)\nend");
}

void function_header()
{
    ast::FunctionHeader header = eval_as<ast::FunctionHeader>(
        "'function test(int a, float, string b , int) -> float'.tokenize.parse-function-header");

    test_assert(header.functionName == "test");
    test_assert(header.arguments[0].type == "int");
    test_assert(header.arguments[0].name == "a");
    test_assert(header.arguments[1].type == "float");
    test_assert(header.arguments[1].name == "");
    test_assert(header.arguments[2].type == "string");
    test_assert(header.arguments[2].name == "b");
    test_assert(header.arguments[3].type == "int");
    test_assert(header.arguments[3].name == "");
    test_assert(header.outputType == "float");
}

void function_decl_parse()
{
    token_stream::TokenStream tokens("function func(string a, int b) -> void\n"
            "concat(a, to-string(b)) -> print\n"
            "end");
    ast::Statement* statement = parser::statement(tokens);

    test_assert(statement->typeName() == "FunctionDecl");

    tokens.reset("function func2()\n"
            "print(\"hello\")\n"
            "end\n"
            "some-function(1,2)\n");

    /*ast::StatementList* statementList = */parser::statementList(tokens);
}

void rebind_operator()
{
    token_stream::TokenStream tokens("@cheese");
    ast::Expression* expr = parser::atom(tokens);

    ast::Identifier* id = dynamic_cast<ast::Identifier*>(expr);
    test_assert(id != NULL);
    test_assert(id->text == "cheese");
    test_assert(id->hasRebindOperator);

    delete expr;

    tokens.reset("gouda");
    expr = parser::atom(tokens);
    id = dynamic_cast<ast::Identifier*>(expr);
    test_assert(id != NULL);
    test_assert(id->text == "gouda");
    test_assert(!id->hasRebindOperator);

    delete expr;

    Branch branch;
    Term* a = int_var(branch, 2, "a");
    Term* b = int_var(branch, 5, "b");

    test_assert(branch.getNamed("a") == a);
    test_assert(branch.getNamed("b") == b);

    Term* result = apply_statement(branch, "add(@a, b)");

    test_assert(branch.getNamed("a") == result);
    test_assert(branch.getNamed("b") == b);
}

void ast_walk()
{
    token_stream::TokenStream tokens("concat(to-string(add(1,2.0)), \"cheese\")");
    ast::Expression* expr = parser::infixExpression(tokens);

    std::vector<std::string> expected;
    expected.push_back("concat(to-string(add(1,2.0)), \"cheese\")");
    expected.push_back("to-string(add(1,2.0))");
    expected.push_back("add(1,2.0)");
    expected.push_back("1");
    expected.push_back("2.0");
    expected.push_back("\"cheese\"");

    struct MyWalker : public ast::ExpressionWalker
    {
        std::vector<std::string> found;

        virtual void visit(ast::Expression* expr)
        {
            found.push_back(expr->toString());
        }
    } walker;

    expr->walk(walker);

    for (int i=0; i < (int) expected.size(); i++) {
        test_assert(expected[i] == walker.found[i]);
    }

    delete expr;
}

void test_eval_statement()
{
    Branch b;

    test_assert(eval_statement(b, "5")->asInt() == 5);
    test_assert(eval_statement(b, "-2")->asInt() == -2);
    test_assert(eval_statement(b, "1.0")->asFloat() == 1.0);
    //test_assert(eval_statement(b, "-.123")->asFloat() == -0.123);FIXME
    test_assert(eval_statement(b, "\"apple\"")->asString() == "apple");
    test_assert(eval_statement(b, "\"\"")->asString() == "");
}

void comment_statement()
{
    token_stream::TokenStream tokens("-- this is a comment");

    ast::Statement* statement = parser::statement(tokens);
    test_assert(dynamic_cast<ast::IgnorableStatement*>(statement) != NULL);

    test_assert(tokens.finished());
}

void test_type_decl()
{
    token_stream::TokenStream tokens("type Vector {\nfloat x\nint y\n}");

    ast::TypeDecl* typeDecl = parser::typeDecl(tokens);
    test_assert(typeDecl != NULL);
    test_assert(typeDecl->name == "Vector");
    test_assert(typeDecl->members[0].type == "float");
    test_assert(typeDecl->members[0].name == "x");
    test_assert(typeDecl->members[1].type == "int");
    test_assert(typeDecl->members[1].name == "y");
}

void test_type_decl_statement()
{
    token_stream::TokenStream tokens("type Mytype\n{\nstring str\nfloat asdf\n}");
    ast::Statement* statement = parser::statement(tokens);

    test_assert(dynamic_cast<ast::TypeDecl*>(statement) != NULL);
}

void test_type_decl_full_trip()
{
    Branch branch;
    Term *t = eval_statement(branch, "type Mytype\n{\nstring s\nfloat a\n}");

    test_assert(!t->hasError());

    test_assert(t->name == "Mytype");
    test_assert(as_type(t).fields[0].type == STRING_TYPE);
    test_assert(as_type(t).fields[0].name == "s");
    test_assert(as_type(t).fields[1].type == FLOAT_TYPE);
    test_assert(as_type(t).fields[1].name == "a");
}

void test_parse_if_block()
{
    token_stream::TokenStream tokens(
            "if (x > 2) \n print('yes') \n else \n print('no') \n end");

    ast::IfStatement* ifStatement = parser::ifStatement(tokens);

    test_assert(ifStatement != NULL);

    // these lines will need to be fixed when there is better whitespace preservation
    test_assert(ifStatement->condition->toString() == "x>2");
    test_assert(ifStatement->positiveBranch->toString() == "print(\"yes\")\n");
    test_assert(ifStatement->negativeBranch->toString() == "print(\"no\")\n");
}

} // namespace parser_tests

void register_parser_tests()
{
    REGISTER_TEST_CASE(parser_tests::literal_float);
    REGISTER_TEST_CASE(parser_tests::literal_string);
    REGISTER_TEST_CASE(parser_tests::function_call);
    REGISTER_TEST_CASE(parser_tests::name_binding_expression);
    REGISTER_TEST_CASE(parser_tests::test_to_string);
    REGISTER_TEST_CASE(parser_tests::create_literals);
    REGISTER_TEST_CASE(parser_tests::create_function_call);
    REGISTER_TEST_CASE(parser_tests::test_apply_statement);
    REGISTER_TEST_CASE(parser_tests::function_decl_ast);
    REGISTER_TEST_CASE(parser_tests::function_header);
    REGISTER_TEST_CASE(parser_tests::function_decl_parse);
    REGISTER_TEST_CASE(parser_tests::rebind_operator);
    REGISTER_TEST_CASE(parser_tests::ast_walk);
    REGISTER_TEST_CASE(parser_tests::test_eval_statement);
    REGISTER_TEST_CASE(parser_tests::comment_statement);
    REGISTER_TEST_CASE(parser_tests::test_type_decl);
    REGISTER_TEST_CASE(parser_tests::test_type_decl_statement);
    REGISTER_TEST_CASE(parser_tests::test_type_decl_full_trip);
    REGISTER_TEST_CASE(parser_tests::test_parse_if_block);
}

} // namespace circa
<commit_msg>Add to if block parsing test<commit_after>// Copyright 2008 Paul Hodge

#include "common_headers.h"

#include "testing.h"
#include "circa.h"

namespace circa {
namespace parser_tests {

void literal_float()
{
    token_stream::TokenStream tokens("1.0");
    ast::Expression *expr = parser::atom(tokens);
    ast::LiteralFloat *literal = dynamic_cast<ast::LiteralFloat*>(expr);
    test_assert(literal != NULL);
    test_assert(literal->text == "1.0");
    test_assert(literal->toString() == "1.0");
    test_assert(literal->hasQuestionMark == false);

    delete expr;

    tokens.reset("5.0?");
    expr = parser::atom(tokens);
    literal = dynamic_cast<ast::LiteralFloat*>(expr);
    test_assert(literal != NULL);
    test_assert(literal->text == "5.0");
    test_assert(literal->toString() == "5.0");
    test_assert(literal->hasQuestionMark == true);

    delete expr;
}

void literal_string()
{
    token_stream::TokenStream tokens("\"quoted string\"");
    ast::Expression *expr = parser::atom(tokens);
    ast::LiteralString *literal = dynamic_cast<ast::LiteralString*>(expr);
    test_assert(literal != NULL);
    test_assert(literal->text == "quoted string");
    test_assert(literal->toString() == "\"quoted string\"");

    delete expr;
}

void function_call()
{
    token_stream::TokenStream tokens("add(1,2)");
    ast::FunctionCall *functionCall = parser::functionCall(tokens);
    test_assert(functionCall != NULL);
    test_assert(functionCall->functionName == "add");

    ast::LiteralInteger *arg1 =
        dynamic_cast<ast::LiteralInteger*>(functionCall->arguments[0]->expression);
    test_assert(arg1 != NULL);
    test_assert(arg1->text == "1");
    ast::LiteralInteger *arg2 =
        dynamic_cast<ast::LiteralInteger*>(functionCall->arguments[1]->expression);
    test_assert(arg2 != NULL);
    test_assert(arg2->text == "2");
    test_assert(functionCall->toString() == "add(1,2)");

    delete functionCall;
}

void name_binding_expression()
{
    token_stream::TokenStream tokens("name = hi(2,u)");
    ast::ExpressionStatement *statement = parser::expressionStatement(tokens);

    test_assert(statement->nameBinding == "name");

    ast::FunctionCall *functionCall =
        dynamic_cast<ast::FunctionCall*>(statement->expression);

    test_assert(functionCall->functionName == "hi");
    test_assert(functionCall->toString() == "hi(2,u)");
    test_assert(statement->toString() == "name = hi(2,u)");

    delete statement;
}

void test_to_string()
{
    // Build an AST programatically so that the toString function
    // can't cheat.
    ast::FunctionCall *functionCall = new ast::FunctionCall("something");
    functionCall->addArgument(new ast::LiteralString("input0"), " ", "");
    functionCall->addArgument(new ast::LiteralInteger("4"), "", " ");
    test_assert(functionCall->toString() == "something( \"input0\",4 )");
    delete functionCall;
}

void create_literals()
{
    Branch branch;

    ast::LiteralInteger *lint = new ast::LiteralInteger("13");
    Term *int_t = lint->createTerm(branch);
    test_assert(int_t->type == INT_TYPE);
    test_assert(as_int(int_t) == 13);
    delete lint;

    ast::LiteralFloat *lfloat = new ast::LiteralFloat("1.432");
    Term *float_t = lfloat->createTerm(branch);
    test_assert(float_t->type == FLOAT_TYPE);
    test_assert(as_float(float_t) > 1.431 && as_float(float_t) < 1.433);
    delete lfloat;

    ast::LiteralString *lstr = new ast::LiteralString("hello");
    Term *str_t = lstr->createTerm(branch);
    test_assert(str_t->type == STRING_TYPE);
    test_assert(as_string(str_t) == "hello");
    delete lstr;
}

void create_function_call()
{
    Branch branch;

    ast::FunctionCall *functionCall = new ast::FunctionCall("add");
    functionCall->addArgument(new ast::LiteralFloat("2"), "", "");
    functionCall->addArgument(new ast::LiteralFloat("3"), "", "");

    Term *term = functionCall->createTerm(branch);
    test_assert(as_function(term->function).name == "add");

    // make sure this term is not evaluated yet
    test_assert(term->needsUpdate);

    evaluate_term(term);

    test_assert(!term->needsUpdate);
    test_assert(as_float(term) == 5);

    delete functionCall;
}

void test_apply_statement()
{
    Branch branch;

    Term* result = apply_statement(branch, "something = add(5.0,3.0)");

    test_assert(result != NULL);
    test_assert(result->name == "something");
    test_assert(as_function(result->function).name == "add");
    test_assert(result->needsUpdate);
    evaluate_term(result);
    test_assert(as_float(result) == 8);
    test_assert(!result->needsUpdate);
}

void function_decl_ast()
{
    // Create a FunctionDecl from scratch
    ast::FunctionDecl decl;

    decl.header = new ast::FunctionHeader();
    decl.header->functionName = "add";
    decl.header->outputType = "float";
    decl.header->addArgument("float","arg1");
    decl.header->addArgument("float","arg2");

    decl.statements = new ast::StatementList();

    token_stream::TokenStream tokens("x = add(arg1,arg2)");
    decl.statements->push(parser::statement(tokens));

    test_assert(decl.toString() == "function add(float arg1, float arg2)\nx = add(arg1,arg2)\nend");
}

void function_header()
{
    ast::FunctionHeader header = eval_as<ast::FunctionHeader>(
        "'function test(int a, float, string b , int) -> float'.tokenize.parse-function-header");

    test_assert(header.functionName == "test");
    test_assert(header.arguments[0].type == "int");
    test_assert(header.arguments[0].name == "a");
    test_assert(header.arguments[1].type == "float");
    test_assert(header.arguments[1].name == "");
    test_assert(header.arguments[2].type == "string");
    test_assert(header.arguments[2].name == "b");
    test_assert(header.arguments[3].type == "int");
    test_assert(header.arguments[3].name == "");
    test_assert(header.outputType == "float");
}

void function_decl_parse()
{
    token_stream::TokenStream tokens("function func(string a, int b) -> void\n"
            "concat(a, to-string(b)) -> print\n"
            "end");
    ast::Statement* statement = parser::statement(tokens);

    test_assert(statement->typeName() == "FunctionDecl");

    tokens.reset("function func2()\n"
            "print(\"hello\")\n"
            "end\n"
            "some-function(1,2)\n");

    /*ast::StatementList* statementList = */parser::statementList(tokens);
}

void rebind_operator()
{
    token_stream::TokenStream tokens("@cheese");
    ast::Expression* expr = parser::atom(tokens);

    ast::Identifier* id = dynamic_cast<ast::Identifier*>(expr);
    test_assert(id != NULL);
    test_assert(id->text == "cheese");
    test_assert(id->hasRebindOperator);

    delete expr;

    tokens.reset("gouda");
    expr = parser::atom(tokens);
    id = dynamic_cast<ast::Identifier*>(expr);
    test_assert(id != NULL);
    test_assert(id->text == "gouda");
    test_assert(!id->hasRebindOperator);

    delete expr;

    Branch branch;
    Term* a = int_var(branch, 2, "a");
    Term* b = int_var(branch, 5, "b");

    test_assert(branch.getNamed("a") == a);
    test_assert(branch.getNamed("b") == b);

    Term* result = apply_statement(branch, "add(@a, b)");

    test_assert(branch.getNamed("a") == result);
    test_assert(branch.getNamed("b") == b);
}

void ast_walk()
{
    token_stream::TokenStream tokens("concat(to-string(add(1,2.0)), \"cheese\")");
    ast::Expression* expr = parser::infixExpression(tokens);

    std::vector<std::string> expected;
    expected.push_back("concat(to-string(add(1,2.0)), \"cheese\")");
    expected.push_back("to-string(add(1,2.0))");
    expected.push_back("add(1,2.0)");
    expected.push_back("1");
    expected.push_back("2.0");
    expected.push_back("\"cheese\"");

    struct MyWalker : public ast::ExpressionWalker
    {
        std::vector<std::string> found;

        virtual void visit(ast::Expression* expr)
        {
            found.push_back(expr->toString());
        }
    } walker;

    expr->walk(walker);

    for (int i=0; i < (int) expected.size(); i++) {
        test_assert(expected[i] == walker.found[i]);
    }

    delete expr;
}

void test_eval_statement()
{
    Branch b;

    test_assert(eval_statement(b, "5")->asInt() == 5);
    test_assert(eval_statement(b, "-2")->asInt() == -2);
    test_assert(eval_statement(b, "1.0")->asFloat() == 1.0);
    //test_assert(eval_statement(b, "-.123")->asFloat() == -0.123);FIXME
    test_assert(eval_statement(b, "\"apple\"")->asString() == "apple");
    test_assert(eval_statement(b, "\"\"")->asString() == "");
}

void comment_statement()
{
    token_stream::TokenStream tokens("-- this is a comment");

    ast::Statement* statement = parser::statement(tokens);
    test_assert(dynamic_cast<ast::IgnorableStatement*>(statement) != NULL);

    test_assert(tokens.finished());
}

void test_type_decl()
{
    token_stream::TokenStream tokens("type Vector {\nfloat x\nint y\n}");

    ast::TypeDecl* typeDecl = parser::typeDecl(tokens);
    test_assert(typeDecl != NULL);
    test_assert(typeDecl->name == "Vector");
    test_assert(typeDecl->members[0].type == "float");
    test_assert(typeDecl->members[0].name == "x");
    test_assert(typeDecl->members[1].type == "int");
    test_assert(typeDecl->members[1].name == "y");
}

void test_type_decl_statement()
{
    token_stream::TokenStream tokens("type Mytype\n{\nstring str\nfloat asdf\n}");
    ast::Statement* statement = parser::statement(tokens);

    test_assert(dynamic_cast<ast::TypeDecl*>(statement) != NULL);
}

void test_type_decl_full_trip()
{
    Branch branch;
    Term *t = eval_statement(branch, "type Mytype\n{\nstring s\nfloat a\n}");

    test_assert(!t->hasError());

    test_assert(t->name == "Mytype");
    test_assert(as_type(t).fields[0].type == STRING_TYPE);
    test_assert(as_type(t).fields[0].name == "s");
    test_assert(as_type(t).fields[1].type == FLOAT_TYPE);
    test_assert(as_type(t).fields[1].name == "a");
}

void test_parse_if_block()
{
    token_stream::TokenStream tokens(
            "if (x > 2) \n print('yes') \n else \n print('no') \n end");

    ast::IfStatement* ifStatement = parser::ifStatement(tokens);

    test_assert(ifStatement != NULL);

    // these lines will need to be fixed when there is better whitespace preservation
    test_assert(ifStatement->condition->toString() == "x>2");
    test_assert(ifStatement->positiveBranch->toString() == "print(\"yes\")\n");
    test_assert(ifStatement->negativeBranch->toString() == "print(\"no\")\n");

    delete ifStatement;

    tokens.reset("if (false) \n print('blah') \n end");

    ifStatement = parser::ifStatement(tokens);

    test_assert(ifStatement->condition->toString() == "false");
    test_assert(ifStatement->positiveBranch->toString() == "print(\"blah\")\n");
    test_assert(ifStatement->negativeBranch == NULL);

    delete ifStatement;
}

} // namespace parser_tests

void register_parser_tests()
{
    REGISTER_TEST_CASE(parser_tests::literal_float);
    REGISTER_TEST_CASE(parser_tests::literal_string);
    REGISTER_TEST_CASE(parser_tests::function_call);
    REGISTER_TEST_CASE(parser_tests::name_binding_expression);
    REGISTER_TEST_CASE(parser_tests::test_to_string);
    REGISTER_TEST_CASE(parser_tests::create_literals);
    REGISTER_TEST_CASE(parser_tests::create_function_call);
    REGISTER_TEST_CASE(parser_tests::test_apply_statement);
    REGISTER_TEST_CASE(parser_tests::function_decl_ast);
    REGISTER_TEST_CASE(parser_tests::function_header);
    REGISTER_TEST_CASE(parser_tests::function_decl_parse);
    REGISTER_TEST_CASE(parser_tests::rebind_operator);
    REGISTER_TEST_CASE(parser_tests::ast_walk);
    REGISTER_TEST_CASE(parser_tests::test_eval_statement);
    REGISTER_TEST_CASE(parser_tests::comment_statement);
    REGISTER_TEST_CASE(parser_tests::test_type_decl);
    REGISTER_TEST_CASE(parser_tests::test_type_decl_statement);
    REGISTER_TEST_CASE(parser_tests::test_type_decl_full_trip);
    REGISTER_TEST_CASE(parser_tests::test_parse_if_block);
}

} // namespace circa
<|endoftext|>


<commit_before>/*! @file selector.cc
 *  @brief Select the specified part of input.
 *  @author Markovtsev Vadim <v.markovtsev@samsung.com>
 *  @version 1.0
 *
 *  @section Notes
 *  This code partially conforms to Google C++ Style Guide.
 *
 *  @section Copyright
 *  Copyright 2013 Samsung R&D Institute Russia
 */

#include "src/transforms/selector.h"
#include <string>

namespace SpeechFeatureExtraction {
namespace Transforms {

Selector::Selector()
  : UniformFormatTransform(SupportedParameters()),
    length_(12),
    from_(ANCHOR_RIGHT) {
}

void Selector::SetParameter(const std::string& name,
                            const std::string& value) {
  if (name == "length") {
    int length = Parse<int>(name, value);
    if (length < 1) {
      throw InvalidParameterValueException(name, value, Name());
    }
    length_ = length;
  } else if (name == "from") {
    if (value == "left") {
      from_ = ANCHOR_LEFT;
    } else if (value == "right") {
      from_ = ANCHOR_RIGHT;
    } else {
      throw InvalidParameterValueException(name, value, Name());
    }
  }
}

void Selector::OnFormatChanged() {
  outputFormat_->SetSize(
      std::min(length_, static_cast<int>(inputFormat_->Size())));
}

void Selector::TypeSafeInitializeBuffers(
    const BuffersBase<Formats::WindowF>& in,
    BuffersBase<Formats::WindowF>* buffers) const noexcept {
  buffers->Initialize(in.Size(), outputFormat_->Size());
}

void Selector::TypeSafeDo(
    const BuffersBase<Formats::WindowF>& in,
    BuffersBase<Formats::WindowF> *out) const noexcept {
  for (size_t i = 0; i < in.Size(); i++) {
    auto input = in[i]->Data.get();
    auto output = (*out)[i]->Data.get();
    if (input != output) {
      memcpy(output,
             input +
               (from_ == ANCHOR_LEFT? 0 : inputFormat_->Size() - length_),
             length_ * sizeof(input[0]));
    } else {
      memmove(output,
              input +
                (from_ == ANCHOR_LEFT? 0 : inputFormat_->Size() - length_),
              length_ * sizeof(input[0]));
    }
  }
}

REGISTER_TRANSFORM(Selector);

}  // namespace Transforms
}  // namespace SpeechFeatureExtraction
<commit_msg>Fixed a bug in Selector<commit_after>/*! @file selector.cc
 *  @brief Select the specified part of input.
 *  @author Markovtsev Vadim <v.markovtsev@samsung.com>
 *  @version 1.0
 *
 *  @section Notes
 *  This code partially conforms to Google C++ Style Guide.
 *
 *  @section Copyright
 *  Copyright 2013 Samsung R&D Institute Russia
 */

#include "src/transforms/selector.h"
#include <string>

namespace SpeechFeatureExtraction {
namespace Transforms {

Selector::Selector()
  : UniformFormatTransform(SupportedParameters()),
    length_(12),
    from_(ANCHOR_RIGHT) {
}

void Selector::SetParameter(const std::string& name,
                            const std::string& value) {
  if (name == "length") {
    int length = Parse<int>(name, value);
    if (length < 1) {
      throw InvalidParameterValueException(name, value, Name());
    }
    length_ = length;
  } else if (name == "from") {
    if (value == "left") {
      from_ = ANCHOR_LEFT;
    } else if (value == "right") {
      from_ = ANCHOR_RIGHT;
    } else {
      throw InvalidParameterValueException(name, value, Name());
    }
  }
}

void Selector::OnFormatChanged() {
  outputFormat_->SetSize(
      std::min(length_, static_cast<int>(inputFormat_->Size())));
}

void Selector::TypeSafeInitializeBuffers(
    const BuffersBase<Formats::WindowF>& in,
    BuffersBase<Formats::WindowF>* buffers) const noexcept {
  buffers->Initialize(in.Size(), outputFormat_->Size());
}

void Selector::TypeSafeDo(
    const BuffersBase<Formats::WindowF>& in,
    BuffersBase<Formats::WindowF> *out) const noexcept {
  int length = outputFormat_->Size();
  for (size_t i = 0; i < in.Size(); i++) {
    auto input = in[i]->Data.get();
    auto output = (*out)[i]->Data.get();
    if (input != output) {
      memcpy(output,
             input + (from_ == ANCHOR_LEFT? 0 : inputFormat_->Size() - length),
             length * sizeof(input[0]));
    } else {
      memmove(output,
              input + (from_ == ANCHOR_LEFT? 0 : inputFormat_->Size() - length),
              length * sizeof(input[0]));
    }
  }
}

REGISTER_TRANSFORM(Selector);

}  // namespace Transforms
}  // namespace SpeechFeatureExtraction
<|endoftext|>


<commit_before>// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.

#include "persistencemessagetracker.h"
#include <vespa/storage/common/vectorprinter.h>
#include <vespa/storageapi/message/persistence.h>
#include "distributor_bucket_space_repo.h"
#include "distributor_bucket_space.h"

#include <vespa/log/log.h>

LOG_SETUP(".persistencemessagetracker");

namespace storage::distributor {

PersistenceMessageTrackerImpl::PersistenceMessageTrackerImpl(
        PersistenceOperationMetricSet& metric,
        std::shared_ptr<api::BucketInfoReply> reply,
        DistributorNodeContext& node_ctx,
        DistributorOperationContext& op_ctx,
        api::Timestamp revertTimestamp)
    : MessageTracker(node_ctx.cluster_name()),
      _metric(metric),
      _reply(std::move(reply)),
      _op_ctx(op_ctx),
      _revertTimestamp(revertTimestamp),
      _trace(_reply->getTrace().getLevel()),
      _requestTimer(node_ctx.clock()),
      _n_persistence_replies_total(0),
      _n_successful_persistence_replies(0),
      _priority(_reply->getPriority()),
      _success(true)
{
}

PersistenceMessageTrackerImpl::~PersistenceMessageTrackerImpl() = default;

void
PersistenceMessageTrackerImpl::updateDB()
{
    for (const auto & entry : _bucketInfo) {
        _op_ctx.update_bucket_database(entry.first, entry.second);
    }

    for (const auto & entry :  _remapBucketInfo){
        _op_ctx.update_bucket_database(entry.first, entry.second, DatabaseUpdate::CREATE_IF_NONEXISTING);
    }
}

void
PersistenceMessageTrackerImpl::updateMetrics()
{
    const api::ReturnCode& result(_reply->getResult());
    _metric.updateFromResult(result);
    _metric.latency.addValue(_requestTimer.getElapsedTimeAsDouble());
}

void
PersistenceMessageTrackerImpl::fail(MessageSender& sender, const api::ReturnCode& result) {
    if (_reply.get()) {
        _reply->setResult(result);
        updateMetrics();
        sender.sendReply(_reply);
        _reply.reset();
    }
}

uint16_t
PersistenceMessageTrackerImpl::receiveReply(
        MessageSender& sender,
        api::BucketInfoReply& reply)
{
    uint16_t node = handleReply(reply);

    if (node != (uint16_t)-1) {
        updateFromReply(sender, reply, node);
    }

    return node;
}

void
PersistenceMessageTrackerImpl::revert(
        MessageSender& sender,
        const std::vector<BucketNodePair>& revertNodes)
{
    if (_revertTimestamp != 0) {
        // Since we're reverting, all received bucket info is voided.
        _bucketInfo.clear();

        std::vector<api::Timestamp> reverts;
        reverts.push_back(_revertTimestamp);

        for (const auto & revertNode : revertNodes) {
            auto toRevert = std::make_shared<api::RevertCommand>(revertNode.first, reverts);
            toRevert->setPriority(_priority);
            queueCommand(std::move(toRevert), revertNode.second);
        }

        flushQueue(sender);
    }
}

void
PersistenceMessageTrackerImpl::queueMessageBatch(const std::vector<MessageTracker::ToSend>& messages) {
    _messageBatches.emplace_back();
    for (const auto & message : messages) {
        if (_reply) {
            message._msg->getTrace().setLevel(_reply->getTrace().getLevel());
        }

        _messageBatches.back().push_back(message._msg->getMsgId());
        queueCommand(message._msg, message._target);
    }
}

bool
PersistenceMessageTrackerImpl::canSendReplyEarly() const
{
    if (!_reply.get() || !_reply->getResult().success()) {
        LOG(spam, "Can't return early because we have already replied or failed");
        return false;
    }
    auto &bucketSpaceRepo(_op_ctx.bucket_space_repo());
    auto &bucketSpace(bucketSpaceRepo.get(_reply->getBucket().getBucketSpace()));
    const lib::Distribution& distribution = bucketSpace.getDistribution();

    if (distribution.getInitialRedundancy() == 0) {
        LOG(spam, "Not returning early because initial redundancy wasn't set");
        return false;
    }

    for (const MessageBatch & batch : _messageBatches) {
        uint32_t messagesDone = 0;

        for (uint32_t i = 0; i < batch.size(); i++) {
            if (_sentMessages.find(batch[i]) == _sentMessages.end()) {
                messagesDone++;
            } else if (distribution.ensurePrimaryPersisted() && i == 0) {
                // Primary must always be written.
                LOG(debug, "Not returning early because primary node wasn't done");
                return false;
            }
        }

        if (messagesDone < distribution.getInitialRedundancy()) {
            LOG(spam, "Not returning early because only %d messages out of %d are done",
                messagesDone, distribution.getInitialRedundancy());
            return false;
        }
    }

    return true;
}

void
PersistenceMessageTrackerImpl::addBucketInfoFromReply(
        uint16_t node,
        const api::BucketInfoReply& reply)
{
    document::Bucket bucket(reply.getBucket());
    const api::BucketInfo& bucketInfo(reply.getBucketInfo());

    if (reply.hasBeenRemapped()) {
        LOG(debug, "Bucket %s: Received remapped bucket info %s from node %d",
            bucket.toString().c_str(),
            bucketInfo.toString().c_str(),
            node);
        _remapBucketInfo[bucket].emplace_back(_op_ctx.generate_unique_timestamp(), node, bucketInfo);
    } else {
        LOG(debug, "Bucket %s: Received bucket info %s from node %d",
            bucket.toString().c_str(),
            bucketInfo.toString().c_str(),
            node);
        _bucketInfo[bucket].emplace_back(_op_ctx.generate_unique_timestamp(), node, bucketInfo);
    }
}

void
PersistenceMessageTrackerImpl::logSuccessfulReply(uint16_t node, const api::BucketInfoReply& reply) const
{
    LOG(spam, "Bucket %s: Received successful reply %s",
        reply.getBucketId().toString().c_str(),
        reply.toString().c_str());

    if (!reply.getBucketInfo().valid()) {
        LOG(error,
            "Reply %s from node %d contained invalid bucket "
            "information %s. This is a bug! Please report "
            "this to the Vespa team",
            reply.toString().c_str(),
            node,
            reply.getBucketInfo().toString().c_str());
    }
}

bool
PersistenceMessageTrackerImpl::shouldRevert() const
{
    return _op_ctx.distributor_config().enable_revert()
            &&  !_revertNodes.empty() && !_success && _reply;
}

bool PersistenceMessageTrackerImpl::has_majority_successful_replies() const noexcept {
    // FIXME this has questionable interaction with early client ACK since we only count
    // the number of observed replies rather than the number of total requests sent.
    // ... but the early ACK-feature dearly needs a redesign anyway.
    return (_n_successful_persistence_replies >= (_n_persistence_replies_total / 2 + 1));
}

bool PersistenceMessageTrackerImpl::has_minority_test_and_set_failure() const noexcept {
    return ((_reply->getResult().getResult() == api::ReturnCode::TEST_AND_SET_CONDITION_FAILED)
            && has_majority_successful_replies());
}

void
PersistenceMessageTrackerImpl::sendReply(MessageSender& sender)
{
    // If we've observed _partial_ TaS failures but have had a majority of good ACKs,
    // treat the reply as successful. This is because the ACKed write(s) will eventually
    // become visible across all nodes.
    if (has_minority_test_and_set_failure()) {
        _reply->setResult(api::ReturnCode());
    }

    updateMetrics();
    _trace.setStrict(false);
    if ( ! _trace.isEmpty()) {
        _reply->getTrace().addChild(std::move(_trace));
    }

    sender.sendReply(_reply);
    _reply = std::shared_ptr<api::BucketInfoReply>();
}

void
PersistenceMessageTrackerImpl::updateFailureResult(const api::BucketInfoReply& reply)
{
    LOG(debug, "Bucket %s: Received failed reply %s with result %s",
        reply.getBucketId().toString().c_str(),
        reply.toString().c_str(),
        reply.getResult().toString().c_str());
    if (reply.getResult().getResult() >
        _reply->getResult().getResult())
    {
        _reply->setResult(reply.getResult());
    }

    _success = false;
}

void
PersistenceMessageTrackerImpl::handleCreateBucketReply(
        api::BucketInfoReply& reply,
        uint16_t node)
{
    LOG(spam, "Received CreateBucket reply for %s from node %u",
        reply.getBucketId().toString().c_str(), node);
    if (!reply.getResult().success()
        && reply.getResult().getResult() != api::ReturnCode::EXISTS)
    {
        LOG(spam, "Create bucket reply failed, so deleting it from bucket db");
        _op_ctx.remove_node_from_bucket_database(reply.getBucket(), node);
    }
}

void
PersistenceMessageTrackerImpl::handlePersistenceReply(
        api::BucketInfoReply& reply,
        uint16_t node)
{
    ++_n_persistence_replies_total;
    if (reply.getBucketInfo().valid()) {
        addBucketInfoFromReply(node, reply);
    }
    if (reply.getResult().success()) {
        logSuccessfulReply(node, reply);
        _revertNodes.emplace_back(reply.getBucket(), node);
        ++_n_successful_persistence_replies;
    } else if (!hasSentReply()) {
        updateFailureResult(reply);
    }
}

void
PersistenceMessageTrackerImpl::updateFromReply(
        MessageSender& sender,
        api::BucketInfoReply& reply,
        uint16_t node)
{
    _trace.addChild(reply.steal_trace());

    if (reply.getType() == api::MessageType::CREATEBUCKET_REPLY) {
        handleCreateBucketReply(reply, node);
    } else {
        handlePersistenceReply(reply, node);
    }

    if (finished()) {
        bool doRevert(shouldRevert());

        updateDB();

        if (!hasSentReply()) {
            sendReply(sender);
        }
        if (doRevert) {
            revert(sender, _revertNodes);
        }
    } else if (canSendReplyEarly()) {
        LOG(debug, "Sending reply early because initial redundancy has been reached");
        sendReply(sender);
    }
}

}
<commit_msg>Avoid constructing a TraceNode (via ensureRoot) when tracing is disabled.<commit_after>// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.

#include "persistencemessagetracker.h"
#include <vespa/storage/common/vectorprinter.h>
#include <vespa/storageapi/message/persistence.h>
#include "distributor_bucket_space_repo.h"
#include "distributor_bucket_space.h"

#include <vespa/log/log.h>

LOG_SETUP(".persistencemessagetracker");

namespace storage::distributor {

PersistenceMessageTrackerImpl::PersistenceMessageTrackerImpl(
        PersistenceOperationMetricSet& metric,
        std::shared_ptr<api::BucketInfoReply> reply,
        DistributorNodeContext& node_ctx,
        DistributorOperationContext& op_ctx,
        api::Timestamp revertTimestamp)
    : MessageTracker(node_ctx.cluster_name()),
      _metric(metric),
      _reply(std::move(reply)),
      _op_ctx(op_ctx),
      _revertTimestamp(revertTimestamp),
      _trace(_reply->getTrace().getLevel()),
      _requestTimer(node_ctx.clock()),
      _n_persistence_replies_total(0),
      _n_successful_persistence_replies(0),
      _priority(_reply->getPriority()),
      _success(true)
{
}

PersistenceMessageTrackerImpl::~PersistenceMessageTrackerImpl() = default;

void
PersistenceMessageTrackerImpl::updateDB()
{
    for (const auto & entry : _bucketInfo) {
        _op_ctx.update_bucket_database(entry.first, entry.second);
    }

    for (const auto & entry :  _remapBucketInfo){
        _op_ctx.update_bucket_database(entry.first, entry.second, DatabaseUpdate::CREATE_IF_NONEXISTING);
    }
}

void
PersistenceMessageTrackerImpl::updateMetrics()
{
    const api::ReturnCode& result(_reply->getResult());
    _metric.updateFromResult(result);
    _metric.latency.addValue(_requestTimer.getElapsedTimeAsDouble());
}

void
PersistenceMessageTrackerImpl::fail(MessageSender& sender, const api::ReturnCode& result) {
    if (_reply.get()) {
        _reply->setResult(result);
        updateMetrics();
        sender.sendReply(_reply);
        _reply.reset();
    }
}

uint16_t
PersistenceMessageTrackerImpl::receiveReply(
        MessageSender& sender,
        api::BucketInfoReply& reply)
{
    uint16_t node = handleReply(reply);

    if (node != (uint16_t)-1) {
        updateFromReply(sender, reply, node);
    }

    return node;
}

void
PersistenceMessageTrackerImpl::revert(
        MessageSender& sender,
        const std::vector<BucketNodePair>& revertNodes)
{
    if (_revertTimestamp != 0) {
        // Since we're reverting, all received bucket info is voided.
        _bucketInfo.clear();

        std::vector<api::Timestamp> reverts;
        reverts.push_back(_revertTimestamp);

        for (const auto & revertNode : revertNodes) {
            auto toRevert = std::make_shared<api::RevertCommand>(revertNode.first, reverts);
            toRevert->setPriority(_priority);
            queueCommand(std::move(toRevert), revertNode.second);
        }

        flushQueue(sender);
    }
}

void
PersistenceMessageTrackerImpl::queueMessageBatch(const std::vector<MessageTracker::ToSend>& messages) {
    _messageBatches.emplace_back();
    for (const auto & message : messages) {
        if (_reply) {
            message._msg->getTrace().setLevel(_reply->getTrace().getLevel());
        }

        _messageBatches.back().push_back(message._msg->getMsgId());
        queueCommand(message._msg, message._target);
    }
}

bool
PersistenceMessageTrackerImpl::canSendReplyEarly() const
{
    if (!_reply.get() || !_reply->getResult().success()) {
        LOG(spam, "Can't return early because we have already replied or failed");
        return false;
    }
    auto &bucketSpaceRepo(_op_ctx.bucket_space_repo());
    auto &bucketSpace(bucketSpaceRepo.get(_reply->getBucket().getBucketSpace()));
    const lib::Distribution& distribution = bucketSpace.getDistribution();

    if (distribution.getInitialRedundancy() == 0) {
        LOG(spam, "Not returning early because initial redundancy wasn't set");
        return false;
    }

    for (const MessageBatch & batch : _messageBatches) {
        uint32_t messagesDone = 0;

        for (uint32_t i = 0; i < batch.size(); i++) {
            if (_sentMessages.find(batch[i]) == _sentMessages.end()) {
                messagesDone++;
            } else if (distribution.ensurePrimaryPersisted() && i == 0) {
                // Primary must always be written.
                LOG(debug, "Not returning early because primary node wasn't done");
                return false;
            }
        }

        if (messagesDone < distribution.getInitialRedundancy()) {
            LOG(spam, "Not returning early because only %d messages out of %d are done",
                messagesDone, distribution.getInitialRedundancy());
            return false;
        }
    }

    return true;
}

void
PersistenceMessageTrackerImpl::addBucketInfoFromReply(
        uint16_t node,
        const api::BucketInfoReply& reply)
{
    document::Bucket bucket(reply.getBucket());
    const api::BucketInfo& bucketInfo(reply.getBucketInfo());

    if (reply.hasBeenRemapped()) {
        LOG(debug, "Bucket %s: Received remapped bucket info %s from node %d",
            bucket.toString().c_str(),
            bucketInfo.toString().c_str(),
            node);
        _remapBucketInfo[bucket].emplace_back(_op_ctx.generate_unique_timestamp(), node, bucketInfo);
    } else {
        LOG(debug, "Bucket %s: Received bucket info %s from node %d",
            bucket.toString().c_str(),
            bucketInfo.toString().c_str(),
            node);
        _bucketInfo[bucket].emplace_back(_op_ctx.generate_unique_timestamp(), node, bucketInfo);
    }
}

void
PersistenceMessageTrackerImpl::logSuccessfulReply(uint16_t node, const api::BucketInfoReply& reply) const
{
    LOG(spam, "Bucket %s: Received successful reply %s",
        reply.getBucketId().toString().c_str(),
        reply.toString().c_str());

    if (!reply.getBucketInfo().valid()) {
        LOG(error,
            "Reply %s from node %d contained invalid bucket "
            "information %s. This is a bug! Please report "
            "this to the Vespa team",
            reply.toString().c_str(),
            node,
            reply.getBucketInfo().toString().c_str());
    }
}

bool
PersistenceMessageTrackerImpl::shouldRevert() const
{
    return _op_ctx.distributor_config().enable_revert()
            &&  !_revertNodes.empty() && !_success && _reply;
}

bool PersistenceMessageTrackerImpl::has_majority_successful_replies() const noexcept {
    // FIXME this has questionable interaction with early client ACK since we only count
    // the number of observed replies rather than the number of total requests sent.
    // ... but the early ACK-feature dearly needs a redesign anyway.
    return (_n_successful_persistence_replies >= (_n_persistence_replies_total / 2 + 1));
}

bool PersistenceMessageTrackerImpl::has_minority_test_and_set_failure() const noexcept {
    return ((_reply->getResult().getResult() == api::ReturnCode::TEST_AND_SET_CONDITION_FAILED)
            && has_majority_successful_replies());
}

void
PersistenceMessageTrackerImpl::sendReply(MessageSender& sender)
{
    // If we've observed _partial_ TaS failures but have had a majority of good ACKs,
    // treat the reply as successful. This is because the ACKed write(s) will eventually
    // become visible across all nodes.
    if (has_minority_test_and_set_failure()) {
        _reply->setResult(api::ReturnCode());
    }

    updateMetrics();
    if ( ! _trace.isEmpty()) {
        _trace.setStrict(false);
        _reply->getTrace().addChild(std::move(_trace));
    }

    sender.sendReply(_reply);
    _reply = std::shared_ptr<api::BucketInfoReply>();
}

void
PersistenceMessageTrackerImpl::updateFailureResult(const api::BucketInfoReply& reply)
{
    LOG(debug, "Bucket %s: Received failed reply %s with result %s",
        reply.getBucketId().toString().c_str(),
        reply.toString().c_str(),
        reply.getResult().toString().c_str());
    if (reply.getResult().getResult() >
        _reply->getResult().getResult())
    {
        _reply->setResult(reply.getResult());
    }

    _success = false;
}

void
PersistenceMessageTrackerImpl::handleCreateBucketReply(
        api::BucketInfoReply& reply,
        uint16_t node)
{
    LOG(spam, "Received CreateBucket reply for %s from node %u",
        reply.getBucketId().toString().c_str(), node);
    if (!reply.getResult().success()
        && reply.getResult().getResult() != api::ReturnCode::EXISTS)
    {
        LOG(spam, "Create bucket reply failed, so deleting it from bucket db");
        _op_ctx.remove_node_from_bucket_database(reply.getBucket(), node);
    }
}

void
PersistenceMessageTrackerImpl::handlePersistenceReply(
        api::BucketInfoReply& reply,
        uint16_t node)
{
    ++_n_persistence_replies_total;
    if (reply.getBucketInfo().valid()) {
        addBucketInfoFromReply(node, reply);
    }
    if (reply.getResult().success()) {
        logSuccessfulReply(node, reply);
        _revertNodes.emplace_back(reply.getBucket(), node);
        ++_n_successful_persistence_replies;
    } else if (!hasSentReply()) {
        updateFailureResult(reply);
    }
}

void
PersistenceMessageTrackerImpl::updateFromReply(
        MessageSender& sender,
        api::BucketInfoReply& reply,
        uint16_t node)
{
    _trace.addChild(reply.steal_trace());

    if (reply.getType() == api::MessageType::CREATEBUCKET_REPLY) {
        handleCreateBucketReply(reply, node);
    } else {
        handlePersistenceReply(reply, node);
    }

    if (finished()) {
        bool doRevert(shouldRevert());

        updateDB();

        if (!hasSentReply()) {
            sendReply(sender);
        }
        if (doRevert) {
            revert(sender, _revertNodes);
        }
    } else if (canSendReplyEarly()) {
        LOG(debug, "Sending reply early because initial redundancy has been reached");
        sendReply(sender);
    }
}

}
<|endoftext|>


<commit_before>/* IBM_PROLOG_BEGIN_TAG                                                   */
/* This is an automatically generated prolog.                             */
/*                                                                        */
/* $Source: src/usr/pnor/pnoraltsync.C $                                  */
/*                                                                        */
/* OpenPOWER HostBoot Project                                             */
/*                                                                        */
/* Contributors Listed Below - COPYRIGHT 2021                             */
/* [+] International Business Machines Corp.                              */
/*                                                                        */
/*                                                                        */
/* Licensed under the Apache License, Version 2.0 (the "License");        */
/* you may not use this file except in compliance with the License.       */
/* You may obtain a copy of the License at                                */
/*                                                                        */
/*     http://www.apache.org/licenses/LICENSE-2.0                         */
/*                                                                        */
/* Unless required by applicable law or agreed to in writing, software    */
/* distributed under the License is distributed on an "AS IS" BASIS,      */
/* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or        */
/* implied. See the License for the specific language governing           */
/* permissions and limitations under the License.                         */
/*                                                                        */
/* IBM_PROLOG_END_TAG                                                     */
/**
 *  @file pnoraltsync.C
 *
 *  @brief Implements PNOR::copyPnorPartitionToAlt() to synchronize
 *         partitions between the active and alternate PNORs
 */

/*****************************************************************************/
// I n c l u d e s
/*****************************************************************************/
#include <trace/interface.H>
#include <devicefw/driverif.H>
#include <errl/errlentry.H>
#include <errl/errlmanager.H>
#include <targeting/common/targetservice.H>
#include <targeting/common/utilFilter.H>
#include <attributeenums.H>
#include "ffs.h"
#include "common/ffs_hb.H"
#include "pnorrp.H"
#include <pnor/pnorif.H>
#include <pnor/pnor_reasoncodes.H>
#include <lpc/lpcif.H>

#if defined(CONFIG_PNORDD_IS_SFC)
#include "pnor_sfcdd.H"
using PnorDD = PnorSfcDD;
#elif (defined(CONFIG_PNORDD_IS_BMCMBOX) || defined(CONFIG_PNORDD_IS_IPMI))
#include "pnor_hiomapdd.H"
using PnorDD = PnorHiomapDD;
#else
#error "No PNOR DD configured"
#endif

extern trace_desc_t* g_trac_pnor;

namespace PNOR
{

/**
 * @brief Copy a given PNOR partition from the active PNOR into the
 *        alternate copy
 */
errlHndl_t copyPnorPartitionToAlt( PNOR::SectionId i_section )
{
    errlHndl_t l_errhdl = nullptr;
    const char* l_sectionName = PNOR::SectionIdToString(i_section);
    TRACFCOMP( g_trac_pnor, ENTER_MRK"PNOR::copyPnorPartitionToAlt(%d=%s)> ",
               i_section, l_sectionName );

    // Algorithm is:
    // 0. Check if there is an alt-pnor to update, exit if not
    // 1. Instantiate an alt-pnordd to access the alternate PNOR.
    // 2. Find the flash offset and full size (including ECC) of the
    //    target section on the active PNOR.
    // 3. Find the flash offset and full size (including ECC) of the
    //    target section on the alternate PNOR.
    // 4. If size+offset match, bulk copy the complete section from
    //    the active PNOR to the alternate.

    PnorDD* altpnordd = nullptr;

    // When reading PNOR TOC assume a single page and no ECC
    uint8_t* tocBuffer = new uint8_t[PAGESIZE];
    const uint64_t toc0_offset = PnorRP::getInstance().getTocOffset(TOC_0);

    do{
        //--------------------------------
        // 0. Check if there is an alt-pnor to update, exit if not

        // Get list of all processors
        TARGETING::TargetHandleList procList;
        TARGETING::getAllChips(procList,
                               TARGETING::TYPE_PROC,
                               true); // true: return functional targets

        if( ( 0 == procList.size() ) ||
            ( nullptr == procList[0] ) )
        {
            TRACFCOMP( g_trac_pnor, INFO_MRK"PNOR::copyPnorPartitionToAlt()> No functional processors Found! Validation skipped" );
            break;
        }

        // Loop through all processors to find the alt-proc
        TARGETING::Target* l_altProc = nullptr;
        for(uint32_t i=0; i<procList.size(); i++)
        {
            // Check if processor is MASTER_CANDIDATE
            TARGETING::ATTR_PROC_MASTER_TYPE_type type_enum =
              procList[i]->getAttr<TARGETING::ATTR_PROC_MASTER_TYPE>();

            if ( type_enum != TARGETING::PROC_MASTER_TYPE_MASTER_CANDIDATE )
            {
                TRACDCOMP( g_trac_pnor, INFO_MRK"PNOR::copyPnorPartitionToAlt> Skipping Processor 0x%X (PROC_MASTER_TYPE=%d)",
                           TARGETING::get_huid(procList[i]), type_enum);
                continue;
            }

            TRACFCOMP( g_trac_pnor, INFO_MRK"PNOR::copyPnorPartitionToAlt> Validating Processor 0x%X (PROC_MASTER_TYPE=%d)",
                       TARGETING::get_huid(procList[i]), type_enum);
            l_altProc = procList[i];
            break;
        }
        if( !l_altProc )
        {
            TRACFCOMP( g_trac_pnor, INFO_MRK"PNOR::copyPnorPartitionToAlt> There are no alternate PNORs to update" );
            break;
        }

        //--------------------------------
        // 1. Instantiate an alt-pnordd to access the alternate PNOR.

        // Create the LPC objects we're going to use
        l_errhdl = LPC::create_altmaster_objects( true, l_altProc );
        if ( l_errhdl )
        {
            // Commit Error Log, but continue the test
            TRACFCOMP( g_trac_pnor, INFO_MRK"PNOR::copyPnorPartitionToAlt> Could not create LPC objects for %.8X.", TARGETING::get_huid(l_altProc) );
            break;
        }

        // Create and initialize custom PNOR DD class for this target
        altpnordd = new PnorDD(l_altProc);

        //--------------------------------
        // 2. Find the flash offset and full size (including ECC) of the
        //    target section on the active PNOR.

        PNOR::SectionInfo_t l_activePnor;
        l_errhdl = PNOR::getSectionInfo(i_section, l_activePnor);
        if( l_errhdl )
        {
            TRACFCOMP( g_trac_pnor, INFO_MRK"PNOR::copyPnorPartitionToAlt> Error getting data for section %d(%s)", i_section, l_sectionName );
            break;
        }
        TRACFCOMP( g_trac_pnor, INFO_MRK"PNOR::copyPnorPartitionToAlt> Section %d(%s) located at 0x%.X for 0x%X bytes", i_section, l_sectionName, l_activePnor.flashAddr, l_activePnor.sizeActual );

        //--------------------------------
        // 3. Find the flash offset and full size (including ECC) of the
        //    target section on the alternate PNOR.

        size_t read_size = PAGESIZE;
        l_errhdl = altpnordd->readFlash(tocBuffer, read_size, toc0_offset);
        if ( l_errhdl )
        {
            TRACFCOMP( g_trac_pnor, INFO_MRK"PNOR::copyPnorPartitionToAlt> readFlash fail for 0x%X.",
                       TARGETING::get_huid(l_altProc) );
            break;
        }

        PNOR::SectionData_t l_altTOC[PNOR::NUM_SECTIONS+1];
        l_errhdl = parseTOC( tocBuffer, l_altTOC );
        if( l_errhdl )
        {
            TRACFCOMP( g_trac_pnor, INFO_MRK"PNOR::copyPnorPartitionToAlt> parseTOC fail for 0x%X.",
                       TARGETING::get_huid(l_altProc) );
            break;
        }

        //--------------------------------
        // 4. If size+offset match, bulk copy the complete section from
        //    the active PNOR to the alternate.
        if( l_activePnor.sizeActual != l_altTOC[i_section].size )
        {
            TRACFCOMP( g_trac_pnor, INFO_MRK"PNOR::copyPnorPartitionToAlt> Size mismatch between active (0x%X) and alt (0x%X) sections",
                       l_activePnor.sizeActual,
                       l_altTOC[i_section].size );
            break;
        }

        TRACFCOMP( g_trac_pnor, INFO_MRK"PNOR::copyPnorPartitionToAlt> About to copy 0x%X bytes", l_altTOC[i_section].size );
        // loop through to write a few KB at a time so we don't
        //  need to allocate a giant chunk of memory
        constexpr size_t TRANSFER_SIZE = 4*PAGESIZE;
        uint8_t l_buffer[TRANSFER_SIZE] = {0};
        for( size_t loop = 0; loop < l_altTOC[i_section].size; loop += TRANSFER_SIZE )
        {
            size_t l_readSize = std::min( TRANSFER_SIZE, l_altTOC[i_section].size-loop );

            // read the data off the active pnor
            l_errhdl = DeviceFW::deviceRead(
                  TARGETING::MASTER_PROCESSOR_CHIP_TARGET_SENTINEL,
                  l_buffer,
                  l_readSize,
                  DEVICE_PNOR_ADDRESS(0,l_activePnor.flashAddr+loop) );
            if( l_errhdl )
            {
                TRACFCOMP( g_trac_pnor, INFO_MRK"PNOR::copyPnorPartitionToAlt> Failed to read 0x%X from active PNOR", loop+l_activePnor.flashAddr );
                break;
            }

            // write the data into the alternate pnor
            l_errhdl = altpnordd->writeFlash(l_buffer,
                                          l_readSize,
                                          l_altTOC[i_section].flashAddr+loop);
            if( l_errhdl )
            {
                TRACFCOMP( g_trac_pnor, INFO_MRK"PNOR::copyPnorPartitionToAlt> Failed to write 0x%X into alternate PNOR", l_altTOC[i_section].flashAddr+loop );
                break;
            }
        }
        if( l_errhdl ) { break; }

    }while(0);

    if( l_errhdl )
    {
        l_errhdl->collectTrace(PNOR_COMP_NAME);
    }

    // Delete PNOR DD class and memory buffer
    if ( altpnordd )
    {
        delete altpnordd;
        altpnordd = nullptr;

        // Delete the LPC objects we used
        errlHndl_t l_errhdl2 = LPC::create_altmaster_objects( false, nullptr );
        if ( l_errhdl2 )
        {
            // Commit Error Log, but don't kill the istep because we can recover from being
            //   out of sync on a later IPL
            TRACFCOMP( g_trac_pnor, INFO_MRK"PNOR::copyPnorPartitionToAlt> Could not delete LPC objects. eid=0x%X, rc=0x%X. Committing log as info", l_errhdl2->eid(), l_errhdl2->reasonCode());

            l_errhdl->collectTrace(PNOR_COMP_NAME);
            l_errhdl2->setSev(ERRORLOG::ERRL_SEV_INFORMATIONAL);
            errlCommit(l_errhdl2,PNOR_COMP_ID);
        }
    }

    if(tocBuffer != nullptr)
    {
        delete[] tocBuffer;
        tocBuffer = nullptr;
    }

    TRACFCOMP( g_trac_pnor, EXIT_MRK"PNOR::copyPnorPartitionToAlt(%d=%s)> ",
               i_section, l_sectionName );
    return l_errhdl;
}

}; //namespace PNOR

<commit_msg>Ignore TOC errors when syncing EECACHE to alternate pnor<commit_after>/* IBM_PROLOG_BEGIN_TAG                                                   */
/* This is an automatically generated prolog.                             */
/*                                                                        */
/* $Source: src/usr/pnor/pnoraltsync.C $                                  */
/*                                                                        */
/* OpenPOWER HostBoot Project                                             */
/*                                                                        */
/* Contributors Listed Below - COPYRIGHT 2021                             */
/* [+] International Business Machines Corp.                              */
/*                                                                        */
/*                                                                        */
/* Licensed under the Apache License, Version 2.0 (the "License");        */
/* you may not use this file except in compliance with the License.       */
/* You may obtain a copy of the License at                                */
/*                                                                        */
/*     http://www.apache.org/licenses/LICENSE-2.0                         */
/*                                                                        */
/* Unless required by applicable law or agreed to in writing, software    */
/* distributed under the License is distributed on an "AS IS" BASIS,      */
/* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or        */
/* implied. See the License for the specific language governing           */
/* permissions and limitations under the License.                         */
/*                                                                        */
/* IBM_PROLOG_END_TAG                                                     */
/**
 *  @file pnoraltsync.C
 *
 *  @brief Implements PNOR::copyPnorPartitionToAlt() to synchronize
 *         partitions between the active and alternate PNORs
 */

/*****************************************************************************/
// I n c l u d e s
/*****************************************************************************/
#include <trace/interface.H>
#include <devicefw/driverif.H>
#include <errl/errlentry.H>
#include <errl/errlmanager.H>
#include <targeting/common/targetservice.H>
#include <targeting/common/utilFilter.H>
#include <attributeenums.H>
#include "ffs.h"
#include "common/ffs_hb.H"
#include "pnorrp.H"
#include <pnor/pnorif.H>
#include <pnor/pnor_reasoncodes.H>
#include <lpc/lpcif.H>

#if defined(CONFIG_PNORDD_IS_SFC)
#include "pnor_sfcdd.H"
using PnorDD = PnorSfcDD;
#elif (defined(CONFIG_PNORDD_IS_BMCMBOX) || defined(CONFIG_PNORDD_IS_IPMI))
#include "pnor_hiomapdd.H"
using PnorDD = PnorHiomapDD;
#else
#error "No PNOR DD configured"
#endif

extern trace_desc_t* g_trac_pnor;

namespace PNOR
{

/**
 * @brief Copy a given PNOR partition from the active PNOR into the
 *        alternate copy
 */
errlHndl_t copyPnorPartitionToAlt( PNOR::SectionId i_section )
{
    errlHndl_t l_errhdl = nullptr;
    const char* l_sectionName = PNOR::SectionIdToString(i_section);
    TRACFCOMP( g_trac_pnor, ENTER_MRK"PNOR::copyPnorPartitionToAlt(%d=%s)> ",
               i_section, l_sectionName );

    // Algorithm is:
    // 0. Check if there is an alt-pnor to update, exit if not
    // 1. Instantiate an alt-pnordd to access the alternate PNOR.
    // 2. Find the flash offset and full size (including ECC) of the
    //    target section on the active PNOR.
    // 3. Find the flash offset and full size (including ECC) of the
    //    target section on the alternate PNOR.
    // 4. If size+offset match, bulk copy the complete section from
    //    the active PNOR to the alternate.

    PnorDD* altpnordd = nullptr;

    // When reading PNOR TOC assume a single page and no ECC
    uint8_t* tocBuffer = new uint8_t[PAGESIZE];
    const uint64_t toc0_offset = PnorRP::getInstance().getTocOffset(TOC_0);

    do{
        //--------------------------------
        // 0. Check if there is an alt-pnor to update, exit if not

        // Get list of all processors
        TARGETING::TargetHandleList procList;
        TARGETING::getAllChips(procList,
                               TARGETING::TYPE_PROC,
                               true); // true: return functional targets

        if( ( 0 == procList.size() ) ||
            ( nullptr == procList[0] ) )
        {
            TRACFCOMP( g_trac_pnor, INFO_MRK"PNOR::copyPnorPartitionToAlt()> No functional processors Found! Validation skipped" );
            break;
        }

        // Loop through all processors to find the alt-proc
        TARGETING::Target* l_altProc = nullptr;
        for(uint32_t i=0; i<procList.size(); i++)
        {
            // Check if processor is MASTER_CANDIDATE
            TARGETING::ATTR_PROC_MASTER_TYPE_type type_enum =
              procList[i]->getAttr<TARGETING::ATTR_PROC_MASTER_TYPE>();

            if ( type_enum != TARGETING::PROC_MASTER_TYPE_MASTER_CANDIDATE )
            {
                TRACDCOMP( g_trac_pnor, INFO_MRK"PNOR::copyPnorPartitionToAlt> Skipping Processor 0x%X (PROC_MASTER_TYPE=%d)",
                           TARGETING::get_huid(procList[i]), type_enum);
                continue;
            }

            TRACFCOMP( g_trac_pnor, INFO_MRK"PNOR::copyPnorPartitionToAlt> Validating Processor 0x%X (PROC_MASTER_TYPE=%d)",
                       TARGETING::get_huid(procList[i]), type_enum);
            l_altProc = procList[i];
            break;
        }
        if( !l_altProc )
        {
            TRACFCOMP( g_trac_pnor, INFO_MRK"PNOR::copyPnorPartitionToAlt> There are no alternate PNORs to update" );
            break;
        }

        //--------------------------------
        // 1. Instantiate an alt-pnordd to access the alternate PNOR.

        // Create the LPC objects we're going to use
        l_errhdl = LPC::create_altmaster_objects( true, l_altProc );
        if ( l_errhdl )
        {
            // Commit Error Log, but continue the test
            TRACFCOMP( g_trac_pnor, INFO_MRK"PNOR::copyPnorPartitionToAlt> Could not create LPC objects for %.8X.", TARGETING::get_huid(l_altProc) );
            break;
        }

        // Create and initialize custom PNOR DD class for this target
        altpnordd = new PnorDD(l_altProc);

        //--------------------------------
        // 2. Find the flash offset and full size (including ECC) of the
        //    target section on the active PNOR.

        PNOR::SectionInfo_t l_activePnor;
        l_errhdl = PNOR::getSectionInfo(i_section, l_activePnor);
        if( l_errhdl )
        {
            TRACFCOMP( g_trac_pnor, INFO_MRK"PNOR::copyPnorPartitionToAlt> Error getting data for section %d(%s)", i_section, l_sectionName );
            break;
        }
        TRACFCOMP( g_trac_pnor, INFO_MRK"PNOR::copyPnorPartitionToAlt> Section %d(%s) located at 0x%.X for 0x%X bytes", i_section, l_sectionName, l_activePnor.flashAddr, l_activePnor.sizeActual );

        //--------------------------------
        // 3. Find the flash offset and full size (including ECC) of the
        //    target section on the alternate PNOR.

        size_t read_size = PAGESIZE;
        l_errhdl = altpnordd->readFlash(tocBuffer, read_size, toc0_offset);
        if ( l_errhdl )
        {
            TRACFCOMP( g_trac_pnor, INFO_MRK"PNOR::copyPnorPartitionToAlt> readFlash fail for 0x%X.",
                       TARGETING::get_huid(l_altProc) );
            break;
        }

        PNOR::SectionData_t l_altTOC[PNOR::NUM_SECTIONS+1];
        l_errhdl = parseTOC( tocBuffer, l_altTOC );
        if( l_errhdl )
        {
            TRACFCOMP( g_trac_pnor, INFO_MRK"PNOR::copyPnorPartitionToAlt> parseTOC fail for 0x%X.",
                       TARGETING::get_huid(l_altProc) );
            // This can be caused by a genesis boot where the alt side has never been IPLed.
            // Need to just ignore this error and move on.
            l_errhdl->setSev(ERRORLOG::ERRL_SEV_INFORMATIONAL);
            errlCommit(l_errhdl,PNOR_COMP_ID);
            break;
        }

        //--------------------------------
        // 4. If size+offset match, bulk copy the complete section from
        //    the active PNOR to the alternate.
        if( l_activePnor.sizeActual != l_altTOC[i_section].size )
        {
            TRACFCOMP( g_trac_pnor, INFO_MRK"PNOR::copyPnorPartitionToAlt> Size mismatch between active (0x%X) and alt (0x%X) sections",
                       l_activePnor.sizeActual,
                       l_altTOC[i_section].size );
            break;
        }

        TRACFCOMP( g_trac_pnor, INFO_MRK"PNOR::copyPnorPartitionToAlt> About to copy 0x%X bytes", l_altTOC[i_section].size );
        // loop through to write a few KB at a time so we don't
        //  need to allocate a giant chunk of memory
        constexpr size_t TRANSFER_SIZE = 4*PAGESIZE;
        uint8_t l_buffer[TRANSFER_SIZE] = {0};
        for( size_t loop = 0; loop < l_altTOC[i_section].size; loop += TRANSFER_SIZE )
        {
            size_t l_readSize = std::min( TRANSFER_SIZE, l_altTOC[i_section].size-loop );

            // read the data off the active pnor
            l_errhdl = DeviceFW::deviceRead(
                  TARGETING::MASTER_PROCESSOR_CHIP_TARGET_SENTINEL,
                  l_buffer,
                  l_readSize,
                  DEVICE_PNOR_ADDRESS(0,l_activePnor.flashAddr+loop) );
            if( l_errhdl )
            {
                TRACFCOMP( g_trac_pnor, INFO_MRK"PNOR::copyPnorPartitionToAlt> Failed to read 0x%X from active PNOR", loop+l_activePnor.flashAddr );
                break;
            }

            // write the data into the alternate pnor
            l_errhdl = altpnordd->writeFlash(l_buffer,
                                          l_readSize,
                                          l_altTOC[i_section].flashAddr+loop);
            if( l_errhdl )
            {
                TRACFCOMP( g_trac_pnor, INFO_MRK"PNOR::copyPnorPartitionToAlt> Failed to write 0x%X into alternate PNOR", l_altTOC[i_section].flashAddr+loop );
                break;
            }
        }
        if( l_errhdl ) { break; }

    }while(0);

    if( l_errhdl )
    {
        l_errhdl->collectTrace(PNOR_COMP_NAME);
    }

    // Delete PNOR DD class and memory buffer
    if ( altpnordd )
    {
        delete altpnordd;
        altpnordd = nullptr;

        // Delete the LPC objects we used
        errlHndl_t l_errhdl2 = LPC::create_altmaster_objects( false, nullptr );
        if ( l_errhdl2 )
        {
            // Commit Error Log, but don't kill the istep because we can recover from being
            //   out of sync on a later IPL
            TRACFCOMP( g_trac_pnor, INFO_MRK"PNOR::copyPnorPartitionToAlt> Could not delete LPC objects. eid=0x%X, rc=0x%X. Committing log as info", l_errhdl2->eid(), l_errhdl2->reasonCode());

            l_errhdl->collectTrace(PNOR_COMP_NAME);
            l_errhdl2->setSev(ERRORLOG::ERRL_SEV_INFORMATIONAL);
            errlCommit(l_errhdl2,PNOR_COMP_ID);
        }
    }

    if(tocBuffer != nullptr)
    {
        delete[] tocBuffer;
        tocBuffer = nullptr;
    }

    TRACFCOMP( g_trac_pnor, EXIT_MRK"PNOR::copyPnorPartitionToAlt(%d=%s)> ",
               i_section, l_sectionName );
    return l_errhdl;
}

}; //namespace PNOR

<|endoftext|>


<commit_before>/* Copyright 2017 The TensorFlow Authors. All Rights Reserved.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
==============================================================================*/
#include <memory>
#include <string>
#include <vector>

#include "tensorflow/contrib/lite/toco/graph_transformations/graph_transformations.h"
#include "tensorflow/contrib/lite/toco/model.h"
#include "tensorflow/contrib/lite/toco/tooling_util.h"

namespace toco {

namespace {

std::vector<std::unique_ptr<Operator>>::iterator FindOperator(
    Model* model, const Operator& op) {
  auto it = model->operators.begin();
  for (; it != model->operators.end(); ++it) {
    if (it->get() == &op) {
      break;
    }
  }
  return it;
}

bool GetStateArrayForBackEdge(const Model& model,
                              const string& back_edge_source_array,
                              string* state_array = nullptr) {
  for (const auto& rnn_state : model.flags.rnn_states()) {
    if (back_edge_source_array == rnn_state.back_edge_source_array()) {
      // Found LSTM cell output
      if (state_array) {
        *state_array = rnn_state.state_array();
      }
      return true;
    }
  }
  return false;
}

// Returns true if the given operator has exactly 1 input, and is connected to
// the given op_type.
// We use kNone to indicate an input unattached to an operator output. Usually
// these are the static input arrays.
bool MatchOperatorInputs(const Operator& op, const Model& model,
                         OperatorType op_type, Operator** connected_op) {
  // Check for required number of inputs
  if (op.inputs.size() != 1) {
    return false;
  }

  // Check if first input is disconnected/connected to an operator
  Operator* x = GetOpWithOutput(model, op.inputs[0]);
  if ((op_type == OperatorType::kNone) && (x != nullptr)) {
    return false;
  }
  if ((op_type != OperatorType::kNone) && (x == nullptr)) {
    return false;
  }

  // Check that first operator, if connected, is of correct type
  if ((x != nullptr) && (x->type != op_type)) {
    return false;
  }

  // Successfully matched. Optionally return matching input operators.
  if (connected_op) {
    *connected_op = x;
  }

  return true;
}

// Returns true if the given operator has exactly 2 inputs, which are connected
// to the given op_types.
// We use kNone to indicate an input unattached to an operator output. Usually
// these are the static input arrays.
bool MatchOperatorInputs(const Operator& op, const Model& model,
                         OperatorType a_op_type, Operator** a_op,
                         OperatorType b_op_type, Operator** b_op) {
  // Check for required number of inputs
  if (op.inputs.size() != 2) {
    return false;
  }

  // Check if first input is disconnected/connected to an operator
  Operator* x = GetOpWithOutput(model, op.inputs[0]);
  if ((a_op_type == OperatorType::kNone) && (x != nullptr)) {
    return false;
  }
  if ((a_op_type != OperatorType::kNone) && (x == nullptr)) {
    return false;
  }

  // Check that first operator, if connected, is of correct type
  if ((x != nullptr) && (x->type != a_op_type)) {
    return false;
  }

  // Check if second input is disconnected/connected to an operator
  Operator* y = GetOpWithOutput(model, op.inputs[1]);
  if ((b_op_type == OperatorType::kNone) && (y != nullptr)) {
    return false;
  }
  if ((b_op_type != OperatorType::kNone) && (y == nullptr)) {
    return false;
  }

  // Check that second operator, if connected, is of correct type
  if ((y != nullptr) && (y->type != b_op_type)) {
    return false;
  }

  // Successfully matched. Optionally return matching input operators.
  if (a_op != nullptr) {
    *a_op = x;
  }
  if (b_op != nullptr) {
    *b_op = y;
  }
  return true;
}

// Returns true if the given operator has exactly 3 inputs, which are connected
// to the given op_types.
// We use kNone to indicate an input unattached to an operator output. Usually
// these are the static input arrays.
bool MatchOperatorInputs(const Operator& op, const Model& model,
                         OperatorType a_op_type, Operator** a_op,
                         OperatorType b_op_type, Operator** b_op,
                         OperatorType c_op_type, Operator** c_op) {
  // Check for required number of inputs
  if (op.inputs.size() != 3) {
    return false;
  }

  // Check if first input is disconnected/connected to an operator
  Operator* x = GetOpWithOutput(model, op.inputs[0]);
  if ((a_op_type == OperatorType::kNone) && (x != nullptr)) {
    return false;
  }
  if ((a_op_type != OperatorType::kNone) && (x == nullptr)) {
    return false;
  }

  // Check that first operator, if connected, is of correct type
  if ((x != nullptr) && (x->type != a_op_type)) {
    return false;
  }

  // Check if second input is disconnected/connected to an operator
  Operator* y = GetOpWithOutput(model, op.inputs[1]);
  if ((b_op_type == OperatorType::kNone) && (y != nullptr)) {
    return false;
  }
  if ((b_op_type != OperatorType::kNone) && (y == nullptr)) {
    return false;
  }

  // Check that second operator, if connected, is of correct type
  if ((y != nullptr) && (y->type != b_op_type)) {
    return false;
  }

  // Check if third input is disconnected/connected to an operator
  Operator* z = GetOpWithOutput(model, op.inputs[2]);
  if ((c_op_type == OperatorType::kNone) && (z != nullptr)) {
    return false;
  }
  if ((c_op_type != OperatorType::kNone) && (z == nullptr)) {
    return false;
  }

  // Check that third operator, if connected, is of correct type
  if ((z != nullptr) && (z->type != c_op_type)) {
    return false;
  }

  // Successfully matched. Optionally return matching input operators.
  if (a_op != nullptr) {
    *a_op = x;
  }
  if (b_op != nullptr) {
    *b_op = y;
  }
  if (c_op != nullptr) {
    *c_op = z;
  }
  return true;
}

}  // namespace

bool IdentifyLstmCell::Run(Model* model, std::size_t op_index) {
  // This LSTM cell identification method is not invariant to commutation of
  // commutative operator inputs. For example, if input[0] and input[1] of the
  // final output multiplication were swapped, this method would not identify it
  // as an LSTM cell. This is OK in most cases, because
  // tf.rnn.contrib.BasicLSTMCell always generates LSTM cells the same way.

  // Final output multiply
  auto op_it = model->operators.begin() + op_index;
  Operator* final_output_mul = op_it->get();
  if (final_output_mul->type != OperatorType::kMul) {
    return false;
  }
  Operator *state_output_tanh, *fc_output_sig;
  if (!MatchOperatorInputs(*final_output_mul, *model, OperatorType::kTanh,
                           &state_output_tanh, OperatorType::kLogistic,
                           &fc_output_sig)) {
    return false;
  }

  // State output TanH
  // (We don't count an operator as ID'd until we verify it has the correct
  // operator types feeding into it.)
  Operator* state_combine_add;
  if (!MatchOperatorInputs(*state_output_tanh, *model, OperatorType::kAdd,
                           &state_combine_add)) {
    return false;
  }
  string prev_state;
  if (!GetStateArrayForBackEdge(*model, state_output_tanh->inputs[0],
                                &prev_state)) {
    return false;
  }

  // State forget & remember addition
  Operator *state_forget_mul, *state_remember_mul;
  if (!MatchOperatorInputs(*state_combine_add, *model, OperatorType::kMul,
                           &state_forget_mul, OperatorType::kMul,
                           &state_remember_mul)) {
    return false;
  }
  if (state_forget_mul->inputs[0] != prev_state) {
    return false;
  }

  // State forget gate
  Operator* state_forget_sig;
  if (!MatchOperatorInputs(*state_forget_mul, *model, OperatorType::kNone,
                           nullptr, OperatorType::kLogistic,
                           &state_forget_sig)) {
    return false;
  }

  // State remember gate
  Operator *state_remember_sig, *state_info_tanh;
  if (!MatchOperatorInputs(*state_remember_mul, *model, OperatorType::kLogistic,
                           &state_remember_sig, OperatorType::kTanh,
                           &state_info_tanh)) {
    return false;
  }

  // State remember "information" activation function
  Operator* fc_output_split;
  if (!MatchOperatorInputs(*state_info_tanh, *model, OperatorType::kSplit,
                           &fc_output_split)) {
    return false;
  }
  // State remember gate activation function
  Operator* tmp;
  if (!MatchOperatorInputs(*state_remember_sig, *model, OperatorType::kSplit,
                           &tmp) ||
      (tmp != fc_output_split)) {
    return false;
  }
  // State forget gate activation function
  if (!MatchOperatorInputs(*state_forget_sig, *model, OperatorType::kSplit,
                           &tmp) ||
      (tmp != fc_output_split)) {
    return false;
  }
  // Fully connected output activation function
  if (!MatchOperatorInputs(*fc_output_sig, *model, OperatorType::kSplit,
                           &tmp) ||
      (tmp != fc_output_split)) {
    return false;
  }
  // Fully connected output split
  Operator* fully_connected;
  if (!MatchOperatorInputs(*fc_output_split, *model, OperatorType::kNone,
                           nullptr, OperatorType::kFullyConnected,
                           &fully_connected)) {
    return false;
  }

  // Fully connected op
  Operator* concat_inputs;
  if (!MatchOperatorInputs(*fully_connected, *model,
                           OperatorType::kConcatenation, &concat_inputs,
                           OperatorType::kNone, nullptr, OperatorType::kNone,
                           nullptr)) {
    return false;
  }

  if (static_cast<FullyConnectedOperator*>(fully_connected)->weights_format !=
      FullyConnectedWeightsFormat::kDefault) {
    // Not yet implemented: experimental shuffled weights in fused LSTM cell.
    return false;
  }

  // Emplace a new LSTM cell operator
  auto* lstm_cell_op = new LstmCellOperator;
  lstm_cell_op->inputs.resize(LstmCellOperator::NUM_INPUTS);
  lstm_cell_op->inputs[LstmCellOperator::DATA_INPUT] = concat_inputs->inputs[0];
  lstm_cell_op->inputs[LstmCellOperator::PREV_ACTIV_INPUT] =
      concat_inputs->inputs[1];
  lstm_cell_op->inputs[LstmCellOperator::WEIGHTS_INPUT] =
      fully_connected->inputs[1];
  lstm_cell_op->inputs[LstmCellOperator::BIASES_INPUT] =
      fully_connected->inputs[2];
  lstm_cell_op->inputs[LstmCellOperator::PREV_STATE_INPUT] = prev_state;
  lstm_cell_op->outputs.resize(LstmCellOperator::NUM_OUTPUTS);
  lstm_cell_op->outputs[LstmCellOperator::STATE_OUTPUT] =
      state_output_tanh->inputs[0];
  lstm_cell_op->outputs[LstmCellOperator::ACTIV_OUTPUT] =
      final_output_mul->outputs[0];
  model->operators.emplace(op_it, lstm_cell_op);
  AddMessageF("Creating %s replacing equivalent subgraph",
              LogName(*lstm_cell_op));

  // Create temp arrays used internally during runtime.
  const string base_name(FindLongestCommonPrefix(
      lstm_cell_op->outputs[LstmCellOperator::STATE_OUTPUT],
      lstm_cell_op->outputs[LstmCellOperator::ACTIV_OUTPUT]));
  const string& concat_temp_array_name =
      AvailableArrayName(*model, base_name + "concat_temp");
  model->GetOrCreateArray(concat_temp_array_name);
  lstm_cell_op->outputs[LstmCellOperator::CONCAT_TEMP] = concat_temp_array_name;
  const string& activ_temp_array_name =
      AvailableArrayName(*model, base_name + "activ_temp");
  model->GetOrCreateArray(activ_temp_array_name);
  lstm_cell_op->outputs[LstmCellOperator::ACTIV_TEMP] = activ_temp_array_name;
  AddMessageF("Created temp outputs %s and %s on operator %s",
              concat_temp_array_name, activ_temp_array_name,
              LogName(*lstm_cell_op));

  // Delete arrays and operators replaced by the LSTM cell operator. Order is
  // important - DeleteArrayIfUnused() only succeeds if dependent operators
  // have been removed first. Start at the output and work towards the input.
  model->operators.erase(FindOperator(model, *final_output_mul));
  DeleteArrayIfUnused(state_output_tanh->outputs[0], model);
  DeleteArrayIfUnused(fc_output_sig->outputs[0], model);
  model->operators.erase(FindOperator(model, *state_output_tanh));
  model->operators.erase(FindOperator(model, *fc_output_sig));
  model->operators.erase(FindOperator(model, *state_combine_add));
  DeleteArrayIfUnused(state_forget_mul->outputs[0], model);
  DeleteArrayIfUnused(state_remember_mul->outputs[0], model);
  model->operators.erase(FindOperator(model, *state_forget_mul));
  model->operators.erase(FindOperator(model, *state_remember_mul));
  DeleteArrayIfUnused(state_forget_sig->outputs[0], model);
  DeleteArrayIfUnused(state_info_tanh->outputs[0], model);
  DeleteArrayIfUnused(state_remember_sig->outputs[0], model);
  model->operators.erase(FindOperator(model, *state_forget_sig));
  model->operators.erase(FindOperator(model, *state_info_tanh));
  model->operators.erase(FindOperator(model, *state_remember_sig));
  DeleteArrayIfUnused(fc_output_split->outputs[0], model);
  DeleteArrayIfUnused(fc_output_split->outputs[1], model);
  DeleteArrayIfUnused(fc_output_split->outputs[2], model);
  DeleteArrayIfUnused(fc_output_split->outputs[3], model);
  string dims_array = fc_output_split->inputs[0];
  model->operators.erase(FindOperator(model, *fc_output_split));
  DeleteArrayIfUnused(dims_array, model);
  DeleteArrayIfUnused(fully_connected->outputs[0], model);
  model->operators.erase(FindOperator(model, *fully_connected));
  DeleteArrayIfUnused(concat_inputs->outputs[0], model);
  model->operators.erase(FindOperator(model, *concat_inputs));
  return true;
}

}  // namespace toco
<commit_msg>make identify_lstm independent of rnn_states<commit_after>/* Copyright 2017 The TensorFlow Authors. All Rights Reserved.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
==============================================================================*/
#include <memory>
#include <string>
#include <vector>

#include "tensorflow/contrib/lite/toco/graph_transformations/graph_transformations.h"
#include "tensorflow/contrib/lite/toco/model.h"
#include "tensorflow/contrib/lite/toco/tooling_util.h"

namespace toco {

namespace {

std::vector<std::unique_ptr<Operator>>::iterator FindOperator(
    Model* model, const Operator& op) {
  auto it = model->operators.begin();
  for (; it != model->operators.end(); ++it) {
    if (it->get() == &op) {
      break;
    }
  }
  return it;
}

// Returns true if the given operator has exactly 1 input, and is connected to
// the given op_type.
// We use kNone to indicate an input unattached to an operator output. Usually
// these are the static input arrays.
bool MatchOperatorInputs(const Operator& op, const Model& model,
                         OperatorType op_type, Operator** connected_op) {
  // Check for required number of inputs
  if (op.inputs.size() != 1) {
    return false;
  }

  // Check if first input is disconnected/connected to an operator
  Operator* x = GetOpWithOutput(model, op.inputs[0]);
  if ((op_type == OperatorType::kNone) && (x != nullptr)) {
    return false;
  }
  if ((op_type != OperatorType::kNone) && (x == nullptr)) {
    return false;
  }

  // Check that first operator, if connected, is of correct type
  if ((x != nullptr) && (x->type != op_type)) {
    return false;
  }

  // Successfully matched. Optionally return matching input operators.
  if (connected_op) {
    *connected_op = x;
  }

  return true;
}

// Returns true if the given operator has exactly 2 inputs, which are connected
// to the given op_types.
// We use kNone to indicate an input unattached to an operator output. Usually
// these are the static input arrays.
bool MatchOperatorInputs(const Operator& op, const Model& model,
                         OperatorType a_op_type, Operator** a_op,
                         OperatorType b_op_type, Operator** b_op) {
  // Check for required number of inputs
  if (op.inputs.size() != 2) {
    return false;
  }

  // Check if first input is disconnected/connected to an operator
  Operator* x = GetOpWithOutput(model, op.inputs[0]);
  if ((a_op_type == OperatorType::kNone) && (x != nullptr)) {
    return false;
  }
  if ((a_op_type != OperatorType::kNone) && (x == nullptr)) {
    return false;
  }

  // Check that first operator, if connected, is of correct type
  if ((x != nullptr) && (x->type != a_op_type)) {
    return false;
  }

  // Check if second input is disconnected/connected to an operator
  Operator* y = GetOpWithOutput(model, op.inputs[1]);
  if ((b_op_type == OperatorType::kNone) && (y != nullptr)) {
    return false;
  }
  if ((b_op_type != OperatorType::kNone) && (y == nullptr)) {
    return false;
  }

  // Check that second operator, if connected, is of correct type
  if ((y != nullptr) && (y->type != b_op_type)) {
    return false;
  }

  // Successfully matched. Optionally return matching input operators.
  if (a_op != nullptr) {
    *a_op = x;
  }
  if (b_op != nullptr) {
    *b_op = y;
  }
  return true;
}

// Returns true if the given operator has exactly 3 inputs, which are connected
// to the given op_types.
// We use kNone to indicate an input unattached to an operator output. Usually
// these are the static input arrays.
bool MatchOperatorInputs(const Operator& op, const Model& model,
                         OperatorType a_op_type, Operator** a_op,
                         OperatorType b_op_type, Operator** b_op,
                         OperatorType c_op_type, Operator** c_op) {
  // Check for required number of inputs
  if (op.inputs.size() != 3) {
    return false;
  }

  // Check if first input is disconnected/connected to an operator
  Operator* x = GetOpWithOutput(model, op.inputs[0]);
  if ((a_op_type == OperatorType::kNone) && (x != nullptr)) {
    return false;
  }
  if ((a_op_type != OperatorType::kNone) && (x == nullptr)) {
    return false;
  }

  // Check that first operator, if connected, is of correct type
  if ((x != nullptr) && (x->type != a_op_type)) {
    return false;
  }

  // Check if second input is disconnected/connected to an operator
  Operator* y = GetOpWithOutput(model, op.inputs[1]);
  if ((b_op_type == OperatorType::kNone) && (y != nullptr)) {
    return false;
  }
  if ((b_op_type != OperatorType::kNone) && (y == nullptr)) {
    return false;
  }

  // Check that second operator, if connected, is of correct type
  if ((y != nullptr) && (y->type != b_op_type)) {
    return false;
  }

  // Check if third input is disconnected/connected to an operator
  Operator* z = GetOpWithOutput(model, op.inputs[2]);
  if ((c_op_type == OperatorType::kNone) && (z != nullptr)) {
    return false;
  }
  if ((c_op_type != OperatorType::kNone) && (z == nullptr)) {
    return false;
  }

  // Check that third operator, if connected, is of correct type
  if ((z != nullptr) && (z->type != c_op_type)) {
    return false;
  }

  // Successfully matched. Optionally return matching input operators.
  if (a_op != nullptr) {
    *a_op = x;
  }
  if (b_op != nullptr) {
    *b_op = y;
  }
  if (c_op != nullptr) {
    *c_op = z;
  }
  return true;
}

}  // namespace

bool IdentifyLstmCell::Run(Model* model, std::size_t op_index) {
  // This LSTM cell identification method is not invariant to commutation of
  // commutative operator inputs. For example, if input[0] and input[1] of the
  // final output multiplication were swapped, this method would not identify it
  // as an LSTM cell. This is OK in most cases, because
  // tf.rnn.contrib.BasicLSTMCell always generates LSTM cells the same way.

  // Final output multiply
  auto op_it = model->operators.begin() + op_index;
  Operator* final_output_mul = op_it->get();
  if (final_output_mul->type != OperatorType::kMul) {
    return false;
  }
  Operator *state_output_tanh, *fc_output_sig;
  if (!MatchOperatorInputs(*final_output_mul, *model, OperatorType::kTanh,
                           &state_output_tanh, OperatorType::kLogistic,
                           &fc_output_sig)) {
    return false;
  }

  // State output TanH
  // (We don't count an operator as ID'd until we verify it has the correct
  // operator types feeding into it.)
  Operator* state_combine_add;
  if (!MatchOperatorInputs(*state_output_tanh, *model, OperatorType::kAdd,
                           &state_combine_add)) {
    return false;
  }

  // State forget & remember addition
  Operator *state_forget_mul, *state_remember_mul;
  if (!MatchOperatorInputs(*state_combine_add, *model, OperatorType::kMul,
                           &state_forget_mul, OperatorType::kMul,
                           &state_remember_mul)) {
    return false;
  }
  const string prev_state = state_forget_mul->inputs[0];

  // State forget gate
  Operator* state_forget_sig;
  if (!MatchOperatorInputs(*state_forget_mul, *model, OperatorType::kNone,
                           nullptr, OperatorType::kLogistic,
                           &state_forget_sig)) {
    return false;
  }

  // State remember gate
  Operator *state_remember_sig, *state_info_tanh;
  if (!MatchOperatorInputs(*state_remember_mul, *model, OperatorType::kLogistic,
                           &state_remember_sig, OperatorType::kTanh,
                           &state_info_tanh)) {
    return false;
  }

  // State remember "information" activation function
  Operator* fc_output_split;
  if (!MatchOperatorInputs(*state_info_tanh, *model, OperatorType::kSplit,
                           &fc_output_split)) {
    return false;
  }
  // State remember gate activation function
  Operator* tmp;
  if (!MatchOperatorInputs(*state_remember_sig, *model, OperatorType::kSplit,
                           &tmp) ||
      (tmp != fc_output_split)) {
    return false;
  }
  // State forget gate activation function
  if (!MatchOperatorInputs(*state_forget_sig, *model, OperatorType::kSplit,
                           &tmp) ||
      (tmp != fc_output_split)) {
    return false;
  }
  // Fully connected output activation function
  if (!MatchOperatorInputs(*fc_output_sig, *model, OperatorType::kSplit,
                           &tmp) ||
      (tmp != fc_output_split)) {
    return false;
  }
  // Fully connected output split
  Operator* fully_connected;
  if (!MatchOperatorInputs(*fc_output_split, *model, OperatorType::kNone,
                           nullptr, OperatorType::kFullyConnected,
                           &fully_connected)) {
    return false;
  }

  // Fully connected op
  Operator* concat_inputs;
  if (!MatchOperatorInputs(*fully_connected, *model,
                           OperatorType::kConcatenation, &concat_inputs,
                           OperatorType::kNone, nullptr, OperatorType::kNone,
                           nullptr)) {
    return false;
  }

  if (static_cast<FullyConnectedOperator*>(fully_connected)->weights_format !=
      FullyConnectedWeightsFormat::kDefault) {
    // Not yet implemented: experimental shuffled weights in fused LSTM cell.
    return false;
  }

  // Emplace a new LSTM cell operator
  auto* lstm_cell_op = new LstmCellOperator;
  lstm_cell_op->inputs.resize(LstmCellOperator::NUM_INPUTS);
  lstm_cell_op->inputs[LstmCellOperator::DATA_INPUT] = concat_inputs->inputs[0];
  lstm_cell_op->inputs[LstmCellOperator::PREV_ACTIV_INPUT] =
      concat_inputs->inputs[1];
  lstm_cell_op->inputs[LstmCellOperator::WEIGHTS_INPUT] =
      fully_connected->inputs[1];
  lstm_cell_op->inputs[LstmCellOperator::BIASES_INPUT] =
      fully_connected->inputs[2];
  lstm_cell_op->inputs[LstmCellOperator::PREV_STATE_INPUT] = prev_state;
  lstm_cell_op->outputs.resize(LstmCellOperator::NUM_OUTPUTS);
  lstm_cell_op->outputs[LstmCellOperator::STATE_OUTPUT] =
      state_output_tanh->inputs[0];
  lstm_cell_op->outputs[LstmCellOperator::ACTIV_OUTPUT] =
      final_output_mul->outputs[0];
  model->operators.emplace(op_it, lstm_cell_op);
  AddMessageF("Creating %s replacing equivalent subgraph",
              LogName(*lstm_cell_op));

  // Create temp arrays used internally during runtime.
  const string base_name(FindLongestCommonPrefix(
      lstm_cell_op->outputs[LstmCellOperator::STATE_OUTPUT],
      lstm_cell_op->outputs[LstmCellOperator::ACTIV_OUTPUT]));
  const string& concat_temp_array_name =
      AvailableArrayName(*model, base_name + "concat_temp");
  model->GetOrCreateArray(concat_temp_array_name);
  lstm_cell_op->outputs[LstmCellOperator::CONCAT_TEMP] = concat_temp_array_name;
  const string& activ_temp_array_name =
      AvailableArrayName(*model, base_name + "activ_temp");
  model->GetOrCreateArray(activ_temp_array_name);
  lstm_cell_op->outputs[LstmCellOperator::ACTIV_TEMP] = activ_temp_array_name;
  AddMessageF("Created temp outputs %s and %s on operator %s",
              concat_temp_array_name, activ_temp_array_name,
              LogName(*lstm_cell_op));

  // Delete arrays and operators replaced by the LSTM cell operator. Order is
  // important - DeleteArrayIfUnused() only succeeds if dependent operators
  // have been removed first. Start at the output and work towards the input.
  model->operators.erase(FindOperator(model, *final_output_mul));
  DeleteArrayIfUnused(state_output_tanh->outputs[0], model);
  DeleteArrayIfUnused(fc_output_sig->outputs[0], model);
  model->operators.erase(FindOperator(model, *state_output_tanh));
  model->operators.erase(FindOperator(model, *fc_output_sig));
  model->operators.erase(FindOperator(model, *state_combine_add));
  DeleteArrayIfUnused(state_forget_mul->outputs[0], model);
  DeleteArrayIfUnused(state_remember_mul->outputs[0], model);
  model->operators.erase(FindOperator(model, *state_forget_mul));
  model->operators.erase(FindOperator(model, *state_remember_mul));
  DeleteArrayIfUnused(state_forget_sig->outputs[0], model);
  DeleteArrayIfUnused(state_info_tanh->outputs[0], model);
  DeleteArrayIfUnused(state_remember_sig->outputs[0], model);
  model->operators.erase(FindOperator(model, *state_forget_sig));
  model->operators.erase(FindOperator(model, *state_info_tanh));
  model->operators.erase(FindOperator(model, *state_remember_sig));
  DeleteArrayIfUnused(fc_output_split->outputs[0], model);
  DeleteArrayIfUnused(fc_output_split->outputs[1], model);
  DeleteArrayIfUnused(fc_output_split->outputs[2], model);
  DeleteArrayIfUnused(fc_output_split->outputs[3], model);
  string dims_array = fc_output_split->inputs[0];
  model->operators.erase(FindOperator(model, *fc_output_split));
  DeleteArrayIfUnused(dims_array, model);
  DeleteArrayIfUnused(fully_connected->outputs[0], model);
  model->operators.erase(FindOperator(model, *fully_connected));
  DeleteArrayIfUnused(concat_inputs->outputs[0], model);
  model->operators.erase(FindOperator(model, *concat_inputs));
  return true;
}

}  // namespace toco
<|endoftext|>