<commit_before>//=======================================================================
// Copyright (c) 2013-2014 Baptiste Wicht.
// Distributed under the terms of the MIT License.
// (See accompanying file LICENSE or copy at
//  http://opensource.org/licenses/MIT)
//=======================================================================

#include <iostream>
#include <fstream>
#include <sstream>

#include <boost/filesystem.hpp>
#include <boost/algorithm/string.hpp>

#include "wishes.hpp"
#include "objectives.hpp"
#include "expenses.hpp"
#include "earnings.hpp"
#include "fortune.hpp"
#include "accounts.hpp"
#include "args.hpp"
#include "data.hpp"
#include "guid.hpp"
#include "config.hpp"
#include "utils.hpp"
#include "console.hpp"
#include "budget_exception.hpp"
#include "compute.hpp"

using namespace budget;

namespace {

static data_handler<wish> wishes;

void list_wishes(){
    if(wishes.data.size() == 0){
        std::cout << "No wishes" << std::endl;
    } else {
        std::vector<std::string> columns = {"ID", "Name", "Amount", "Paid"};
        std::vector<std::vector<std::string>> contents;

        money total;
        money unpaid_total;
        for(auto& wish : wishes.data){
            contents.push_back({to_string(wish.id), wish.name, to_string(wish.amount), wish.paid ? to_string(wish.paid_amount) : "No"});

            total += wish.amount;
            if(!wish.paid){
                unpaid_total += wish.amount;
            }
        }

        contents.push_back({"", "", "", ""});
        contents.push_back({"", "Total", to_string(total), ""});
        contents.push_back({"", "Unpaid Total", to_string(unpaid_total), ""});

        display_table(columns, contents);
    }
}

void status_wishes(){
    std::cout << "Wishes" << std::endl << std::endl;

    size_t width = 0;
    for(auto& wish : wishes.data){
        if(wish.paid){
            continue;
        }

        width = std::max(rsize(wish.name), width);
    }

    auto today = boost::gregorian::day_clock::local_day();
    auto month_status = budget::compute_month_status(today.year(), today.month());
    auto year_status = budget::compute_year_status(today.year(), today.month());

    auto fortune_amount = budget::current_fortune();

    for(auto& wish : wishes.data){
        if(wish.paid){
            continue;
        }

        auto amount = wish.amount;

        std::cout << "  ";
        print_minimum(wish.name, width);
        std::cout << "  ";

        size_t monthly_breaks = 0;
        size_t yearly_breaks = 0;

        bool month_objective = true;
        bool year_objective = true;

        for(auto& objective : all_objectives()){
            if(objective.type == "monthly"){
                auto success_before = budget::compute_success(month_status, objective);
                auto success_after = budget::compute_success(month_status.add_expense(amount), objective);

                if(success_before >= 100 && success_after < 100){
                    ++monthly_breaks;
                }

                if(success_after < 100){
                    month_objective = false;
                }
            } else if(objective.type == "yearly"){
                auto success_before = budget::compute_success(year_status, objective);
                auto success_after = budget::compute_success(year_status.add_expense(amount), objective);

                if(success_before >= 100 && success_after < 100){
                    ++yearly_breaks;
                }

                if(success_after < 100){
                    year_objective = false;
                }
            }
        }

        if(fortune_amount < wish.amount){
            std::cout << "Impossible (not enough fortune)";
        } else {
            if(month_status.balance > wish.amount){
                if(!all_objectives().empty()){
                    if(month_objective && year_objective){
                        std::cout << "Perfect (On month balance, all objectives fullfilled)";
                    } else if(month_objective){
                        std::cout << "Good (On month balance, month objectives fullfilled)";
                    } else if(yearly_breaks > 0 || monthly_breaks > 0){
                        std::cout << "OK (On month balance, " << (yearly_breaks + monthly_breaks) << " objectives broken)";
                    } else if(yearly_breaks == 0 && monthly_breaks == 0){
                        std::cout << "Warning (On month balance, objectives not fullfilled)";
                    }
                } else {
                    std::cout << "OK (on month balance)";
                }
            } else if(year_status.balance > wish.amount){
                if(!all_objectives().empty()){
                    if(month_objective && year_objective){
                        std::cout << "Perfect (On year balance, all objectives fullfilled)";
                    } else if(month_objective){
                        std::cout << "Good (On year balance, month objectives fullfilled)";
                    } else if(yearly_breaks > 0 || monthly_breaks > 0){
                        std::cout << "OK (On year balance, " << (yearly_breaks + monthly_breaks) << " objectives broken)";
                    } else if(yearly_breaks == 0 && monthly_breaks == 0){
                        std::cout << "Warning (On year balance, objectives not fullfilled)";
                    }
                } else {
                    std::cout << "OK (on year balance)";
                }
            } else {
                std::cout << "Warning (on fortune only)";
            }
        }

        std::cout << std::endl;
    }
}

void edit(budget::wish& wish){
    edit_string(wish.name, "Name", not_empty_checker());
    edit_money(wish.amount, "Amount", not_negative_checker(), not_zero_checker());
}

} //end of anonymous namespace

void budget::wishes_module::load(){
    load_expenses();
    load_earnings();
    load_accounts();
    load_fortunes();
    load_objectives();
    load_wishes();
}

void budget::wishes_module::unload(){
    save_wishes();
}

void budget::wishes_module::handle(const std::vector<std::string>& args){
    if(args.size() == 1){
        status_wishes();
    } else {
        auto& subcommand = args[1];

        if(subcommand == "list"){
            list_wishes();
        } else if(subcommand == "status"){
            status_wishes();
        } else if(subcommand == "add"){
            wish wish;
            wish.guid = generate_guid();
            wish.date = boost::gregorian::day_clock::local_day();

            edit(wish);

            auto id = add_data(wishes, std::move(wish));
            std::cout << "Wish " << id << " has been created" << std::endl;
        } else if(subcommand == "delete"){
            enough_args(args, 3);

            std::size_t id = to_number<std::size_t>(args[2]);

            if(!exists(wishes, id)){
                throw budget_exception("There are no wish with id " + args[2]);
            }

            remove(wishes, id);

            std::cout << "wish " << id << " has been deleted" << std::endl;
        } else if(subcommand == "edit"){
            enough_args(args, 3);

            std::size_t id = to_number<std::size_t>(args[2]);

            if(!exists(wishes, id)){
                throw budget_exception("There are no wish with id " + args[2]);
            }

            auto& wish = get(wishes, id);

            edit(wish);

            set_wishes_changed();

            std::cout << "wish " << id << " has been modified" << std::endl;
        } else if(subcommand == "paid"){
            enough_args(args, 3);

            std::size_t id = to_number<std::size_t>(args[2]);

            if(!exists(wishes, id)){
                throw budget_exception("There are no wish with id " + args[2]);
            }

            auto& wish = get(wishes, id);

            edit_money(wish.paid_amount, "Paid Amount", not_negative_checker(), not_zero_checker());

            wish.paid = true;

            set_wishes_changed();

            std::cout << "wish " << id << " has been marked as paid" << std::endl;
        } else {
            throw budget_exception("Invalid subcommand \"" + subcommand + "\"");
        }
    }
}

void budget::load_wishes(){
    load_data(wishes, "wishes.data");
}

void budget::save_wishes(){
    save_data(wishes, "wishes.data");
}

void budget::add_wish(budget::wish&& wish){
    add_data(wishes, std::forward<budget::wish>(wish));
}

std::ostream& budget::operator<<(std::ostream& stream, const wish& wish){
    return stream
        << wish.id  << ':'
        << wish.guid << ':'
        << wish.name << ':'
        << wish.amount << ':'
        << to_string(wish.date) << ':'
        << static_cast<size_t>(wish.paid) << ':'
        << wish.paid_amount;
}

void budget::operator>>(const std::vector<std::string>& parts, wish& wish){
    wish.id = to_number<std::size_t>(parts[0]);
    wish.guid = parts[1];
    wish.name = parts[2];
    wish.amount = parse_money(parts[3]);
    wish.date = boost::gregorian::from_string(parts[4]);
    wish.paid = to_number<std::size_t>(parts[5]) == 1;
    wish.paid_amount = parse_money(parts[6]);
}

std::vector<wish>& budget::all_wishes(){
    return wishes.data;
}

void budget::set_wishes_changed(){
    wishes.changed = true;
}

void budget::migrate_wishes_2_to_3(){
    load_data(wishes, "wishes.data", [](const std::vector<std::string>& parts, wish& wish){
        wish.id = to_number<std::size_t>(parts[0]);
        wish.guid = parts[1];
        wish.name = parts[2];
        wish.amount = parse_money(parts[3]);
        wish.date = boost::gregorian::from_string(parts[4]);
        wish.paid = false;
        wish.paid_amount = budget::money(0,0);
        });

    set_wishes_changed();

    save_data(wishes, "wishes.data");

}
<commit_msg>Display warning if early in the month<commit_after>//=======================================================================
// Copyright (c) 2013-2014 Baptiste Wicht.
// Distributed under the terms of the MIT License.
// (See accompanying file LICENSE or copy at
//  http://opensource.org/licenses/MIT)
//=======================================================================

#include <iostream>
#include <fstream>
#include <sstream>

#include <boost/filesystem.hpp>
#include <boost/algorithm/string.hpp>

#include "wishes.hpp"
#include "objectives.hpp"
#include "expenses.hpp"
#include "earnings.hpp"
#include "fortune.hpp"
#include "accounts.hpp"
#include "args.hpp"
#include "data.hpp"
#include "guid.hpp"
#include "config.hpp"
#include "utils.hpp"
#include "console.hpp"
#include "budget_exception.hpp"
#include "compute.hpp"

using namespace budget;

namespace {

static data_handler<wish> wishes;

void list_wishes(){
    if(wishes.data.size() == 0){
        std::cout << "No wishes" << std::endl;
    } else {
        std::vector<std::string> columns = {"ID", "Name", "Amount", "Paid"};
        std::vector<std::vector<std::string>> contents;

        money total;
        money unpaid_total;
        for(auto& wish : wishes.data){
            contents.push_back({to_string(wish.id), wish.name, to_string(wish.amount), wish.paid ? to_string(wish.paid_amount) : "No"});

            total += wish.amount;
            if(!wish.paid){
                unpaid_total += wish.amount;
            }
        }

        contents.push_back({"", "", "", ""});
        contents.push_back({"", "Total", to_string(total), ""});
        contents.push_back({"", "Unpaid Total", to_string(unpaid_total), ""});

        display_table(columns, contents);
    }
}

void status_wishes(){
    auto today = boost::gregorian::day_clock::local_day();

    if(today.day() < 12){
        std::cout << "WARNING: It is early in the month, no one can know what may happen ;)" << std::endl << std::endl;
    }

    std::cout << "Wishes" << std::endl << std::endl;

    size_t width = 0;
    for(auto& wish : wishes.data){
        if(wish.paid){
            continue;
        }

        width = std::max(rsize(wish.name), width);
    }

    auto month_status = budget::compute_month_status(today.year(), today.month());
    auto year_status = budget::compute_year_status(today.year(), today.month());

    auto fortune_amount = budget::current_fortune();

    for(auto& wish : wishes.data){
        if(wish.paid){
            continue;
        }

        auto amount = wish.amount;

        std::cout << "  ";
        print_minimum(wish.name, width);
        std::cout << "  ";

        size_t monthly_breaks = 0;
        size_t yearly_breaks = 0;

        bool month_objective = true;
        bool year_objective = true;

        for(auto& objective : all_objectives()){
            if(objective.type == "monthly"){
                auto success_before = budget::compute_success(month_status, objective);
                auto success_after = budget::compute_success(month_status.add_expense(amount), objective);

                if(success_before >= 100 && success_after < 100){
                    ++monthly_breaks;
                }

                if(success_after < 100){
                    month_objective = false;
                }
            } else if(objective.type == "yearly"){
                auto success_before = budget::compute_success(year_status, objective);
                auto success_after = budget::compute_success(year_status.add_expense(amount), objective);

                if(success_before >= 100 && success_after < 100){
                    ++yearly_breaks;
                }

                if(success_after < 100){
                    year_objective = false;
                }
            }
        }

        if(fortune_amount < wish.amount){
            std::cout << "Impossible (not enough fortune)";
        } else {
            if(month_status.balance > wish.amount){
                if(!all_objectives().empty()){
                    if(month_objective && year_objective){
                        std::cout << "Perfect (On month balance, all objectives fullfilled)";
                    } else if(month_objective){
                        std::cout << "Good (On month balance, month objectives fullfilled)";
                    } else if(yearly_breaks > 0 || monthly_breaks > 0){
                        std::cout << "OK (On month balance, " << (yearly_breaks + monthly_breaks) << " objectives broken)";
                    } else if(yearly_breaks == 0 && monthly_breaks == 0){
                        std::cout << "Warning (On month balance, objectives not fullfilled)";
                    }
                } else {
                    std::cout << "OK (on month balance)";
                }
            } else if(year_status.balance > wish.amount){
                if(!all_objectives().empty()){
                    if(month_objective && year_objective){
                        std::cout << "Perfect (On year balance, all objectives fullfilled)";
                    } else if(month_objective){
                        std::cout << "Good (On year balance, month objectives fullfilled)";
                    } else if(yearly_breaks > 0 || monthly_breaks > 0){
                        std::cout << "OK (On year balance, " << (yearly_breaks + monthly_breaks) << " objectives broken)";
                    } else if(yearly_breaks == 0 && monthly_breaks == 0){
                        std::cout << "Warning (On year balance, objectives not fullfilled)";
                    }
                } else {
                    std::cout << "OK (on year balance)";
                }
            } else {
                std::cout << "Warning (on fortune only)";
            }
        }

        std::cout << std::endl;
    }
}

void edit(budget::wish& wish){
    edit_string(wish.name, "Name", not_empty_checker());
    edit_money(wish.amount, "Amount", not_negative_checker(), not_zero_checker());
}

} //end of anonymous namespace

void budget::wishes_module::load(){
    load_expenses();
    load_earnings();
    load_accounts();
    load_fortunes();
    load_objectives();
    load_wishes();
}

void budget::wishes_module::unload(){
    save_wishes();
}

void budget::wishes_module::handle(const std::vector<std::string>& args){
    if(args.size() == 1){
        status_wishes();
    } else {
        auto& subcommand = args[1];

        if(subcommand == "list"){
            list_wishes();
        } else if(subcommand == "status"){
            status_wishes();
        } else if(subcommand == "add"){
            wish wish;
            wish.guid = generate_guid();
            wish.date = boost::gregorian::day_clock::local_day();

            edit(wish);

            auto id = add_data(wishes, std::move(wish));
            std::cout << "Wish " << id << " has been created" << std::endl;
        } else if(subcommand == "delete"){
            enough_args(args, 3);

            std::size_t id = to_number<std::size_t>(args[2]);

            if(!exists(wishes, id)){
                throw budget_exception("There are no wish with id " + args[2]);
            }

            remove(wishes, id);

            std::cout << "wish " << id << " has been deleted" << std::endl;
        } else if(subcommand == "edit"){
            enough_args(args, 3);

            std::size_t id = to_number<std::size_t>(args[2]);

            if(!exists(wishes, id)){
                throw budget_exception("There are no wish with id " + args[2]);
            }

            auto& wish = get(wishes, id);

            edit(wish);

            set_wishes_changed();

            std::cout << "wish " << id << " has been modified" << std::endl;
        } else if(subcommand == "paid"){
            enough_args(args, 3);

            std::size_t id = to_number<std::size_t>(args[2]);

            if(!exists(wishes, id)){
                throw budget_exception("There are no wish with id " + args[2]);
            }

            auto& wish = get(wishes, id);

            edit_money(wish.paid_amount, "Paid Amount", not_negative_checker(), not_zero_checker());

            wish.paid = true;

            set_wishes_changed();

            std::cout << "wish " << id << " has been marked as paid" << std::endl;
        } else {
            throw budget_exception("Invalid subcommand \"" + subcommand + "\"");
        }
    }
}

void budget::load_wishes(){
    load_data(wishes, "wishes.data");
}

void budget::save_wishes(){
    save_data(wishes, "wishes.data");
}

void budget::add_wish(budget::wish&& wish){
    add_data(wishes, std::forward<budget::wish>(wish));
}

std::ostream& budget::operator<<(std::ostream& stream, const wish& wish){
    return stream
        << wish.id  << ':'
        << wish.guid << ':'
        << wish.name << ':'
        << wish.amount << ':'
        << to_string(wish.date) << ':'
        << static_cast<size_t>(wish.paid) << ':'
        << wish.paid_amount;
}

void budget::operator>>(const std::vector<std::string>& parts, wish& wish){
    wish.id = to_number<std::size_t>(parts[0]);
    wish.guid = parts[1];
    wish.name = parts[2];
    wish.amount = parse_money(parts[3]);
    wish.date = boost::gregorian::from_string(parts[4]);
    wish.paid = to_number<std::size_t>(parts[5]) == 1;
    wish.paid_amount = parse_money(parts[6]);
}

std::vector<wish>& budget::all_wishes(){
    return wishes.data;
}

void budget::set_wishes_changed(){
    wishes.changed = true;
}

void budget::migrate_wishes_2_to_3(){
    load_data(wishes, "wishes.data", [](const std::vector<std::string>& parts, wish& wish){
        wish.id = to_number<std::size_t>(parts[0]);
        wish.guid = parts[1];
        wish.name = parts[2];
        wish.amount = parse_money(parts[3]);
        wish.date = boost::gregorian::from_string(parts[4]);
        wish.paid = false;
        wish.paid_amount = budget::money(0,0);
        });

    set_wishes_changed();

    save_data(wishes, "wishes.data");

}
<|endoftext|>


<commit_before>/******************************************************************************
 *                    _   _____   __________                                  *
 *                   | | / / _ | / __/_  __/     Visibility                   *
 *                   | |/ / __ |_\ \  / /          Across                     *
 *                   |___/_/ |_/___/ /_/       Space and Time                 *
 *                                                                            *
 * This file is part of VAST. It is subject to the license terms in the       *
 * LICENSE file found in the top-level directory of this distribution and at  *
 * http://vast.io/license. No part of VAST, including this file, may be       *
 * copied, modified, propagated, or distributed except according to the terms *
 * contained in the LICENSE file.                                             *
 ******************************************************************************/

#define SUITE table_index
#include "test.hpp"

#include "fixtures/events.hpp"
#include "fixtures/filesystem.hpp"

#include "vast/bitmap.hpp"
#include "vast/concept/parseable/to.hpp"
#include "vast/concept/parseable/vast/expression.hpp"
#include "vast/concept/printable/stream.hpp"
#include "vast/concept/printable/vast/error.hpp"
#include "vast/concept/printable/vast/event.hpp"
#include "vast/concept/printable/vast/expression.hpp"
#include "vast/system/table_index.hpp"

using namespace vast;
using namespace vast::system;

namespace {

struct fixture : fixtures::events, fixtures::filesystem {
  template <class T>
  T unbox(expected<T> x) {
    if (!x)
      FAIL("error: " << x.error());
    return std::move(*x);
  }

  ids query(std::string_view what) {
    return unbox(tbl->lookup(unbox(to<expression>(what))));
  }

  void reset(table_index&& new_tbl) {
    tbl = std::make_unique<table_index>(std::move(new_tbl));
  }

  void reset(expected<table_index>&& new_tbl) {
    if (!new_tbl)
      FAIL("error: " << new_tbl.error());
    reset(std::move(*new_tbl));
  }

  void add(event x) {
    tbl->add(std::move(x));
  }

  std::unique_ptr<table_index> tbl;
};

} // namespace <anonymous>

FIXTURE_SCOPE(table_index_tests, fixture)

TEST(flat type) {
  MESSAGE("generate table layout for flat integer type");
  reset(make_table_index(directory, integer_type{}));
  MESSAGE("ingest test data (integers)");
  std::vector<int> xs{1, 2, 3, 1, 2, 3, 1, 2, 3};
  for (size_t i = 0; i < xs.size(); ++i) {
    event x{xs[i]};
    x.id(i);
    add(std::move(x));
  }
  auto res = [&](auto... args) {
    return make_ids({args...}, xs.size());
  };
  MESSAGE("verify table index");
  auto verify = [&] {
    CHECK_EQUAL(query(":int == +1"), res(0, 3, 6));
    CHECK_EQUAL(query(":int == +2"), res(1, 4, 7));
    CHECK_EQUAL(query(":int == +3"), res(2, 5, 8));
    CHECK_EQUAL(query(":int == +4"), res());
    CHECK_EQUAL(query(":int != +1"), res(1, 2, 4, 5, 7, 8));
    CHECK_EQUAL(query("!(:int == +1)"), res(1, 2, 4, 5, 7, 8));
    CHECK_EQUAL(query(":int > +1 && :int < +3"), res(1, 4, 7));
  };
  verify();
  MESSAGE("(automatically) persist table index and restore from disk");
  reset(make_table_index(directory, integer_type{}));
  MESSAGE("verify table index again");
  verify();
}

TEST(record type) {
  MESSAGE("generate table layout for record type");
  auto tbl_type = record_type{
    {"x", record_type{
      {"a", integer_type{}},
      {"b", boolean_type{}}
    }},
    {"y", record_type{
      {"a", string_type{}}
    }}
  };
  reset(make_table_index(directory, tbl_type));
  MESSAGE("ingest test data (records)");
  auto mk_row = [&](int x, bool y, std::string z) {
    return value::make(vector{vector{x, y}, vector{std::move(z)}}, tbl_type);
  };
  // Some test data.
  std::vector<value> xs{mk_row(1, true, "abc"),     mk_row(10, false, "def"),
                        mk_row(5, true, "hello"),   mk_row(1, true, "d e f"),
                        mk_row(15, true, "world"),  mk_row(5, true, "bar"),
                        mk_row(10, false, "a b c"), mk_row(10, false, "baz"),
                        mk_row(5, false, "foo"),    mk_row(1, true, "test")};
  for (size_t i = 0; i < xs.size(); ++i) {
    event x{xs[i]};
    x.id(i);
    add(std::move(x));
  }
  auto res = [&](auto... args) {
    return make_ids({args...}, xs.size());
  };
  MESSAGE("verify table index");
  auto verify = [&] {
    CHECK_EQUAL(query("x.a == +1"), res(0, 3, 9));
    CHECK_EQUAL(query("x.a > +1"), res(1, 2, 4, 5, 6, 7, 8));
    CHECK_EQUAL(query("x.a > +1 && x.b == T"), res(2, 4, 5));
  };
  verify();
  MESSAGE("(automatically) persist table index and restore from disk");
  reset(make_table_index(directory, tbl_type));
  MESSAGE("verify table index again");
  verify();
}

TEST(bro conn logs) {
  MESSAGE("generate table layout for bro conn logs");
  auto tbl_type = bro_conn_log[0].type();
  reset(make_table_index(directory, tbl_type));
  CHECK_EQUAL(tbl->num_meta_columns(), 2u);
  MESSAGE("ingest test data (bro conn log)");
  for (auto& entry : bro_conn_log) {
    add(entry);
  }
  MESSAGE("verify table index");
  auto verify = [&] {
    CHECK_EQUAL(rank(query("id.resp_p == 995/?")), 53u);
    CHECK_EQUAL(rank(query("id.resp_p == 5355/?")), 49u);
    CHECK_EQUAL(rank(query("id.resp_p == 995/? || id.resp_p == 5355/?")), 102u);
    CHECK_EQUAL(rank(query("&time > 1970-01-01")), bro_conn_log.size());
    CHECK_EQUAL(rank(query("proto == \"udp\"")), 5306u);
    CHECK_EQUAL(rank(query("proto == \"tcp\"")), 3135u);
    CHECK_EQUAL(rank(query("uid == \"nkCxlvNN8pi\"")), 1u);
    CHECK_EQUAL(rank(query("orig_bytes < 400")), 5332u);
    CHECK_EQUAL(rank(query("orig_bytes < 400 && proto == \"udp\"")), 4357u);
  };
  verify();
  MESSAGE("(automatically) persist table index and restore from disk");
  reset(make_table_index(directory, tbl_type));
  MESSAGE("verify table index again");
  verify();
}

FIXTURE_SCOPE_END()
<commit_msg>Generate events with proper type<commit_after>/******************************************************************************
 *                    _   _____   __________                                  *
 *                   | | / / _ | / __/_  __/     Visibility                   *
 *                   | |/ / __ |_\ \  / /          Across                     *
 *                   |___/_/ |_/___/ /_/       Space and Time                 *
 *                                                                            *
 * This file is part of VAST. It is subject to the license terms in the       *
 * LICENSE file found in the top-level directory of this distribution and at  *
 * http://vast.io/license. No part of VAST, including this file, may be       *
 * copied, modified, propagated, or distributed except according to the terms *
 * contained in the LICENSE file.                                             *
 ******************************************************************************/

#define SUITE table_index
#include "test.hpp"

#include "fixtures/events.hpp"
#include "fixtures/filesystem.hpp"

#include "vast/bitmap.hpp"
#include "vast/concept/parseable/to.hpp"
#include "vast/concept/parseable/vast/expression.hpp"
#include "vast/concept/printable/stream.hpp"
#include "vast/concept/printable/vast/error.hpp"
#include "vast/concept/printable/vast/event.hpp"
#include "vast/concept/printable/vast/expression.hpp"
#include "vast/system/table_index.hpp"

using namespace vast;
using namespace vast::system;

namespace {

struct fixture : fixtures::events, fixtures::filesystem {
  template <class T>
  T unbox(expected<T> x) {
    if (!x)
      FAIL("error: " << x.error());
    return std::move(*x);
  }

  ids query(std::string_view what) {
    return unbox(tbl->lookup(unbox(to<expression>(what))));
  }

  void reset(table_index&& new_tbl) {
    tbl = std::make_unique<table_index>(std::move(new_tbl));
  }

  void reset(expected<table_index>&& new_tbl) {
    if (!new_tbl)
      FAIL("error: " << new_tbl.error());
    reset(std::move(*new_tbl));
  }

  void add(event x) {
    auto err = tbl->add(std::move(x));
    if (err)
      FAIL("error: " << err);
  }

  std::unique_ptr<table_index> tbl;
};

} // namespace <anonymous>

FIXTURE_SCOPE(table_index_tests, fixture)

TEST(flat type) {
  MESSAGE("generate table layout for flat integer type");
  integer_type layout;
  reset(make_table_index(directory, layout));
  MESSAGE("ingest test data (integers)");
  std::vector<int> xs{1, 2, 3, 1, 2, 3, 1, 2, 3};
  size_t next_id = 0;
  for (auto i : xs)
    add(event::make(i, layout, next_id++));
  auto res = [&](auto... args) {
    return make_ids({args...}, xs.size());
  };
  MESSAGE("verify table index");
  auto verify = [&] {
    CHECK_EQUAL(query(":int == +1"), res(0, 3, 6));
    CHECK_EQUAL(query(":int == +2"), res(1, 4, 7));
    CHECK_EQUAL(query(":int == +3"), res(2, 5, 8));
    CHECK_EQUAL(query(":int == +4"), res());
    CHECK_EQUAL(query(":int != +1"), res(1, 2, 4, 5, 7, 8));
    CHECK_EQUAL(query("!(:int == +1)"), res(1, 2, 4, 5, 7, 8));
    CHECK_EQUAL(query(":int > +1 && :int < +3"), res(1, 4, 7));
  };
  verify();
  MESSAGE("(automatically) persist table index and restore from disk");
  reset(make_table_index(directory, integer_type{}));
  MESSAGE("verify table index again");
  verify();
}

TEST(record type) {
  MESSAGE("generate table layout for record type");
  auto tbl_type = record_type{
    {"x", record_type{
      {"a", integer_type{}},
      {"b", boolean_type{}}
    }},
    {"y", record_type{
      {"a", string_type{}}
    }}
  };
  reset(make_table_index(directory, tbl_type));
  MESSAGE("ingest test data (records)");
  auto mk_row = [&](int x, bool y, std::string z) {
    return value::make(vector{vector{x, y}, vector{std::move(z)}}, tbl_type);
  };
  // Some test data.
  std::vector<value> xs{mk_row(1, true, "abc"),     mk_row(10, false, "def"),
                        mk_row(5, true, "hello"),   mk_row(1, true, "d e f"),
                        mk_row(15, true, "world"),  mk_row(5, true, "bar"),
                        mk_row(10, false, "a b c"), mk_row(10, false, "baz"),
                        mk_row(5, false, "foo"),    mk_row(1, true, "test")};
  for (size_t i = 0; i < xs.size(); ++i) {
    event x{xs[i]};
    x.id(i);
    add(std::move(x));
  }
  auto res = [&](auto... args) {
    return make_ids({args...}, xs.size());
  };
  MESSAGE("verify table index");
  auto verify = [&] {
    CHECK_EQUAL(query("x.a == +1"), res(0, 3, 9));
    CHECK_EQUAL(query("x.a > +1"), res(1, 2, 4, 5, 6, 7, 8));
    CHECK_EQUAL(query("x.a > +1 && x.b == T"), res(2, 4, 5));
  };
  verify();
  MESSAGE("(automatically) persist table index and restore from disk");
  reset(make_table_index(directory, tbl_type));
  MESSAGE("verify table index again");
  verify();
}

TEST(bro conn logs) {
  MESSAGE("generate table layout for bro conn logs");
  auto tbl_type = bro_conn_log[0].type();
  reset(make_table_index(directory, tbl_type));
  CHECK_EQUAL(tbl->num_meta_columns(), 2u);
  MESSAGE("ingest test data (bro conn log)");
  for (auto& entry : bro_conn_log) {
    add(entry);
  }
  MESSAGE("verify table index");
  auto verify = [&] {
    CHECK_EQUAL(rank(query("id.resp_p == 995/?")), 53u);
    CHECK_EQUAL(rank(query("id.resp_p == 5355/?")), 49u);
    CHECK_EQUAL(rank(query("id.resp_p == 995/? || id.resp_p == 5355/?")), 102u);
    CHECK_EQUAL(rank(query("&time > 1970-01-01")), bro_conn_log.size());
    CHECK_EQUAL(rank(query("proto == \"udp\"")), 5306u);
    CHECK_EQUAL(rank(query("proto == \"tcp\"")), 3135u);
    CHECK_EQUAL(rank(query("uid == \"nkCxlvNN8pi\"")), 1u);
    CHECK_EQUAL(rank(query("orig_bytes < 400")), 5332u);
    CHECK_EQUAL(rank(query("orig_bytes < 400 && proto == \"udp\"")), 4357u);
  };
  verify();
  MESSAGE("(automatically) persist table index and restore from disk");
  reset(make_table_index(directory, tbl_type));
  MESSAGE("verify table index again");
  verify();
}

FIXTURE_SCOPE_END()
<|endoftext|>


<commit_before>#include <build-bot/worker.h>

#include <boost/algorithm/string.hpp>
#include <boost/filesystem.hpp>
#include <boost/property_tree/ptree.hpp>
#include <boost/property_tree/ini_parser.hpp>
#include <boost/random/random_device.hpp>
#include <boost/random/uniform_int_distribution.hpp>

#include <boost/process.hpp>

namespace fs = boost::filesystem;

namespace dsn {
namespace build_bot {
    namespace priv {
        class Worker : public dsn::log::Base<Worker> {
        private:
            std::string m_url;
            std::string m_branch;
            std::string m_revision;
            std::string m_configFile;
            std::string m_profileName;
            std::string m_buildDir;
            std::string m_repoName;
            std::string m_macroFile;

            std::string generateBuildId()
            {
                std::string res;
                boost::random::random_device rng;
                boost::random::uniform_int_distribution<> dist(0, BUILD_ID_CHARS.size() - 1);

                for (size_t i = 0; i < BUILD_ID_LENGTH; i++)
                    res += BUILD_ID_CHARS[dist(rng)];

                return res;
            }

            std::string m_buildId;

            static const std::string BUILD_ID_CHARS;
            static const size_t BUILD_ID_LENGTH;

            std::string m_toplevelDirectory;

            bool initToplevelDirectory()
            {
                std::stringstream ss;
                ss << m_buildDir << "/" << m_profileName << "/" << m_repoName << "/" << m_buildId;
                m_toplevelDirectory = ss.str();
                fs::path path(m_toplevelDirectory);

                BOOST_LOG_SEV(log, severity::info) << "Toplevel build directory is " << m_toplevelDirectory;

                if (!fs::exists(path)) {
                    try {
                        fs::create_directories(path);
                    }

                    catch (boost::system::system_error& ex) {
                        BOOST_LOG_SEV(log, severity::error) << "Failed to create toplevel directory " << m_toplevelDirectory << ": " << ex.what();
                        return false;
                    }
                }

                return true;
            }

            std::string m_gitExecutable;
            bool findGitExecutable()
            {
                try {
                    m_gitExecutable = boost::process::search_path("git");
                }
                catch (std::runtime_error& ex) {
                    BOOST_LOG_SEV(log, severity::error) << "Failed to locate git executable in PATH: " << ex.what();
                    return false;
                }

                if (m_gitExecutable.size() == 0) {
                    BOOST_LOG_SEV(log, severity::error) << "No git executable found in PATH!";
                    return false;
                }

                BOOST_LOG_SEV(log, severity::debug) << "Using git executable from " << m_gitExecutable;
                return true;
            }

            std::string m_sourceDirectory;
            bool checkoutSources()
            {
                m_sourceDirectory = m_toplevelDirectory + "/repo";
                BOOST_LOG_SEV(log, severity::info) << "Checking out sources from " << m_url << " to " << m_sourceDirectory;

                try {
                    std::stringstream ssArgs;
                    ssArgs << m_gitExecutable << " clone --recursive -b " << m_branch << " " << m_url << " " << m_sourceDirectory;
                    std::string args = ssArgs.str();
                    BOOST_LOG_SEV(log, severity::debug) << "Git command line is " << args;

                    boost::process::child child = boost::process::execute(boost::process::initializers::run_exe(m_gitExecutable),
                                                                          boost::process::initializers::set_cmd_line(args));
                    auto exit_code = boost::process::wait_for_exit(child);

                    if (exit_code != 0) {
                        BOOST_LOG_SEV(log, severity::error) << "Got non-zero exit status from '" << args << "': " << exit_code;
                        return false;
                    }
                }

                catch (boost::system::system_error& ex) {
                    BOOST_LOG_SEV(log, severity::error) << "Failed to run git to check out sources: " << ex.what();
                    return false;
                }

                BOOST_LOG_SEV(log, severity::info) << "Repository cloned successfully. Checking out revision " << m_revision;

                try {
                    std::stringstream ssArgs;
                    ssArgs << m_gitExecutable << " checkout " << m_revision << " .";
                    std::string args = ssArgs.str();
                    BOOST_LOG_SEV(log, severity::debug) << "Git command line is " << args;
                    boost::process::child child = boost::process::execute(boost::process::initializers::run_exe(m_gitExecutable),
                                                                          boost::process::initializers::set_cmd_line(args),
                                                                          boost::process::initializers::start_in_dir(m_sourceDirectory));
                    auto exit_code = boost::process::wait_for_exit(child);
                    if (exit_code != 0) {
                        BOOST_LOG_SEV(log, severity::error) << "Got non-zero exit status from '" << args << "': " << exit_code;
                        return false;
                    }
                }

                catch (boost::system::system_error& ex) {
                    BOOST_LOG_SEV(log, severity::error) << "Failed to run git to check out revision " << m_revision << ": " << ex.what();
                    return false;
                }

                BOOST_LOG_SEV(log, severity::info) << "Checked out revision " << m_revision;

                return true;
            }

            boost::property_tree::ptree m_macros;
            bool loadMacroFile()
            {
                fs::path path(m_macroFile);
                if (!fs::exists(path)) {
                    BOOST_LOG_SEV(log, severity::warning) << "Macro file " << m_macroFile << " doesn't exist!";
                    return true;
                }

                if (!fs::is_regular_file(path)) {
                    BOOST_LOG_SEV(log, severity::error) << "Macro configuration " << m_macroFile << " exists but isn't a regular file!";
                    return false;
                }

                try {
                    boost::property_tree::read_ini(m_macroFile, m_macros);
                }

                catch (boost::property_tree::ptree_error& ex) {
                    BOOST_LOG_SEV(log, severity::error) << "Failed to parse macros from " << m_macroFile << ": " << ex.what();
                    return false;
                }

                return true;
            }

            boost::property_tree::ptree m_buildSettings;
            bool loadBuildConfig()
            {
                std::string configFile = m_sourceDirectory + "/" + m_configFile;
                fs::path path(configFile);
                if (!fs::exists(path)) {
                    BOOST_LOG_SEV(log, severity::error) << "Build config file " << m_configFile << " doesn't exist in " << m_sourceDirectory;
                    return false;
                }

                if (!fs::is_regular_file(path)) {
                    BOOST_LOG_SEV(log, severity::error) << "Build config file " << configFile << " exists but isn't a regular file!";
                    return false;
                }

                try {
                    boost::property_tree::read_ini(configFile, m_buildSettings);
                }

                catch (boost::property_tree::ini_parser_error& ex) {
                    BOOST_LOG_SEV(log, severity::error) << "Failed to parse build configuration in " << m_configFile << ": " << ex.what();
                    return false;
                }

                return true;
            }

            std::string m_binaryDir;
            bool createBinaryDir()
            {
                m_binaryDir = m_toplevelDirectory + "/build";
                fs::path path(m_binaryDir);

                if (!fs::exists(path)) {
                    BOOST_LOG_SEV(log, severity::info) << "Creating binary dir: " << m_binaryDir;
                    try {
                        fs::create_directories(path);
                    }
                    catch (boost::system::system_error& ex) {
                        BOOST_LOG_SEV(log, severity::error) << "Failed to create binary dir " << m_binaryDir << ": " << ex.what();
                        return false;
                    }
                }

                return true;
            }

            bool replaceMacros(std::string& str)
            {
                for (auto& kv : m_macros) {
                    try {
                        std::string key = kv.first;
                        std::string val = m_macros.get<std::string>(key);
                        std::string token = "@" + key + "@";
                        boost::algorithm::replace_all(str, token, val);
                    }

                    catch (boost::property_tree::ptree_error& ex) {
                        BOOST_LOG_SEV(log, severity::error) << "Failed to replace macro: " << ex.what();
                        return false;
                    }
                }

                return true;
            }

            bool configureSources()
            {
                BOOST_LOG_SEV(log, severity::info) << "Trying to configure sources";
                std::string configureCommand;
                try {
                    configureCommand = m_buildSettings.get<std::string>(m_profileName + ".cmd_configure");
                }

                catch (boost::property_tree::ptree_error& ex) {
                    BOOST_LOG_SEV(log, severity::error) << "Failed to get configure command from build settings: " << ex.what();
                    return false;
                }

                if (configureCommand.size() == 0) {
                    BOOST_LOG_SEV(log, severity::warning) << "Configure command is empty; continuing build!";
                    return true;
                }

                BOOST_LOG_SEV(log, severity::debug) << "Configure command is: " << configureCommand;

                if (!replaceMacros(configureCommand)) {
                    BOOST_LOG_SEV(log, severity::error) << "Macro expansion failed for configure command!";
                    return false;
                }

                BOOST_LOG_SEV(log, severity::debug) << "Configure command after macro expansion is: " << configureCommand;

                return true;
            }

        public:
            Worker(const std::string& macro_file, const std::string& build_directory,
                   const std::string& repo_name,
                   const std::string& url, const std::string& branch, const std::string& revision,
                   const std::string& config_file, const std::string& profile_name)
                : m_buildDir(build_directory)
                , m_macroFile(macro_file)
                , m_url(url)
                , m_branch(branch)
                , m_revision(revision)
                , m_configFile(config_file)
                , m_profileName(profile_name)
                , m_repoName(repo_name)
            {
            }

            void run()
            {
                if (!findGitExecutable()) {
                    BOOST_LOG_SEV(log, severity::error) << "Failed to find git; build FAILED!";
                    return;
                }

                m_buildId = generateBuildId();
                BOOST_LOG_SEV(log, severity::info) << "Worker started for repo " << m_repoName
                                                   << " (profile: " << m_profileName << ", config: " << m_configFile << ") - Build ID: " << m_buildId;
                if (!initToplevelDirectory()) {
                    BOOST_LOG_SEV(log, severity::error) << "Unable to create build directory; build FAILED!";
                    return;
                }

                if (!loadMacroFile()) {
                    BOOST_LOG_SEV(log, severity::error) << "Failed to load macro file; build FAILED!";
                    return;
                }

                if (!checkoutSources()) {
                    BOOST_LOG_SEV(log, severity::error) << "Failed to checkout sources from " << m_url << "; build FAILED!";
                    return;
                }

                if (!createBinaryDir()) {
                    BOOST_LOG_SEV(log, severity::error) << "Failed to create build directory; build FAILED!";
                    return;
                }

                if (!loadBuildConfig()) {
                    BOOST_LOG_SEV(log, severity::error) << "Failed to load build configuration; build FAILED!";
                    return;
                }

                if (!configureSources()) {
                    BOOST_LOG_SEV(log, severity::error) << "Configure step aborted; build FAILED!";
                    return;
                }
            }
        };

        const std::string Worker::BUILD_ID_CHARS{ "0123456789abcdef" };
        const size_t Worker::BUILD_ID_LENGTH{ 8 };
    }
}
}

using namespace dsn::build_bot;

Worker::Worker(const std::string& macro_file, const std::string& build_directory,
               const std::string& repo_name,
               const std::string& url, const std::string& branch, const std::string& revision,
               const std::string& config_file, const std::string& profile_name)
    : m_impl(new priv::Worker(macro_file, build_directory, repo_name, url, branch, revision, config_file, profile_name))
{
}

Worker::~Worker()
{
}

void Worker::run()
{
    return m_impl->run();
}
<commit_msg>Add source directory to macro expansion list<commit_after>#include <build-bot/worker.h>

#include <boost/algorithm/string.hpp>
#include <boost/filesystem.hpp>
#include <boost/property_tree/ptree.hpp>
#include <boost/property_tree/ini_parser.hpp>
#include <boost/random/random_device.hpp>
#include <boost/random/uniform_int_distribution.hpp>

#include <boost/process.hpp>

namespace fs = boost::filesystem;

namespace dsn {
namespace build_bot {
    namespace priv {
        class Worker : public dsn::log::Base<Worker> {
        private:
            std::string m_url;
            std::string m_branch;
            std::string m_revision;
            std::string m_configFile;
            std::string m_profileName;
            std::string m_buildDir;
            std::string m_repoName;
            std::string m_macroFile;

            std::string generateBuildId()
            {
                std::string res;
                boost::random::random_device rng;
                boost::random::uniform_int_distribution<> dist(0, BUILD_ID_CHARS.size() - 1);

                for (size_t i = 0; i < BUILD_ID_LENGTH; i++)
                    res += BUILD_ID_CHARS[dist(rng)];

                return res;
            }

            std::string m_buildId;

            static const std::string BUILD_ID_CHARS;
            static const size_t BUILD_ID_LENGTH;

            std::string m_toplevelDirectory;

            bool initToplevelDirectory()
            {
                std::stringstream ss;
                ss << m_buildDir << "/" << m_profileName << "/" << m_repoName << "/" << m_buildId;
                m_toplevelDirectory = ss.str();
                fs::path path(m_toplevelDirectory);

                BOOST_LOG_SEV(log, severity::info) << "Toplevel build directory is " << m_toplevelDirectory;

                if (!fs::exists(path)) {
                    try {
                        fs::create_directories(path);
                    }

                    catch (boost::system::system_error& ex) {
                        BOOST_LOG_SEV(log, severity::error) << "Failed to create toplevel directory " << m_toplevelDirectory << ": " << ex.what();
                        return false;
                    }
                }

                return true;
            }

            std::string m_gitExecutable;
            bool findGitExecutable()
            {
                try {
                    m_gitExecutable = boost::process::search_path("git");
                }
                catch (std::runtime_error& ex) {
                    BOOST_LOG_SEV(log, severity::error) << "Failed to locate git executable in PATH: " << ex.what();
                    return false;
                }

                if (m_gitExecutable.size() == 0) {
                    BOOST_LOG_SEV(log, severity::error) << "No git executable found in PATH!";
                    return false;
                }

                BOOST_LOG_SEV(log, severity::debug) << "Using git executable from " << m_gitExecutable;
                return true;
            }

            std::string m_sourceDirectory;
            bool checkoutSources()
            {
                m_sourceDirectory = m_toplevelDirectory + "/repo";
                BOOST_LOG_SEV(log, severity::info) << "Checking out sources from " << m_url << " to " << m_sourceDirectory;

                try {
                    std::stringstream ssArgs;
                    ssArgs << m_gitExecutable << " clone --recursive -b " << m_branch << " " << m_url << " " << m_sourceDirectory;
                    std::string args = ssArgs.str();
                    BOOST_LOG_SEV(log, severity::debug) << "Git command line is " << args;

                    boost::process::child child = boost::process::execute(boost::process::initializers::run_exe(m_gitExecutable),
                                                                          boost::process::initializers::set_cmd_line(args));
                    auto exit_code = boost::process::wait_for_exit(child);

                    if (exit_code != 0) {
                        BOOST_LOG_SEV(log, severity::error) << "Got non-zero exit status from '" << args << "': " << exit_code;
                        return false;
                    }
                }

                catch (boost::system::system_error& ex) {
                    BOOST_LOG_SEV(log, severity::error) << "Failed to run git to check out sources: " << ex.what();
                    return false;
                }

                BOOST_LOG_SEV(log, severity::info) << "Repository cloned successfully. Checking out revision " << m_revision;

                try {
                    std::stringstream ssArgs;
                    ssArgs << m_gitExecutable << " checkout " << m_revision << " .";
                    std::string args = ssArgs.str();
                    BOOST_LOG_SEV(log, severity::debug) << "Git command line is " << args;
                    boost::process::child child = boost::process::execute(boost::process::initializers::run_exe(m_gitExecutable),
                                                                          boost::process::initializers::set_cmd_line(args),
                                                                          boost::process::initializers::start_in_dir(m_sourceDirectory));
                    auto exit_code = boost::process::wait_for_exit(child);
                    if (exit_code != 0) {
                        BOOST_LOG_SEV(log, severity::error) << "Got non-zero exit status from '" << args << "': " << exit_code;
                        return false;
                    }
                }

                catch (boost::system::system_error& ex) {
                    BOOST_LOG_SEV(log, severity::error) << "Failed to run git to check out revision " << m_revision << ": " << ex.what();
                    return false;
                }

                BOOST_LOG_SEV(log, severity::info) << "Checked out revision " << m_revision;

                m_macros.put<std::string>("CMAKE_SOURCE_DIRECTORY", m_sourceDirectory);
                return true;
            }

            boost::property_tree::ptree m_macros;
            bool loadMacroFile()
            {
                fs::path path(m_macroFile);
                if (!fs::exists(path)) {
                    BOOST_LOG_SEV(log, severity::warning) << "Macro file " << m_macroFile << " doesn't exist!";
                    return true;
                }

                if (!fs::is_regular_file(path)) {
                    BOOST_LOG_SEV(log, severity::error) << "Macro configuration " << m_macroFile << " exists but isn't a regular file!";
                    return false;
                }

                try {
                    boost::property_tree::read_ini(m_macroFile, m_macros);
                }

                catch (boost::property_tree::ptree_error& ex) {
                    BOOST_LOG_SEV(log, severity::error) << "Failed to parse macros from " << m_macroFile << ": " << ex.what();
                    return false;
                }

                return true;
            }

            boost::property_tree::ptree m_buildSettings;
            bool loadBuildConfig()
            {
                std::string configFile = m_sourceDirectory + "/" + m_configFile;
                fs::path path(configFile);
                if (!fs::exists(path)) {
                    BOOST_LOG_SEV(log, severity::error) << "Build config file " << m_configFile << " doesn't exist in " << m_sourceDirectory;
                    return false;
                }

                if (!fs::is_regular_file(path)) {
                    BOOST_LOG_SEV(log, severity::error) << "Build config file " << configFile << " exists but isn't a regular file!";
                    return false;
                }

                try {
                    boost::property_tree::read_ini(configFile, m_buildSettings);
                }

                catch (boost::property_tree::ini_parser_error& ex) {
                    BOOST_LOG_SEV(log, severity::error) << "Failed to parse build configuration in " << m_configFile << ": " << ex.what();
                    return false;
                }

                return true;
            }

            std::string m_binaryDir;
            bool createBinaryDir()
            {
                m_binaryDir = m_toplevelDirectory + "/build";
                fs::path path(m_binaryDir);

                if (!fs::exists(path)) {
                    BOOST_LOG_SEV(log, severity::info) << "Creating binary dir: " << m_binaryDir;
                    try {
                        fs::create_directories(path);
                    }
                    catch (boost::system::system_error& ex) {
                        BOOST_LOG_SEV(log, severity::error) << "Failed to create binary dir " << m_binaryDir << ": " << ex.what();
                        return false;
                    }
                }

                return true;
            }

            bool replaceMacros(std::string& str)
            {
                for (auto& kv : m_macros) {
                    try {
                        std::string key = kv.first;
                        std::string val = m_macros.get<std::string>(key);
                        std::string token = "@" + key + "@";
                        boost::algorithm::replace_all(str, token, val);
                    }

                    catch (boost::property_tree::ptree_error& ex) {
                        BOOST_LOG_SEV(log, severity::error) << "Failed to replace macro: " << ex.what();
                        return false;
                    }
                }

                return true;
            }

            bool configureSources()
            {
                BOOST_LOG_SEV(log, severity::info) << "Trying to configure sources";
                std::string configureCommand;
                try {
                    configureCommand = m_buildSettings.get<std::string>(m_profileName + ".cmd_configure");
                }

                catch (boost::property_tree::ptree_error& ex) {
                    BOOST_LOG_SEV(log, severity::error) << "Failed to get configure command from build settings: " << ex.what();
                    return false;
                }

                if (configureCommand.size() == 0) {
                    BOOST_LOG_SEV(log, severity::warning) << "Configure command is empty; continuing build!";
                    return true;
                }

                BOOST_LOG_SEV(log, severity::debug) << "Configure command is: " << configureCommand;

                if (!replaceMacros(configureCommand)) {
                    BOOST_LOG_SEV(log, severity::error) << "Macro expansion failed for configure command!";
                    return false;
                }

                BOOST_LOG_SEV(log, severity::debug) << "Configure command after macro expansion is: " << configureCommand;

                return true;
            }

        public:
            Worker(const std::string& macro_file, const std::string& build_directory,
                   const std::string& repo_name,
                   const std::string& url, const std::string& branch, const std::string& revision,
                   const std::string& config_file, const std::string& profile_name)
                : m_buildDir(build_directory)
                , m_macroFile(macro_file)
                , m_url(url)
                , m_branch(branch)
                , m_revision(revision)
                , m_configFile(config_file)
                , m_profileName(profile_name)
                , m_repoName(repo_name)
            {
            }

            void run()
            {
                if (!findGitExecutable()) {
                    BOOST_LOG_SEV(log, severity::error) << "Failed to find git; build FAILED!";
                    return;
                }

                m_buildId = generateBuildId();
                BOOST_LOG_SEV(log, severity::info) << "Worker started for repo " << m_repoName
                                                   << " (profile: " << m_profileName << ", config: " << m_configFile << ") - Build ID: " << m_buildId;
                if (!initToplevelDirectory()) {
                    BOOST_LOG_SEV(log, severity::error) << "Unable to create build directory; build FAILED!";
                    return;
                }

                if (!loadMacroFile()) {
                    BOOST_LOG_SEV(log, severity::error) << "Failed to load macro file; build FAILED!";
                    return;
                }

                if (!checkoutSources()) {
                    BOOST_LOG_SEV(log, severity::error) << "Failed to checkout sources from " << m_url << "; build FAILED!";
                    return;
                }

                if (!createBinaryDir()) {
                    BOOST_LOG_SEV(log, severity::error) << "Failed to create build directory; build FAILED!";
                    return;
                }

                if (!loadBuildConfig()) {
                    BOOST_LOG_SEV(log, severity::error) << "Failed to load build configuration; build FAILED!";
                    return;
                }

                if (!configureSources()) {
                    BOOST_LOG_SEV(log, severity::error) << "Configure step aborted; build FAILED!";
                    return;
                }
            }
        };

        const std::string Worker::BUILD_ID_CHARS{ "0123456789abcdef" };
        const size_t Worker::BUILD_ID_LENGTH{ 8 };
    }
}
}

using namespace dsn::build_bot;

Worker::Worker(const std::string& macro_file, const std::string& build_directory,
               const std::string& repo_name,
               const std::string& url, const std::string& branch, const std::string& revision,
               const std::string& config_file, const std::string& profile_name)
    : m_impl(new priv::Worker(macro_file, build_directory, repo_name, url, branch, revision, config_file, profile_name))
{
}

Worker::~Worker()
{
}

void Worker::run()
{
    return m_impl->run();
}
<|endoftext|>


<commit_before>#include "Scene.h"
#include"ObjectBehaviour.h"

namespace AEngine::Game
{
	void Scene::OnInit()
	{
		for (var i : m_objects)
		{
			i->OnInit();
		}
	}

	void Scene::OnUpdate()
	{
	}

	void Scene::OnRelease()
	{
		for (var i : m_objects)
		{
			RemoveObject(dynamic_cast<GameObject*>(dynamic_cast<ObjectBehaviour*>(i)));
		}
	}

	void Scene::AddObject(GameObject * obj)
	{
		for (var i : obj->GetChildren())
		{
			AddObject(i);
		}

		var behaviour = dynamic_cast<ObjectBehaviour*>(obj);
		for (var i : behaviour->GetComponents())
		{
			if (i)
			{
				m_objects.emplace_back(dynamic_cast<BaseBehaviour*>(i));
			}
		}
		m_objects.emplace_back(dynamic_cast<BaseBehaviour*>(behaviour));
	}

	void Scene::RemoveObject(GameObject * obj)
	{
		lock_guard<std::mutex> lock(m_mutex);
		for (var i : obj->GetChildren())
		{
			RemoveObject(i);
		}
		var behaviour = dynamic_cast<BaseBehaviour*>(dynamic_cast<ObjectBehaviour*>(obj));
		behaviour->OnRelease();
	}
}<commit_msg>Try my best to avoid deadlock<commit_after>#include "Scene.h"
#include"ObjectBehaviour.h"

namespace AEngine::Game
{
	void Scene::OnInit()
	{
		for (var i : m_objects)
		{
			i->OnInit();
		}
	}

	void Scene::OnUpdate()
	{
	}

	void Scene::OnRelease()
	{
		for (var i : m_objects)
		{
			RemoveObject(dynamic_cast<GameObject*>(dynamic_cast<ObjectBehaviour*>(i)));
		}
	}

	void Scene::AddObject(GameObject * obj)
	{
		for (var i : obj->GetChildren())
		{
			AddObject(i);
		}

		var behaviour = dynamic_cast<ObjectBehaviour*>(obj);
		for (var i : behaviour->GetComponents())
		{
			if (i)
			{
				m_objects.emplace_back(dynamic_cast<BaseBehaviour*>(i));
			}
		}
		m_objects.emplace_back(dynamic_cast<BaseBehaviour*>(behaviour));
	}

	void Scene::RemoveObject(GameObject * obj)
	{
		for (var i : obj->GetChildren())
		{
			RemoveObject(i);
		}
		lock_guard<std::mutex> lock(m_mutex);
		var behaviour = dynamic_cast<BaseBehaviour*>(dynamic_cast<ObjectBehaviour*>(obj));
		behaviour->OnRelease();
	}
}<|endoftext|>


<commit_before>#include <sstream>

#include <pcl/features/fpfh.h>
#include <pcl/io/pcd_io.h>
#include <pcl/keypoints/uniform_sampling.h>
#include <pcl/registration/icp.h>
#include <pcl/registration/ia_ransac.h>
#include <pcl/common/time.h>

#include "proctor/detector.h"
#include "proctor/proctor.h"
#include "proctor/ia_ransac_sub.h"

namespace pcl {
  namespace proctor {
    /** create the viewports to be used by top candidates */
    class create_viewports {
    public:
      create_viewports(PointCloud<PointNormal>::ConstPtr sentinel) : sentinel(sentinel) {}
      void operator()(visualization::PCLVisualizer &v) {
        for (int ci = 0; ci < Detector::num_registration; ci++) {
          int vp;
          v.createViewPort(double(ci) / Detector::num_registration, 0, double(ci + 1) / Detector::num_registration, 1, vp);
          {
            stringstream ss;
            ss << "candidate_" << ci;
            v.addPointCloud(sentinel, visualization::PointCloudColorHandlerCustom<PointNormal>(sentinel, 0xc0, 0x00, 0x40), ss.str(), vp);
          }
          {
            stringstream ss;
            ss << "aligned_" << ci;
            v.addPointCloud(sentinel, visualization::PointCloudColorHandlerCustom<PointNormal>(sentinel, 0xc0, 0x00, 0x40), ss.str(), vp);
          }
        }
      }
    private:
      PointCloud<PointNormal>::ConstPtr sentinel;
    };

    /** show a candidate model as the specified candidate */
    class show_candidate {
    public:
      show_candidate(int ci, PointCloud<PointNormal>::ConstPtr candidate) : ci(ci), candidate(candidate) {}
      void operator()(visualization::PCLVisualizer &v) {
        {
          stringstream ss;
          ss << "candidate_" << ci;
          v.updatePointCloud(candidate, visualization::PointCloudColorHandlerCustom<PointNormal>(candidate, 0xc0, 0x00, 0x40), ss.str());
        }
        {
          stringstream ss;
          ss << "aligned_" << ci;
          v.updatePointCloud(candidate, visualization::PointCloudColorHandlerCustom<PointNormal>(candidate, 0xc0, 0x00, 0x40), ss.str());
        }
      }
    private:
      int ci;
      PointCloud<PointNormal>::ConstPtr candidate;
    };

    /** show an aligned point cloud in the specified viewport */
    class show_aligned {
    public:
      show_aligned(int ci, PointCloud<PointNormal>::ConstPtr aligned) : ci(ci), aligned(aligned) {}
      void operator()(visualization::PCLVisualizer &v) {
        stringstream ss;
        ss << "aligned_" << ci;
        v.updatePointCloud(aligned, visualization::PointCloudColorHandlerCustom<PointNormal>(aligned, 0xff, 0xff, 0xff), ss.str());
      }
    private:
      int ci;
      PointCloud<PointNormal>::ConstPtr aligned;
    };

    class visualization_callback {
    public:
      visualization_callback(int ci, visualization::CloudViewer *vis) : ci(ci), vis(vis) {}
      void operator()(const PointCloud<PointNormal> &cloud_src,
                      const vector<int> &indices_src,
                      const PointCloud<PointNormal> &cloud_tgt,
                      const vector<int> &indices_tgt) {
        // make a copy of the cloud. expensive.
        PointCloud<PointNormal>::ConstPtr aligned (new PointCloud<PointNormal>(cloud_src));
        vis->runOnVisualizationThreadOnce(show_aligned(ci, aligned));
      }
    private:
      int ci;
      visualization::CloudViewer *vis;
    };

    /** Detector */
    const double Detector::keypoint_separation = 0.1;
    const int Detector::max_votes = 100;
    const int Detector::num_registration = 4;

    /* Trains a single model */
    void Detector::train(Scene &scene) {
      srand(0);
      IndicesPtr keypoints = computeKeypoints(scene.cloud);
      PointCloud<Detector::Signature>::Ptr features = obtainFeatures(scene, keypoints, false);

      Entry e;
      e.cloud = scene.cloud;
      e.indices = keypoints;
      e.features = features;
      e.tree = KdTree<Signature>::Ptr(new KdTreeFLANN<Signature>());
      e.tree->setInputCloud(e.features);

      database[scene.id] = e;
    }

    typedef struct {
      std::string id;
      float votes;
    } Candidate;

    bool operator<(const Candidate &a, const Candidate &b) {
      return a.votes > b.votes; // sort descending
    }

    double Detector::get_votes(Entry &query, Entry &match) {
      double votes = 0;
      for (unsigned int pi = 0; pi < query.indices->size(); pi++) {
        vector<int> indices;
        vector<float> distances;

        clock_t start = clock();
        StopWatch s;
        match.tree->nearestKSearch(*query.features, pi, max_votes, indices, distances);

        for (int ri = 0; ri < max_votes; ri++) {
          //votes += 1. / (distances[ri] + numeric_limits<float>::epsilon());
          votes -= distances[ri];
        }
      }

      return votes;
    }

    // TODO scene as a reference
    std::string Detector::query(Scene scene, float *classifier, double *registration) {
      cout << "Running a test on model " << scene.id << endl;

      Entry e;
      e.cloud = scene.cloud;
      timer.start();
      e.indices = computeKeypoints(e.cloud);
      timer.stop(KEYPOINTS_TESTING);
      timer.start();
      e.features = obtainFeatures(scene, e.indices, true);
      timer.stop(COMPUTE_FEATURES_TESTING);
      // let each point cast votes
      timer.start();
      memset(classifier, 0, Config::num_models * sizeof(*classifier));
      clock_t totalTime = 0;
      float time = 0;

      StopWatch s;

      std::map<std::string, Entry>::iterator it;

      // get top candidates
      vector<Candidate> ballot;
      for ( it = database.begin() ; it != database.end(); it++ ) {
        std::string target_id = (*it).first;
        Entry target = (*it).second;

        Candidate* candidate = new Candidate;
        candidate->id = target_id;
        candidate->votes = get_votes(e, target);
        ballot.push_back(*candidate);
      }

      sort(ballot.begin(), ballot.end());

      time += s.getTimeSeconds();
      //cout << "Total K-Nearest Search Time for Query: " << time << endl;

      //timer.stop(VOTING_CLASSIFIER);
      //if (vis.get()) {
        //for (int ci = 0; ci < num_registration; ci++) {
          //vis->runOnVisualizationThreadOnce(show_candidate(ci, database[ballot[ci].mi].cloud));
        //}
      //}
       //run registration on top candidates
      //memset(registration, 0, Config::num_models * sizeof(*registration));
      std::map<std::string, double> reg_scores;
      double best = numeric_limits<double>::infinity();
      std::string guessed_id = "";
      for (int ci = 0; ci < num_registration; ci++) {
        std::string id = ballot[ci].id;
        flush(cout << id << ": " << ballot[ci].votes);
        reg_scores[id] = computeRegistration(e, id, ci);
        cout << " / " << reg_scores[id] << endl;
        if (reg_scores[id] < best) {
          guessed_id = id;
          best = reg_scores[id];
        }
      }
      return guessed_id;
    }

    void Detector::enableVisualization() {
      vis.reset(new visualization::CloudViewer("Detector Visualization"));
      PointCloud<PointNormal>::Ptr sentinel (new PointCloud<PointNormal>());
      sentinel->points.push_back(PointNormal());
      vis->runOnVisualizationThreadOnce(create_viewports(sentinel));
    }

    void Detector::printTimer() {
      printf(
        "select training keypoints: %10.3f sec\n"
        "obtain training features:  %10.3f sec\n"
        "build feature tree:        %10.3f sec\n"
        "select testing keypoints:  %10.3f sec\n"
        "compute testing features:  %10.3f sec\n"
        "voting classifier:         %10.3f sec\n"
        "initial alignment:         %10.3f sec\n"
        "ICP:                       %10.3f sec\n",
        timer[KEYPOINTS_TRAINING],
        timer[OBTAIN_FEATURES_TRAINING],
        timer[BUILD_TREE],
        timer[KEYPOINTS_TESTING],
        timer[COMPUTE_FEATURES_TESTING],
        timer[VOTING_CLASSIFIER],
        timer[IA_RANSAC],
        timer[ICP]
      );
    }

    IndicesPtr Detector::computeKeypoints(PointCloud<PointNormal>::Ptr cloud) {
      IndicesPtr indices (new vector<int>());
      PointCloud<int> leaves;
      UniformSampling<PointNormal> us;
      us.setRadiusSearch(keypoint_separation);
      us.setInputCloud(cloud);
      us.compute(leaves);
      indices->assign(leaves.points.begin(), leaves.points.end()); // can't use operator=, probably because of different allocators

      return indices;

      srand(0);
      int r = 200;
      IndicesPtr subset (new vector<int>());
      for (int i = 0; i < r; i++) {
        if (indices->size() == 0) {
          break;
        }
        int pick = rand() % indices->size();
        subset->push_back(indices->at(pick));
        indices->erase(indices->begin() + pick);
      }
      cout << "Subset Size: " << subset->size() << endl;
      return subset;
    }

    PointCloud<Detector::Signature>::Ptr Detector::computeFeatures(PointCloud<PointNormal>::Ptr cloud, IndicesPtr indices) {
      cout << "computing features on " << indices->size() << " points" << endl;
      PointCloud<Signature>::Ptr features (new PointCloud<Signature>());
      FPFHEstimation<PointNormal, PointNormal, Signature> fpfh;
      fpfh.setRadiusSearch(0.1);
      fpfh.setInputCloud(cloud);
      fpfh.setIndices(indices);
      search::KdTree<PointNormal>::Ptr kdt (new search::KdTree<PointNormal>());
      fpfh.setSearchMethod(kdt);
      fpfh.setInputNormals(cloud);
      fpfh.compute(*features);
      if (features->points.size() != indices->size())
        cout << "got " << features->points.size() << " features from " << indices->size() << " points" << endl;
      return features;
    }

    // TODO Enum for is_test_phase
    PointCloud<Detector::Signature>::Ptr Detector::obtainFeatures(Scene &scene, IndicesPtr indices, bool is_test_phase) {
      std::string name_str = std::string("feature_") + scene.id;

      if (is_test_phase) {
        name_str += "_test";
      }
      else {
        name_str += "_train";
      }

      name_str += ".pcd";

      const char *name = name_str.c_str();

      if (ifstream(name)) {
        PointCloud<Signature>::Ptr features (new PointCloud<Signature>());
        io::loadPCDFile(name, *features);
        //if (features->points.size() != indices->size())
          //cout << "got " << features->points.size() << " features from " << indices->size() << " points" << endl;
        return features;
      } else {
        PointCloud<Signature>::Ptr features = computeFeatures(scene.cloud, indices);
        io::savePCDFileBinary(name, *features);
        return features;
      }
    }

    double Detector::computeRegistration(Entry &source, std::string id, int ci) {
      Entry& target = database[id];
      typedef boost::function<void(const PointCloud<PointNormal> &cloud_src,
                                  const vector<int> &indices_src,
                                  const PointCloud<PointNormal> &cloud_tgt,
                                  const vector<int> &indices_tgt)> f;

      //cout << "Source: " << id << "\tSize: " << source.cloud->size() << "\tIndices Size: " << source.indices->size() << "\tFeature Size: " << source.features->size() << endl;
      //cout << "Target: " << id << "\tSize: " << target.cloud->size() << "\tIndices Size: " << target.indices->size() << "\tFeature Size: " << target.features->size() << endl;
      // Copy the source/target clouds with only the subset of points that
      // also features calculated for them.
      PointCloud<PointNormal>::Ptr source_subset(
          new PointCloud<PointNormal>(*(source.cloud), *(source.indices)));
      PointCloud<PointNormal>::Ptr target_subset(
          new PointCloud<PointNormal>(*(target.cloud), *(target.indices)));

      timer.start();

      // TODO Filtering the source cloud makes it much faster when computing the
      // error metric, but may not be as good
      SampleConsensusInitialAlignment<PointNormal, PointNormal, Signature> ia_ransac;
      ia_ransac.setMinSampleDistance(0.05);
      ia_ransac.setMaxCorrespondenceDistance(0.5);
      ia_ransac.setMaximumIterations(256);
      ia_ransac.setInputCloud(source_subset);
      ia_ransac.setSourceFeatures(source.features);
      //ia_ransac.setSourceIndices(source.indices);
      ia_ransac.setInputTarget(target_subset);
      ia_ransac.setTargetFeatures(target.features);
      //ia_ransac.setTargetIndices(target.indices);

      if (vis.get()) {
        f updater (visualization_callback(ci, vis.get()));
        ia_ransac.registerVisualizationCallback(updater);
      }

      PointCloud<PointNormal>::Ptr aligned (new PointCloud<PointNormal>());
      ia_ransac.align(*aligned);
      timer.stop(IA_RANSAC);

      timer.start();
      IterativeClosestPoint<PointNormal, PointNormal> icp;
      PointCloud<PointNormal>::Ptr aligned2 (new PointCloud<PointNormal>());
      icp.setInputCloud(aligned);
      icp.setInputTarget(target.cloud);
      icp.setMaxCorrespondenceDistance(0.1);
      icp.setMaximumIterations(64);
      if (vis.get()) {
        f updater (visualization_callback(ci, vis.get()));
        icp.registerVisualizationCallback(updater);
      }
      icp.align(*aligned2);
      timer.stop(ICP);
      return icp.getFitnessScore();
    }
  }
}
<commit_msg>Fixed voting bug, where the number of votes is not kept track of.<commit_after>#include <sstream>

#include <pcl/features/fpfh.h>
#include <pcl/io/pcd_io.h>
#include <pcl/keypoints/uniform_sampling.h>
#include <pcl/registration/icp.h>
#include <pcl/registration/ia_ransac.h>
#include <pcl/common/time.h>

#include "proctor/detector.h"
#include "proctor/proctor.h"
#include "proctor/ia_ransac_sub.h"

namespace pcl {
  namespace proctor {
    /** create the viewports to be used by top candidates */
    class create_viewports {
    public:
      create_viewports(PointCloud<PointNormal>::ConstPtr sentinel) : sentinel(sentinel) {}
      void operator()(visualization::PCLVisualizer &v) {
        for (int ci = 0; ci < Detector::num_registration; ci++) {
          int vp;
          v.createViewPort(double(ci) / Detector::num_registration, 0, double(ci + 1) / Detector::num_registration, 1, vp);
          {
            stringstream ss;
            ss << "candidate_" << ci;
            v.addPointCloud(sentinel, visualization::PointCloudColorHandlerCustom<PointNormal>(sentinel, 0xc0, 0x00, 0x40), ss.str(), vp);
          }
          {
            stringstream ss;
            ss << "aligned_" << ci;
            v.addPointCloud(sentinel, visualization::PointCloudColorHandlerCustom<PointNormal>(sentinel, 0xc0, 0x00, 0x40), ss.str(), vp);
          }
        }
      }
    private:
      PointCloud<PointNormal>::ConstPtr sentinel;
    };

    /** show a candidate model as the specified candidate */
    class show_candidate {
    public:
      show_candidate(int ci, PointCloud<PointNormal>::ConstPtr candidate) : ci(ci), candidate(candidate) {}
      void operator()(visualization::PCLVisualizer &v) {
        {
          stringstream ss;
          ss << "candidate_" << ci;
          v.updatePointCloud(candidate, visualization::PointCloudColorHandlerCustom<PointNormal>(candidate, 0xc0, 0x00, 0x40), ss.str());
        }
        {
          stringstream ss;
          ss << "aligned_" << ci;
          v.updatePointCloud(candidate, visualization::PointCloudColorHandlerCustom<PointNormal>(candidate, 0xc0, 0x00, 0x40), ss.str());
        }
      }
    private:
      int ci;
      PointCloud<PointNormal>::ConstPtr candidate;
    };

    /** show an aligned point cloud in the specified viewport */
    class show_aligned {
    public:
      show_aligned(int ci, PointCloud<PointNormal>::ConstPtr aligned) : ci(ci), aligned(aligned) {}
      void operator()(visualization::PCLVisualizer &v) {
        stringstream ss;
        ss << "aligned_" << ci;
        v.updatePointCloud(aligned, visualization::PointCloudColorHandlerCustom<PointNormal>(aligned, 0xff, 0xff, 0xff), ss.str());
      }
    private:
      int ci;
      PointCloud<PointNormal>::ConstPtr aligned;
    };

    class visualization_callback {
    public:
      visualization_callback(int ci, visualization::CloudViewer *vis) : ci(ci), vis(vis) {}
      void operator()(const PointCloud<PointNormal> &cloud_src,
                      const vector<int> &indices_src,
                      const PointCloud<PointNormal> &cloud_tgt,
                      const vector<int> &indices_tgt) {
        // make a copy of the cloud. expensive.
        PointCloud<PointNormal>::ConstPtr aligned (new PointCloud<PointNormal>(cloud_src));
        vis->runOnVisualizationThreadOnce(show_aligned(ci, aligned));
      }
    private:
      int ci;
      visualization::CloudViewer *vis;
    };

    /** Detector */
    const double Detector::keypoint_separation = 0.1;
    const int Detector::max_votes = 100;
    const int Detector::num_registration = 4;

    /* Trains a single model */
    void Detector::train(Scene &scene) {
      srand(0);
      IndicesPtr keypoints = computeKeypoints(scene.cloud);
      PointCloud<Detector::Signature>::Ptr features = obtainFeatures(scene, keypoints, false);

      Entry e;
      e.cloud = scene.cloud;
      e.indices = keypoints;
      e.features = features;
      e.tree = KdTree<Signature>::Ptr(new KdTreeFLANN<Signature>());
      e.tree->setInputCloud(e.features);

      database[scene.id] = e;
    }

    typedef struct {
      std::string id;
      float votes;
    } Candidate;

    bool operator<(const Candidate &a, const Candidate &b) {
      return a.votes > b.votes; // sort descending
    }

    double Detector::get_votes(Entry &query, Entry &match) {
      double votes = 0;
      for (unsigned int pi = 0; pi < query.indices->size(); pi++) {
        vector<int> indices;
        vector<float> distances;

        clock_t start = clock();
        StopWatch s;
        int num_found = match.tree->nearestKSearch(*query.features, pi, max_votes, indices, distances);

        for (int ri = 0; ri < num_found; ri++) {
          votes += 1. / (distances[ri] + numeric_limits<float>::epsilon());
          //votes -= distances[ri];
        }
      }

      return votes;
    }

    // TODO scene as a reference
    std::string Detector::query(Scene scene, float *classifier, double *registration) {
      cout << "Running a test on model " << scene.id << endl;

      Entry e;
      e.cloud = scene.cloud;
      timer.start();
      e.indices = computeKeypoints(e.cloud);
      timer.stop(KEYPOINTS_TESTING);
      timer.start();
      e.features = obtainFeatures(scene, e.indices, true);
      timer.stop(COMPUTE_FEATURES_TESTING);
      // let each point cast votes
      timer.start();
      memset(classifier, 0, Config::num_models * sizeof(*classifier));
      clock_t totalTime = 0;
      float time = 0;

      StopWatch s;

      std::map<std::string, Entry>::iterator it;

      // get top candidates
      vector<Candidate> ballot;
      for ( it = database.begin() ; it != database.end(); it++ ) {
        std::string target_id = (*it).first;
        Entry target = (*it).second;

        Candidate* candidate = new Candidate;
        candidate->id = target_id;
        candidate->votes = get_votes(e, target);
        ballot.push_back(*candidate);
      }

      sort(ballot.begin(), ballot.end());

      time += s.getTimeSeconds();
      //cout << "Total K-Nearest Search Time for Query: " << time << endl;

      //timer.stop(VOTING_CLASSIFIER);
      //if (vis.get()) {
        //for (int ci = 0; ci < num_registration; ci++) {
          //vis->runOnVisualizationThreadOnce(show_candidate(ci, database[ballot[ci].mi].cloud));
        //}
      //}
       //run registration on top candidates
      //memset(registration, 0, Config::num_models * sizeof(*registration));
      std::map<std::string, double> reg_scores;
      double best = numeric_limits<double>::infinity();
      std::string guessed_id = "";
      for (int ci = 0; ci < num_registration; ci++) {
        std::string id = ballot[ci].id;
        flush(cout << id << ": " << ballot[ci].votes);
        reg_scores[id] = computeRegistration(e, id, ci);
        cout << " / " << reg_scores[id] << endl;
        if (reg_scores[id] < best) {
          guessed_id = id;
          best = reg_scores[id];
        }
      }
      return guessed_id;
    }

    void Detector::enableVisualization() {
      vis.reset(new visualization::CloudViewer("Detector Visualization"));
      PointCloud<PointNormal>::Ptr sentinel (new PointCloud<PointNormal>());
      sentinel->points.push_back(PointNormal());
      vis->runOnVisualizationThreadOnce(create_viewports(sentinel));
    }

    void Detector::printTimer() {
      printf(
        "select training keypoints: %10.3f sec\n"
        "obtain training features:  %10.3f sec\n"
        "build feature tree:        %10.3f sec\n"
        "select testing keypoints:  %10.3f sec\n"
        "compute testing features:  %10.3f sec\n"
        "voting classifier:         %10.3f sec\n"
        "initial alignment:         %10.3f sec\n"
        "ICP:                       %10.3f sec\n",
        timer[KEYPOINTS_TRAINING],
        timer[OBTAIN_FEATURES_TRAINING],
        timer[BUILD_TREE],
        timer[KEYPOINTS_TESTING],
        timer[COMPUTE_FEATURES_TESTING],
        timer[VOTING_CLASSIFIER],
        timer[IA_RANSAC],
        timer[ICP]
      );
    }

    IndicesPtr Detector::computeKeypoints(PointCloud<PointNormal>::Ptr cloud) {
      IndicesPtr indices (new vector<int>());
      PointCloud<int> leaves;
      UniformSampling<PointNormal> us;
      us.setRadiusSearch(keypoint_separation);
      us.setInputCloud(cloud);
      us.compute(leaves);
      indices->assign(leaves.points.begin(), leaves.points.end()); // can't use operator=, probably because of different allocators

      return indices;

      srand(0);
      int r = 200;
      IndicesPtr subset (new vector<int>());
      for (int i = 0; i < r; i++) {
        if (indices->size() == 0) {
          break;
        }
        int pick = rand() % indices->size();
        subset->push_back(indices->at(pick));
        indices->erase(indices->begin() + pick);
      }
      cout << "Subset Size: " << subset->size() << endl;
      return subset;
    }

    PointCloud<Detector::Signature>::Ptr Detector::computeFeatures(PointCloud<PointNormal>::Ptr cloud, IndicesPtr indices) {
      cout << "computing features on " << indices->size() << " points" << endl;
      PointCloud<Signature>::Ptr features (new PointCloud<Signature>());
      FPFHEstimation<PointNormal, PointNormal, Signature> fpfh;
      fpfh.setRadiusSearch(0.1);
      fpfh.setInputCloud(cloud);
      fpfh.setIndices(indices);
      search::KdTree<PointNormal>::Ptr kdt (new search::KdTree<PointNormal>());
      fpfh.setSearchMethod(kdt);
      fpfh.setInputNormals(cloud);
      fpfh.compute(*features);
      if (features->points.size() != indices->size())
        cout << "got " << features->points.size() << " features from " << indices->size() << " points" << endl;
      return features;
    }

    // TODO Enum for is_test_phase
    PointCloud<Detector::Signature>::Ptr Detector::obtainFeatures(Scene &scene, IndicesPtr indices, bool is_test_phase) {
      std::string name_str = std::string("feature_") + scene.id;

      if (is_test_phase) {
        name_str += "_test";
      }
      else {
        name_str += "_train";
      }

      name_str += ".pcd";

      const char *name = name_str.c_str();

      if (ifstream(name)) {
        PointCloud<Signature>::Ptr features (new PointCloud<Signature>());
        io::loadPCDFile(name, *features);
        //if (features->points.size() != indices->size())
          //cout << "got " << features->points.size() << " features from " << indices->size() << " points" << endl;
        return features;
      } else {
        PointCloud<Signature>::Ptr features = computeFeatures(scene.cloud, indices);
        io::savePCDFileBinary(name, *features);
        return features;
      }
    }

    double Detector::computeRegistration(Entry &source, std::string id, int ci) {
      Entry& target = database[id];
      typedef boost::function<void(const PointCloud<PointNormal> &cloud_src,
                                  const vector<int> &indices_src,
                                  const PointCloud<PointNormal> &cloud_tgt,
                                  const vector<int> &indices_tgt)> f;

      //cout << "Source: " << id << "\tSize: " << source.cloud->size() << "\tIndices Size: " << source.indices->size() << "\tFeature Size: " << source.features->size() << endl;
      //cout << "Target: " << id << "\tSize: " << target.cloud->size() << "\tIndices Size: " << target.indices->size() << "\tFeature Size: " << target.features->size() << endl;
      // Copy the source/target clouds with only the subset of points that
      // also features calculated for them.
      PointCloud<PointNormal>::Ptr source_subset(
          new PointCloud<PointNormal>(*(source.cloud), *(source.indices)));
      PointCloud<PointNormal>::Ptr target_subset(
          new PointCloud<PointNormal>(*(target.cloud), *(target.indices)));

      timer.start();

      // TODO Filtering the source cloud makes it much faster when computing the
      // error metric, but may not be as good
      SampleConsensusInitialAlignment<PointNormal, PointNormal, Signature> ia_ransac;
      ia_ransac.setMinSampleDistance(0.05);
      ia_ransac.setMaxCorrespondenceDistance(0.5);
      ia_ransac.setMaximumIterations(256);
      ia_ransac.setInputCloud(source_subset);
      ia_ransac.setSourceFeatures(source.features);
      //ia_ransac.setSourceIndices(source.indices);
      ia_ransac.setInputTarget(target_subset);
      ia_ransac.setTargetFeatures(target.features);
      //ia_ransac.setTargetIndices(target.indices);

      if (vis.get()) {
        f updater (visualization_callback(ci, vis.get()));
        ia_ransac.registerVisualizationCallback(updater);
      }

      PointCloud<PointNormal>::Ptr aligned (new PointCloud<PointNormal>());
      ia_ransac.align(*aligned);
      timer.stop(IA_RANSAC);

      timer.start();
      IterativeClosestPoint<PointNormal, PointNormal> icp;
      PointCloud<PointNormal>::Ptr aligned2 (new PointCloud<PointNormal>());
      icp.setInputCloud(aligned);
      icp.setInputTarget(target.cloud);
      icp.setMaxCorrespondenceDistance(0.1);
      icp.setMaximumIterations(64);
      if (vis.get()) {
        f updater (visualization_callback(ci, vis.get()));
        icp.registerVisualizationCallback(updater);
      }
      icp.align(*aligned2);
      timer.stop(ICP);
      return icp.getFitnessScore();
    }
  }
}
<|endoftext|>


<commit_before>#include "Engine.h"
#include "Game.h"
#include "ObjectDummy.h"
#include "BodyRigid.h"
#include "BodyDummy.h"
#include "Physics.h"
#include "ShapeCapsule.h"
#include "ActorBase.h"
#include "Visualizer.h"
#include "sys/SysControl.h"

using namespace MathLib;

#ifdef MEMORY_INFO
#define new new(__FILE__, __LINE__)
#endif // MEMORY_INFO

/*
*/
#define ACTOR_BASE_IFPS             (1.0f / 60.0f)
#define ACTOR_BASE_CLAMP            89.9f
#define ACTOR_BASE_COLLISIONS       4

/*
*/
CActorBase::CActorBase()
{
	m_vUp = vec3(0.0f, 0.0f, 1.0f);
	m_pObject = new CObjectDummy();
	m_pDummy = new CBodyDummy();
	m_pShape = new CShapeCapsule(1.0f, 1.0f);

	m_nFlush = 0;
	m_vPosition = Vec3_zero;
	m_vVelocity = Vec3_zero;
	m_fPhiAngle = 0.0f;			//倾斜角		二维平面坐标系中，直线向Y周延伸的方向与X轴正向之间的夹角
	m_fThetaAngle = 0.0f;			//方位角，正北方那条线与当前线条按照顺时针走过的角度

	for (int i = 0; i < NUM_STATES; i++)
	{
		m_pStates[i] = 0;
		m_pTimes[i] = 0.0f;
	}
	m_pDummy->SetEnabled(1);
	m_pObject->SetBody(NULL);
	m_pObject->SetBody(m_pDummy);
	m_pShape->SetBody(NULL);
	m_pShape->SetBody(m_pDummy);

	m_pObject->SetWorldTransform(Get_Body_Transform());
	m_pShape->SetRestitution(0.0f);
	m_pShape->SetCollisionMask(2);

	SetEnabled(1);
	SetViewDirection(vec3(0.0f, 1.0f, 0.0f));
	SetCollision(1);
	SetCollisionRadius(0.3f);
	SetCollisionHeight(1.0f);
	SetFriction(2.0f);
	SetMinVelocity(2.0f);
	SetMaxVelocity(4.0f);
	SetAcceleration(8.0f);
	SetDamping(8.0f);
	SetJumping(1.5f);

	SetGround(0);
	SetCeiling(0);
}

CActorBase::~CActorBase()
{
	m_pDummy->SetObject(NULL);
	delete m_pObject;
	delete m_pDummy;
}

void CActorBase::SetEnabled(int enable)
{
	m_nEnable = enable;
	m_pDummy->SetEnabled(m_nEnable);
}

int CActorBase::IsEnabled() const
{
	return m_nEnable;
}
void CActorBase::Update(float ifps)
{
	if (!m_nEnable)
	{
		return;
	}

	// impulse
	vec3 impulse = vec3_zero;

	// ortho basis
	vec3 tangent, binormal;
	OrthoBasis(m_vUp, tangent, binormal);

	// current basis
	vec3 x = quat(m_vUp, -m_fPhiAngle) * binormal;
	vec3 y = Normalize(Cross(m_vUp, x));
	vec3 z = Normalize(Cross(x, y));

	handle states
		Update_States(1, ifps);

	// old velocity
	float x_velocity = Dot(x, m_vVelocity);
	float y_velocity = Dot(y, m_vVelocity);
	float z_velocity = Dot(z, m_vVelocity);
	// movement
	if (m_pStates[STATE_FORWARD]) impulse += x;
	if (m_pStates[STATE_BACKWARD]) impulse -= x;
	if (m_pStates[STATE_MOVE_LEFT]) impulse += y;
	if (m_pStates[STATE_MOVE_RIGHT]) impulse -= y;
	impulse.normalize();
	//velocity
	if (m_pStates[STATE_RUN])
		impulse *= m_fMaxVelocity;
	else
		impulse *= m_fMinVelocity;
	// jump
	if (m_pStates[STATE_JUMP] == STATE_BEGIN)
	{
		impulse += z * CMathCore::Sqrt(2.0f * 9.8f * m_fJumping) / (m_fAcceleration * ifps);
	}

	// rotate velocity
	if (GetGround())
	{
		m_vVelocity = x * x_velocity + y * y_velocity + z * z_velocity;
	}

	// time
	float time = ifps * g_Engine.pPhysics->GetScale();

	// target velocity
	float target_velocity = Length(vec2(Dot(x, impulse), Dot(y, impulse)));

	// penetration tolerance
	float penetration = g_Engine.pPhysics->GetPenetrationTolerance();
	float penetration_2 = penetration * 2.0f;

	// frozen linear velocity
	float frozen_velocity = g_Engine.pPhysics->GetFrozenLinearVelocity();

	// friction
	float friction = 0.0f;
	if (target_velocity < EPSILON)
	{
		friction = m_fFriction;
	}

	//clear collision flags
	if (GetCollision())
	{
		m_nGround = 0;
		m_nCeiling = 0;
	}
	// movement
	do
	{
		// adaptive time step
		float ifps = Min(time, ACTOR_BASE_IFPS);
		time -= ifps;

		// save old velocity
		float old_velocity = Length(vec2(Dot(x, m_vVelocity), Dot(y, m_vVelocity)));

		// integrate velocity
		m_vVelocity += impulse * (m_fAcceleration * ifps);
		m_vVelocity += g_Engine.pPhysics->GetGravity() * ifps;

		// damping
		float current_velocity = Length(vec2(Dot(x, m_vVelocity), Dot(y, m_vVelocity)));
		if (target_velocity < EPSILON || current_velocity > target_velocity)
		{
			m_vVelocity = (x * Dot(x, m_vVelocity) + y * Dot(y, m_vVelocity)) * CMathCore::Exp(-m_fDamping * ifps) + z * Dot(z, m_vVelocity);
		}

		// clamp maximum velocity
		current_velocity = Length(vec2(Dot(x, m_vVelocity), Dot(y, m_vVelocity)));
		if (current_velocity > old_velocity)
		{
			if (current_velocity > target_velocity)
			{
				m_vVelocity = (x * Dot(x, m_vVelocity) + y * Dot(y, m_vVelocity)) * target_velocity / current_velocity + z * Dot(z, m_vVelocity);
			}
		}

		// frozen velocity
		int is_frozen = 0;
		if (current_velocity < frozen_velocity)
		{
			m_vVelocity = z * Dot(z, m_vVelocity);
			is_frozen = 1;
		}

	}

}<commit_msg>Signed-off-by: mrlitong <litongtongxue@gmail.com><commit_after>#include "Engine.h"
#include "Game.h"
#include "ObjectDummy.h"
#include "BodyRigid.h"
#include "BodyDummy.h"
#include "Physics.h"
#include "ShapeCapsule.h"
#include "ActorBase.h"
#include "Visualizer.h"
#include "sys/SysControl.h"

using namespace MathLib;

#ifdef MEMORY_INFO
#define new new(__FILE__, __LINE__)
#endif // MEMORY_INFO

/*
*/
#define ACTOR_BASE_IFPS             (1.0f / 60.0f)
#define ACTOR_BASE_CLAMP            89.9f
#define ACTOR_BASE_COLLISIONS       4

/*
*/
CActorBase::CActorBase()
{
	m_vUp = vec3(0.0f, 0.0f, 1.0f);
	m_pObject = new CObjectDummy();
	m_pDummy = new CBodyDummy();
	m_pShape = new CShapeCapsule(1.0f, 1.0f);

	m_nFlush = 0;
	m_vPosition = Vec3_zero;
	m_vVelocity = Vec3_zero;
	m_fPhiAngle = 0.0f;			//倾斜角		二维平面坐标系中，直线向Y周延伸的方向与X轴正向之间的夹角
	m_fThetaAngle = 0.0f;			//方位角，正北方那条线与当前线条按照顺时针走过的角度

	for (int i = 0; i < NUM_STATES; i++)
	{
		m_pStates[i] = 0;
		m_pTimes[i] = 0.0f;
	}
	m_pDummy->SetEnabled(1);
	m_pObject->SetBody(NULL);
	m_pObject->SetBody(m_pDummy);
	m_pShape->SetBody(NULL);
	m_pShape->SetBody(m_pDummy);

	m_pObject->SetWorldTransform(Get_Body_Transform());
	m_pShape->SetRestitution(0.0f);
	m_pShape->SetCollisionMask(2);

	SetEnabled(1);
	SetViewDirection(vec3(0.0f, 1.0f, 0.0f));
	SetCollision(1);
	SetCollisionRadius(0.3f);
	SetCollisionHeight(1.0f);
	SetFriction(2.0f);
	SetMinVelocity(2.0f);
	SetMaxVelocity(4.0f);
	SetAcceleration(8.0f);
	SetDamping(8.0f);
	SetJumping(1.5f);

	SetGround(0);
	SetCeiling(0);
}

CActorBase::~CActorBase()
{
	m_pDummy->SetObject(NULL);
	delete m_pObject;
	delete m_pDummy;
}

void CActorBase::SetEnabled(int enable)
{
	m_nEnable = enable;
	m_pDummy->SetEnabled(m_nEnable);
}

int CActorBase::IsEnabled() const
{
	return m_nEnable;
}
void CActorBase::Update(float ifps)
{
	if (!m_nEnable)
	{
		return;
	}

	// impulse
	vec3 impulse = vec3_zero;

	// ortho basis
	vec3 tangent, binormal;
	OrthoBasis(m_vUp, tangent, binormal);

	// current basis
	vec3 x = quat(m_vUp, -m_fPhiAngle) * binormal;
	vec3 y = Normalize(Cross(m_vUp, x));
	vec3 z = Normalize(Cross(x, y));

	handle states
		Update_States(1, ifps);

	// old velocity
	float x_velocity = Dot(x, m_vVelocity);
	float y_velocity = Dot(y, m_vVelocity);
	float z_velocity = Dot(z, m_vVelocity);
	// movement
	if (m_pStates[STATE_FORWARD]) impulse += x;
	if (m_pStates[STATE_BACKWARD]) impulse -= x;
	if (m_pStates[STATE_MOVE_LEFT]) impulse += y;
	if (m_pStates[STATE_MOVE_RIGHT]) impulse -= y;
	impulse.normalize();
	//velocity
	if (m_pStates[STATE_RUN])
		impulse *= m_fMaxVelocity;
	else
		impulse *= m_fMinVelocity;
	// jump
	if (m_pStates[STATE_JUMP] == STATE_BEGIN)
	{
		impulse += z * CMathCore::Sqrt(2.0f * 9.8f * m_fJumping) / (m_fAcceleration * ifps);
	}

	// rotate velocity
	if (GetGround())
	{
		m_vVelocity = x * x_velocity + y * y_velocity + z * z_velocity;
	}

	// time
	float time = ifps * g_Engine.pPhysics->GetScale();

	// target velocity
	float target_velocity = Length(vec2(Dot(x, impulse), Dot(y, impulse)));

	// penetration tolerance
	float penetration = g_Engine.pPhysics->GetPenetrationTolerance();
	float penetration_2 = penetration * 2.0f;

	// frozen linear velocity
	float frozen_velocity = g_Engine.pPhysics->GetFrozenLinearVelocity();

	// friction
	float friction = 0.0f;
	if (target_velocity < EPSILON)
	{
		friction = m_fFriction;
	}

	//clear collision flags
	if (GetCollision())
	{
		m_nGround = 0;
		m_nCeiling = 0;
	}
	// movement
	do
	{
		// adaptive time step
		float ifps = Min(time, ACTOR_BASE_IFPS);
		time -= ifps;

		// save old velocity
		float old_velocity = Length(vec2(Dot(x, m_vVelocity), Dot(y, m_vVelocity)));

		// integrate velocity
		m_vVelocity += impulse * (m_fAcceleration * ifps);
		m_vVelocity += g_Engine.pPhysics->GetGravity() * ifps;

		// damping
		float current_velocity = Length(vec2(Dot(x, m_vVelocity), Dot(y, m_vVelocity)));
		if (target_velocity < EPSILON || current_velocity > target_velocity)
		{
			m_vVelocity = (x * Dot(x, m_vVelocity) + y * Dot(y, m_vVelocity)) * CMathCore::Exp(-m_fDamping * ifps) + z * Dot(z, m_vVelocity);
		}

		// clamp maximum velocity
		current_velocity = Length(vec2(Dot(x, m_vVelocity), Dot(y, m_vVelocity)));
		if (current_velocity > old_velocity)
		{
			if (current_velocity > target_velocity)
			{
				m_vVelocity = (x * Dot(x, m_vVelocity) + y * Dot(y, m_vVelocity)) * target_velocity / current_velocity + z * Dot(z, m_vVelocity);
			}
		}

		// frozen velocity
		int is_frozen = 0;
		if (current_velocity < frozen_velocity)
		{
			m_vVelocity = z * Dot(z, m_vVelocity);
			is_frozen = 1;
		}

		// integrate position
		//m_vPosition += Vec3(m_vVelocity * ifps);

		// world collision

	}

}<|endoftext|>


<commit_before>#include "Advertisement.h"

Advertisement::Advertisement(void) {
	path = "advertisement.txt";
}

Advertisement::~Advertisement(void) {};

string Advertisement::getAd(void) {
	int numLines = 0;
    string line;
    ifstream file(path.c_str());

    //count number of lines
    while (getline(file, line))
        ++numLines;

    //go back to begining of file
    file.clear();
    file.seekg(0, ios::beg);

    string temp;
    for (int i = 0; i < (rand() % (numLines + 1)); i++) {
    	getline(file, line);
    }
    file.close();

	return line;
}
<commit_msg>fixed random line retrieval<commit_after>#include "Advertisement.h"

Advertisement::Advertisement(void) {
	path = "advertisement.txt";
    srand(time(NULL));
}

Advertisement::~Advertisement(void) {};

string Advertisement::getAd(void) {
	int numLines = 0;
    string randLine;
    ifstream file(path.c_str());

    if(file.is_open()) {
        //count number of lines
        while (getline(file, randLine))
            numLines++;

        //go back to begining of file
        file.clear();
        file.seekg(0, ios::beg);

        //select a random line from file
        string temp;

        int randNum = rand();
        for (int i = 0; i < randNum % numLines + 1; i++) {
        	getline(file, randLine);
        }
        file.close();
    } else
        return "Error in retrieving advertisement";
	return randLine;
}
<|endoftext|>