<commit_before>/* Copyright © 2001-2014, Canal TP and/or its affiliates. All rights reserved.

This file is part of Navitia,
    the software to build cool stuff with public transport.

Hope you'll enjoy and contribute to this project,
    powered by Canal TP (www.canaltp.fr).
Help us simplify mobility and open public transport:
    a non ending quest to the responsive locomotion way of traveling!

LICENCE: This program is free software; you can redistribute it and/or modify
it under the terms of the GNU Affero General Public License as published by
the Free Software Foundation, either version 3 of the License, or
(at your option) any later version.

This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU Affero General Public License for more details.

You should have received a copy of the GNU Affero General Public License
along with this program. If not, see <http://www.gnu.org/licenses/>.

Stay tuned using
twitter @navitia
IRC #navitia on freenode
https://groups.google.com/d/forum/navitia
www.navitia.io
*/

#include "route_schedules.h"
#include "thermometer.h"
#include "request_handle.h"
#include "type/pb_converter.h"
#include "ptreferential/ptreferential.h"
#include "utils/paginate.h"
#include "type/datetime.h"
#include "routing/best_stoptime.h"
#include <boost/range/adaptor/reversed.hpp>

namespace pt = boost::posix_time;

namespace navitia { namespace timetables {

std::vector<std::vector<datetime_stop_time> >
get_all_stop_times(const vector_idx &journey_patterns,
                   const DateTime &dateTime,
                   const DateTime &max_datetime, const type::Data &d, bool disruption_active) {
    std::vector<std::vector<datetime_stop_time> > result;

    //On cherche les premiers journey_pattern_points
    //de tous les journey_patterns
    std::vector<type::idx_t> first_journey_pattern_points;
    for(type::idx_t jp_idx : journey_patterns) {
        auto jpp = d.pt_data->journey_patterns[jp_idx];
        auto first_jpp_idx = jpp->journey_pattern_point_list.front()->idx;
        first_journey_pattern_points.push_back(first_jpp_idx);
    }

    //On fait un best_stop_time sur ces journey_pattern points
    auto first_dt_st = get_stop_times(first_journey_pattern_points,
                                      dateTime, max_datetime,
                                      std::numeric_limits<int>::max(), d, disruption_active);

    //On va chercher tous les prochains horaires
    for(auto ho : first_dt_st) {
        result.push_back(std::vector<datetime_stop_time>());
        DateTime dt = ho.first;
        for(const type::StopTime* stop_time : ho.second->vehicle_journey->stop_time_list) {
            if(!stop_time->is_frequency()) {
                DateTimeUtils::update(dt, stop_time->departure_time);
            } else {
                DateTimeUtils::update(dt, routing::f_departure_time(dt, stop_time));
            }
            result.back().push_back(std::make_pair(dt, stop_time));
        }
    }
    return result;
}

std::vector<datetime_stop_time>::const_iterator get_first_dt_st_after(
        const std::vector<datetime_stop_time>& v, size_t order) {
    if (order < v.size()) {
        return std::find_if(v.begin() + order, v.end(),
                            [](datetime_stop_time dt_st) {
                                return dt_st.second != nullptr;
                            });
    }
    return v.end();
}

std::vector<datetime_stop_time>::const_iterator get_first_dt_st_before(
        const std::vector<datetime_stop_time>& v, size_t order) {
    if (order < v.size()) {
        for (auto it = v.rbegin()+order; it != v.rend(); ++ it) {
            if (it->second != nullptr) {
                return it.base() - 1;
            }
        }
    }
    return v.end();
}

/*
 *  We want to compare the first filled dt_st of v1, named d1 with order o1
 *  with the first filled dt_st of v2 having an order >= o1
 *  If cannot find such an element in v2, we will compare with the last filled
 *  dt_st in v2.
 *  We want to compare v2, with the first st in v1 before o2.
 */
bool compare(const std::vector<datetime_stop_time>& v1,
        const std::vector<datetime_stop_time>& v2) {
    if (v1.empty()) {
        return false;
    }
    if (v2.empty()) {
        return true;
    }
    auto first_st_a_it = get_first_dt_st_after(v1, 0);
    if (first_st_a_it == v1.end()) {
        return false;
    }
    auto first_st_b_it = v2.end();
    size_t order = std::distance(first_st_a_it, v1.begin());
    if (order < v2.size()) {
        first_st_b_it = get_first_dt_st_after(v2, order);
    }
    if (first_st_b_it == v2.end()) {
        first_st_b_it = get_first_dt_st_before(v2, 0);
        if (first_st_b_it == v2.end()) {
            return true;
        }
    }
    auto r_first_st_b_it = std::reverse_iterator<std::vector<datetime_stop_time>::const_iterator>(first_st_b_it);
    size_t distance = std::distance(v2.rbegin(), r_first_st_b_it);
    // Even if r_first_st_b_it == v2.rbegin(), distance will be 1... So we need to decrease it.
    --distance;
    first_st_a_it = get_first_dt_st_before(v1, distance);
    return first_st_a_it->first < first_st_b_it->first;
}

std::vector<std::vector<datetime_stop_time> >
make_matrice(const std::vector<std::vector<datetime_stop_time> >& stop_times,
             const Thermometer &thermometer, const type::Data &) {
    // result group stop_times by stop_point, tmp by vj.
    std::vector<std::vector<datetime_stop_time> > result, tmp;
    const size_t thermometer_size = thermometer.get_thermometer().size();
    for (size_t i=0; i < stop_times.size(); ++i) {
        tmp.push_back(std::vector<datetime_stop_time>(thermometer_size));
    }

    // We match every stop_time with the journey pattern
    int y=0;
    for(const std::vector<datetime_stop_time>& vec : stop_times) {
        auto journey_pattern = vec.front().second->vehicle_journey->journey_pattern;
        std::vector<uint32_t> orders = thermometer.match_journey_pattern(*journey_pattern);
        int order = 0;
        for(datetime_stop_time dt_stop_time : vec) {
            tmp[y][orders[order]] = dt_stop_time;
            ++order;
        }
        ++y;
    }

    std::sort(tmp.begin(), tmp.end(), compare);

    for(unsigned int i=0; i<thermometer_size; ++i) {
        result.push_back(std::vector<datetime_stop_time>());
        result.back().resize(stop_times.size());
    }
    // We rotate the matrice, so it can be handle more easily in route_schedule
    for (size_t i=0; i<tmp.size(); ++i) {
        for (size_t j=0; j<tmp[i].size(); ++j) {
            result[j][i] = tmp[i][j];
        }
    }
    return result;
}

pbnavitia::Response
route_schedule(const std::string& filter,
               const std::vector<std::string>& forbidden_uris,
               const pt::ptime datetime,
               uint32_t duration, uint32_t interface_version,
               const uint32_t max_depth, int count, int start_page,
               const type::Data &d, bool disruption_active, const bool show_codes) {
    RequestHandle handler("ROUTE_SCHEDULE", filter, forbidden_uris, datetime, duration, d, {});

    if(handler.pb_response.has_error()) {
        return handler.pb_response;
    }
    auto now = pt::second_clock::local_time();
    auto pt_datetime = to_posix_time(handler.date_time, d);
    auto pt_max_datetime = to_posix_time(handler.max_datetime, d);
    pt::time_period action_period(pt_datetime, pt_max_datetime);
    Thermometer thermometer;
    auto routes_idx = ptref::make_query(type::Type_e::Route, filter, forbidden_uris, d);
    size_t total_result = routes_idx.size();
    routes_idx = paginate(routes_idx, count, start_page);
    for(type::idx_t route_idx : routes_idx) {
        auto route = d.pt_data->routes[route_idx];
        auto jps =  ptref::make_query(type::Type_e::JourneyPattern, filter+" and route.uri="+route->uri, forbidden_uris, d);
        //On récupère les stop_times
        auto stop_times = get_all_stop_times(jps, handler.date_time,
                                             handler.max_datetime, d, disruption_active);
        std::vector<vector_idx> stop_points;
        for(auto jp_idx : jps) {
            auto jp = d.pt_data->journey_patterns[jp_idx];
            stop_points.push_back(vector_idx());
            for(auto jpp : jp->journey_pattern_point_list) {
                stop_points.back().push_back(jpp->stop_point->idx);
            }
        }
        thermometer.generate_thermometer(stop_points);
        //On génère la matrice
        auto  matrice = make_matrice(stop_times, thermometer, d);
        auto schedule = handler.pb_response.add_route_schedules();
        pbnavitia::Table *table = schedule->mutable_table();
        auto m_pt_display_informations = schedule->mutable_pt_display_informations();
        fill_pb_object(route, d, m_pt_display_informations, 0, now, action_period);

        std::vector<bool> is_vj_set(stop_times.size(), false);
        for(unsigned int i=0; i < thermometer.get_thermometer().size(); ++i) {
            type::idx_t spidx=thermometer.get_thermometer()[i];
            const type::StopPoint* sp = d.pt_data->stop_points[spidx];
            //version v1
            pbnavitia::RouteScheduleRow* row = table->add_rows();
            fill_pb_object(sp, d, row->mutable_stop_point(), max_depth,
                           now, action_period, show_codes);
            for(unsigned int j=0; j<stop_times.size(); ++j) {
                datetime_stop_time dt_stop_time  = matrice[i][j];
                if (!is_vj_set[j] && dt_stop_time.second != nullptr) {
                    pbnavitia::Header* header = table->add_headers();
                    pbnavitia::PtDisplayInfo* vj_display_information = header->mutable_pt_display_informations();
                    pbnavitia::addInfoVehicleJourney* add_info_vehicle_journey = header->mutable_add_info_vehicle_journey();
                    auto vj = dt_stop_time.second->vehicle_journey;
                    fill_pb_object(vj, d, vj_display_information, 0, now, action_period);
                    fill_pb_object(vj, d, {}, add_info_vehicle_journey, 0, now, action_period);
                    is_vj_set[j] = true;
                }
                if(interface_version == 1) {
                    auto pb_dt = row->add_date_times();
                    fill_pb_object(dt_stop_time.second, d, pb_dt, max_depth,
                                   now, action_period, dt_stop_time.first);
                } else if(interface_version == 0) {
                    row->add_stop_times(navitia::iso_string(dt_stop_time.first, d));
                }
            }
        }
        fill_pb_object(route->shape, schedule->mutable_geojson());
    }
    auto pagination = handler.pb_response.mutable_pagination();
    pagination->set_totalresult(total_result);
    pagination->set_startpage(start_page);
    pagination->set_itemsperpage(count);
    pagination->set_itemsonpage(handler.pb_response.route_schedules_size());
    return handler.pb_response;
}
}}
<commit_msg>Kraken: better init of vector<commit_after>/* Copyright © 2001-2014, Canal TP and/or its affiliates. All rights reserved.

This file is part of Navitia,
    the software to build cool stuff with public transport.

Hope you'll enjoy and contribute to this project,
    powered by Canal TP (www.canaltp.fr).
Help us simplify mobility and open public transport:
    a non ending quest to the responsive locomotion way of traveling!

LICENCE: This program is free software; you can redistribute it and/or modify
it under the terms of the GNU Affero General Public License as published by
the Free Software Foundation, either version 3 of the License, or
(at your option) any later version.

This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU Affero General Public License for more details.

You should have received a copy of the GNU Affero General Public License
along with this program. If not, see <http://www.gnu.org/licenses/>.

Stay tuned using
twitter @navitia
IRC #navitia on freenode
https://groups.google.com/d/forum/navitia
www.navitia.io
*/

#include "route_schedules.h"
#include "thermometer.h"
#include "request_handle.h"
#include "type/pb_converter.h"
#include "ptreferential/ptreferential.h"
#include "utils/paginate.h"
#include "type/datetime.h"
#include "routing/best_stoptime.h"
#include <boost/range/adaptor/reversed.hpp>

namespace pt = boost::posix_time;

namespace navitia { namespace timetables {

std::vector<std::vector<datetime_stop_time> >
get_all_stop_times(const vector_idx &journey_patterns,
                   const DateTime &dateTime,
                   const DateTime &max_datetime, const type::Data &d, bool disruption_active) {
    std::vector<std::vector<datetime_stop_time> > result;

    //On cherche les premiers journey_pattern_points
    //de tous les journey_patterns
    std::vector<type::idx_t> first_journey_pattern_points;
    for(type::idx_t jp_idx : journey_patterns) {
        auto jpp = d.pt_data->journey_patterns[jp_idx];
        auto first_jpp_idx = jpp->journey_pattern_point_list.front()->idx;
        first_journey_pattern_points.push_back(first_jpp_idx);
    }

    //On fait un best_stop_time sur ces journey_pattern points
    auto first_dt_st = get_stop_times(first_journey_pattern_points,
                                      dateTime, max_datetime,
                                      std::numeric_limits<int>::max(), d, disruption_active);

    //On va chercher tous les prochains horaires
    for(auto ho : first_dt_st) {
        result.push_back(std::vector<datetime_stop_time>());
        DateTime dt = ho.first;
        for(const type::StopTime* stop_time : ho.second->vehicle_journey->stop_time_list) {
            if(!stop_time->is_frequency()) {
                DateTimeUtils::update(dt, stop_time->departure_time);
            } else {
                DateTimeUtils::update(dt, routing::f_departure_time(dt, stop_time));
            }
            result.back().push_back(std::make_pair(dt, stop_time));
        }
    }
    return result;
}

std::vector<datetime_stop_time>::const_iterator get_first_dt_st_after(
        const std::vector<datetime_stop_time>& v, size_t order) {
    if (order < v.size()) {
        return std::find_if(v.begin() + order, v.end(),
                            [](datetime_stop_time dt_st) {
                                return dt_st.second != nullptr;
                            });
    }
    return v.end();
}

std::vector<datetime_stop_time>::const_iterator get_first_dt_st_before(
        const std::vector<datetime_stop_time>& v, size_t order) {
    if (order < v.size()) {
        for (auto it = v.rbegin()+order; it != v.rend(); ++ it) {
            if (it->second != nullptr) {
                return it.base() - 1;
            }
        }
    }
    return v.end();
}

/*
 *  We want to compare the first filled dt_st of v1, named d1 with order o1
 *  with the first filled dt_st of v2 having an order >= o1
 *  If cannot find such an element in v2, we will compare with the last filled
 *  dt_st in v2.
 *  We want to compare v2, with the first st in v1 before o2.
 */
bool compare(const std::vector<datetime_stop_time>& v1,
        const std::vector<datetime_stop_time>& v2) {
    if (v1.empty()) {
        return false;
    }
    if (v2.empty()) {
        return true;
    }
    auto first_st_a_it = get_first_dt_st_after(v1, 0);
    if (first_st_a_it == v1.end()) {
        return false;
    }
    auto first_st_b_it = v2.end();
    size_t order = std::distance(first_st_a_it, v1.begin());
    if (order < v2.size()) {
        first_st_b_it = get_first_dt_st_after(v2, order);
    }
    if (first_st_b_it == v2.end()) {
        first_st_b_it = get_first_dt_st_before(v2, 0);
        if (first_st_b_it == v2.end()) {
            return true;
        }
    }
    auto r_first_st_b_it = std::reverse_iterator<std::vector<datetime_stop_time>::const_iterator>(first_st_b_it);
    size_t distance = std::distance(v2.rbegin(), r_first_st_b_it);
    // Even if r_first_st_b_it == v2.rbegin(), distance will be 1... So we need to decrease it.
    --distance;
    first_st_a_it = get_first_dt_st_before(v1, distance);
    return first_st_a_it->first < first_st_b_it->first;
}

std::vector<std::vector<datetime_stop_time> >
make_matrice(const std::vector<std::vector<datetime_stop_time> >& stop_times,
             const Thermometer &thermometer, const type::Data &) {
    // result group stop_times by stop_point, tmp by vj.
    const size_t thermometer_size = thermometer.get_thermometer().size();
    std::vector<std::vector<datetime_stop_time> >
        result(thermometer_size, std::vector<datetime_stop_time>(stop_times.size())),
        tmp(stop_times.size(), std::vector<datetime_stop_time>(thermometer_size));
    // We match every stop_time with the journey pattern
    int y=0;
    for(const std::vector<datetime_stop_time>& vec : stop_times) {
        auto journey_pattern = vec.front().second->vehicle_journey->journey_pattern;
        std::vector<uint32_t> orders = thermometer.match_journey_pattern(*journey_pattern);
        int order = 0;
        for(datetime_stop_time dt_stop_time : vec) {
            tmp[y][orders[order]] = dt_stop_time;
            ++order;
        }
        ++y;
    }

    std::sort(tmp.begin(), tmp.end(), compare);
    // We rotate the matrice, so it can be handle more easily in route_schedule
    for (size_t i=0; i<tmp.size(); ++i) {
        for (size_t j=0; j<tmp[i].size(); ++j) {
            result[j][i] = tmp[i][j];
        }
    }
    return result;
}

pbnavitia::Response
route_schedule(const std::string& filter,
               const std::vector<std::string>& forbidden_uris,
               const pt::ptime datetime,
               uint32_t duration, uint32_t interface_version,
               const uint32_t max_depth, int count, int start_page,
               const type::Data &d, bool disruption_active, const bool show_codes) {
    RequestHandle handler("ROUTE_SCHEDULE", filter, forbidden_uris, datetime, duration, d, {});

    if(handler.pb_response.has_error()) {
        return handler.pb_response;
    }
    auto now = pt::second_clock::local_time();
    auto pt_datetime = to_posix_time(handler.date_time, d);
    auto pt_max_datetime = to_posix_time(handler.max_datetime, d);
    pt::time_period action_period(pt_datetime, pt_max_datetime);
    Thermometer thermometer;
    auto routes_idx = ptref::make_query(type::Type_e::Route, filter, forbidden_uris, d);
    size_t total_result = routes_idx.size();
    routes_idx = paginate(routes_idx, count, start_page);
    for(type::idx_t route_idx : routes_idx) {
        auto route = d.pt_data->routes[route_idx];
        auto jps =  ptref::make_query(type::Type_e::JourneyPattern, filter+" and route.uri="+route->uri, forbidden_uris, d);
        //On récupère les stop_times
        auto stop_times = get_all_stop_times(jps, handler.date_time,
                                             handler.max_datetime, d, disruption_active);
        std::vector<vector_idx> stop_points;
        for(auto jp_idx : jps) {
            auto jp = d.pt_data->journey_patterns[jp_idx];
            stop_points.push_back(vector_idx());
            for(auto jpp : jp->journey_pattern_point_list) {
                stop_points.back().push_back(jpp->stop_point->idx);
            }
        }
        thermometer.generate_thermometer(stop_points);
        //On génère la matrice
        auto  matrice = make_matrice(stop_times, thermometer, d);
        auto schedule = handler.pb_response.add_route_schedules();
        pbnavitia::Table *table = schedule->mutable_table();
        auto m_pt_display_informations = schedule->mutable_pt_display_informations();
        fill_pb_object(route, d, m_pt_display_informations, 0, now, action_period);

        std::vector<bool> is_vj_set(stop_times.size(), false);
        for(unsigned int i=0; i < thermometer.get_thermometer().size(); ++i) {
            type::idx_t spidx=thermometer.get_thermometer()[i];
            const type::StopPoint* sp = d.pt_data->stop_points[spidx];
            //version v1
            pbnavitia::RouteScheduleRow* row = table->add_rows();
            fill_pb_object(sp, d, row->mutable_stop_point(), max_depth,
                           now, action_period, show_codes);
            for(unsigned int j=0; j<stop_times.size(); ++j) {
                datetime_stop_time dt_stop_time  = matrice[i][j];
                if (!is_vj_set[j] && dt_stop_time.second != nullptr) {
                    pbnavitia::Header* header = table->add_headers();
                    pbnavitia::PtDisplayInfo* vj_display_information = header->mutable_pt_display_informations();
                    pbnavitia::addInfoVehicleJourney* add_info_vehicle_journey = header->mutable_add_info_vehicle_journey();
                    auto vj = dt_stop_time.second->vehicle_journey;
                    fill_pb_object(vj, d, vj_display_information, 0, now, action_period);
                    fill_pb_object(vj, d, {}, add_info_vehicle_journey, 0, now, action_period);
                    is_vj_set[j] = true;
                }
                if(interface_version == 1) {
                    auto pb_dt = row->add_date_times();
                    fill_pb_object(dt_stop_time.second, d, pb_dt, max_depth,
                                   now, action_period, dt_stop_time.first);
                } else if(interface_version == 0) {
                    row->add_stop_times(navitia::iso_string(dt_stop_time.first, d));
                }
            }
        }
        fill_pb_object(route->shape, schedule->mutable_geojson());
    }
    auto pagination = handler.pb_response.mutable_pagination();
    pagination->set_totalresult(total_result);
    pagination->set_startpage(start_page);
    pagination->set_itemsperpage(count);
    pagination->set_itemsonpage(handler.pb_response.route_schedules_size());
    return handler.pb_response;
}
}}
<|endoftext|>


<commit_before><commit_msg>dtc: fix bootstrapping from 8.2-STABLE<commit_after><|endoftext|>


<commit_before>#include <Salamandre-daemon/Daemon.hpp>
#include <Salamandre-daemon/ServerBroadcast.hpp>

#include <utils/log.hpp>

#include <iostream>
#include <csignal>

#define GUI_PORT 1
#define SERVER_PORT 2
#define LOCAL_ONLY 3
#define NB_ARGS (LOCAL_ONLY + 1)

namespace salamandre
{
    Daemon* daemon = nullptr;
}

void stop_server_handler(int sig)
{
    utils::log::info("Stop","Recv signal",sig,"Stoping server.\n Please wait.");
    if(salamandre::daemon)
        salamandre::daemon->stop();
}

int main(int argc,char* argv[])
{
    int gui_port = 3842;
    int server_port = 3843;
    int broadcast_port = 5001;
    bool local_only = false;

    if(argc < NB_ARGS)
    {
        utils::log::error("Usage",argv[0],"<gui port> <server port> <local only>");
        if(argc <= GUI_PORT)
            utils::log::info("Useage",gui_port,"is used as default gui port");
        if(argc <= SERVER_PORT)
            utils::log::info("Usage",server_port,"is use as default server port");
        if(argc <= LOCAL_ONLY)
            utils::log::info("Useage",local_only,"is use as default local only value");
    }
    if(argc > GUI_PORT)
        gui_port = atoi(argv[GUI_PORT]);
    if(argc > SERVER_PORT)
        server_port = atoi(argv[SERVER_PORT]);
    if(argc > LOCAL_ONLY)
        local_only = atoi(argv[LOCAL_ONLY]);

    std::cout<<"Daemon start on:"
        <<"\n\tgui listen on port : "<<gui_port
        <<"\n\tfile server listen on port : "<<server_port
        <<"\n\tbrodcast server listen on port: "<<broadcast_port
        <<"\n\tlocal only : "<<local_only
        <<"\nPress ^C to exit"
        <<std::endl<<std::endl;
    std::signal(SIGINT, stop_server_handler);

    try
    {
        salamandre::Daemon::init();

        salamandre::daemon = new salamandre::Daemon(gui_port,server_port,broadcast_port,local_only);
        salamandre::daemon->start();
        salamandre::daemon->wait();

        delete salamandre::daemon;
        salamandre::daemon = nullptr;

        salamandre::Daemon::close();

    }
    catch(ntw::SocketExeption& e)
    {
        utils::log::error("Socket error",e.what());
    }

    utils::log::info("End","Daemon is now close, Good bye");

    return 0;
}
<commit_msg>[daemon] Use TCLAP to parse command line arguments<commit_after>#include <Salamandre-daemon/Daemon.hpp>
#include <Salamandre-daemon/ServerBroadcast.hpp>

#include <utils/log.hpp>

#include <iostream>
#include <csignal>
#include <tclap/CmdLine.h>

#define GUI_PORT 1
#define SERVER_PORT 2
#define LOCAL_ONLY 3
#define NB_ARGS (LOCAL_ONLY + 1)

namespace salamandre
{
    Daemon* daemon = nullptr;
}

void stop_server_handler(int sig)
{
    utils::log::info("Stop","Recv signal",sig,"Stoping server.\n Please wait.");
    if(salamandre::daemon)
        salamandre::daemon->stop();
}

int main(int argc,char* argv[])
{
    int gui_port = 3842;
    int server_port = 3843;
    int broadcast_port = 5001;
    bool local_only = false;
    try {
    TCLAP::CmdLine cmd("Salamandre Daemon", ' ', "0.1");
    TCLAP::ValueArg<int> gui_port_arg("g", "gui-port", "Port for the GUI", false, gui_port, "int", cmd);
    TCLAP::ValueArg<int> server_port_arg("s", "server-port", "Port for the server to listen", false, server_port, "int", cmd);
    TCLAP::SwitchArg local_switch("l", "local", "Whether daemon sould run stricly locally", cmd, false);

    cmd.parse(argc, argv);

    local_only = local_switch.getValue();
    gui_port = gui_port_arg.getValue();
    server_port = server_port_arg.getValue();

    } catch(TCLAP::ArgException &e) {
        std::cerr << "error: " << e.error() << " for arg " << e.argId() << std::endl;
        exit(1);
    }

    std::cout<<"Daemon start on:"
        <<"\n\tgui listen on port : "<<gui_port
        <<"\n\tfile server listen on port : "<<server_port
        <<"\n\tbrodcast server listen on port: "<<broadcast_port
        <<"\n\tlocal only : "<<local_only
        <<"\nPress ^C to exit"
        <<std::endl<<std::endl;
    std::signal(SIGINT, stop_server_handler);

    try
    {
        salamandre::Daemon::init();

        salamandre::daemon = new salamandre::Daemon(gui_port,server_port,broadcast_port,local_only);
        salamandre::daemon->start();
        salamandre::daemon->wait();

        delete salamandre::daemon;
        salamandre::daemon = nullptr;

        salamandre::Daemon::close();

    }
    catch(ntw::SocketExeption& e)
    {
        utils::log::error("Socket error",e.what());
    }

    utils::log::info("End","Daemon is now close, Good bye");

    return 0;
}
<|endoftext|>


<commit_before>// -*- mode:c++; tab-width:2; indent-tabs-mode:nil; c-basic-offset:2 -*-
/*
 *  Copyright 2010 ZXing authors. All rights reserved.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *      http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

#include <zxing/common/BitArray.h>
#include <zxing/common/Array.h>
#include <cstring>

using std::vector;
using zxing::BitArray;

// VC++
using zxing::Ref;

int BitArray::makeArraySize(int size) {
    return (size + bitsPerWord-1) >> logBits;
}

BitArray::BitArray(): size(0), bits(1) {}

BitArray::BitArray(int size_)
    : size(size_), bits(makeArraySize(size)) {}

BitArray::~BitArray() {
}

int BitArray::getSize() const {
    return size;
}

void BitArray::setBulk(int i, int newBits) {
    bits[i >> logBits] = newBits;
}

void BitArray::clear() {
    int max = bits->size();
    for (int i = 0; i < max; i++) {
        bits[i] = 0;
    }
}

bool BitArray::isRange(int start, int end, bool value) {
    if (end < start) {
        throw IllegalArgumentException();
    }
    if (end == start) {
        return true; // empty range matches
    }
    end--; // will be easier to treat this as the last actually set bit -- inclusive
    int firstInt = start >> logBits;
    int lastInt = end >> logBits;
    for (int i = firstInt; i <= lastInt; i++) {
        int firstBit = i > firstInt ? 0 : start & bitsMask;
        int lastBit = i < lastInt ? (bitsPerWord-1) : end & bitsMask;
        int mask;
        if (firstBit == 0 && lastBit == (bitsPerWord-1)) {
            mask = -1;
        } else {
            mask = 0;
            for (int j = firstBit; j <= lastBit; j++) {
                mask |= 1 << j;
            }
        }

        // Return false if we're looking for 1s and the masked bits[i] isn't all 1s (that is,
        // equals the mask, or we're looking for 0s and the masked portion is not all 0s
        if ((bits[i] & mask) != (value ? mask : 0)) {
            return false;
        }
    }
    return true;
}

vector<int>& BitArray::getBitArray() {
    return bits->values();
}

void BitArray::reverse() {
    ArrayRef<int> newBits(bits->size());
    int size = this->size;
    for (int i = 0; i < size; i++) {
        if (get(size - i - 1)) {
            newBits[i >> logBits] |= 1 << (i & bitsMask);
        }
    }
    bits = newBits;
}

BitArray::Reverse::Reverse(Ref<BitArray> array_) : array(array_) {
    array->reverse();
}

BitArray::Reverse::~Reverse() {
    array->reverse();
}

namespace {
// N.B.: This only works for 32 bit ints ...
int numberOfTrailingZeros(int i) {
    // HD, Figure 5-14
    int y;
    if (i == 0) return 32;
    int n = 31;
    y = i <<16; if (y != 0) { n = n -16; i = y; }
    y = i << 8; if (y != 0) { n = n - 8; i = y; }
    y = i << 4; if (y != 0) { n = n - 4; i = y; }
    y = i << 2; if (y != 0) { n = n - 2; i = y; }
    return n - (((unsigned int)(i << 1)) >> 31);
}
}

int BitArray::getNextSet(int from) {
    if (from >= size) {
        return size;
    }
    int bitsOffset = from >> logBits;
    int currentBits = bits[bitsOffset];
    // mask off lesser bits first
    currentBits &= ~((1 << (from & bitsMask)) - 1);
    while (currentBits == 0) {
        if (++bitsOffset == (int)bits->size()) {
            return size;
        }
        currentBits = bits[bitsOffset];
    }
    int result = (bitsOffset << logBits) + numberOfTrailingZeros(currentBits);
    return result > size ? size : result;
}

int BitArray::getNextUnset(int from) {
    if (from >= size) {
        return size;
    }
    int bitsOffset = from >> logBits;
    int currentBits = ~bits[bitsOffset];
    // mask off lesser bits first
    currentBits &= ~((1 << (from & bitsMask)) - 1);
    while (currentBits == 0) {
        if (++bitsOffset == (int)bits->size()) {
            return size;
        }
        currentBits = ~bits[bitsOffset];
    }
    int result = (bitsOffset << logBits) + numberOfTrailingZeros(currentBits);
    return result > size ? size : result;
}

void BitArray::appendBit(bool bit)
{
    ensureCapacity(size + 1);
    if (bit) {
        bits[size / 32] |= 1 << (size & 0x1F);
    }
    size++;
}

void BitArray::appendBits(int value, int numBits)
{
    if (numBits < 0 || numBits > 32) {
        throw IllegalArgumentException("Num bits must be between 0 and 32");
    }
    ensureCapacity(size + numBits);
    for (int numBitsLeft = numBits; numBitsLeft > 0; numBitsLeft--) {
        appendBit(((value >> (numBitsLeft - 1)) & 0x01) == 1);
    }
}

void BitArray::ensureCapacity(int size)
{
    if (size > bits->size() * 32)
    {
        ArrayRef<int> newBits = makeArray(size);
        //memcpy(bits, newBits->, bits->size());
        for (size_t i=0; i<bits->size(); ++i) {
            newBits[i] = bits[i];
        }
        bits = newBits;
    }
}

void BitArray::xor_(const BitArray& other)
{
    if (bits->size() != other.bits->size()) {
        throw IllegalArgumentException("Sizes don't match");
    }
    for (int i = 0; i < bits->size(); i++) {
        // The last byte could be incomplete (i.e. not have 8 bits in
        // it) but there is no problem since 0 XOR 0 == 0.
        bits[i] ^= other.bits[i];
    }
}

void BitArray::toBytes(int bitOffset, std::vector<char>& array, int offset, int numBytes)
{
    if(array.size() < numBytes)
        array.resize(numBytes);

    for (int i = 0; i < numBytes; i++) {
        int theByte = 0;
        for (int j = 0; j < 8; j++) {
            if (get(bitOffset)) {
                theByte |= 1 << (7 - j);
            }
            bitOffset++;
        }
        array[offset + i] = (char) theByte;
    }
}
<commit_msg>fix to the toBytes at initial check.<commit_after>// -*- mode:c++; tab-width:2; indent-tabs-mode:nil; c-basic-offset:2 -*-
/*
 *  Copyright 2010 ZXing authors. All rights reserved.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *      http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

#include <zxing/common/BitArray.h>
#include <zxing/common/Array.h>
#include <cstring>

using std::vector;
using zxing::BitArray;

// VC++
using zxing::Ref;

int BitArray::makeArraySize(int size) {
    return (size + bitsPerWord-1) >> logBits;
}

BitArray::BitArray(): size(0), bits(1) {}

BitArray::BitArray(int size_)
    : size(size_), bits(makeArraySize(size)) {}

BitArray::~BitArray() {
}

int BitArray::getSize() const {
    return size;
}

void BitArray::setBulk(int i, int newBits) {
    bits[i >> logBits] = newBits;
}

void BitArray::clear() {
    int max = bits->size();
    for (int i = 0; i < max; i++) {
        bits[i] = 0;
    }
}

bool BitArray::isRange(int start, int end, bool value) {
    if (end < start) {
        throw IllegalArgumentException();
    }
    if (end == start) {
        return true; // empty range matches
    }
    end--; // will be easier to treat this as the last actually set bit -- inclusive
    int firstInt = start >> logBits;
    int lastInt = end >> logBits;
    for (int i = firstInt; i <= lastInt; i++) {
        int firstBit = i > firstInt ? 0 : start & bitsMask;
        int lastBit = i < lastInt ? (bitsPerWord-1) : end & bitsMask;
        int mask;
        if (firstBit == 0 && lastBit == (bitsPerWord-1)) {
            mask = -1;
        } else {
            mask = 0;
            for (int j = firstBit; j <= lastBit; j++) {
                mask |= 1 << j;
            }
        }

        // Return false if we're looking for 1s and the masked bits[i] isn't all 1s (that is,
        // equals the mask, or we're looking for 0s and the masked portion is not all 0s
        if ((bits[i] & mask) != (value ? mask : 0)) {
            return false;
        }
    }
    return true;
}

vector<int>& BitArray::getBitArray() {
    return bits->values();
}

void BitArray::reverse() {
    ArrayRef<int> newBits(bits->size());
    int size = this->size;
    for (int i = 0; i < size; i++) {
        if (get(size - i - 1)) {
            newBits[i >> logBits] |= 1 << (i & bitsMask);
        }
    }
    bits = newBits;
}

BitArray::Reverse::Reverse(Ref<BitArray> array_) : array(array_) {
    array->reverse();
}

BitArray::Reverse::~Reverse() {
    array->reverse();
}

namespace {
// N.B.: This only works for 32 bit ints ...
int numberOfTrailingZeros(int i) {
    // HD, Figure 5-14
    int y;
    if (i == 0) return 32;
    int n = 31;
    y = i <<16; if (y != 0) { n = n -16; i = y; }
    y = i << 8; if (y != 0) { n = n - 8; i = y; }
    y = i << 4; if (y != 0) { n = n - 4; i = y; }
    y = i << 2; if (y != 0) { n = n - 2; i = y; }
    return n - (((unsigned int)(i << 1)) >> 31);
}
}

int BitArray::getNextSet(int from) {
    if (from >= size) {
        return size;
    }
    int bitsOffset = from >> logBits;
    int currentBits = bits[bitsOffset];
    // mask off lesser bits first
    currentBits &= ~((1 << (from & bitsMask)) - 1);
    while (currentBits == 0) {
        if (++bitsOffset == (int)bits->size()) {
            return size;
        }
        currentBits = bits[bitsOffset];
    }
    int result = (bitsOffset << logBits) + numberOfTrailingZeros(currentBits);
    return result > size ? size : result;
}

int BitArray::getNextUnset(int from) {
    if (from >= size) {
        return size;
    }
    int bitsOffset = from >> logBits;
    int currentBits = ~bits[bitsOffset];
    // mask off lesser bits first
    currentBits &= ~((1 << (from & bitsMask)) - 1);
    while (currentBits == 0) {
        if (++bitsOffset == (int)bits->size()) {
            return size;
        }
        currentBits = ~bits[bitsOffset];
    }
    int result = (bitsOffset << logBits) + numberOfTrailingZeros(currentBits);
    return result > size ? size : result;
}

void BitArray::appendBit(bool bit)
{
    ensureCapacity(size + 1);
    if (bit) {
        bits[size / 32] |= 1 << (size & 0x1F);
    }
    size++;
}

void BitArray::appendBits(int value, int numBits)
{
    if (numBits < 0 || numBits > 32) {
        throw IllegalArgumentException("Num bits must be between 0 and 32");
    }
    ensureCapacity(size + numBits);
    for (int numBitsLeft = numBits; numBitsLeft > 0; numBitsLeft--) {
        appendBit(((value >> (numBitsLeft - 1)) & 0x01) == 1);
    }
}

void BitArray::ensureCapacity(int size)
{
    if (size > bits->size() * 32)
    {
        ArrayRef<int> newBits = makeArray(size);
        //memcpy(bits, newBits->, bits->size());
        for (size_t i=0; i<bits->size(); ++i) {
            newBits[i] = bits[i];
        }
        bits = newBits;
    }
}

void BitArray::xor_(const BitArray& other)
{
    if (bits->size() != other.bits->size()) {
        throw IllegalArgumentException("Sizes don't match");
    }
    for (int i = 0; i < bits->size(); i++) {
        // The last byte could be incomplete (i.e. not have 8 bits in
        // it) but there is no problem since 0 XOR 0 == 0.
        bits[i] ^= other.bits[i];
    }
}

void BitArray::toBytes(int bitOffset, std::vector<char>& array, int offset, int numBytes)
{
    if(array.size() < (numBytes + offset))
        array.resize(numBytes + offset);

    for (int i = 0; i < numBytes; i++) {
        int theByte = 0;
        for (int j = 0; j < 8; j++) {
            if (get(bitOffset)) {
                theByte |= 1 << (7 - j);
            }
            bitOffset++;
        }
        array[offset + i] = (char) theByte;
    }
}
<|endoftext|>


<commit_before>#include "Table.hpp"
#include "Record.hpp"
#include "Page.hpp"
#include "InsertMap.hpp"

#include <memory>

namespace tell {
namespace store {
namespace deltamain {

Table::Iterator::Iterator(
        const std::shared_ptr<allocator>& alloc,
        const PageList* pages,
        size_t pageIdx,
        const LogIterator& logIter,
        const LogIterator& logEnd,
        PageManager* pageManager,
        const Record* record)
    : mAllocator(alloc)
    , pages(pages)
    , pageIdx(pageIdx)
    , logIter(logIter)
    , logEnd(logEnd)
    , pageManager(pageManager)
    , record(record)
    , pageIter(Page(*pageManager, (*pages)[pageIdx]).begin())
    , pageEnd (Page(*pageManager, (*pages)[pageIdx]).end())
{
    setCurrentEntry();
}

Table::Iterator::Iterator(const Iterator& other)
    : mAllocator(other.mAllocator)
    , pages(other.pages)
    , pageIdx(other.pageIdx)
    , logIter(other.logIter)
    , logEnd(other.logEnd)
    , pageManager(other.pageManager)
    , record(other.record)
    , pageIter(other.pageIter)
    , pageEnd(other.pageEnd)
{
    setCurrentEntry();
}

auto Table::Iterator::operator=(const Iterator& o) -> Iterator&
{
    mAllocator = o.mAllocator;
    pages = o.pages;
    pageIdx = o.pageIdx;
    logIter = o.logIter;
    logEnd = o.logEnd;
    pageManager = o.pageManager;
    record = o.record;
    pageIter = o.pageIter;
    pageEnd = o.pageEnd;
    setCurrentEntry();
    return *this;
}

bool Table::Iterator::operator==(const Iterator& o) const
{
    return record == o.record &&
        pages == o.pages &&
        pageIdx == o.pageIdx &&
        logIter == o.logIter &&
        logEnd == o.logEnd &&
        pageIter == o.pageIter &&
        pageEnd == o.pageEnd &&
        currVersionIter.isValid() == currVersionIter.isValid();
}

auto Table::Iterator::operator++() -> Iterator&
{
    if (currVersionIter.isValid() && (++currVersionIter).isValid()) {
        ++currVersionIter;
        return *this;
    }
    if (logIter != logEnd) ++logIter;
    else if (pageIter != pageEnd) {
        ++pageIter;
    } else {
        if (++pageIdx == pages->size()) return *this;
        Page p(*pageManager, (*pages)[pageIdx]);
        pageIter = p.begin();
        pageEnd = p.end();
    }
    setCurrentEntry();
    return *this;
}

auto Table::Iterator::operator++(int) -> Iterator
{
    auto res = *this;
    ++res;
    return res;
}

const Table::Iterator::IteratorEntry& Table::Iterator::operator*() const {
    return *currVersionIter;
}

const Table::Iterator::IteratorEntry* Table::Iterator::operator->() const
{
    return &(this->operator*());
}

void Table::Iterator::setCurrentEntry()
{
    while (logIter != logEnd) {
        if (logIter->sealed()) {
            CDMRecord rec(logIter->data());
            if (rec.isValidDataRecord()) {
                currVersionIter = rec.getVersionIterator(record);
                return;
            }
        }
        ++logIter;
    }
    while (true) {
        while (pageIter != pageEnd) {
            CDMRecord rec(*pageIter);
            if (rec.isValidDataRecord()) {
                currVersionIter = rec.getVersionIterator(record);
                return;
            }
            ++pageIter;
        }
        ++pageIdx;
        if (pageIdx == pages->size()) return;
        Page p(*pageManager, (*pages)[pageIdx]);
        pageIter = p.begin();
        pageEnd = p.end();
    }
}

Table::Table(PageManager& pageManager, const Schema& schema, uint64_t /* idx */)
    : mPageManager(pageManager)
    , mSchema(schema)
    , mRecord(schema)
    , mHashTable(new (allocator::malloc(sizeof(CuckooTable))) CuckooTable(pageManager))
    , mInsertLog(pageManager)
    , mUpdateLog(pageManager)
    , mPages(new (allocator::malloc(sizeof(std::vector<char*>))) std::vector<char*>())
{}

bool Table::get(uint64_t key,
                size_t& size,
                const char*& data,
                const SnapshotDescriptor& snapshot,
                bool& isNewest) const {
    auto ptr = mHashTable.load()->get(key);
    if (ptr) {
        CDMRecord rec(reinterpret_cast<char*>(ptr));
        bool wasDeleted;
        bool isValid;
        data = rec.data(snapshot, size, isNewest, isValid, &wasDeleted);
        // if the newest version is a delete, it might be that there is
        // a new insert in the insert log
        if (isValid && !(wasDeleted && isNewest)) {
            return !wasDeleted;
        }
    }
    // in this case we need to scan through the insert log
    auto iterEnd = mInsertLog.end();
    for (auto iter = mInsertLog.begin(); iter != iterEnd; ++iter) {
        if (!iter->sealed()) continue;
        CDMRecord rec(iter->data());
        if (rec.isValidDataRecord() && rec.key() == key) {
            bool wasDeleted;
            bool isValid;
            data = rec.data(snapshot, size, isNewest, isValid, &wasDeleted);
            if (isNewest && wasDeleted) {
                // same as above, it could be that the record was inserted and
                // then updated - in this case we to continue scanning
                continue;
            }
            return !wasDeleted;
        }
    }
    // in this case the tuple does not exist
    return false;
}

bool Table::getNewest(uint64_t key,
                      size_t& size,
                      const char*& data,
                      uint64_t& version) const {
    // TODO Implement
    return false;
}

void Table::insert(uint64_t key,
                   size_t size,
                   const char* const data,
                   const SnapshotDescriptor& snapshot,
                   bool* succeeded /*= nullptr*/) {
    // we need to get the iterator as a first step to make
    // sure to check the part of the log that was visible
    // at this point in time
    auto iter = mInsertLog.begin();
    auto ptr = mHashTable.load()->get(key);
    if (ptr) {
        // the key exists... but it could be, that it got deleted
        CDMRecord rec(reinterpret_cast<const char*>(ptr));
        bool wasDeleted, isNewest;
        size_t s;
        bool isValid;
        rec.data(snapshot, s, isNewest, isValid, &wasDeleted);
        if (isValid && !(wasDeleted && isNewest)) {
            if (succeeded) *succeeded = false;
            return;
        }
        // the tuple was deleted/reverted and we don't have a
        // write-write conflict, therefore we can continue
        // with the insert
    }
    // To do an insert, we optimistically append it to the log.
    // Then we check for conflicts iff the user wants to know whether
    // the insert succeeded.
    auto logEntrySize = size + DMRecord::spaceOverhead(DMRecord::Type::LOG_INSERT);
    auto entry = mInsertLog.append(logEntrySize);
    // We do this in another scope, after this scope is closed, the log
    // is read only (when seal is called)
    auto iterEnd = mInsertLog.end();
    DMRecord insertRecord(entry->data());
    insertRecord.setType(DMRecord::Type::LOG_INSERT);
    insertRecord.writeKey(key);
    insertRecord.writeVersion(snapshot.version());
    insertRecord.writePrevious(nullptr);
    insertRecord.writeData(size, data);
    while (iter != iterEnd) {
        // we busy wait if the entry was not sealed
        while (iter->data() != entry->data() && !iter->sealed()) {}
        const LogEntry* en = iter.operator->();
        if (en == entry) {
            entry->seal();
            if (succeeded) *succeeded = true;
            return;
        }
        CDMRecord rec(en->data());
        if (rec.isValidDataRecord() && rec.key() == key) {
            insertRecord.revert(snapshot.version());
            entry->seal();
            if (succeeded) *succeeded = false;
            return;
        }
    }
    LOG_ASSERT(false, "We should never reach this point");
}

void Table::insert(uint64_t key,
                   const GenericTuple& tuple,
                   const SnapshotDescriptor& snapshot,
                   bool* succeeded /*= nullptr*/)
{
    size_t size;
    std::unique_ptr<char[]> rec(mRecord.create(tuple, size));
    insert(key, size, rec.get(), snapshot, succeeded);
}

bool Table::update(uint64_t key,
                   size_t size,
                   const char* const data,
                   const SnapshotDescriptor& snapshot)
{
    auto fun = [this, key, size, data, &snapshot]()
    {
        auto logEntrySize = size + DMRecord::spaceOverhead(DMRecord::Type::LOG_UPDATE);
        auto entry = mUpdateLog.append(logEntrySize);
        {
            DMRecord updateRecord(entry->data());
            updateRecord.setType(DMRecord::Type::LOG_UPDATE);
            updateRecord.writeKey(key);
            updateRecord.writeVersion(snapshot.version());
            updateRecord.writePrevious(nullptr);
            updateRecord.writeData(size, data);
        }
        return entry->data();
    };
    return genericUpdate(fun, key, snapshot);
}

bool Table::remove(uint64_t key, const SnapshotDescriptor& snapshot) {
    auto fun = [this, key, &snapshot]() {
        auto logEntrySize = DMRecord::spaceOverhead(DMRecord::Type::LOG_DELETE);
        auto entry = mUpdateLog.append(logEntrySize);
        DMRecord rmRecord(entry->data());
        rmRecord.setType(DMRecord::Type::LOG_DELETE);
        rmRecord.writeKey(key);
        rmRecord.writeVersion(snapshot.version());
        rmRecord.writePrevious(nullptr);
        return entry->data();
    };
    return genericUpdate(fun, key, snapshot);
}

bool Table::revert(uint64_t key, const SnapshotDescriptor& snapshot) {
    // TODO Implement
    return false;
}

template<class Fun>
bool Table::genericUpdate(const Fun& appendFun,
                          uint64_t key,
                          const SnapshotDescriptor& snapshot)
{
    auto iter = mInsertLog.begin();
    auto iterEnd = mInsertLog.end();
    auto ptr = mHashTable.load()->get(key);
    if (!ptr) {
        while (iter != iterEnd) {
            CDMRecord rec(iter->data());
            if (rec.isValidDataRecord() && rec.key() == key) {
                // we found it!
                ptr = iter->data();
                break;
            }
        }
    }
    if (!ptr) {
        // no record with key exists
        return false;
    }
    // now we found it. Therefore we first append the
    // update optimistaically
    char* nextPtr = appendFun();
    DMRecord rec(reinterpret_cast<char*>(ptr));
    bool isValid;
    return rec.update(nextPtr, isValid, snapshot);
}

auto Table::startScan(int numThreads) const -> std::vector<std::pair<Iterator, Iterator>>
{
    auto alloc = std::make_shared<allocator>();
    auto insIter = mInsertLog.begin();
    auto endIns = mInsertLog.end();
    const auto* pages = mPages.load();
    auto numPages = pages->size();
    std::vector<std::pair<Iterator, Iterator>> result(numThreads);
    size_t beginIdx = 0;
    auto mod = numPages % numThreads;
    for (decltype(numPages) i = 0; i < numPages; ++i) {
        result[i].first = Iterator(alloc, pages, beginIdx, endIns, endIns, &mPageManager, &mRecord);
        beginIdx += numPages / numThreads + (i < mod ? 1 : 0);
        result[i].second = Iterator(alloc, pages, beginIdx, endIns, endIns, &mPageManager, &mRecord);
        auto& item = result[i];
        item.first.mAllocator = alloc;
        item.first.pages = pages;
        item.first.pageIdx = beginIdx;
        item.first.logIter = insIter;
        item.second.mAllocator = alloc;
        item.second.pages = pages;
        item.second.pageIdx = beginIdx;
        if (i == numPages - 1) {
            // add log iterators here
            item.first.logIter = insIter;
            item.second.logIter = endIns;
        } else {
            item.first.logIter = endIns;
            item.second.logIter = endIns;
        }
    }
    return std::vector<std::pair<Iterator, Iterator>>();
}

void Table::runGC(uint64_t minVersion) {
    allocator _;
    auto hashTable = mHashTable.load()->modifier(_);
    // we need to process the insert-log first. There might be delted
    // records which have an insert
    auto insBegin = mInsertLog.begin();
    auto insIter = insBegin;
    auto end = mInsertLog.end();
    auto updateBegin = mUpdateLog.end();
    auto updateEnd = mUpdateLog.end();
    InsertMap insertMap;
    for (; insIter != end && insIter->sealed(); ++insIter) {
        CDMRecord rec(insIter->data());
        if (!rec.isValidDataRecord()) continue;
        auto k = rec.key();
        insertMap[InsertMapKey(k)].push_back(insIter->data());
    }
    auto& roPages = *mPages.load();
    auto nPagesPtr = new (malloc(sizeof(PageList))) PageList(roPages);
    auto& nPages = *nPagesPtr;
    auto fillPage = reinterpret_cast<char*>(mPageManager.alloc());
    PageList newPages;
    // this loop just iterates over all pages
    for (size_t i = 0; i < nPages.size(); ++i) {
        Page page(mPageManager, nPages[i]);
        bool done;
        nPages[i] = page.gc(minVersion, insertMap, fillPage, done, hashTable);
        while (!done) {
            if (nPages[i]) {
                newPages.push_back(nPages[i]);
            }
            fillPage = reinterpret_cast<char*>(mPageManager.alloc());
            nPages[i] = page.gc(minVersion, insertMap, fillPage, done, hashTable);
        }
        if (nPages[i] == nullptr) {
            // This means that this page got merged with the older page.
            // Therefore we can remove it from the list
            nPages.erase(nPages.begin() + i);
        }
    }
    // now we can process the inserts
    while (!insertMap.empty()) {
        fillPage = reinterpret_cast<char*>(mPageManager.alloc());
        Page::fillWithInserts(minVersion, insertMap, fillPage, hashTable);
        nPages.push_back(fillPage);
    }
    // The garbage collection is finished - we can now reset the read only table
    mPages.store(nPagesPtr);
    {
        auto ht = mHashTable.load();
        mHashTable.store(hashTable.done());
        allocator::free(ht, [ht](){ ht->~CuckooTable(); });
    }
    while (!mInsertLog.truncateLog(insIter.page(), end.page()));
    while (!mUpdateLog.truncateLog(updateBegin.page(), updateEnd.page()));
}

void GarbageCollector::run(const std::vector<Table*>& tables, uint64_t minVersion) {
    for (auto table : tables) {
        table->runGC(minVersion);
    }
}

} // namespace deltamain

StoreImpl<Implementation::DELTA_MAIN_REWRITE>::StoreImpl(const StorageConfig& config)
    : pageManager(new (allocator::malloc(sizeof(PageManager))) PageManager(config.totalMemory), [](PageManager* p){ allocator::free_in_order(p, [p](){p->~PageManager();}); })
    , tableManager(*pageManager, config, gc, commitManager)
{
}

StoreImpl<Implementation::DELTA_MAIN_REWRITE>::StoreImpl(const StorageConfig& config, size_t totalMem)
    : pageManager(new (allocator::malloc(sizeof(PageManager))) PageManager(config.totalMemory), [](PageManager* p){ allocator::free_in_order(p, [p](){p->~PageManager();}); })
    , tableManager(*pageManager, config, gc, commitManager)
{
}

} // namespace store
} // namespace tell

<commit_msg>Increment the iterator when searching for an element in the log<commit_after>#include "Table.hpp"
#include "Record.hpp"
#include "Page.hpp"
#include "InsertMap.hpp"

#include <memory>

namespace tell {
namespace store {
namespace deltamain {

Table::Iterator::Iterator(
        const std::shared_ptr<allocator>& alloc,
        const PageList* pages,
        size_t pageIdx,
        const LogIterator& logIter,
        const LogIterator& logEnd,
        PageManager* pageManager,
        const Record* record)
    : mAllocator(alloc)
    , pages(pages)
    , pageIdx(pageIdx)
    , logIter(logIter)
    , logEnd(logEnd)
    , pageManager(pageManager)
    , record(record)
    , pageIter(Page(*pageManager, (*pages)[pageIdx]).begin())
    , pageEnd (Page(*pageManager, (*pages)[pageIdx]).end())
{
    setCurrentEntry();
}

Table::Iterator::Iterator(const Iterator& other)
    : mAllocator(other.mAllocator)
    , pages(other.pages)
    , pageIdx(other.pageIdx)
    , logIter(other.logIter)
    , logEnd(other.logEnd)
    , pageManager(other.pageManager)
    , record(other.record)
    , pageIter(other.pageIter)
    , pageEnd(other.pageEnd)
{
    setCurrentEntry();
}

auto Table::Iterator::operator=(const Iterator& o) -> Iterator&
{
    mAllocator = o.mAllocator;
    pages = o.pages;
    pageIdx = o.pageIdx;
    logIter = o.logIter;
    logEnd = o.logEnd;
    pageManager = o.pageManager;
    record = o.record;
    pageIter = o.pageIter;
    pageEnd = o.pageEnd;
    setCurrentEntry();
    return *this;
}

bool Table::Iterator::operator==(const Iterator& o) const
{
    return record == o.record &&
        pages == o.pages &&
        pageIdx == o.pageIdx &&
        logIter == o.logIter &&
        logEnd == o.logEnd &&
        pageIter == o.pageIter &&
        pageEnd == o.pageEnd &&
        currVersionIter.isValid() == currVersionIter.isValid();
}

auto Table::Iterator::operator++() -> Iterator&
{
    if (currVersionIter.isValid() && (++currVersionIter).isValid()) {
        ++currVersionIter;
        return *this;
    }
    if (logIter != logEnd) ++logIter;
    else if (pageIter != pageEnd) {
        ++pageIter;
    } else {
        if (++pageIdx == pages->size()) return *this;
        Page p(*pageManager, (*pages)[pageIdx]);
        pageIter = p.begin();
        pageEnd = p.end();
    }
    setCurrentEntry();
    return *this;
}

auto Table::Iterator::operator++(int) -> Iterator
{
    auto res = *this;
    ++res;
    return res;
}

const Table::Iterator::IteratorEntry& Table::Iterator::operator*() const {
    return *currVersionIter;
}

const Table::Iterator::IteratorEntry* Table::Iterator::operator->() const
{
    return &(this->operator*());
}

void Table::Iterator::setCurrentEntry()
{
    while (logIter != logEnd) {
        if (logIter->sealed()) {
            CDMRecord rec(logIter->data());
            if (rec.isValidDataRecord()) {
                currVersionIter = rec.getVersionIterator(record);
                return;
            }
        }
        ++logIter;
    }
    while (true) {
        while (pageIter != pageEnd) {
            CDMRecord rec(*pageIter);
            if (rec.isValidDataRecord()) {
                currVersionIter = rec.getVersionIterator(record);
                return;
            }
            ++pageIter;
        }
        ++pageIdx;
        if (pageIdx == pages->size()) return;
        Page p(*pageManager, (*pages)[pageIdx]);
        pageIter = p.begin();
        pageEnd = p.end();
    }
}

Table::Table(PageManager& pageManager, const Schema& schema, uint64_t /* idx */)
    : mPageManager(pageManager)
    , mSchema(schema)
    , mRecord(schema)
    , mHashTable(new (allocator::malloc(sizeof(CuckooTable))) CuckooTable(pageManager))
    , mInsertLog(pageManager)
    , mUpdateLog(pageManager)
    , mPages(new (allocator::malloc(sizeof(std::vector<char*>))) std::vector<char*>())
{}

bool Table::get(uint64_t key,
                size_t& size,
                const char*& data,
                const SnapshotDescriptor& snapshot,
                bool& isNewest) const {
    auto ptr = mHashTable.load()->get(key);
    if (ptr) {
        CDMRecord rec(reinterpret_cast<char*>(ptr));
        bool wasDeleted;
        bool isValid;
        data = rec.data(snapshot, size, isNewest, isValid, &wasDeleted);
        // if the newest version is a delete, it might be that there is
        // a new insert in the insert log
        if (isValid && !(wasDeleted && isNewest)) {
            return !wasDeleted;
        }
    }
    // in this case we need to scan through the insert log
    auto iterEnd = mInsertLog.end();
    for (auto iter = mInsertLog.begin(); iter != iterEnd; ++iter) {
        if (!iter->sealed()) continue;
        CDMRecord rec(iter->data());
        if (rec.isValidDataRecord() && rec.key() == key) {
            bool wasDeleted;
            bool isValid;
            data = rec.data(snapshot, size, isNewest, isValid, &wasDeleted);
            if (isNewest && wasDeleted) {
                // same as above, it could be that the record was inserted and
                // then updated - in this case we to continue scanning
                continue;
            }
            return !wasDeleted;
        }
    }
    // in this case the tuple does not exist
    return false;
}

bool Table::getNewest(uint64_t key,
                      size_t& size,
                      const char*& data,
                      uint64_t& version) const {
    // TODO Implement
    return false;
}

void Table::insert(uint64_t key,
                   size_t size,
                   const char* const data,
                   const SnapshotDescriptor& snapshot,
                   bool* succeeded /*= nullptr*/) {
    // we need to get the iterator as a first step to make
    // sure to check the part of the log that was visible
    // at this point in time
    auto iter = mInsertLog.begin();
    auto ptr = mHashTable.load()->get(key);
    if (ptr) {
        // the key exists... but it could be, that it got deleted
        CDMRecord rec(reinterpret_cast<const char*>(ptr));
        bool wasDeleted, isNewest;
        size_t s;
        bool isValid;
        rec.data(snapshot, s, isNewest, isValid, &wasDeleted);
        if (isValid && !(wasDeleted && isNewest)) {
            if (succeeded) *succeeded = false;
            return;
        }
        // the tuple was deleted/reverted and we don't have a
        // write-write conflict, therefore we can continue
        // with the insert
    }
    // To do an insert, we optimistically append it to the log.
    // Then we check for conflicts iff the user wants to know whether
    // the insert succeeded.
    auto logEntrySize = size + DMRecord::spaceOverhead(DMRecord::Type::LOG_INSERT);
    auto entry = mInsertLog.append(logEntrySize);
    // We do this in another scope, after this scope is closed, the log
    // is read only (when seal is called)
    auto iterEnd = mInsertLog.end();
    DMRecord insertRecord(entry->data());
    insertRecord.setType(DMRecord::Type::LOG_INSERT);
    insertRecord.writeKey(key);
    insertRecord.writeVersion(snapshot.version());
    insertRecord.writePrevious(nullptr);
    insertRecord.writeData(size, data);
    for (; iter != iterEnd; ++iter) {
        // we busy wait if the entry was not sealed
        while (iter->data() != entry->data() && !iter->sealed()) {}
        const LogEntry* en = iter.operator->();
        if (en == entry) {
            entry->seal();
            if (succeeded) *succeeded = true;
            return;
        }
        CDMRecord rec(en->data());
        if (rec.isValidDataRecord() && rec.key() == key) {
            insertRecord.revert(snapshot.version());
            entry->seal();
            if (succeeded) *succeeded = false;
            return;
        }
    }
    LOG_ASSERT(false, "We should never reach this point");
}

void Table::insert(uint64_t key,
                   const GenericTuple& tuple,
                   const SnapshotDescriptor& snapshot,
                   bool* succeeded /*= nullptr*/)
{
    size_t size;
    std::unique_ptr<char[]> rec(mRecord.create(tuple, size));
    insert(key, size, rec.get(), snapshot, succeeded);
}

bool Table::update(uint64_t key,
                   size_t size,
                   const char* const data,
                   const SnapshotDescriptor& snapshot)
{
    auto fun = [this, key, size, data, &snapshot]()
    {
        auto logEntrySize = size + DMRecord::spaceOverhead(DMRecord::Type::LOG_UPDATE);
        auto entry = mUpdateLog.append(logEntrySize);
        {
            DMRecord updateRecord(entry->data());
            updateRecord.setType(DMRecord::Type::LOG_UPDATE);
            updateRecord.writeKey(key);
            updateRecord.writeVersion(snapshot.version());
            updateRecord.writePrevious(nullptr);
            updateRecord.writeData(size, data);
        }
        return entry->data();
    };
    return genericUpdate(fun, key, snapshot);
}

bool Table::remove(uint64_t key, const SnapshotDescriptor& snapshot) {
    auto fun = [this, key, &snapshot]() {
        auto logEntrySize = DMRecord::spaceOverhead(DMRecord::Type::LOG_DELETE);
        auto entry = mUpdateLog.append(logEntrySize);
        DMRecord rmRecord(entry->data());
        rmRecord.setType(DMRecord::Type::LOG_DELETE);
        rmRecord.writeKey(key);
        rmRecord.writeVersion(snapshot.version());
        rmRecord.writePrevious(nullptr);
        return entry->data();
    };
    return genericUpdate(fun, key, snapshot);
}

bool Table::revert(uint64_t key, const SnapshotDescriptor& snapshot) {
    // TODO Implement
    return false;
}

template<class Fun>
bool Table::genericUpdate(const Fun& appendFun,
                          uint64_t key,
                          const SnapshotDescriptor& snapshot)
{
    auto iter = mInsertLog.begin();
    auto iterEnd = mInsertLog.end();
    auto ptr = mHashTable.load()->get(key);
    if (!ptr) {
        for (; iter != iterEnd; ++iter) {
            CDMRecord rec(iter->data());
            if (rec.isValidDataRecord() && rec.key() == key) {
                // we found it!
                ptr = iter->data();
                break;
            }
        }
    }
    if (!ptr) {
        // no record with key exists
        return false;
    }
    // now we found it. Therefore we first append the
    // update optimistaically
    char* nextPtr = appendFun();
    DMRecord rec(reinterpret_cast<char*>(ptr));
    bool isValid;
    return rec.update(nextPtr, isValid, snapshot);
}

auto Table::startScan(int numThreads) const -> std::vector<std::pair<Iterator, Iterator>>
{
    auto alloc = std::make_shared<allocator>();
    auto insIter = mInsertLog.begin();
    auto endIns = mInsertLog.end();
    const auto* pages = mPages.load();
    auto numPages = pages->size();
    std::vector<std::pair<Iterator, Iterator>> result(numThreads);
    size_t beginIdx = 0;
    auto mod = numPages % numThreads;
    for (decltype(numPages) i = 0; i < numPages; ++i) {
        result[i].first = Iterator(alloc, pages, beginIdx, endIns, endIns, &mPageManager, &mRecord);
        beginIdx += numPages / numThreads + (i < mod ? 1 : 0);
        result[i].second = Iterator(alloc, pages, beginIdx, endIns, endIns, &mPageManager, &mRecord);
        auto& item = result[i];
        item.first.mAllocator = alloc;
        item.first.pages = pages;
        item.first.pageIdx = beginIdx;
        item.first.logIter = insIter;
        item.second.mAllocator = alloc;
        item.second.pages = pages;
        item.second.pageIdx = beginIdx;
        if (i == numPages - 1) {
            // add log iterators here
            item.first.logIter = insIter;
            item.second.logIter = endIns;
        } else {
            item.first.logIter = endIns;
            item.second.logIter = endIns;
        }
    }
    return std::vector<std::pair<Iterator, Iterator>>();
}

void Table::runGC(uint64_t minVersion) {
    allocator _;
    auto hashTable = mHashTable.load()->modifier(_);
    // we need to process the insert-log first. There might be delted
    // records which have an insert
    auto insBegin = mInsertLog.begin();
    auto insIter = insBegin;
    auto end = mInsertLog.end();
    auto updateBegin = mUpdateLog.end();
    auto updateEnd = mUpdateLog.end();
    InsertMap insertMap;
    for (; insIter != end && insIter->sealed(); ++insIter) {
        CDMRecord rec(insIter->data());
        if (!rec.isValidDataRecord()) continue;
        auto k = rec.key();
        insertMap[InsertMapKey(k)].push_back(insIter->data());
    }
    auto& roPages = *mPages.load();
    auto nPagesPtr = new (malloc(sizeof(PageList))) PageList(roPages);
    auto& nPages = *nPagesPtr;
    auto fillPage = reinterpret_cast<char*>(mPageManager.alloc());
    PageList newPages;
    // this loop just iterates over all pages
    for (size_t i = 0; i < nPages.size(); ++i) {
        Page page(mPageManager, nPages[i]);
        bool done;
        nPages[i] = page.gc(minVersion, insertMap, fillPage, done, hashTable);
        while (!done) {
            if (nPages[i]) {
                newPages.push_back(nPages[i]);
            }
            fillPage = reinterpret_cast<char*>(mPageManager.alloc());
            nPages[i] = page.gc(minVersion, insertMap, fillPage, done, hashTable);
        }
        if (nPages[i] == nullptr) {
            // This means that this page got merged with the older page.
            // Therefore we can remove it from the list
            nPages.erase(nPages.begin() + i);
        }
    }
    // now we can process the inserts
    while (!insertMap.empty()) {
        fillPage = reinterpret_cast<char*>(mPageManager.alloc());
        Page::fillWithInserts(minVersion, insertMap, fillPage, hashTable);
        nPages.push_back(fillPage);
    }
    // The garbage collection is finished - we can now reset the read only table
    mPages.store(nPagesPtr);
    {
        auto ht = mHashTable.load();
        mHashTable.store(hashTable.done());
        allocator::free(ht, [ht](){ ht->~CuckooTable(); });
    }
    while (!mInsertLog.truncateLog(insIter.page(), end.page()));
    while (!mUpdateLog.truncateLog(updateBegin.page(), updateEnd.page()));
}

void GarbageCollector::run(const std::vector<Table*>& tables, uint64_t minVersion) {
    for (auto table : tables) {
        table->runGC(minVersion);
    }
}

} // namespace deltamain

StoreImpl<Implementation::DELTA_MAIN_REWRITE>::StoreImpl(const StorageConfig& config)
    : pageManager(new (allocator::malloc(sizeof(PageManager))) PageManager(config.totalMemory), [](PageManager* p){ allocator::free_in_order(p, [p](){p->~PageManager();}); })
    , tableManager(*pageManager, config, gc, commitManager)
{
}

StoreImpl<Implementation::DELTA_MAIN_REWRITE>::StoreImpl(const StorageConfig& config, size_t totalMem)
    : pageManager(new (allocator::malloc(sizeof(PageManager))) PageManager(config.totalMemory), [](PageManager* p){ allocator::free_in_order(p, [p](){p->~PageManager();}); })
    , tableManager(*pageManager, config, gc, commitManager)
{
}

} // namespace store
} // namespace tell

<|endoftext|>


<commit_before>/**
 * @file cosine_tree_test.cpp
 * @author Siddharth Agrawal
 *
 * Test file for CosineTree class.
 */

#include <mlpack/core.hpp>
#include <mlpack/core/tree/cosine_tree/cosine_tree.hpp>

#include <boost/test/unit_test.hpp>
#include "old_boost_test_definitions.hpp"

BOOST_AUTO_TEST_SUITE(CosineTreeTest);

using namespace mlpack;
using namespace mlpack::tree;

/**
 * Constructs a cosine tree with epsilon = 1. Checks if the root node is split
 * further, as it shouldn't be.
 */
BOOST_AUTO_TEST_CASE(CosineTreeNoSplit)
{
  // Initialize constants required for the test.
  const size_t numRows = 10;
  const size_t numCols = 15;
  const double epsilon = 1;
  const double delta = 0.1;

  // Make a random dataset.
  arma::mat data = arma::randu(numRows, numCols);

  // Make a cosine tree, with the generated dataset and the defined constants.
  // Note that the value of epsilon is one.
  CosineTree ctree(data, epsilon, delta);
  arma::mat basis;
  ctree.GetFinalBasis(basis);

  // Since epsilon is one, there should be no splitting and the only vector in
  // the basis should come from the root node.
  BOOST_REQUIRE_EQUAL(basis.n_cols, 1);
}

/**
 * Checks CosineTree::CosineNodeSplit() by doing a depth first search on a
 * random dataset and checking if it satisfies the split condition.
 */
BOOST_AUTO_TEST_CASE(CosineNodeCosineSplit)
{
  // Intialize constants required for the test.
  const size_t numRows = 500;
  const size_t numCols = 1000;

  // Make a random dataset and the root object.
  arma::mat data = arma::randu(numRows, numCols);
  CosineTree root(data);

  // Stack for depth first search of the tree.
  std::vector<CosineTree*> nodeStack;
  nodeStack.push_back(&root);

  // While stack is not empty.
  while (nodeStack.size())
  {
    // Pop a node from the stack and split it.
    CosineTree *currentNode, *currentLeft, *currentRight;
    currentNode = nodeStack.back();
    currentNode->CosineNodeSplit();
    nodeStack.pop_back();

    // Obtain pointers to the children of the node.
    currentLeft = currentNode->Left();
    currentRight = currentNode->Right();

    // If children exist.
    if (currentLeft && currentRight)
    {
      // Push the child nodes on to the stack.
      nodeStack.push_back(currentLeft);
      nodeStack.push_back(currentRight);

      // Obtain the split point of the popped node.
      arma::vec splitPoint = data.col(currentNode->SplitPointIndex());

      // Column indices of the the child nodes.
      std::vector<size_t> leftIndices, rightIndices;
      leftIndices = currentLeft->VectorIndices();
      rightIndices = currentRight->VectorIndices();

      // The columns in the popped should be split into left and right nodes.
      BOOST_REQUIRE_EQUAL(currentNode->NumColumns(), leftIndices.size() +
          rightIndices.size());

      // Calculate the cosine values for each of the columns in the node.
      arma::vec cosines;
      cosines.zeros(currentNode->NumColumns());

      size_t i, j, k;
      for (i = 0; i < leftIndices.size(); i++)
        cosines(i) = arma::norm_dot(data.col(leftIndices[i]), splitPoint);

      for (j = 0, k = i; j < rightIndices.size(); j++, k++)
        cosines(k) = arma::norm_dot(data.col(rightIndices[j]), splitPoint);

      // Check if the columns assigned to the children agree with the splitting
      // condition.
//      double cosineMax = arma::max(cosines % (cosines < 1));
//      double cosineMin = arma::min(cosines);

      for (i = 0; i < leftIndices.size(); i++)
        BOOST_CHECK_LT(cosineMax - cosines(i), cosines(i) - cosineMin);

      for (j = 0, k = i; j < rightIndices.size(); j++, k++)
        BOOST_CHECK_GT(cosineMax - cosines(k), cosines(k) - cosineMin);
    }
  }
}

/**
 * Checks CosineTree::ModifiedGramSchmidt() by creating a random basis for the
 * vector subspace and checking if all the vectors are orthogonal to each other.
 */
BOOST_AUTO_TEST_CASE(CosineTreeModifiedGramSchmidt)
{
  // Initialize constants required for the test.
  const size_t numRows = 100;
  const size_t numCols = 50;
  const double epsilon = 1;
  const double delta = 0.1;

  // Make a random dataset.
  arma::mat data = arma::randu(numRows, numCols);

  // Declare a queue and a dummy CosineTree object.
  CosineNodeQueue basisQueue;
  CosineTree dummyTree(data, epsilon, delta);

  for(size_t i = 0; i < numCols; i++)
  {
    // Make a new CosineNode object.
    CosineTree* basisNode;
    basisNode = new CosineTree(data);

    // Use the columns of the dataset as random centroids.
    arma::vec centroid = data.col(i);
    arma::vec newBasisVector;

    // Obtain the orthonormalized version of the centroid.
    dummyTree.ModifiedGramSchmidt(basisQueue, centroid, newBasisVector);

    // Check if the obtained vector is orthonormal to the basis vectors.
    CosineNodeQueue::const_iterator j = basisQueue.begin();
    CosineTree* currentNode;

    for(; j != basisQueue.end(); j++)
    {
      currentNode = *j;
      BOOST_REQUIRE_SMALL(arma::dot(currentNode->BasisVector(), newBasisVector),
                          1e-5);
    }

    // Add the obtained vector to the basis.
    basisNode->BasisVector(newBasisVector);
    basisNode->L2Error(arma::randu());
    basisQueue.push(basisNode);
  }

  // Deallocate memory given to the objects.
  for(size_t i = 0; i < numCols; i++)
  {
    CosineTree* currentNode;
    currentNode = basisQueue.top();
    basisQueue.pop();

    delete currentNode;
  }
}

BOOST_AUTO_TEST_SUITE_END();
<commit_msg>Why did I merge that change in?  It broke everything.  Revert...<commit_after>/**
 * @file cosine_tree_test.cpp
 * @author Siddharth Agrawal
 *
 * Test file for CosineTree class.
 */

#include <mlpack/core.hpp>
#include <mlpack/core/tree/cosine_tree/cosine_tree.hpp>

#include <boost/test/unit_test.hpp>
#include "old_boost_test_definitions.hpp"

BOOST_AUTO_TEST_SUITE(CosineTreeTest);

using namespace mlpack;
using namespace mlpack::tree;

/**
 * Constructs a cosine tree with epsilon = 1. Checks if the root node is split
 * further, as it shouldn't be.
 */
BOOST_AUTO_TEST_CASE(CosineTreeNoSplit)
{
  // Initialize constants required for the test.
  const size_t numRows = 10;
  const size_t numCols = 15;
  const double epsilon = 1;
  const double delta = 0.1;

  // Make a random dataset.
  arma::mat data = arma::randu(numRows, numCols);

  // Make a cosine tree, with the generated dataset and the defined constants.
  // Note that the value of epsilon is one.
  CosineTree ctree(data, epsilon, delta);
  arma::mat basis;
  ctree.GetFinalBasis(basis);

  // Since epsilon is one, there should be no splitting and the only vector in
  // the basis should come from the root node.
  BOOST_REQUIRE_EQUAL(basis.n_cols, 1);
}

/**
 * Checks CosineTree::CosineNodeSplit() by doing a depth first search on a
 * random dataset and checking if it satisfies the split condition.
 */
BOOST_AUTO_TEST_CASE(CosineNodeCosineSplit)
{
  // Intialize constants required for the test.
  const size_t numRows = 500;
  const size_t numCols = 1000;

  // Make a random dataset and the root object.
  arma::mat data = arma::randu(numRows, numCols);
  CosineTree root(data);

  // Stack for depth first search of the tree.
  std::vector<CosineTree*> nodeStack;
  nodeStack.push_back(&root);

  // While stack is not empty.
  while (nodeStack.size())
  {
    // Pop a node from the stack and split it.
    CosineTree *currentNode, *currentLeft, *currentRight;
    currentNode = nodeStack.back();
    currentNode->CosineNodeSplit();
    nodeStack.pop_back();

    // Obtain pointers to the children of the node.
    currentLeft = currentNode->Left();
    currentRight = currentNode->Right();

    // If children exist.
    if (currentLeft && currentRight)
    {
      // Push the child nodes on to the stack.
      nodeStack.push_back(currentLeft);
      nodeStack.push_back(currentRight);

      // Obtain the split point of the popped node.
      arma::vec splitPoint = data.col(currentNode->SplitPointIndex());

      // Column indices of the the child nodes.
      std::vector<size_t> leftIndices, rightIndices;
      leftIndices = currentLeft->VectorIndices();
      rightIndices = currentRight->VectorIndices();

      // The columns in the popped should be split into left and right nodes.
      BOOST_REQUIRE_EQUAL(currentNode->NumColumns(), leftIndices.size() +
          rightIndices.size());

      // Calculate the cosine values for each of the columns in the node.
      arma::vec cosines;
      cosines.zeros(currentNode->NumColumns());

      size_t i, j, k;
      for (i = 0; i < leftIndices.size(); i++)
        cosines(i) = arma::norm_dot(data.col(leftIndices[i]), splitPoint);

      for (j = 0, k = i; j < rightIndices.size(); j++, k++)
        cosines(k) = arma::norm_dot(data.col(rightIndices[j]), splitPoint);

      // Check if the columns assigned to the children agree with the splitting
      // condition.
      double cosineMax = arma::max(cosines % (cosines < 1));
      double cosineMin = arma::min(cosines);

      for (i = 0; i < leftIndices.size(); i++)
        BOOST_CHECK_LT(cosineMax - cosines(i), cosines(i) - cosineMin);

      for (j = 0, k = i; j < rightIndices.size(); j++, k++)
        BOOST_CHECK_GT(cosineMax - cosines(k), cosines(k) - cosineMin);
    }
  }
}

/**
 * Checks CosineTree::ModifiedGramSchmidt() by creating a random basis for the
 * vector subspace and checking if all the vectors are orthogonal to each other.
 */
BOOST_AUTO_TEST_CASE(CosineTreeModifiedGramSchmidt)
{
  // Initialize constants required for the test.
  const size_t numRows = 100;
  const size_t numCols = 50;
  const double epsilon = 1;
  const double delta = 0.1;

  // Make a random dataset.
  arma::mat data = arma::randu(numRows, numCols);

  // Declare a queue and a dummy CosineTree object.
  CosineNodeQueue basisQueue;
  CosineTree dummyTree(data, epsilon, delta);

  for(size_t i = 0; i < numCols; i++)
  {
    // Make a new CosineNode object.
    CosineTree* basisNode;
    basisNode = new CosineTree(data);

    // Use the columns of the dataset as random centroids.
    arma::vec centroid = data.col(i);
    arma::vec newBasisVector;

    // Obtain the orthonormalized version of the centroid.
    dummyTree.ModifiedGramSchmidt(basisQueue, centroid, newBasisVector);

    // Check if the obtained vector is orthonormal to the basis vectors.
    CosineNodeQueue::const_iterator j = basisQueue.begin();
    CosineTree* currentNode;

    for(; j != basisQueue.end(); j++)
    {
      currentNode = *j;
      BOOST_REQUIRE_SMALL(arma::dot(currentNode->BasisVector(), newBasisVector),
                          1e-5);
    }

    // Add the obtained vector to the basis.
    basisNode->BasisVector(newBasisVector);
    basisNode->L2Error(arma::randu());
    basisQueue.push(basisNode);
  }

  // Deallocate memory given to the objects.
  for(size_t i = 0; i < numCols; i++)
  {
    CosineTree* currentNode;
    currentNode = basisQueue.top();
    basisQueue.pop();

    delete currentNode;
  }
}

BOOST_AUTO_TEST_SUITE_END();
<|endoftext|>


<commit_before>#include "acmacs-base/string-join.hh"
#include "seqdb-3/seqdb.hh"
#include "acmacs-map-draw/report-antigens.hh"
#include "acmacs-map-draw/chart-select-interface.hh"

// ----------------------------------------------------------------------

std::string report_antigens(const acmacs::chart::PointIndexList& indexes, const ChartSelectInterface& aChartSelectInterface, size_t threshold)
{
    using namespace std::string_literals;

    fmt::memory_buffer output;
    if (threshold >= indexes.size()) {
        const auto& chart = aChartSelectInterface.chart();
        const auto& seqdb = acmacs::seqdb::get();
        const auto& matched_seqdb = aChartSelectInterface.chart(0).match_seqdb();
        auto layout = aChartSelectInterface.chart(0).modified_layout();
        auto antigens = chart.antigens();
        auto titers = chart.titers();
        auto info = chart.info();
        fmt::format_to(output, "  AG ({}) {}\n", indexes.size(), acmacs::string::join(acmacs::string::join_comma, std::begin(indexes), std::end(indexes)));

        size_t seq_max{0}, clades_max{0};
        for (auto index : indexes) {
            if (const auto& ref = matched_seqdb[index]; ref) {
                seq_max = std::max(seq_max, ref.seq_id().size());
                clades_max = std::max(clades_max, fmt::format("{}", ref.seq_with_sequence(seqdb).clades).size());
            }
        }

        const auto full_name_max{acmacs::chart::max_full_name(*antigens, indexes)};
        for (auto index : indexes) {
            const auto antigen = antigens->at(index);
            const auto coord = layout->at(index);
            fmt::format_to(output, "  AG {:5d} {: <{}} {:10s} {: <6s}", index, fmt::format("\"{}\"", antigen->name_full()), full_name_max + 2, antigen->date(), antigen->passage().passage_type());
            if (const auto& ref = matched_seqdb[index]; ref)
                fmt::format_to(output, " {:<{}s} {:<{}s}", ref.seq_id(), seq_max, fmt::format("{}", ref.seq_with_sequence(seqdb).clades), clades_max);
            else
                fmt::format_to(output, " {:{}c}", ' ', seq_max + clades_max + 1);
            if (coord.exists())
                fmt::format_to(output, " {:8.4f}", coord);
            else
                fmt::format_to(output, "    <disconnected>   ");
            if (titers->number_of_layers() > 1) {
                std::vector<std::string> layers;
                for (size_t layer_no : titers->layers_with_antigen(index))
                    layers.emplace_back(info->source(layer_no)->date());
                fmt::format_to(output, " layers:{}{}", layers.size(), layers);
            }
            fmt::format_to(output, "\n");
        }
    }
    return fmt::to_string(output);
}

// ----------------------------------------------------------------------

std::string report_sera(const acmacs::chart::PointIndexList& indexes, const ChartSelectInterface& aChartSelectInterface, size_t threshold)
{
    fmt::memory_buffer output;
    if (threshold >= indexes.size()) {
        const auto& seqdb = acmacs::seqdb::get();
        const auto& chart = aChartSelectInterface.chart();
        chart.set_homologous(acmacs::chart::find_homologous::all);
        auto sera = chart.sera();
        const auto number_of_antigens = chart.number_of_antigens();
        const auto full_name_max{acmacs::chart::max_full_name(*sera, indexes)};
        auto layout = aChartSelectInterface.chart(0).modified_layout();
        auto titers = chart.titers();
        auto info = chart.info();

        const auto& matched_seqdb = aChartSelectInterface.chart(0).match_seqdb();
        using seq_data_t = std::pair<acmacs::seqdb::seq_id_t, std::string>;
        acmacs::small_map_with_unique_keys_t<size_t, seq_data_t> seq_clades;
        size_t seq_max{0}, clades_max{0};
        for (auto index : indexes) {
            for (const auto antigen_index : sera->at(index)->homologous_antigens()) {
                if (const auto& ref = matched_seqdb[antigen_index]; ref) {
                    const auto seq_id = ref.seq_id();
                    seq_max = std::max(seq_max, seq_id.size());
                    const auto clades = fmt::format("{}", ref.seq_with_sequence(seqdb).clades);
                    clades_max = std::max(clades_max, clades.size());
                    seq_clades.emplace_or_replace(index, seq_id, clades);
                    break;
                }
            }
        }

        fmt::format_to(output, "  SR ({}) {}\n", indexes.size(), acmacs::string::join(acmacs::string::join_comma, std::begin(indexes), std::end(indexes)));
        for (auto index : indexes) {
            const auto serum = sera->at(index);
            const auto coord = layout->at(index + number_of_antigens);
            fmt::format_to(output, "  SR {:5d} {: <{}} {: <6s}", index, fmt::format("\"{}\"", serum->name_full()), full_name_max + 2, serum->passage().passage_type());
            const auto& seq_data = seq_clades.get_or(index, seq_data_t{});
            fmt::format_to(output, " {:<{}s} {:<{}s}", seq_data.first, seq_max, seq_data.second, clades_max);
            if (coord.exists())
                fmt::format_to(output, " {:8.4f}", coord);
            else
                fmt::format_to(output, "    <disconnected>   ");
            if (titers->number_of_layers() > 1) {
                std::vector<std::string> layers;
                for (size_t layer_no : titers->layers_with_serum(index))
                    layers.emplace_back(info->source(layer_no)->date());
                fmt::format_to(output, " {:3d}{}", layers.size(), layers);
            }
            fmt::format_to(output, "\n");
        }
    }
    return fmt::to_string(output);
}

// ----------------------------------------------------------------------
/// Local Variables:
/// eval: (if (fboundp 'eu-rename-buffer) (eu-rename-buffer))
/// End:
<commit_msg>avoid seqdb_match (currently information is unused)<commit_after>#include "acmacs-base/string-join.hh"
#include "seqdb-3/seqdb.hh"
#include "acmacs-map-draw/report-antigens.hh"
#include "acmacs-map-draw/chart-select-interface.hh"

// ----------------------------------------------------------------------

std::string report_antigens(const acmacs::chart::PointIndexList& indexes, const ChartSelectInterface& aChartSelectInterface, size_t threshold)
{
    using namespace std::string_literals;

    fmt::memory_buffer output;
    if (threshold >= indexes.size()) {
        const auto& chart = aChartSelectInterface.chart();
        const auto& seqdb = acmacs::seqdb::get();
        // const auto& matched_seqdb = aChartSelectInterface.chart(0).match_seqdb();
        auto layout = aChartSelectInterface.chart(0).modified_layout();
        auto antigens = chart.antigens();
        auto titers = chart.titers();
        auto info = chart.info();
        fmt::format_to(output, "  AG ({}) {}\n", indexes.size(), acmacs::string::join(acmacs::string::join_comma, std::begin(indexes), std::end(indexes)));

        size_t seq_max{0}, clades_max{0};
        // for (auto index : indexes) {
        //     if (const auto& ref = matched_seqdb[index]; ref) {
        //         seq_max = std::max(seq_max, ref.seq_id().size());
        //         clades_max = std::max(clades_max, fmt::format("{}", ref.seq_with_sequence(seqdb).clades).size());
        //     }
        // }

        const auto full_name_max{acmacs::chart::max_full_name(*antigens, indexes)};
        for (auto index : indexes) {
            const auto antigen = antigens->at(index);
            const auto coord = layout->at(index);
            fmt::format_to(output, "  AG {:5d} {: <{}} {:10s} {: <6s}", index, fmt::format("\"{}\"", antigen->name_full()), full_name_max + 2, antigen->date(), antigen->passage().passage_type());
            // if (const auto& ref = matched_seqdb[index]; ref)
            //     fmt::format_to(output, " {:<{}s} {:<{}s}", ref.seq_id(), seq_max, fmt::format("{}", ref.seq_with_sequence(seqdb).clades), clades_max);
            // else
            //     fmt::format_to(output, " {:{}c}", ' ', seq_max + clades_max + 1);
            if (coord.exists())
                fmt::format_to(output, " {:8.4f}", coord);
            else
                fmt::format_to(output, "    <disconnected>   ");
            if (titers->number_of_layers() > 1) {
                std::vector<std::string> layers;
                for (size_t layer_no : titers->layers_with_antigen(index))
                    layers.emplace_back(info->source(layer_no)->date());
                fmt::format_to(output, " layers:{}{}", layers.size(), layers);
            }
            fmt::format_to(output, "\n");
        }
    }
    return fmt::to_string(output);
}

// ----------------------------------------------------------------------

std::string report_sera(const acmacs::chart::PointIndexList& indexes, const ChartSelectInterface& aChartSelectInterface, size_t threshold)
{
    fmt::memory_buffer output;
    if (threshold >= indexes.size()) {
        const auto& seqdb = acmacs::seqdb::get();
        const auto& chart = aChartSelectInterface.chart();
        chart.set_homologous(acmacs::chart::find_homologous::all);
        auto sera = chart.sera();
        const auto number_of_antigens = chart.number_of_antigens();
        const auto full_name_max{acmacs::chart::max_full_name(*sera, indexes)};
        auto layout = aChartSelectInterface.chart(0).modified_layout();
        auto titers = chart.titers();
        auto info = chart.info();

        const auto& matched_seqdb = aChartSelectInterface.chart(0).match_seqdb();
        using seq_data_t = std::pair<acmacs::seqdb::seq_id_t, std::string>;
        acmacs::small_map_with_unique_keys_t<size_t, seq_data_t> seq_clades;
        size_t seq_max{0}, clades_max{0};
        for (auto index : indexes) {
            for (const auto antigen_index : sera->at(index)->homologous_antigens()) {
                if (const auto& ref = matched_seqdb[antigen_index]; ref) {
                    const auto seq_id = ref.seq_id();
                    seq_max = std::max(seq_max, seq_id.size());
                    const auto clades = fmt::format("{}", ref.seq_with_sequence(seqdb).clades);
                    clades_max = std::max(clades_max, clades.size());
                    seq_clades.emplace_or_replace(index, seq_id, clades);
                    break;
                }
            }
        }

        fmt::format_to(output, "  SR ({}) {}\n", indexes.size(), acmacs::string::join(acmacs::string::join_comma, std::begin(indexes), std::end(indexes)));
        for (auto index : indexes) {
            const auto serum = sera->at(index);
            const auto coord = layout->at(index + number_of_antigens);
            fmt::format_to(output, "  SR {:5d} {: <{}} {: <6s}", index, fmt::format("\"{}\"", serum->name_full()), full_name_max + 2, serum->passage().passage_type());
            const auto& seq_data = seq_clades.get_or(index, seq_data_t{});
            fmt::format_to(output, " {:<{}s} {:<{}s}", seq_data.first, seq_max, seq_data.second, clades_max);
            if (coord.exists())
                fmt::format_to(output, " {:8.4f}", coord);
            else
                fmt::format_to(output, "    <disconnected>   ");
            if (titers->number_of_layers() > 1) {
                std::vector<std::string> layers;
                for (size_t layer_no : titers->layers_with_serum(index))
                    layers.emplace_back(info->source(layer_no)->date());
                fmt::format_to(output, " {:3d}{}", layers.size(), layers);
            }
            fmt::format_to(output, "\n");
        }
    }
    return fmt::to_string(output);
}

// ----------------------------------------------------------------------
/// Local Variables:
/// eval: (if (fboundp 'eu-rename-buffer) (eu-rename-buffer))
/// End:
<|endoftext|>