<commit_before>#include "wvtest.h"
#include "tripgraph.h"

using namespace std;

WVTEST_MAIN("basic_graph_pathfinding")
{
    TripGraph g;

    // simple path, just walking
    g.add_tripstop(0, TripStop::OSM, 0.0f, 0.0f);
    g.add_tripstop(1, TripStop::OSM, 1.0f, 0.0f);
    g.add_walkhop(0, 1);

    {
        TripPath *p = g.find_path(0, false, 0.0, 0.0, 1.0, 0.0);

        std::list<TripAction> actions = p->get_actions();
        WVPASSEQ(actions.size(), 1);

        TripAction action = actions.front();
        WVPASSEQ(action.src_id, 0);
        WVPASSEQ(action.dest_id, 1);
    }

    // take the triphop if we have it
    {
        ServicePeriod s("all", 0, 0, 0, 7, 0, 100, 2000, true, true, true);
        g.add_service_period(s);
        g.add_triphop(500, 1000, 0, 1, 1, 1, "all");
    }

    {
        TripPath *p = g.find_path(0, false, 0.0, 0.0, 1.0, 0.0);

        std::list<TripAction> actions = p->get_actions();
        WVPASSEQ(actions.size(), 1);

        TripAction action = actions.front();
        WVPASSEQ(action.src_id, 0);
        WVPASSEQ(action.dest_id, 1);
        WVPASSEQ(action.start_time, 500.0f);
        WVPASSEQ(action.end_time, 1000.0f);

        delete p;
    }
}

WVTEST_MAIN("basic_graph_saveload")
{
    TripGraph g;
    g.add_tripstop(0, TripStop::OSM, 0.0f, 0.0f);
    g.add_tripstop(1, TripStop::OSM, 1.0f, 0.0f);
    g.add_walkhop(0, 1);

    ServicePeriod s("all", 1, 0, 0, 7, 0, 100, 2000, true, true, true);
    g.add_service_period(s);
    g.add_triphop(500, 1000, 0, 1, 1, 1, "all");

    char *tmpgraphname = tmpnam(NULL); // security issues in unit tests? bah.
    unlink(tmpgraphname);
    g.save(tmpgraphname);

    TripGraph g2;
    g2.load(tmpgraphname);

    // verify that we have two tripstops
    for (int i=0; i<2; i++)
    {
        TripStop ts = g2.get_tripstop(1);
        WVPASSEQ(ts.type, TripStop::OSM);
    }

    // verify that we can still solve a basic path
    {
        TripPath *p = g.find_path(0, false, 0.0, 0.0, 1.0, 0.0);

        std::list<TripAction> actions = p->get_actions();
        WVPASSEQ(actions.size(), 1);

        TripAction action = actions.front();
        WVPASSEQ(action.src_id, 0);
        WVPASSEQ(action.dest_id, 1);
        WVPASSEQ(action.start_time, 500.0f);
        WVPASSEQ(action.end_time, 1000.0f);

        delete p;
    }
}

WVTEST_MAIN("impossible_path")
{
    TripGraph g;
    g.add_tripstop(0, TripStop::OSM, 0.0f, 0.0f);
    g.add_tripstop(1, TripStop::OSM, 1.0f, 0.0f);
    g.add_tripstop(2, TripStop::OSM, 0.0f, 1.0f);
    g.add_walkhop(0, 1);

    TripPath *p = g.find_path(0, false, 0.0, 0.0, 0.0, 1.0);
    WVPASS(!p);
}

WVTEST_MAIN("tripstops_in_range")
{
    TripGraph g;
    // north and agricola
    g.add_tripstop(0, TripStop::GTFS, 44.6554236f, -63.5936968f);
    // north and robie (just north of north&agricola)
    g.add_tripstop(1, TripStop::OSM, 44.6546407f, -63.5948438f);
    // north and northwood (just south of north&agricola)
    g.add_tripstop(2, TripStop::GTFS, 44.6567144f, -63.5919115f);
    // Quinpool and Connaught (a few kms away from north&agricola)
    g.add_tripstop(3, TripStop::GTFS, 44.6432423f, -63.6045261f);

    {
        vector<TripStop> v = g.find_tripstops_in_range(44.6554236f,
                                                       -63.5936968f,
                                                       TripStop::GTFS,
                                                       500.0f);
        WVPASSEQ(v.size(), 2);
        WVPASS(v[0].id == 0 || v[0].id == 2);
        WVPASS(v[1].id == 0 || v[1].id == 2);
    }
}

static time_t get_time_t(int tm_mday, int tm_mon, int tm_year)
{
    struct tm t;
    t.tm_sec = 0;
    t.tm_min = 0;
    t.tm_hour = 0;
    t.tm_mday = tm_mday;
    t.tm_mon = tm_mon;
    t.tm_year = tm_year;
    t.tm_wday = -1;
    t.tm_yday = -1;
    t.tm_isdst = -1;

    return mktime(&t);
}

WVTEST_MAIN("service_periods")
{
    TripGraph g;

    // from the 1st to the 7th (i.e. 1st saturday only)
    {
        ServicePeriod s("saturday_2008", 1, 0, 108, 7, 0, 108, 2000, false, true, false);
        g.add_service_period(s);
    }

    // test something that's within a supported service period
    // Saturday Midnight Jan 5th 2008
    {
        vector<pair<string, int> > vsp = g.get_service_period_ids_for_time(get_time_t(5, 0, 108));
        WVPASSEQ(vsp.size(), 1);
        WVPASSEQ(vsp[0].first, string("saturday_2008"));
    }
    // test something outside a supported service period: day
    // Saturday Midnight Jan 11th 2008
    {
        vector<pair<string, int> > vsp = g.get_service_period_ids_for_time(get_time_t(11, 0, 108));
        WVPASSEQ(vsp.size(), 0);
    }
    // test something outside a supported service period: month
    // Saturday Midnight Feb 5th 2008
    {
        vector<pair<string, int> > vsp = g.get_service_period_ids_for_time(get_time_t(5, 1, 108));
        WVPASSEQ(vsp.size(), 0);
    }

    // test something outside a supported service period: year
    // Saturday Midnight Jan 11th 2009
    {
        vector<pair<string, int> > vsp = g.get_service_period_ids_for_time(get_time_t(5, 1, 109));
        WVPASSEQ(vsp.size(), 0);
    }

    // add another service period (saturdays for month of january)
    {
        ServicePeriod s("saturday_jan_2008", 1, 0, 108, 31, 0, 108, 2000, false, true,
                        false);
        g.add_service_period(s);
    }

    // test something that's within _two_ supported service periods
    // Saturday Midnight Jan 5th 2008
    {
        vector<pair<string, int> > vsp = g.get_service_period_ids_for_time(get_time_t(5, 0, 108));
        WVPASSEQ(vsp.size(), 2);
        WVPASS(vsp[0].first==string("saturday_2008") || vsp[0].first==string("saturday_jan_2008"));
        WVPASS(vsp[1].first==string("saturday_2008") || vsp[1].first==string("saturday_jan_2008"));
        WVFAILEQ(vsp[0].first, vsp[1].first);
    }

    // save graph, reload, make sure service periods are still there
}

WVTEST_MAIN("service_periods_overlapping")
{
    TripGraph g;

    // from the 1st to the 7th (i.e. 1st saturday only)
    {
        ServicePeriod s1("saturday_2008", 1, 0, 108, 7, 0, 108, 90000, false, true, false);
        g.add_service_period(s1);
        ServicePeriod s2("weekday_2008", 1, 0, 108, 7, 0, 108, 90000, true, false, false);
        g.add_service_period(s2);
    }

    vector<pair<string, int> > vsp = g.get_service_period_ids_for_time(get_time_t(5, 0, 108));
    WVPASSEQ(vsp.size(), 2);
    WVPASS(vsp[0].first==string("saturday_2008") || vsp[0].first==string("weekday_2008"));
    WVPASS(vsp[1].first==string("saturday_2008") || vsp[1].first==string("weekday_2008"));
    WVFAILEQ(vsp[0].first, vsp[1].first);

    int weekday_index = (vsp[0].first == string("weekday_2008")) ? 0 : 1;
    WVPASSEQ(vsp[weekday_index].second, 86400);
}

WVTEST_MAIN("service_periods_turned_on_or_off")
{
    TripGraph g;

    // from the 1st to the 7th (i.e. 1st saturday only)
    // turn off weekday service on the 2nd (wednesday)
    // turn on saturday service on the 3rd (keeping weekday service)
    {
        ServicePeriod s1("saturday_2008", 1, 0, 108, 7, 0, 108, 80000, false, true, false);
        s1.add_exception_on(3, 0, 108);
        WVPASSEQ(s1.is_turned_on(3, 0, 108), true);
        WVPASSEQ(s1.is_turned_on(4, 0, 108), false);
        g.add_service_period(s1);
        ServicePeriod s2("weekday_2008", 1, 0, 108, 7, 0, 108, 80000, true, false, false);
        s2.add_exception_off(2, 0, 108);
        WVPASSEQ(s2.is_turned_off(2, 0, 108), true);
        WVPASSEQ(s2.is_turned_off(3, 0, 108), false);
        g.add_service_period(s2);
    }

    {
        // should be no service on the 2nd
        vector<pair<string, int> > vsp = g.get_service_period_ids_for_time(get_time_t(2, 0, 108));
        WVPASSEQ(vsp.size(), 0);
    }

    {
        // should be two service periods on the 3rd (saturday and weekday)
        vector<pair<string, int> > vsp = g.get_service_period_ids_for_time(get_time_t(3, 0, 108));
        WVPASSEQ(vsp.size(), 2);
        WVPASS(vsp[0].first==string("saturday_2008") || vsp[0].first==string("weekday_2008"));
        WVPASS(vsp[1].first==string("saturday_2008") || vsp[1].first==string("weekday_2008"));
        WVFAILEQ(vsp[0].first, vsp[1].first);
    }
}

WVTEST_MAIN("service_periods_save_load")
{
    TripGraph g;

    // use the same setup as the previous test: saturday and weekday schedules
    // with a few exceptions

    // from the 1st to the 7th (i.e. 1st saturday only)
    // turn off weekday service on the 2nd (wednesday)
    // turn on saturday service on the 3rd (keeping weekday service)
    {
        ServicePeriod s1("saturday_2008", 1, 0, 108, 7, 0, 108, 80000, false, true, false);
        s1.add_exception_on(3, 0, 108);
        g.add_service_period(s1);
        ServicePeriod s2("weekday_2008", 1, 0, 108, 7, 0, 108, 80000, true, false, false);
        s2.add_exception_off(2, 0, 108);
        g.add_service_period(s2);
    }

    char *tmpgraphname = tmpnam(NULL); // security issues in unit tests? bah.
    unlink(tmpgraphname);
    g.save(tmpgraphname);

    TripGraph g2;
    g2.load(tmpgraphname);

    {
        // should be no service on the 2nd
        vector<pair<string, int> > vsp = g2.get_service_period_ids_for_time(get_time_t(2, 0, 108));
        WVPASSEQ(vsp.size(), 0);
    }

    {
        // should be two service periods on the 3rd (saturday and weekday)
        vector<pair<string, int> > vsp = g2.get_service_period_ids_for_time(get_time_t(3, 0, 108));
        WVPASSEQ(vsp.size(), 2);
        WVPASS(vsp[0].first==string("saturday_2008") || vsp[0].first==string("weekday_2008"));
        WVPASS(vsp[1].first==string("saturday_2008") || vsp[1].first==string("weekday_2008"));
        WVFAILEQ(vsp[0].first, vsp[1].first);
    }
}
<commit_msg>Fix minor memory leak in tripgraph unit test<commit_after>#include "wvtest.h"
#include "tripgraph.h"

using namespace std;

WVTEST_MAIN("basic_graph_pathfinding")
{
    TripGraph g;

    // simple path, just walking
    g.add_tripstop(0, TripStop::OSM, 0.0f, 0.0f);
    g.add_tripstop(1, TripStop::OSM, 1.0f, 0.0f);
    g.add_walkhop(0, 1);

    {
        TripPath *p = g.find_path(0, false, 0.0, 0.0, 1.0, 0.0);

        std::list<TripAction> actions = p->get_actions();
        WVPASSEQ(actions.size(), 1);

        TripAction action = actions.front();
        WVPASSEQ(action.src_id, 0);
        WVPASSEQ(action.dest_id, 1);

        delete p;
    }

    // take the triphop if we have it
    {
        ServicePeriod s("all", 0, 0, 0, 7, 0, 100, 2000, true, true, true);
        g.add_service_period(s);
        g.add_triphop(500, 1000, 0, 1, 1, 1, "all");
    }

    {
        TripPath *p = g.find_path(0, false, 0.0, 0.0, 1.0, 0.0);

        std::list<TripAction> actions = p->get_actions();
        WVPASSEQ(actions.size(), 1);

        TripAction action = actions.front();
        WVPASSEQ(action.src_id, 0);
        WVPASSEQ(action.dest_id, 1);
        WVPASSEQ(action.start_time, 500.0f);
        WVPASSEQ(action.end_time, 1000.0f);

        delete p;
    }
}

WVTEST_MAIN("basic_graph_saveload")
{
    TripGraph g;
    g.add_tripstop(0, TripStop::OSM, 0.0f, 0.0f);
    g.add_tripstop(1, TripStop::OSM, 1.0f, 0.0f);
    g.add_walkhop(0, 1);

    ServicePeriod s("all", 1, 0, 0, 7, 0, 100, 2000, true, true, true);
    g.add_service_period(s);
    g.add_triphop(500, 1000, 0, 1, 1, 1, "all");

    char *tmpgraphname = tmpnam(NULL); // security issues in unit tests? bah.
    unlink(tmpgraphname);
    g.save(tmpgraphname);

    TripGraph g2;
    g2.load(tmpgraphname);

    // verify that we have two tripstops
    for (int i=0; i<2; i++)
    {
        TripStop ts = g2.get_tripstop(1);
        WVPASSEQ(ts.type, TripStop::OSM);
    }

    // verify that we can still solve a basic path
    {
        TripPath *p = g.find_path(0, false, 0.0, 0.0, 1.0, 0.0);

        std::list<TripAction> actions = p->get_actions();
        WVPASSEQ(actions.size(), 1);

        TripAction action = actions.front();
        WVPASSEQ(action.src_id, 0);
        WVPASSEQ(action.dest_id, 1);
        WVPASSEQ(action.start_time, 500.0f);
        WVPASSEQ(action.end_time, 1000.0f);

        delete p;
    }
}

WVTEST_MAIN("impossible_path")
{
    TripGraph g;
    g.add_tripstop(0, TripStop::OSM, 0.0f, 0.0f);
    g.add_tripstop(1, TripStop::OSM, 1.0f, 0.0f);
    g.add_tripstop(2, TripStop::OSM, 0.0f, 1.0f);
    g.add_walkhop(0, 1);

    TripPath *p = g.find_path(0, false, 0.0, 0.0, 0.0, 1.0);
    WVPASS(!p);
}

WVTEST_MAIN("tripstops_in_range")
{
    TripGraph g;
    // north and agricola
    g.add_tripstop(0, TripStop::GTFS, 44.6554236f, -63.5936968f);
    // north and robie (just north of north&agricola)
    g.add_tripstop(1, TripStop::OSM, 44.6546407f, -63.5948438f);
    // north and northwood (just south of north&agricola)
    g.add_tripstop(2, TripStop::GTFS, 44.6567144f, -63.5919115f);
    // Quinpool and Connaught (a few kms away from north&agricola)
    g.add_tripstop(3, TripStop::GTFS, 44.6432423f, -63.6045261f);

    {
        vector<TripStop> v = g.find_tripstops_in_range(44.6554236f,
                                                       -63.5936968f,
                                                       TripStop::GTFS,
                                                       500.0f);
        WVPASSEQ(v.size(), 2);
        WVPASS(v[0].id == 0 || v[0].id == 2);
        WVPASS(v[1].id == 0 || v[1].id == 2);
    }
}

static time_t get_time_t(int tm_mday, int tm_mon, int tm_year)
{
    struct tm t;
    t.tm_sec = 0;
    t.tm_min = 0;
    t.tm_hour = 0;
    t.tm_mday = tm_mday;
    t.tm_mon = tm_mon;
    t.tm_year = tm_year;
    t.tm_wday = -1;
    t.tm_yday = -1;
    t.tm_isdst = -1;

    return mktime(&t);
}

WVTEST_MAIN("service_periods")
{
    TripGraph g;

    // from the 1st to the 7th (i.e. 1st saturday only)
    {
        ServicePeriod s("saturday_2008", 1, 0, 108, 7, 0, 108, 2000, false, true, false);
        g.add_service_period(s);
    }

    // test something that's within a supported service period
    // Saturday Midnight Jan 5th 2008
    {
        vector<pair<string, int> > vsp = g.get_service_period_ids_for_time(get_time_t(5, 0, 108));
        WVPASSEQ(vsp.size(), 1);
        WVPASSEQ(vsp[0].first, string("saturday_2008"));
    }
    // test something outside a supported service period: day
    // Saturday Midnight Jan 11th 2008
    {
        vector<pair<string, int> > vsp = g.get_service_period_ids_for_time(get_time_t(11, 0, 108));
        WVPASSEQ(vsp.size(), 0);
    }
    // test something outside a supported service period: month
    // Saturday Midnight Feb 5th 2008
    {
        vector<pair<string, int> > vsp = g.get_service_period_ids_for_time(get_time_t(5, 1, 108));
        WVPASSEQ(vsp.size(), 0);
    }

    // test something outside a supported service period: year
    // Saturday Midnight Jan 11th 2009
    {
        vector<pair<string, int> > vsp = g.get_service_period_ids_for_time(get_time_t(5, 1, 109));
        WVPASSEQ(vsp.size(), 0);
    }

    // add another service period (saturdays for month of january)
    {
        ServicePeriod s("saturday_jan_2008", 1, 0, 108, 31, 0, 108, 2000, false, true,
                        false);
        g.add_service_period(s);
    }

    // test something that's within _two_ supported service periods
    // Saturday Midnight Jan 5th 2008
    {
        vector<pair<string, int> > vsp = g.get_service_period_ids_for_time(get_time_t(5, 0, 108));
        WVPASSEQ(vsp.size(), 2);
        WVPASS(vsp[0].first==string("saturday_2008") || vsp[0].first==string("saturday_jan_2008"));
        WVPASS(vsp[1].first==string("saturday_2008") || vsp[1].first==string("saturday_jan_2008"));
        WVFAILEQ(vsp[0].first, vsp[1].first);
    }

    // save graph, reload, make sure service periods are still there
}

WVTEST_MAIN("service_periods_overlapping")
{
    TripGraph g;

    // from the 1st to the 7th (i.e. 1st saturday only)
    {
        ServicePeriod s1("saturday_2008", 1, 0, 108, 7, 0, 108, 90000, false, true, false);
        g.add_service_period(s1);
        ServicePeriod s2("weekday_2008", 1, 0, 108, 7, 0, 108, 90000, true, false, false);
        g.add_service_period(s2);
    }

    vector<pair<string, int> > vsp = g.get_service_period_ids_for_time(get_time_t(5, 0, 108));
    WVPASSEQ(vsp.size(), 2);
    WVPASS(vsp[0].first==string("saturday_2008") || vsp[0].first==string("weekday_2008"));
    WVPASS(vsp[1].first==string("saturday_2008") || vsp[1].first==string("weekday_2008"));
    WVFAILEQ(vsp[0].first, vsp[1].first);

    int weekday_index = (vsp[0].first == string("weekday_2008")) ? 0 : 1;
    WVPASSEQ(vsp[weekday_index].second, 86400);
}

WVTEST_MAIN("service_periods_turned_on_or_off")
{
    TripGraph g;

    // from the 1st to the 7th (i.e. 1st saturday only)
    // turn off weekday service on the 2nd (wednesday)
    // turn on saturday service on the 3rd (keeping weekday service)
    {
        ServicePeriod s1("saturday_2008", 1, 0, 108, 7, 0, 108, 80000, false, true, false);
        s1.add_exception_on(3, 0, 108);
        WVPASSEQ(s1.is_turned_on(3, 0, 108), true);
        WVPASSEQ(s1.is_turned_on(4, 0, 108), false);
        g.add_service_period(s1);
        ServicePeriod s2("weekday_2008", 1, 0, 108, 7, 0, 108, 80000, true, false, false);
        s2.add_exception_off(2, 0, 108);
        WVPASSEQ(s2.is_turned_off(2, 0, 108), true);
        WVPASSEQ(s2.is_turned_off(3, 0, 108), false);
        g.add_service_period(s2);
    }

    {
        // should be no service on the 2nd
        vector<pair<string, int> > vsp = g.get_service_period_ids_for_time(get_time_t(2, 0, 108));
        WVPASSEQ(vsp.size(), 0);
    }

    {
        // should be two service periods on the 3rd (saturday and weekday)
        vector<pair<string, int> > vsp = g.get_service_period_ids_for_time(get_time_t(3, 0, 108));
        WVPASSEQ(vsp.size(), 2);
        WVPASS(vsp[0].first==string("saturday_2008") || vsp[0].first==string("weekday_2008"));
        WVPASS(vsp[1].first==string("saturday_2008") || vsp[1].first==string("weekday_2008"));
        WVFAILEQ(vsp[0].first, vsp[1].first);
    }
}

WVTEST_MAIN("service_periods_save_load")
{
    TripGraph g;

    // use the same setup as the previous test: saturday and weekday schedules
    // with a few exceptions

    // from the 1st to the 7th (i.e. 1st saturday only)
    // turn off weekday service on the 2nd (wednesday)
    // turn on saturday service on the 3rd (keeping weekday service)
    {
        ServicePeriod s1("saturday_2008", 1, 0, 108, 7, 0, 108, 80000, false, true, false);
        s1.add_exception_on(3, 0, 108);
        g.add_service_period(s1);
        ServicePeriod s2("weekday_2008", 1, 0, 108, 7, 0, 108, 80000, true, false, false);
        s2.add_exception_off(2, 0, 108);
        g.add_service_period(s2);
    }

    char *tmpgraphname = tmpnam(NULL); // security issues in unit tests? bah.
    unlink(tmpgraphname);
    g.save(tmpgraphname);

    TripGraph g2;
    g2.load(tmpgraphname);

    {
        // should be no service on the 2nd
        vector<pair<string, int> > vsp = g2.get_service_period_ids_for_time(get_time_t(2, 0, 108));
        WVPASSEQ(vsp.size(), 0);
    }

    {
        // should be two service periods on the 3rd (saturday and weekday)
        vector<pair<string, int> > vsp = g2.get_service_period_ids_for_time(get_time_t(3, 0, 108));
        WVPASSEQ(vsp.size(), 2);
        WVPASS(vsp[0].first==string("saturday_2008") || vsp[0].first==string("weekday_2008"));
        WVPASS(vsp[1].first==string("saturday_2008") || vsp[1].first==string("weekday_2008"));
        WVFAILEQ(vsp[0].first, vsp[1].first);
    }
}
<|endoftext|>


<commit_before>/********************************************************
 * Ville Hyvönen & Teemu Pitkänen                       *
 * HIIT / University of Helsinki                        *
 * ville.o.hyvonen<at>helsinki.fi                       *
 * teemu.pitkanen<at>cs.helsinki.fi                     *
 * 2016                                                 *
 ********************************************************/

#include "armadillo"
#include <cstdlib>
#include <queue>
#include "Mrpt.h"

using namespace arma;

/**
 * The constructor of the index. The inputs are the data for which the index
 * will be built and additional parameters that affect the accuracy of the NN
 * approximation. Concisely, larger n_trees_ or smaller depth values improve
 * accuracy but slow down the queries. A general rule for the right balance is
 * not known. The constructor does not actually build the trees, but that is
 * done by a separate function 'grow' that has to be called before queries can
 * be made.
 * @param X_ - The data to be indexed. Samples as columns, features as rows.
 * @param n_trees_ - The number of trees to be used in the index.
 * @param depth_ - The depth of the trees
 * @param id_ - A name used for filenames when saving.
 */
Mrpt::Mrpt(const fmat& X_, int n_trees_, int depth_, std::string id_) : X(X_), n_trees(n_trees_), depth(depth_), id(id_){
    n_samples = X.n_cols;
    dim = X.n_rows;
    n_pool = n_trees * depth;
    n_array = pow(2, depth + 1);
    split_points = fmat();
    random_matrix = fmat();
}

Mrpt::Mrpt(const std::string filename, int n_trees_, int depth_, std::string id_) : n_trees(n_trees_), depth(depth_), id(id_){
    X = fmat();
    X.load(filename);
    n_samples = X.n_cols;
    dim = X.n_rows;
    n_pool = n_trees * depth;
    n_array = pow(2, depth + 1);
    split_points = fmat();
    random_matrix = fmat();
}

/**
 * The function whose call starts the actual index construction. Initializes
 * arrays to store the tree structures and computes all the projections needed
 * later. Then repeatedly calls method grow_subtree that builds a single
 * RP-tree.
 */
void Mrpt::grow() {
    split_points = zeros<fmat>(n_array, n_trees);
    uvec indices = linspace<uvec>(0, n_samples - 1, n_samples);

    // generate the random matrix and project the data set onto it
    random_matrix = conv_to<fmat>::from(randn(n_pool, dim));
    projected_data = random_matrix * X;

    // Grow the trees
    for (int n_tree = 0; n_tree < n_trees; n_tree++) {
        first_idx = n_tree * depth;
        std::vector<uvec> t = grow_subtree(indices, 0, 0, n_tree);
        tree_leaves.push_back(t);
    }
}

/**
 * Builds a single random projection tree. The tree is constructed by recursively
 * projecting the data on a random vector and splitting into two by the median.
 * @param indices - The indices left in this branch
 * @param tree_level - The level in tree where the recursion is at
 * @param i - The index within the tree where we are at
 * @param n_tree - The index of the tree within the index
 * @return The leaves as a vector of arma::uvecs
 */
std::vector<uvec> Mrpt::grow_subtree(const uvec &indices, int tree_level, int i, uword n_tree) {
    int n = indices.size();
    int idx_left = 2 * i + 1;
    int idx_right = idx_left + 1;

    if (tree_level == depth) {
        std::vector<uvec> v;
        v.push_back(indices);
        return v;
    }

    uvec level = {first_idx + tree_level};
    frowvec projection = projected_data(level, indices);
    uvec ordered = sort_index(projection);

    int split_point = n % 2 ? n / 2 : n / 2 - 1; // median split
    int idx_split_point = ordered(split_point);
    int idx_split_point2 = ordered(split_point + 1);

    split_points(i, n_tree) = n % 2 ? projection(idx_split_point) : (projection(idx_split_point) + projection(idx_split_point2)) / 2;
    uvec left_indices = ordered.subvec(0, split_point);
    uvec right_indices = ordered.subvec(split_point + 1, n - 1);

    std::vector<uvec> v = grow_subtree(indices.elem(left_indices), tree_level + 1, idx_left, n_tree);
    std::vector<uvec> w = grow_subtree(indices.elem(right_indices), tree_level + 1, idx_right, n_tree);
    v.insert( v.end(), w.begin(), w.end() );
    return v;
}

/**
 * This function finds the k approximate nearest neighbors of the query object
 * q. The accuracy of the query depends on both the parameters used for index
 * construction and additional parameters given to this function. This
 * function implements two tricks to improve performance. The voting trick
 * interprets each index object in leaves returned by tree traversals as votes,
 * and only performs the final linear search with the 'elect' most voted
 * objects. The priority queue trick keeps track of nodes where the split value
 * was close to the projection so that we can split the tree traversal to both
 * subtrees if we want.
 * @param q - The query object whose neighbors the function finds.
 * @param k - The number of neighbors the user wants the function to return
 * @param votes_required - The number of votes required for an object to be included in the linear search step
 * @param branches - The number of extra branches explored in the priority queue trick
 * @return The indices of the k approximate nearest neighbors in the original
 * data set for which the index was built.
 */
uvec Mrpt::query(const fvec& q, int k, int votes_required, int branches) {

    fvec projected_query = random_matrix * q;
    uvec votes = zeros<uvec>(n_samples);
    std::priority_queue<Gap, std::vector<Gap>, std::greater<Gap>> pq;

    /*
     * The following loops over all trees, and routes the query to exactly one
     * leaf in each.
     */
    int j = 0; // Used to find the correct projection value, increases through all trees
    for (int n_tree = 0; n_tree < n_trees; n_tree++) {
        const fvec& tree = split_points.unsafe_col(n_tree);

        double split_point = tree[0];
        int idx_left, idx_right;
        int idx_tree = 0;

        while (split_point) {
            idx_left = 2 * idx_tree + 1;
            idx_right = idx_left + 1;
            if (projected_query(j) <= split_point) {
                idx_tree = idx_left;
                pq.push(Gap(n_tree, idx_right, j+1, split_point-projected_query(j)));
            } else {
                idx_tree = idx_right;
                pq.push(Gap(n_tree, idx_left, j+1, projected_query(j)-split_point));
            }
            j++;
            split_point = tree[idx_tree];
        }
        const uvec& idx_one_tree = tree_leaves[n_tree][idx_tree - pow(2, depth) + 1];
        for (int idx : idx_one_tree)
            votes[idx]++;
    }

    /*
     * The following loop routes the query to extra leaves in the same trees
     * handled already once above. The extra branches are popped from the
     * priority queue and routed down the tree just as new root-to-leaf queries.
     */
    for (int b = 0; b < branches; b++){
        if (pq.empty()) break;
        Gap gap = pq.top();
        pq.pop();

        const fvec& tree = split_points.unsafe_col(gap.tree);
        int idx_tree = gap.node;
        int idx_left, idx_right;
        j = gap.level;
        double split_point = tree[idx_tree];

        while (split_point) {
            idx_left = 2 * idx_tree + 1;
            idx_right = idx_left + 1;
            if (projected_query(j) <= split_point) {
                idx_tree = idx_left;
                pq.push(Gap(gap.tree, idx_right, j+1, split_point-projected_query(j)));
            } else {
                idx_tree = idx_right;
                pq.push(Gap(gap.tree, idx_left, j+1, projected_query(j)-split_point));
            }
            j++;
            split_point = tree[idx_tree];
        }

        const uvec& idx_one_tree = tree_leaves[gap.tree][idx_tree - pow(2, depth) + 1];
        for (int idx : idx_one_tree)
            votes[idx]++;
    }

    // Choose the objects with enough votes...
    uvec elected(n_samples);
    do {
        j = 0;
        for (int i=0; i<n_samples; i++){
            if (votes[i] >= votes_required){
                elected[j] = i;
                j++;
            }
        }
        votes_required--; // If not enough objects with enough votes, decrease the requirement
    } while(j < k);
    elected.resize(j);
    return exact_knn(X, q, k, elected);
}

/**
 * This function implements the barebones MRPT algorithm and finds the k
 * approximate nearest neighbors of the query object q using multiple random
 * projection trees. The accuracy of the query depends on the parameters used
 * for index construction. Separated from the function above for optimal performance.
 * @param q - The query object whose neighbors the function finds.
 * @param k - The number of neighbors the user wants the function to return
 * @return The indices of the k approximate nearest neighbors in the original
 * data set for which the index was built.
 */
uvec Mrpt::query(const fvec& q, int k) {
    fvec projected_query = random_matrix * q;
    std::vector<int> idx_canditates(n_trees * (n_samples/pow(2, depth)));
    int j = 0;

    for (int n_tree = 0; n_tree < n_trees; n_tree++) {
        const fvec& tree = split_points.unsafe_col(n_tree);

        double split_point = tree[0];
        int idx_left, idx_right;
        int idx_tree = 0;

        while (split_point) {
            idx_left = 2 * idx_tree + 1;
            idx_right = idx_left + 1;
            idx_tree = projected_query(j++) <= split_point ? idx_left : idx_right;
            split_point = tree[idx_tree];
        }

        const uvec& idx_one_tree = tree_leaves[n_tree][idx_tree - pow(2, depth) + 1];
        idx_canditates.insert(idx_canditates.begin(), idx_one_tree.begin(), idx_one_tree.end());
    }

    std::sort(idx_canditates.begin(), idx_canditates.end());
    auto last = std::unique(idx_canditates.begin(), idx_canditates.end());
    idx_canditates.erase(last, idx_canditates.end());

    return exact_knn(X, q, k, conv_to<uvec>::from(idx_canditates));
}

/**
 * find k nearest neighbors from data for the query point
 * @param X - data matrix, row = data point, col = dimension
 * @param q - query point as a row matrix
 * @param k - number of neighbors searched for
 * @param indices - indices of the points in the original matrix where search is made
 * @return - indices of nearest neighbors in data matrix X as a column vector
 */
uvec exact_knn(const fmat& D, const fvec& q, uword k, uvec indices) {
    int n_cols = indices.size();
    fvec distances = fvec(n_cols);
    for (int i = 0; i < n_cols; i++)
        distances[i] = sum(pow((D.col(indices(i)) - q), 2));

    if(k == 1) {
        uvec ret(1);
        distances.min(ret[0]);
        return ret;
    }

    uvec sorted_indices = indices(sort_index(distances));
    return sorted_indices.size() > k ? sorted_indices.head(k): sorted_indices;
}<commit_msg>Cleaned some copypaste by combining both tree tarversal loops in queries<commit_after>/********************************************************
 * Ville Hyvönen & Teemu Pitkänen                       *
 * HIIT / University of Helsinki                        *
 * ville.o.hyvonen<at>helsinki.fi                       *
 * teemu.pitkanen<at>cs.helsinki.fi                     *
 * 2016                                                 *
 ********************************************************/

#include "armadillo"
#include <cstdlib>
#include <queue>
#include "Mrpt.h"

using namespace arma;

/**
 * The constructor of the index. The inputs are the data for which the index
 * will be built and additional parameters that affect the accuracy of the NN
 * approximation. Concisely, larger n_trees_ or smaller depth values improve
 * accuracy but slow down the queries. A general rule for the right balance is
 * not known. The constructor does not actually build the trees, but that is
 * done by a separate function 'grow' that has to be called before queries can
 * be made.
 * @param X_ - The data to be indexed. Samples as columns, features as rows.
 * @param n_trees_ - The number of trees to be used in the index.
 * @param depth_ - The depth of the trees
 * @param id_ - A name used for filenames when saving.
 */
Mrpt::Mrpt(const fmat& X_, int n_trees_, int depth_, std::string id_) : X(X_), n_trees(n_trees_), depth(depth_), id(id_){
    n_samples = X.n_cols;
    dim = X.n_rows;
    n_pool = n_trees * depth;
    n_array = pow(2, depth + 1);
    split_points = fmat();
    random_matrix = fmat();
}

Mrpt::Mrpt(const std::string filename, int n_trees_, int depth_, std::string id_) : n_trees(n_trees_), depth(depth_), id(id_){
    X = fmat();
    X.load(filename);
    n_samples = X.n_cols;
    dim = X.n_rows;
    n_pool = n_trees * depth;
    n_array = pow(2, depth + 1);
    split_points = fmat();
    random_matrix = fmat();
}

/**
 * The function whose call starts the actual index construction. Initializes
 * arrays to store the tree structures and computes all the projections needed
 * later. Then repeatedly calls method grow_subtree that builds a single
 * RP-tree.
 */
void Mrpt::grow() {
    split_points = zeros<fmat>(n_array, n_trees);
    uvec indices = linspace<uvec>(0, n_samples - 1, n_samples);

    // generate the random matrix and project the data set onto it
    random_matrix = conv_to<fmat>::from(randn(n_pool, dim));
    projected_data = random_matrix * X;

    // Grow the trees
    for (int n_tree = 0; n_tree < n_trees; n_tree++) {
        first_idx = n_tree * depth;
        std::vector<uvec> t = grow_subtree(indices, 0, 0, n_tree);
        tree_leaves.push_back(t);
    }
}

/**
 * Builds a single random projection tree. The tree is constructed by recursively
 * projecting the data on a random vector and splitting into two by the median.
 * @param indices - The indices left in this branch
 * @param tree_level - The level in tree where the recursion is at
 * @param i - The index within the tree where we are at
 * @param n_tree - The index of the tree within the index
 * @return The leaves as a vector of arma::uvecs
 */
std::vector<uvec> Mrpt::grow_subtree(const uvec &indices, int tree_level, int i, uword n_tree) {
    int n = indices.size();
    int idx_left = 2 * i + 1;
    int idx_right = idx_left + 1;

    if (tree_level == depth) {
        std::vector<uvec> v;
        v.push_back(indices);
        return v;
    }

    uvec level = {first_idx + tree_level};
    frowvec projection = projected_data(level, indices);
    uvec ordered = sort_index(projection);

    int split_point = n % 2 ? n / 2 : n / 2 - 1; // median split
    int idx_split_point = ordered(split_point);
    int idx_split_point2 = ordered(split_point + 1);

    split_points(i, n_tree) = n % 2 ? projection(idx_split_point) : (projection(idx_split_point) + projection(idx_split_point2)) / 2;
    uvec left_indices = ordered.subvec(0, split_point);
    uvec right_indices = ordered.subvec(split_point + 1, n - 1);

    std::vector<uvec> v = grow_subtree(indices.elem(left_indices), tree_level + 1, idx_left, n_tree);
    std::vector<uvec> w = grow_subtree(indices.elem(right_indices), tree_level + 1, idx_right, n_tree);
    v.insert( v.end(), w.begin(), w.end() );
    return v;
}

/**
 * This function finds the k approximate nearest neighbors of the query object
 * q. The accuracy of the query depends on both the parameters used for index
 * construction and additional parameters given to this function. This
 * function implements two tricks to improve performance. The voting trick
 * interprets each index object in leaves returned by tree traversals as votes,
 * and only performs the final linear search with the 'elect' most voted
 * objects. The priority queue trick keeps track of nodes where the split value
 * was close to the projection so that we can split the tree traversal to both
 * subtrees if we want.
 * @param q - The query object whose neighbors the function finds.
 * @param k - The number of neighbors the user wants the function to return
 * @param votes_required - The number of votes required for an object to be included in the linear search step
 * @param branches - The number of extra branches explored in the priority queue trick
 * @return The indices of the k approximate nearest neighbors in the original
 * data set for which the index was built.
 */
uvec Mrpt::query(const fvec& q, int k, int votes_required, int branches) {
    // Compute projections for all levels in all trees
    // initialize pq and vote-count vector
    fvec projected_query = random_matrix * q;
    uvec votes = zeros<uvec>(n_samples);
    std::priority_queue<Gap, std::vector<Gap>, std::greater<Gap>> pq;
    int j;

    // First push RP tree roots to priority queue with negative gap widths
    // (will be handled before extra branches)
    for (int n_tree = 0; n_tree < n_trees; n_tree++){
        pq.push(Gap(n_tree, 0, n_tree*depth, -1));
    }

    // The tree traversal phase. At first the top of the queue will contain the
    // 'n_trees' tree roots, after which 'b' extra branches will be explored
    for (int b = 0; b < branches + n_trees; b++){
        if (pq.empty()) break;
        Gap gap = pq.top();
        pq.pop();

        const fvec& tree = split_points.unsafe_col(gap.tree);
        int idx_tree = gap.node;
        int idx_left, idx_right;
        j = gap.level;
        double split_point = tree[idx_tree];

        while (split_point) {
            idx_left = 2 * idx_tree + 1;
            idx_right = idx_left + 1;
            if (projected_query(j) <= split_point) {
                idx_tree = idx_left;
                pq.push(Gap(gap.tree, idx_right, j+1, split_point-projected_query(j)));
            } else {
                idx_tree = idx_right;
                pq.push(Gap(gap.tree, idx_left, j+1, projected_query(j)-split_point));
            }
            j++;
            split_point = tree[idx_tree];
        }

        const uvec& idx_one_tree = tree_leaves[gap.tree][idx_tree - pow(2, depth) + 1];
        for (int idx : idx_one_tree)
            votes[idx]++;
    }

    // The election phase. Choose the objects with enough votes.
    uvec elected(n_samples);
    do {
        j = 0;
        for (int i=0; i<n_samples; i++){
            if (votes[i] >= votes_required){
                elected[j] = i;
                j++;
            }
        }
        // If not enough objects (at least k required) with enough votes
        // decrease vote count requirement
        votes_required--;
    } while(j < k);
    elected.resize(j);

    // The linear search phase
    return exact_knn(X, q, k, elected);
}

/**
 * This function implements the barebones MRPT algorithm and finds the k
 * approximate nearest neighbors of the query object q using multiple random
 * projection trees. The accuracy of the query depends on the parameters used
 * for index construction. Separated from the function above for optimal performance.
 * @param q - The query object whose neighbors the function finds.
 * @param k - The number of neighbors the user wants the function to return
 * @return The indices of the k approximate nearest neighbors in the original
 * data set for which the index was built.
 */
uvec Mrpt::query(const fvec& q, int k) {
    fvec projected_query = random_matrix * q;
    std::vector<int> idx_canditates(n_trees * (n_samples/pow(2, depth)));
    int j = 0;

    for (int n_tree = 0; n_tree < n_trees; n_tree++) {
        const fvec& tree = split_points.unsafe_col(n_tree);

        double split_point = tree[0];
        int idx_left, idx_right;
        int idx_tree = 0;

        while (split_point) {
            idx_left = 2 * idx_tree + 1;
            idx_right = idx_left + 1;
            idx_tree = projected_query(j++) <= split_point ? idx_left : idx_right;
            split_point = tree[idx_tree];
        }

        const uvec& idx_one_tree = tree_leaves[n_tree][idx_tree - pow(2, depth) + 1];
        idx_canditates.insert(idx_canditates.begin(), idx_one_tree.begin(), idx_one_tree.end());
    }

    std::sort(idx_canditates.begin(), idx_canditates.end());
    auto last = std::unique(idx_canditates.begin(), idx_canditates.end());
    idx_canditates.erase(last, idx_canditates.end());

    return exact_knn(X, q, k, conv_to<uvec>::from(idx_canditates));
}

/**
 * find k nearest neighbors from data for the query point
 * @param X - data matrix, row = data point, col = dimension
 * @param q - query point as a row matrix
 * @param k - number of neighbors searched for
 * @param indices - indices of the points in the original matrix where search is made
 * @return - indices of nearest neighbors in data matrix X as a column vector
 */
uvec exact_knn(const fmat& D, const fvec& q, uword k, uvec indices) {
    int n_cols = indices.size();
    fvec distances = fvec(n_cols);
    for (int i = 0; i < n_cols; i++)
        distances[i] = sum(pow((D.col(indices(i)) - q), 2));

    if(k == 1) {
        uvec ret(1);
        distances.min(ret[0]);
        return ret;
    }

    uvec sorted_indices = indices(sort_index(distances));
    return sorted_indices.size() > k ? sorted_indices.head(k): sorted_indices;
}<|endoftext|>


<commit_before>/*
 * Licensed to the Apache Software Foundation (ASF) under one
 * or more contributor license agreements.  See the NOTICE file
 * distributed with this work for additional information
 * regarding copyright ownership.  The ASF licenses this file
 * to you under the Apache License, Version 2.0 (the
 * "License"); you may not use this file except in compliance
 * with the License.  You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

/*
 * Copyright 2015 Cloudius Systems
 *
 * Modified by Cloudius Systems
 */

#ifndef CQL3_MAPS_HH
#define CQL3_MAPS_HH

#include "cql3/abstract_marker.hh"
#include "cql3/term.hh"

namespace cql3 {

#if 0
package org.apache.cassandra.cql3;

import java.nio.ByteBuffer;
import java.util.ArrayList;
import java.util.Comparator;
import java.util.HashMap;
import java.util.Iterator;
import java.util.LinkedHashMap;
import java.util.List;
import java.util.Map;
import java.util.TreeMap;

import org.apache.cassandra.config.ColumnDefinition;
import org.apache.cassandra.db.ColumnFamily;
import org.apache.cassandra.db.composites.CellName;
import org.apache.cassandra.db.composites.Composite;
import org.apache.cassandra.db.marshal.MapType;
import org.apache.cassandra.exceptions.InvalidRequestException;
import org.apache.cassandra.serializers.CollectionSerializer;
import org.apache.cassandra.serializers.MarshalException;
import org.apache.cassandra.transport.Server;
import org.apache.cassandra.utils.FBUtilities;
import org.apache.cassandra.utils.Pair;
#endif

/**
 * Static helper methods and classes for maps.
 */
class maps {
private:
    maps() = delete;
public:
    static shared_ptr<column_specification> key_spec_of(column_specification& column) {
        return ::make_shared<column_specification>(column.ks_name, column.cf_name,
                ::make_shared<column_identifier>(sprint("key(%s)", *column.name), true),
                dynamic_pointer_cast<map_type_impl>(column.type)->get_keys_type());
    }

    static shared_ptr<column_specification> value_spec_of(column_specification& column) {
        return ::make_shared<column_specification>(column.ks_name, column.cf_name,
                ::make_shared<column_identifier>(sprint("value(%s)", *column.name), true),
                dynamic_pointer_cast<map_type_impl>(column.type)->get_values_type());
    }

    class literal : public term::raw {
    public:
        const std::vector<std::pair<::shared_ptr<term::raw>, ::shared_ptr<term::raw>>> entries;

        literal(const std::vector<std::pair<::shared_ptr<term::raw>, ::shared_ptr<term::raw>>>& entries_)
            : entries{entries_}
        { }

        virtual ::shared_ptr<term> prepare(const sstring& keyspace, ::shared_ptr<column_specification> receiver) override {
            throw std::runtime_error("not implemented");
#if 0
            validateAssignableTo(keyspace, receiver);

            ColumnSpecification keySpec = Maps.keySpecOf(receiver);
            ColumnSpecification valueSpec = Maps.valueSpecOf(receiver);
            Map<Term, Term> values = new HashMap<Term, Term>(entries.size());
            boolean allTerminal = true;
            for (Pair<Term.Raw, Term.Raw> entry : entries)
            {
                Term k = entry.left.prepare(keyspace, keySpec);
                Term v = entry.right.prepare(keyspace, valueSpec);

                if (k.containsBindMarker() || v.containsBindMarker())
                    throw new InvalidRequestException(String.format("Invalid map literal for %s: bind variables are not supported inside collection literals", receiver.name));

                if (k instanceof Term.NonTerminal || v instanceof Term.NonTerminal)
                    allTerminal = false;

                values.put(k, v);
            }
            DelayedValue value = new DelayedValue(((MapType)receiver.type).getKeysType(), values);
            return allTerminal ? value.bind(QueryOptions.DEFAULT) : value;
#endif
        }

#if 0
        private void validateAssignableTo(String keyspace, ColumnSpecification receiver) throws InvalidRequestException
        {
            if (!(receiver.type instanceof MapType))
                throw new InvalidRequestException(String.format("Invalid map literal for %s of type %s", receiver.name, receiver.type.asCQL3Type()));

            ColumnSpecification keySpec = Maps.keySpecOf(receiver);
            ColumnSpecification valueSpec = Maps.valueSpecOf(receiver);
            for (Pair<Term.Raw, Term.Raw> entry : entries)
            {
                if (!entry.left.testAssignment(keyspace, keySpec).isAssignable())
                    throw new InvalidRequestException(String.format("Invalid map literal for %s: key %s is not of type %s", receiver.name, entry.left, keySpec.type.asCQL3Type()));
                if (!entry.right.testAssignment(keyspace, valueSpec).isAssignable())
                    throw new InvalidRequestException(String.format("Invalid map literal for %s: value %s is not of type %s", receiver.name, entry.right, valueSpec.type.asCQL3Type()));
            }
        }
#endif

        virtual assignment_testable::test_result test_assignment(const sstring& keyspace, ::shared_ptr<column_specification> receiver) override {
            throw std::runtime_error("not implemented");
#if 0
            if (!(receiver.type instanceof MapType))
                return AssignmentTestable.TestResult.NOT_ASSIGNABLE;

            // If there is no elements, we can't say it's an exact match (an empty map if fundamentally polymorphic).
            if (entries.isEmpty())
                return AssignmentTestable.TestResult.WEAKLY_ASSIGNABLE;

            ColumnSpecification keySpec = Maps.keySpecOf(receiver);
            ColumnSpecification valueSpec = Maps.valueSpecOf(receiver);
            // It's an exact match if all are exact match, but is not assignable as soon as any is non assignable.
            AssignmentTestable.TestResult res = AssignmentTestable.TestResult.EXACT_MATCH;
            for (Pair<Term.Raw, Term.Raw> entry : entries)
            {
                AssignmentTestable.TestResult t1 = entry.left.testAssignment(keyspace, keySpec);
                AssignmentTestable.TestResult t2 = entry.right.testAssignment(keyspace, valueSpec);
                if (t1 == AssignmentTestable.TestResult.NOT_ASSIGNABLE || t2 == AssignmentTestable.TestResult.NOT_ASSIGNABLE)
                    return AssignmentTestable.TestResult.NOT_ASSIGNABLE;
                if (t1 != AssignmentTestable.TestResult.EXACT_MATCH || t2 != AssignmentTestable.TestResult.EXACT_MATCH)
                    res = AssignmentTestable.TestResult.WEAKLY_ASSIGNABLE;
            }
            return res;
#endif
        }

        virtual sstring to_string() override {
            sstring result = "{";
            for (size_t i = 0; i < entries.size(); i++) {
                if (i > 0) {
                    result += ", ";
                }
                result += entries[i].first->to_string();
                result += ":";
                result += entries[i].second->to_string();
            }
            result += "}";
            return result;
        }
    };

#if 0
    public static class Value extends Term.Terminal implements Term.CollectionTerminal
    {
        public final Map<ByteBuffer, ByteBuffer> map;

        public Value(Map<ByteBuffer, ByteBuffer> map)
        {
            this.map = map;
        }

        public static Value fromSerialized(ByteBuffer value, MapType type, int version) throws InvalidRequestException
        {
            try
            {
                // Collections have this small hack that validate cannot be called on a serialized object,
                // but compose does the validation (so we're fine).
                Map<?, ?> m = (Map<?, ?>)type.getSerializer().deserializeForNativeProtocol(value, version);
                Map<ByteBuffer, ByteBuffer> map = new LinkedHashMap<ByteBuffer, ByteBuffer>(m.size());
                for (Map.Entry<?, ?> entry : m.entrySet())
                    map.put(type.getKeysType().decompose(entry.getKey()), type.getValuesType().decompose(entry.getValue()));
                return new Value(map);
            }
            catch (MarshalException e)
            {
                throw new InvalidRequestException(e.getMessage());
            }
        }

        public ByteBuffer get(QueryOptions options)
        {
            return getWithProtocolVersion(options.getProtocolVersion());
        }

        public ByteBuffer getWithProtocolVersion(int protocolVersion)
        {
            List<ByteBuffer> buffers = new ArrayList<>(2 * map.size());
            for (Map.Entry<ByteBuffer, ByteBuffer> entry : map.entrySet())
            {
                buffers.add(entry.getKey());
                buffers.add(entry.getValue());
            }
            return CollectionSerializer.pack(buffers, map.size(), protocolVersion);
        }

        public boolean equals(MapType mt, Value v)
        {
            if (map.size() != v.map.size())
                return false;

            // We use the fact that we know the maps iteration will both be in comparator order
            Iterator<Map.Entry<ByteBuffer, ByteBuffer>> thisIter = map.entrySet().iterator();
            Iterator<Map.Entry<ByteBuffer, ByteBuffer>> thatIter = v.map.entrySet().iterator();
            while (thisIter.hasNext())
            {
                Map.Entry<ByteBuffer, ByteBuffer> thisEntry = thisIter.next();
                Map.Entry<ByteBuffer, ByteBuffer> thatEntry = thatIter.next();
                if (mt.getKeysType().compare(thisEntry.getKey(), thatEntry.getKey()) != 0 || mt.getValuesType().compare(thisEntry.getValue(), thatEntry.getValue()) != 0)
                    return false;
            }

            return true;
        }
    }

    // See Lists.DelayedValue
    public static class DelayedValue extends Term.NonTerminal
    {
        private final Comparator<ByteBuffer> comparator;
        private final Map<Term, Term> elements;

        public DelayedValue(Comparator<ByteBuffer> comparator, Map<Term, Term> elements)
        {
            this.comparator = comparator;
            this.elements = elements;
        }

        public boolean containsBindMarker()
        {
            // False since we don't support them in collection
            return false;
        }

        public void collectMarkerSpecification(VariableSpecifications boundNames)
        {
        }

        public Value bind(QueryOptions options) throws InvalidRequestException
        {
            Map<ByteBuffer, ByteBuffer> buffers = new TreeMap<ByteBuffer, ByteBuffer>(comparator);
            for (Map.Entry<Term, Term> entry : elements.entrySet())
            {
                // We don't support values > 64K because the serialization format encode the length as an unsigned short.
                ByteBuffer keyBytes = entry.getKey().bindAndGet(options);
                if (keyBytes == null)
                    throw new InvalidRequestException("null is not supported inside collections");
                if (keyBytes.remaining() > FBUtilities.MAX_UNSIGNED_SHORT)
                    throw new InvalidRequestException(String.format("Map key is too long. Map keys are limited to %d bytes but %d bytes keys provided",
                                                                    FBUtilities.MAX_UNSIGNED_SHORT,
                                                                    keyBytes.remaining()));

                ByteBuffer valueBytes = entry.getValue().bindAndGet(options);
                if (valueBytes == null)
                    throw new InvalidRequestException("null is not supported inside collections");
                if (valueBytes.remaining() > FBUtilities.MAX_UNSIGNED_SHORT)
                    throw new InvalidRequestException(String.format("Map value is too long. Map values are limited to %d bytes but %d bytes value provided",
                                                                    FBUtilities.MAX_UNSIGNED_SHORT,
                                                                    valueBytes.remaining()));

                buffers.put(keyBytes, valueBytes);
            }
            return new Value(buffers);
        }
    }
#endif

    class marker : public abstract_marker {
    public:
        marker(int32_t bind_index, ::shared_ptr<column_specification> receiver)
            : abstract_marker{bind_index, std::move(receiver)}
        { }
#if 0
        protected Marker(int bindIndex, ColumnSpecification receiver)
        {
            super(bindIndex, receiver);
            assert receiver.type instanceof MapType;
        }
#endif

        virtual ::shared_ptr<terminal> bind(const query_options& options) override {
            throw std::runtime_error("");
        }
#if 0
        public Value bind(QueryOptions options) throws InvalidRequestException
        {
            ByteBuffer value = options.getValues().get(bindIndex);
            return value == null ? null : Value.fromSerialized(value, (MapType)receiver.type, options.getProtocolVersion());
        }
#endif
    };

#if 0
    public static class Setter extends Operation
    {
        public Setter(ColumnDefinition column, Term t)
        {
            super(column, t);
        }

        public void execute(ByteBuffer rowKey, ColumnFamily cf, Composite prefix, UpdateParameters params) throws InvalidRequestException
        {
            if (column.type.isMultiCell())
            {
                // delete + put
                CellName name = cf.getComparator().create(prefix, column);
                cf.addAtom(params.makeTombstoneForOverwrite(name.slice()));
            }
            Putter.doPut(t, cf, prefix, column, params);
        }
    }

    public static class SetterByKey extends Operation
    {
        private final Term k;

        public SetterByKey(ColumnDefinition column, Term k, Term t)
        {
            super(column, t);
            this.k = k;
        }

        @Override
        public void collectMarkerSpecification(VariableSpecifications boundNames)
        {
            super.collectMarkerSpecification(boundNames);
            k.collectMarkerSpecification(boundNames);
        }

        public void execute(ByteBuffer rowKey, ColumnFamily cf, Composite prefix, UpdateParameters params) throws InvalidRequestException
        {
            assert column.type.isMultiCell() : "Attempted to set a value for a single key on a frozen map";
            ByteBuffer key = k.bindAndGet(params.options);
            ByteBuffer value = t.bindAndGet(params.options);
            if (key == null)
                throw new InvalidRequestException("Invalid null map key");

            CellName cellName = cf.getComparator().create(prefix, column, key);

            if (value == null)
            {
                cf.addColumn(params.makeTombstone(cellName));
            }
            else
            {
                // We don't support value > 64K because the serialization format encode the length as an unsigned short.
                if (value.remaining() > FBUtilities.MAX_UNSIGNED_SHORT)
                    throw new InvalidRequestException(String.format("Map value is too long. Map values are limited to %d bytes but %d bytes value provided",
                                                                    FBUtilities.MAX_UNSIGNED_SHORT,
                                                                    value.remaining()));

                cf.addColumn(params.makeColumn(cellName, value));
            }
        }
    }

    public static class Putter extends Operation
    {
        public Putter(ColumnDefinition column, Term t)
        {
            super(column, t);
        }

        public void execute(ByteBuffer rowKey, ColumnFamily cf, Composite prefix, UpdateParameters params) throws InvalidRequestException
        {
            assert column.type.isMultiCell() : "Attempted to add items to a frozen map";
            doPut(t, cf, prefix, column, params);
        }

        static void doPut(Term t, ColumnFamily cf, Composite prefix, ColumnDefinition column, UpdateParameters params) throws InvalidRequestException
        {
            Term.Terminal value = t.bind(params.options);
            Maps.Value mapValue = (Maps.Value) value;
            if (column.type.isMultiCell())
            {
                if (value == null)
                    return;

                for (Map.Entry<ByteBuffer, ByteBuffer> entry : mapValue.map.entrySet())
                {
                    CellName cellName = cf.getComparator().create(prefix, column, entry.getKey());
                    cf.addColumn(params.makeColumn(cellName, entry.getValue()));
                }
            }
            else
            {
                // for frozen maps, we're overwriting the whole cell
                CellName cellName = cf.getComparator().create(prefix, column);
                if (value == null)
                    cf.addAtom(params.makeTombstone(cellName));
                else
                    cf.addColumn(params.makeColumn(cellName, mapValue.getWithProtocolVersion(Server.CURRENT_VERSION)));
            }
        }
    }

    public static class DiscarderByKey extends Operation
    {
        public DiscarderByKey(ColumnDefinition column, Term k)
        {
            super(column, k);
        }

        public void execute(ByteBuffer rowKey, ColumnFamily cf, Composite prefix, UpdateParameters params) throws InvalidRequestException
        {
            assert column.type.isMultiCell() : "Attempted to delete a single key in a frozen map";
            Term.Terminal key = t.bind(params.options);
            if (key == null)
                throw new InvalidRequestException("Invalid null map key");
            assert key instanceof Constants.Value;

            CellName cellName = cf.getComparator().create(prefix, column, ((Constants.Value)key).bytes);
            cf.addColumn(params.makeTombstone(cellName));
        }
    }
#endif
};

}

#endif
<commit_msg>cql3: convert maps::setter_by_key<commit_after>/*
 * Licensed to the Apache Software Foundation (ASF) under one
 * or more contributor license agreements.  See the NOTICE file
 * distributed with this work for additional information
 * regarding copyright ownership.  The ASF licenses this file
 * to you under the Apache License, Version 2.0 (the
 * "License"); you may not use this file except in compliance
 * with the License.  You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

/*
 * Copyright 2015 Cloudius Systems
 *
 * Modified by Cloudius Systems
 */

#ifndef CQL3_MAPS_HH
#define CQL3_MAPS_HH

#include "cql3/abstract_marker.hh"
#include "cql3/term.hh"
#include "operation.hh"
#include "update_parameters.hh"

namespace cql3 {

#if 0
package org.apache.cassandra.cql3;

import java.nio.ByteBuffer;
import java.util.ArrayList;
import java.util.Comparator;
import java.util.HashMap;
import java.util.Iterator;
import java.util.LinkedHashMap;
import java.util.List;
import java.util.Map;
import java.util.TreeMap;

import org.apache.cassandra.config.ColumnDefinition;
import org.apache.cassandra.db.ColumnFamily;
import org.apache.cassandra.db.composites.CellName;
import org.apache.cassandra.db.composites.Composite;
import org.apache.cassandra.db.marshal.MapType;
import org.apache.cassandra.exceptions.InvalidRequestException;
import org.apache.cassandra.serializers.CollectionSerializer;
import org.apache.cassandra.serializers.MarshalException;
import org.apache.cassandra.transport.Server;
import org.apache.cassandra.utils.FBUtilities;
import org.apache.cassandra.utils.Pair;
#endif

/**
 * Static helper methods and classes for maps.
 */
class maps {
private:
    maps() = delete;
public:
    static shared_ptr<column_specification> key_spec_of(column_specification& column) {
        return ::make_shared<column_specification>(column.ks_name, column.cf_name,
                ::make_shared<column_identifier>(sprint("key(%s)", *column.name), true),
                dynamic_pointer_cast<map_type_impl>(column.type)->get_keys_type());
    }

    static shared_ptr<column_specification> value_spec_of(column_specification& column) {
        return ::make_shared<column_specification>(column.ks_name, column.cf_name,
                ::make_shared<column_identifier>(sprint("value(%s)", *column.name), true),
                dynamic_pointer_cast<map_type_impl>(column.type)->get_values_type());
    }

    class literal : public term::raw {
    public:
        const std::vector<std::pair<::shared_ptr<term::raw>, ::shared_ptr<term::raw>>> entries;

        literal(const std::vector<std::pair<::shared_ptr<term::raw>, ::shared_ptr<term::raw>>>& entries_)
            : entries{entries_}
        { }

        virtual ::shared_ptr<term> prepare(const sstring& keyspace, ::shared_ptr<column_specification> receiver) override {
            throw std::runtime_error("not implemented");
#if 0
            validateAssignableTo(keyspace, receiver);

            ColumnSpecification keySpec = Maps.keySpecOf(receiver);
            ColumnSpecification valueSpec = Maps.valueSpecOf(receiver);
            Map<Term, Term> values = new HashMap<Term, Term>(entries.size());
            boolean allTerminal = true;
            for (Pair<Term.Raw, Term.Raw> entry : entries)
            {
                Term k = entry.left.prepare(keyspace, keySpec);
                Term v = entry.right.prepare(keyspace, valueSpec);

                if (k.containsBindMarker() || v.containsBindMarker())
                    throw new InvalidRequestException(String.format("Invalid map literal for %s: bind variables are not supported inside collection literals", receiver.name));

                if (k instanceof Term.NonTerminal || v instanceof Term.NonTerminal)
                    allTerminal = false;

                values.put(k, v);
            }
            DelayedValue value = new DelayedValue(((MapType)receiver.type).getKeysType(), values);
            return allTerminal ? value.bind(QueryOptions.DEFAULT) : value;
#endif
        }

#if 0
        private void validateAssignableTo(String keyspace, ColumnSpecification receiver) throws InvalidRequestException
        {
            if (!(receiver.type instanceof MapType))
                throw new InvalidRequestException(String.format("Invalid map literal for %s of type %s", receiver.name, receiver.type.asCQL3Type()));

            ColumnSpecification keySpec = Maps.keySpecOf(receiver);
            ColumnSpecification valueSpec = Maps.valueSpecOf(receiver);
            for (Pair<Term.Raw, Term.Raw> entry : entries)
            {
                if (!entry.left.testAssignment(keyspace, keySpec).isAssignable())
                    throw new InvalidRequestException(String.format("Invalid map literal for %s: key %s is not of type %s", receiver.name, entry.left, keySpec.type.asCQL3Type()));
                if (!entry.right.testAssignment(keyspace, valueSpec).isAssignable())
                    throw new InvalidRequestException(String.format("Invalid map literal for %s: value %s is not of type %s", receiver.name, entry.right, valueSpec.type.asCQL3Type()));
            }
        }
#endif

        virtual assignment_testable::test_result test_assignment(const sstring& keyspace, ::shared_ptr<column_specification> receiver) override {
            throw std::runtime_error("not implemented");
#if 0
            if (!(receiver.type instanceof MapType))
                return AssignmentTestable.TestResult.NOT_ASSIGNABLE;

            // If there is no elements, we can't say it's an exact match (an empty map if fundamentally polymorphic).
            if (entries.isEmpty())
                return AssignmentTestable.TestResult.WEAKLY_ASSIGNABLE;

            ColumnSpecification keySpec = Maps.keySpecOf(receiver);
            ColumnSpecification valueSpec = Maps.valueSpecOf(receiver);
            // It's an exact match if all are exact match, but is not assignable as soon as any is non assignable.
            AssignmentTestable.TestResult res = AssignmentTestable.TestResult.EXACT_MATCH;
            for (Pair<Term.Raw, Term.Raw> entry : entries)
            {
                AssignmentTestable.TestResult t1 = entry.left.testAssignment(keyspace, keySpec);
                AssignmentTestable.TestResult t2 = entry.right.testAssignment(keyspace, valueSpec);
                if (t1 == AssignmentTestable.TestResult.NOT_ASSIGNABLE || t2 == AssignmentTestable.TestResult.NOT_ASSIGNABLE)
                    return AssignmentTestable.TestResult.NOT_ASSIGNABLE;
                if (t1 != AssignmentTestable.TestResult.EXACT_MATCH || t2 != AssignmentTestable.TestResult.EXACT_MATCH)
                    res = AssignmentTestable.TestResult.WEAKLY_ASSIGNABLE;
            }
            return res;
#endif
        }

        virtual sstring to_string() override {
            sstring result = "{";
            for (size_t i = 0; i < entries.size(); i++) {
                if (i > 0) {
                    result += ", ";
                }
                result += entries[i].first->to_string();
                result += ":";
                result += entries[i].second->to_string();
            }
            result += "}";
            return result;
        }
    };

#if 0
    public static class Value extends Term.Terminal implements Term.CollectionTerminal
    {
        public final Map<ByteBuffer, ByteBuffer> map;

        public Value(Map<ByteBuffer, ByteBuffer> map)
        {
            this.map = map;
        }

        public static Value fromSerialized(ByteBuffer value, MapType type, int version) throws InvalidRequestException
        {
            try
            {
                // Collections have this small hack that validate cannot be called on a serialized object,
                // but compose does the validation (so we're fine).
                Map<?, ?> m = (Map<?, ?>)type.getSerializer().deserializeForNativeProtocol(value, version);
                Map<ByteBuffer, ByteBuffer> map = new LinkedHashMap<ByteBuffer, ByteBuffer>(m.size());
                for (Map.Entry<?, ?> entry : m.entrySet())
                    map.put(type.getKeysType().decompose(entry.getKey()), type.getValuesType().decompose(entry.getValue()));
                return new Value(map);
            }
            catch (MarshalException e)
            {
                throw new InvalidRequestException(e.getMessage());
            }
        }

        public ByteBuffer get(QueryOptions options)
        {
            return getWithProtocolVersion(options.getProtocolVersion());
        }

        public ByteBuffer getWithProtocolVersion(int protocolVersion)
        {
            List<ByteBuffer> buffers = new ArrayList<>(2 * map.size());
            for (Map.Entry<ByteBuffer, ByteBuffer> entry : map.entrySet())
            {
                buffers.add(entry.getKey());
                buffers.add(entry.getValue());
            }
            return CollectionSerializer.pack(buffers, map.size(), protocolVersion);
        }

        public boolean equals(MapType mt, Value v)
        {
            if (map.size() != v.map.size())
                return false;

            // We use the fact that we know the maps iteration will both be in comparator order
            Iterator<Map.Entry<ByteBuffer, ByteBuffer>> thisIter = map.entrySet().iterator();
            Iterator<Map.Entry<ByteBuffer, ByteBuffer>> thatIter = v.map.entrySet().iterator();
            while (thisIter.hasNext())
            {
                Map.Entry<ByteBuffer, ByteBuffer> thisEntry = thisIter.next();
                Map.Entry<ByteBuffer, ByteBuffer> thatEntry = thatIter.next();
                if (mt.getKeysType().compare(thisEntry.getKey(), thatEntry.getKey()) != 0 || mt.getValuesType().compare(thisEntry.getValue(), thatEntry.getValue()) != 0)
                    return false;
            }

            return true;
        }
    }

    // See Lists.DelayedValue
    public static class DelayedValue extends Term.NonTerminal
    {
        private final Comparator<ByteBuffer> comparator;
        private final Map<Term, Term> elements;

        public DelayedValue(Comparator<ByteBuffer> comparator, Map<Term, Term> elements)
        {
            this.comparator = comparator;
            this.elements = elements;
        }

        public boolean containsBindMarker()
        {
            // False since we don't support them in collection
            return false;
        }

        public void collectMarkerSpecification(VariableSpecifications boundNames)
        {
        }

        public Value bind(QueryOptions options) throws InvalidRequestException
        {
            Map<ByteBuffer, ByteBuffer> buffers = new TreeMap<ByteBuffer, ByteBuffer>(comparator);
            for (Map.Entry<Term, Term> entry : elements.entrySet())
            {
                // We don't support values > 64K because the serialization format encode the length as an unsigned short.
                ByteBuffer keyBytes = entry.getKey().bindAndGet(options);
                if (keyBytes == null)
                    throw new InvalidRequestException("null is not supported inside collections");
                if (keyBytes.remaining() > FBUtilities.MAX_UNSIGNED_SHORT)
                    throw new InvalidRequestException(String.format("Map key is too long. Map keys are limited to %d bytes but %d bytes keys provided",
                                                                    FBUtilities.MAX_UNSIGNED_SHORT,
                                                                    keyBytes.remaining()));

                ByteBuffer valueBytes = entry.getValue().bindAndGet(options);
                if (valueBytes == null)
                    throw new InvalidRequestException("null is not supported inside collections");
                if (valueBytes.remaining() > FBUtilities.MAX_UNSIGNED_SHORT)
                    throw new InvalidRequestException(String.format("Map value is too long. Map values are limited to %d bytes but %d bytes value provided",
                                                                    FBUtilities.MAX_UNSIGNED_SHORT,
                                                                    valueBytes.remaining()));

                buffers.put(keyBytes, valueBytes);
            }
            return new Value(buffers);
        }
    }
#endif

    class marker : public abstract_marker {
    public:
        marker(int32_t bind_index, ::shared_ptr<column_specification> receiver)
            : abstract_marker{bind_index, std::move(receiver)}
        { }
#if 0
        protected Marker(int bindIndex, ColumnSpecification receiver)
        {
            super(bindIndex, receiver);
            assert receiver.type instanceof MapType;
        }
#endif

        virtual ::shared_ptr<terminal> bind(const query_options& options) override {
            throw std::runtime_error("");
        }
#if 0
        public Value bind(QueryOptions options) throws InvalidRequestException
        {
            ByteBuffer value = options.getValues().get(bindIndex);
            return value == null ? null : Value.fromSerialized(value, (MapType)receiver.type, options.getProtocolVersion());
        }
#endif
    };

#if 0
    public static class Setter extends Operation
    {
        public Setter(ColumnDefinition column, Term t)
        {
            super(column, t);
        }

        public void execute(ByteBuffer rowKey, ColumnFamily cf, Composite prefix, UpdateParameters params) throws InvalidRequestException
        {
            if (column.type.isMultiCell())
            {
                // delete + put
                CellName name = cf.getComparator().create(prefix, column);
                cf.addAtom(params.makeTombstoneForOverwrite(name.slice()));
            }
            Putter.doPut(t, cf, prefix, column, params);
        }
    }
#endif

    class setter_by_key : public operation {
        const shared_ptr<term> _k;
    public:
        setter_by_key(column_definition& column, shared_ptr<term> k, shared_ptr<term> t)
            : operation(column, std::move(t)), _k(std::move(k)) {
        }

        virtual void collect_marker_specification(shared_ptr<variable_specifications> bound_names) override {
            operation::collect_marker_specification(bound_names);
            _k->collect_marker_specification(bound_names);
        }

        virtual void execute(mutation& m, const clustering_prefix& prefix, const update_parameters& params) override {
            using exceptions::invalid_request_exception;
            assert(column.type->is_multi_cell()); // "Attempted to set a value for a single key on a frozen map"m
            bytes_opt key = _k->bind_and_get(params._options);
            bytes_opt value = _t->bind_and_get(params._options);
            if (!key) {
                throw invalid_request_exception("Invalid null map key");
            }
            if (value && value->size() >= std::numeric_limits<uint16_t>::max()) {
                throw invalid_request_exception(
                        sprint("Map value is too long. Map values are limited to %d bytes but %d bytes value provided",
                               std::numeric_limits<uint16_t>::max(),
                               value->size()));
            }
            auto avalue = value ? params.make_cell(*value) : params.make_dead_cell();
            map_type_impl::mutation update = { { std::move(*key), std::move(avalue) } };
            // should have been verified as map earlier?
            auto ctype = static_pointer_cast<map_type_impl>(column.type);
            auto col_mut = ctype->serialize_mutation_form(std::move(update));
            if (column.is_static()) {
                m.set_static_cell(column, std::move(col_mut));
            } else {
                m.set_clustered_cell(prefix, column, std::move(col_mut));
            }
        }
    };
#if 0
    public static class Putter extends Operation
    {
        public Putter(ColumnDefinition column, Term t)
        {
            super(column, t);
        }

        public void execute(ByteBuffer rowKey, ColumnFamily cf, Composite prefix, UpdateParameters params) throws InvalidRequestException
        {
            assert column.type.isMultiCell() : "Attempted to add items to a frozen map";
            doPut(t, cf, prefix, column, params);
        }

        static void doPut(Term t, ColumnFamily cf, Composite prefix, ColumnDefinition column, UpdateParameters params) throws InvalidRequestException
        {
            Term.Terminal value = t.bind(params.options);
            Maps.Value mapValue = (Maps.Value) value;
            if (column.type.isMultiCell())
            {
                if (value == null)
                    return;

                for (Map.Entry<ByteBuffer, ByteBuffer> entry : mapValue.map.entrySet())
                {
                    CellName cellName = cf.getComparator().create(prefix, column, entry.getKey());
                    cf.addColumn(params.makeColumn(cellName, entry.getValue()));
                }
            }
            else
            {
                // for frozen maps, we're overwriting the whole cell
                CellName cellName = cf.getComparator().create(prefix, column);
                if (value == null)
                    cf.addAtom(params.makeTombstone(cellName));
                else
                    cf.addColumn(params.makeColumn(cellName, mapValue.getWithProtocolVersion(Server.CURRENT_VERSION)));
            }
        }
    }

    public static class DiscarderByKey extends Operation
    {
        public DiscarderByKey(ColumnDefinition column, Term k)
        {
            super(column, k);
        }

        public void execute(ByteBuffer rowKey, ColumnFamily cf, Composite prefix, UpdateParameters params) throws InvalidRequestException
        {
            assert column.type.isMultiCell() : "Attempted to delete a single key in a frozen map";
            Term.Terminal key = t.bind(params.options);
            if (key == null)
                throw new InvalidRequestException("Invalid null map key");
            assert key instanceof Constants.Value;

            CellName cellName = cf.getComparator().create(prefix, column, ((Constants.Value)key).bytes);
            cf.addColumn(params.makeTombstone(cellName));
        }
    }
#endif
};

}

#endif
<|endoftext|>


<commit_before>/*
 * OXT - OS eXtensions for boosT
 * Provides important functionality necessary for writing robust server software.
 *
 * Copyright (c) 2008-2012 Phusion
 *
 * Permission is hereby granted, free of charge, to any person obtaining a copy
 * of this software and associated documentation files (the "Software"), to deal
 * in the Software without restriction, including without limitation the rights
 * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
 * copies of the Software, and to permit persons to whom the Software is
 * furnished to do so, subject to the following conditions:
 *
 * The above copyright notice and this permission notice shall be included in
 * all copies or substantial portions of the Software.
 *
 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
 * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
 * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
 * THE SOFTWARE.
 */

#include <boost/thread/mutex.hpp>
#include <boost/shared_ptr.hpp>
#include <boost/make_shared.hpp>
#include "tracable_exception.hpp"
#include "backtrace.hpp"
#include "macros.hpp"
#include "thread.hpp"
#include "spin_lock.hpp"
#include "detail/context.hpp"
#ifdef __linux__
	#include <sys/syscall.h>
#endif

#ifdef OXT_THREAD_LOCAL_KEYWORD_SUPPORTED
	#include <cassert>
	#include <sstream>
	#include <cstring>
#endif

namespace oxt {

using namespace std;
using namespace boost;

static global_context_t *global_context = NULL;

/*
 * boost::thread_specific_storage is pretty expensive. So we use the __thread
 * keyword whenever possible - that's almost free.
 */
#ifdef OXT_THREAD_LOCAL_KEYWORD_SUPPORTED
	static __thread thread_local_context_ptr *local_context = NULL;

	static void
	set_thread_local_context(const thread_local_context_ptr &ctx) {
		local_context = new thread_local_context_ptr(ctx);
	}

	static void
	free_thread_local_context() {
		delete local_context;
		local_context = NULL;
	}

	thread_local_context *
	get_thread_local_context() {
		if (OXT_LIKELY(local_context != NULL)) {
			return local_context->get();
		} else {
			return NULL;
		}
	}
#else
	/*
	 * This is a *pointer* to a thread_specific_ptr because, once
	 * we've created a thread_specific_ptr, we never want to destroy it
	 * in order to avoid C++'s global variable destruction. All kinds
	 * of cleanup code may depend on local_context and because global
	 * variable destruction order is undefined, we just want to keep
	 * this object alive until the OS cleans it up.
	 */
	static thread_specific_ptr<thread_local_context_ptr> *local_context = NULL;

	static void
	set_thread_local_context(const thread_local_context_ptr &ctx) {
		local_context = new thread_specific_ptr<thread_local_context_ptr>();
		local_context->reset(new thread_local_context_ptr(ctx));
	}

	static void
	free_thread_local_context() {
		local_context->reset();
	}

	thread_local_context *
	get_thread_local_context() {
		if (OXT_LIKELY(local_context != NULL)) {
			thread_local_context_ptr *pointer = local_context->get();
			if (OXT_LIKELY(pointer != NULL)) {
				return pointer->get();
			} else {
				return NULL;
			}
		} else {
			return NULL;
		}
	}
#endif

#ifdef OXT_BACKTRACE_IS_ENABLED

trace_point::trace_point(const char *_function, const char *_source, unsigned int _line)
	: function(_function),
	  source(_source),
	  line(_line),
	  m_detached(false)
{
	thread_local_context *ctx = get_thread_local_context();
	if (OXT_LIKELY(ctx != NULL)) {
		spin_lock::scoped_lock l(ctx->backtrace_lock);
		ctx->backtrace_list.push_back(this);
	} else {
		m_detached = true;
	}
}

trace_point::trace_point(const char *_function, const char *_source, unsigned int _line, bool detached)
	: function(_function),
	  source(_source),
	  line(_line),
	  m_detached(true)
{ }

trace_point::~trace_point() {
	if (OXT_LIKELY(!m_detached)) {
		thread_local_context *ctx = get_thread_local_context();
		if (OXT_LIKELY(ctx != NULL)) {
			spin_lock::scoped_lock l(ctx->backtrace_lock);
			assert(!ctx->backtrace_list.empty());
			ctx->backtrace_list.pop_back();
		}
	}
}

void
trace_point::update(const char *source, unsigned int line) {
	this->source = source;
	this->line = line;
}

tracable_exception::tracable_exception() {
	thread_local_context *ctx = get_thread_local_context();
	if (OXT_LIKELY(ctx != NULL)) {
		spin_lock::scoped_lock l(ctx->backtrace_lock);
		vector<trace_point *>::const_iterator it, end = ctx->backtrace_list.end();

		backtrace_copy.reserve(ctx->backtrace_list.size());
		for (it = ctx->backtrace_list.begin(); it != end; it++) {
			trace_point *p = new trace_point(
				(*it)->function,
				(*it)->source,
				(*it)->line,
				true);
			backtrace_copy.push_back(p);
		}
	}
}

tracable_exception::tracable_exception(const tracable_exception &other)
	: std::exception()
{
	vector<trace_point *>::const_iterator it, end = other.backtrace_copy.end();
	backtrace_copy.reserve(other.backtrace_copy.size());
	for (it = other.backtrace_copy.begin(); it != end; it++) {
		trace_point *p = new trace_point(
			(*it)->function,
			(*it)->source,
			(*it)->line,
			true);
		backtrace_copy.push_back(p);
	}
}

tracable_exception::~tracable_exception() throw() {
	vector<trace_point *>::iterator it, end = backtrace_copy.end();
	for (it = backtrace_copy.begin(); it != end; it++) {
		delete *it;
	}
}

template<typename Collection>
static string
format_backtrace(const Collection &backtrace_list) {
	if (backtrace_list.empty()) {
		return "     (empty)";
	} else {
		backtrace_list.rbegin();
		stringstream result;
		typename Collection::const_reverse_iterator it;

		for (it = backtrace_list.rbegin(); it != backtrace_list.rend(); it++) {
			const trace_point *p = *it;

			result << "     in '" << p->function << "'";
			if (p->source != NULL) {
				const char *source = strrchr(p->source, '/');
				if (source != NULL) {
					source++;
				} else {
					source = p->source;
				}
				result << " (" << source << ":" << p->line << ")";
			}
			result << endl;
		}
		return result.str();
	}
}

string
tracable_exception::backtrace() const throw() {
	return format_backtrace< vector<trace_point *> >(backtrace_copy);
}

const char *
tracable_exception::what() const throw() {
	return "oxt::tracable_exception";
}

#endif /* OXT_BACKTRACE_IS_ENABLED */

void initialize() {
	global_context = new global_context_t();
	// For some reason make_shared() crashes here when compiled with clang 3.2 on OS X.
	// Clang bug? We use 'new' to work around it.
	thread_local_context_ptr ctx = thread_local_context::make_shared_ptr();
	ctx->thread_number = 1;
	ctx->thread_name = "Main thread";
	set_thread_local_context(ctx);
}

global_context_t::global_context_t()
	: next_thread_number(2)
{ }

thread_local_context_ptr
thread_local_context::make_shared_ptr() {
	// For some reason make_shared() crashes here when compiled with clang 3.2 on OS X.
	// Clang bug? We use 'new' to work around it.
	return thread_local_context_ptr(new thread_local_context());
}

thread_local_context::thread_local_context()
	: thread_number(0)
{
	thread = pthread_self();
	#ifdef __linux__
		tid = syscall(SYS_gettid);
	#endif
	syscall_interruption_lock.lock();
	#ifdef OXT_BACKTRACE_IS_ENABLED
		backtrace_list.reserve(50);
	#endif
}

string
thread::make_thread_name(const string &given_name) {
	if (given_name.empty()) {
		if (OXT_LIKELY(global_context != NULL)) {
			stringstream str;
			str << "Thread #";
			{
				lock_guard<boost::mutex> l(global_context->thread_registration_mutex);
				str << global_context->next_thread_number;
			}
			return str.str();
		} else {
			return "(unknown)";
		}
	} else {
		return given_name;
	}
}

void
thread::thread_main(const boost::function<void ()> func, thread_local_context_ptr ctx) {
	set_thread_local_context(ctx);

	if (OXT_LIKELY(global_context != NULL)) {
		lock_guard<boost::mutex> l(global_context->thread_registration_mutex);

		ctx->thread = pthread_self();
		global_context->next_thread_number++;
		global_context->registered_threads.push_back(ctx);
		ctx->iterator = global_context->registered_threads.end();
		ctx->iterator--;
		// Set this after setting 'iterator' to indicate
		// that push_back() has succeeded.
		ctx->thread_number = global_context->next_thread_number;
	}

	try {
		func();
	} catch (const thread_interrupted &) {
		// Do nothing.
	}
	// We don't care about other exceptions because they'll crash the process anyway.

	if (OXT_LIKELY(global_context != NULL)) {
		lock_guard<boost::mutex> l(global_context->thread_registration_mutex);
		thread_local_context *ctx = get_thread_local_context();
		if (ctx->thread_number != 0) {
			global_context->registered_threads.erase(ctx->iterator);
			ctx->thread_number = 0;
		}
	}
	free_thread_local_context();
}

std::string
thread::name() const throw() {
	return context->thread_name;
}

std::string
thread::backtrace() const throw() {
	#ifdef OXT_BACKTRACE_IS_ENABLED
		spin_lock::scoped_lock l(context->backtrace_lock);
		return format_backtrace(context->backtrace_list);
	#else
		return "    (backtrace support disabled during compile time)";
	#endif
}

string
thread::all_backtraces() throw() {
	#ifdef OXT_BACKTRACE_IS_ENABLED
		if (OXT_LIKELY(global_context != NULL)) {
			lock_guard<boost::mutex> l(global_context->thread_registration_mutex);
			list<thread_local_context_ptr>::const_iterator it;
			std::stringstream result;

			for (it = global_context->registered_threads.begin();
			     it != global_context->registered_threads.end();
			     it++)
			{
				thread_local_context_ptr ctx = *it;
				result << "Thread '" << ctx->thread_name <<
					"' (" << hex << showbase << ctx->thread << dec;
				#ifdef __linux__
					result << ", LWP " << ctx->tid;
				#endif
				result << "):" << endl;

				spin_lock::scoped_lock l(ctx->backtrace_lock);
				std::string bt = format_backtrace(ctx->backtrace_list);
				result << bt;
				if (bt.empty() || bt[bt.size() - 1] != '\n') {
					result << endl;
				}
				result << endl;
			}
			return result.str();
		} else {
			return "(OXT not initialized)";
		}
	#else
		return "(backtrace support disabled during compile time)";
	#endif
}

void
thread::interrupt(bool interruptSyscalls) {
	int ret;

	boost::thread::interrupt();
	if (interruptSyscalls && context->syscall_interruption_lock.try_lock()) {
		do {
			ret = pthread_kill(native_handle(),
				INTERRUPTION_SIGNAL);
		} while (ret == EINTR);
		context->syscall_interruption_lock.unlock();
	}
}

} // namespace oxt
<commit_msg>Fixed a NULL pointer crash in OXT.<commit_after>/*
 * OXT - OS eXtensions for boosT
 * Provides important functionality necessary for writing robust server software.
 *
 * Copyright (c) 2008-2012 Phusion
 *
 * Permission is hereby granted, free of charge, to any person obtaining a copy
 * of this software and associated documentation files (the "Software"), to deal
 * in the Software without restriction, including without limitation the rights
 * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
 * copies of the Software, and to permit persons to whom the Software is
 * furnished to do so, subject to the following conditions:
 *
 * The above copyright notice and this permission notice shall be included in
 * all copies or substantial portions of the Software.
 *
 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
 * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
 * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
 * THE SOFTWARE.
 */

#include <boost/thread/mutex.hpp>
#include <boost/shared_ptr.hpp>
#include <boost/make_shared.hpp>
#include "tracable_exception.hpp"
#include "backtrace.hpp"
#include "macros.hpp"
#include "thread.hpp"
#include "spin_lock.hpp"
#include "detail/context.hpp"
#ifdef __linux__
	#include <sys/syscall.h>
#endif

#ifdef OXT_THREAD_LOCAL_KEYWORD_SUPPORTED
	#include <cassert>
	#include <sstream>
	#include <cstring>
#endif

namespace oxt {

using namespace std;
using namespace boost;

static global_context_t *global_context = NULL;

/*
 * boost::thread_specific_storage is pretty expensive. So we use the __thread
 * keyword whenever possible - that's almost free.
 */
#ifdef OXT_THREAD_LOCAL_KEYWORD_SUPPORTED
	static __thread thread_local_context_ptr *local_context = NULL;

	static void
	set_thread_local_context(const thread_local_context_ptr &ctx) {
		local_context = new thread_local_context_ptr(ctx);
	}

	static void
	free_thread_local_context() {
		delete local_context;
		local_context = NULL;
	}

	thread_local_context *
	get_thread_local_context() {
		if (OXT_LIKELY(local_context != NULL)) {
			return local_context->get();
		} else {
			return NULL;
		}
	}
#else
	/*
	 * This is a *pointer* to a thread_specific_ptr because, once
	 * we've created a thread_specific_ptr, we never want to destroy it
	 * in order to avoid C++'s global variable destruction. All kinds
	 * of cleanup code may depend on local_context and because global
	 * variable destruction order is undefined, we just want to keep
	 * this object alive until the OS cleans it up.
	 */
	static thread_specific_ptr<thread_local_context_ptr> *local_context = NULL;

	static void
	set_thread_local_context(const thread_local_context_ptr &ctx) {
		local_context = new thread_specific_ptr<thread_local_context_ptr>();
		local_context->reset(new thread_local_context_ptr(ctx));
	}

	static void
	free_thread_local_context() {
		local_context->reset();
	}

	thread_local_context *
	get_thread_local_context() {
		if (OXT_LIKELY(local_context != NULL)) {
			thread_local_context_ptr *pointer = local_context->get();
			if (OXT_LIKELY(pointer != NULL)) {
				return pointer->get();
			} else {
				return NULL;
			}
		} else {
			return NULL;
		}
	}
#endif

#ifdef OXT_BACKTRACE_IS_ENABLED

trace_point::trace_point(const char *_function, const char *_source, unsigned int _line)
	: function(_function),
	  source(_source),
	  line(_line),
	  m_detached(false)
{
	thread_local_context *ctx = get_thread_local_context();
	if (OXT_LIKELY(ctx != NULL)) {
		spin_lock::scoped_lock l(ctx->backtrace_lock);
		ctx->backtrace_list.push_back(this);
	} else {
		m_detached = true;
	}
}

trace_point::trace_point(const char *_function, const char *_source, unsigned int _line, bool detached)
	: function(_function),
	  source(_source),
	  line(_line),
	  m_detached(true)
{ }

trace_point::~trace_point() {
	if (OXT_LIKELY(!m_detached)) {
		thread_local_context *ctx = get_thread_local_context();
		if (OXT_LIKELY(ctx != NULL)) {
			spin_lock::scoped_lock l(ctx->backtrace_lock);
			assert(!ctx->backtrace_list.empty());
			ctx->backtrace_list.pop_back();
		}
	}
}

void
trace_point::update(const char *source, unsigned int line) {
	this->source = source;
	this->line = line;
}

tracable_exception::tracable_exception() {
	thread_local_context *ctx = get_thread_local_context();
	if (OXT_LIKELY(ctx != NULL)) {
		spin_lock::scoped_lock l(ctx->backtrace_lock);
		vector<trace_point *>::const_iterator it, end = ctx->backtrace_list.end();

		backtrace_copy.reserve(ctx->backtrace_list.size());
		for (it = ctx->backtrace_list.begin(); it != end; it++) {
			trace_point *p = new trace_point(
				(*it)->function,
				(*it)->source,
				(*it)->line,
				true);
			backtrace_copy.push_back(p);
		}
	}
}

tracable_exception::tracable_exception(const tracable_exception &other)
	: std::exception()
{
	vector<trace_point *>::const_iterator it, end = other.backtrace_copy.end();
	backtrace_copy.reserve(other.backtrace_copy.size());
	for (it = other.backtrace_copy.begin(); it != end; it++) {
		trace_point *p = new trace_point(
			(*it)->function,
			(*it)->source,
			(*it)->line,
			true);
		backtrace_copy.push_back(p);
	}
}

tracable_exception::~tracable_exception() throw() {
	vector<trace_point *>::iterator it, end = backtrace_copy.end();
	for (it = backtrace_copy.begin(); it != end; it++) {
		delete *it;
	}
}

template<typename Collection>
static string
format_backtrace(const Collection &backtrace_list) {
	if (backtrace_list.empty()) {
		return "     (empty)";
	} else {
		backtrace_list.rbegin();
		stringstream result;
		typename Collection::const_reverse_iterator it;

		for (it = backtrace_list.rbegin(); it != backtrace_list.rend(); it++) {
			const trace_point *p = *it;

			result << "     in '" << p->function << "'";
			if (p->source != NULL) {
				const char *source = strrchr(p->source, '/');
				if (source != NULL) {
					source++;
				} else {
					source = p->source;
				}
				result << " (" << source << ":" << p->line << ")";
			}
			result << endl;
		}
		return result.str();
	}
}

string
tracable_exception::backtrace() const throw() {
	return format_backtrace< vector<trace_point *> >(backtrace_copy);
}

const char *
tracable_exception::what() const throw() {
	return "oxt::tracable_exception";
}

#endif /* OXT_BACKTRACE_IS_ENABLED */

void initialize() {
	global_context = new global_context_t();
	// For some reason make_shared() crashes here when compiled with clang 3.2 on OS X.
	// Clang bug? We use 'new' to work around it.
	thread_local_context_ptr ctx = thread_local_context::make_shared_ptr();
	ctx->thread_number = 1;
	ctx->thread_name = "Main thread";
	set_thread_local_context(ctx);
}

global_context_t::global_context_t()
	: next_thread_number(2)
{ }

thread_local_context_ptr
thread_local_context::make_shared_ptr() {
	// For some reason make_shared() crashes here when compiled with clang 3.2 on OS X.
	// Clang bug? We use 'new' to work around it.
	return thread_local_context_ptr(new thread_local_context());
}

thread_local_context::thread_local_context()
	: thread_number(0)
{
	thread = pthread_self();
	#ifdef __linux__
		tid = syscall(SYS_gettid);
	#endif
	syscall_interruption_lock.lock();
	#ifdef OXT_BACKTRACE_IS_ENABLED
		backtrace_list.reserve(50);
	#endif
}

string
thread::make_thread_name(const string &given_name) {
	if (given_name.empty()) {
		if (OXT_LIKELY(global_context != NULL)) {
			stringstream str;
			str << "Thread #";
			{
				lock_guard<boost::mutex> l(global_context->thread_registration_mutex);
				str << global_context->next_thread_number;
			}
			return str.str();
		} else {
			return "(unknown)";
		}
	} else {
		return given_name;
	}
}

void
thread::thread_main(const boost::function<void ()> func, thread_local_context_ptr ctx) {
	set_thread_local_context(ctx);

	if (OXT_LIKELY(global_context != NULL)) {
		lock_guard<boost::mutex> l(global_context->thread_registration_mutex);

		ctx->thread = pthread_self();
		global_context->next_thread_number++;
		global_context->registered_threads.push_back(ctx);
		ctx->iterator = global_context->registered_threads.end();
		ctx->iterator--;
		// Set this after setting 'iterator' to indicate
		// that push_back() has succeeded.
		ctx->thread_number = global_context->next_thread_number;
	}

	try {
		func();
	} catch (const thread_interrupted &) {
		// Do nothing.
	}
	// We don't care about other exceptions because they'll crash the process anyway.

	if (OXT_LIKELY(global_context != NULL)) {
		lock_guard<boost::mutex> l(global_context->thread_registration_mutex);
		thread_local_context *ctx = get_thread_local_context();
		if (ctx != 0 && ctx->thread_number != 0) {
			global_context->registered_threads.erase(ctx->iterator);
			ctx->thread_number = 0;
		}
	}
	free_thread_local_context();
}

std::string
thread::name() const throw() {
	return context->thread_name;
}

std::string
thread::backtrace() const throw() {
	#ifdef OXT_BACKTRACE_IS_ENABLED
		spin_lock::scoped_lock l(context->backtrace_lock);
		return format_backtrace(context->backtrace_list);
	#else
		return "    (backtrace support disabled during compile time)";
	#endif
}

string
thread::all_backtraces() throw() {
	#ifdef OXT_BACKTRACE_IS_ENABLED
		if (OXT_LIKELY(global_context != NULL)) {
			lock_guard<boost::mutex> l(global_context->thread_registration_mutex);
			list<thread_local_context_ptr>::const_iterator it;
			std::stringstream result;

			for (it = global_context->registered_threads.begin();
			     it != global_context->registered_threads.end();
			     it++)
			{
				thread_local_context_ptr ctx = *it;
				result << "Thread '" << ctx->thread_name <<
					"' (" << hex << showbase << ctx->thread << dec;
				#ifdef __linux__
					result << ", LWP " << ctx->tid;
				#endif
				result << "):" << endl;

				spin_lock::scoped_lock l(ctx->backtrace_lock);
				std::string bt = format_backtrace(ctx->backtrace_list);
				result << bt;
				if (bt.empty() || bt[bt.size() - 1] != '\n') {
					result << endl;
				}
				result << endl;
			}
			return result.str();
		} else {
			return "(OXT not initialized)";
		}
	#else
		return "(backtrace support disabled during compile time)";
	#endif
}

void
thread::interrupt(bool interruptSyscalls) {
	int ret;

	boost::thread::interrupt();
	if (interruptSyscalls && context->syscall_interruption_lock.try_lock()) {
		do {
			ret = pthread_kill(native_handle(),
				INTERRUPTION_SIGNAL);
		} while (ret == EINTR);
		context->syscall_interruption_lock.unlock();
	}
}

} // namespace oxt
<|endoftext|>