<commit_before>// This unit test will test the spawning mechanisms of the simulator

#include <stdio.h>
#include <unistd.h>
#include <assert.h>
#include "carbon_user.h"

void* thread_func(void *threadid);
void* thread_func_simple(void *threadid);

void do_a_sim()
{
   const unsigned int numThreads = 5;
   int threads[numThreads];

   for(unsigned int j = 0; j < numThreads; j++)
       threads[j] = CarbonSpawnThread(thread_func, (void *) (j + 1));

   for(unsigned int j = 0; j < numThreads; j++)
       CarbonJoinThread(threads[j]);

}

int main(int argc, char **argv)
{
   int i = 0;
   CarbonStartSim(argc, argv);

   for(i = 0; i < 120; i++)
   {
       do_a_sim();
   }
   CarbonStopSim();

   fprintf(stderr, "done.\n");
   return 0;
}

void* thread_func(void *threadid)
{
//   int tid = CarbonSpawnThread(thread_func_simple, (void*)1);
//   CarbonJoinThread(tid);
   return 0;
}

void* thread_func_simple(void *threadid)
{
//   usleep(250);
   return 0;
}
<commit_msg>[tests/unit/spawn] Reduce no. of iterations.<commit_after>// This unit test will test the spawning mechanisms of the simulator

#include <stdio.h>
#include <unistd.h>
#include <assert.h>
#include "carbon_user.h"

void* thread_func(void *threadid);
void* thread_func_simple(void *threadid);

void do_a_sim()
{
   const unsigned int numThreads = 5;
   int threads[numThreads];

   for(unsigned int j = 0; j < numThreads; j++)
       threads[j] = CarbonSpawnThread(thread_func, (void *) (j + 1));

   for(unsigned int j = 0; j < numThreads; j++)
       CarbonJoinThread(threads[j]);

}

int main(int argc, char **argv)
{
   int i = 0;
   CarbonStartSim(argc, argv);

   for(i = 0; i < 20; i++)
   {
       fprintf(stderr, "%d...", i);
       do_a_sim();
   }
   CarbonStopSim();

   fprintf(stderr, "done.\n");
   return 0;
}

void* thread_func(void *threadid)
{
//   int tid = CarbonSpawnThread(thread_func_simple, (void*)1);
//   CarbonJoinThread(tid);
   return 0;
}

void* thread_func_simple(void *threadid)
{
//   usleep(250);
   return 0;
}
<|endoftext|>


<commit_before>#ifndef LIBTEN_HTTP_SERVER_HH
#define LIBTEN_HTTP_SERVER_HH

#include <fnmatch.h>
#include <boost/algorithm/string/predicate.hpp>
#include <boost/algorithm/string/compare.hpp>
#include <chrono>
#include <functional>
#include <netinet/tcp.h>

#include "ten/buffer.hh"
#include "ten/logging.hh"
#include "ten/task.hh"
#include "ten/net.hh"
#include "ten/http/http_message.hh"
#include "ten/uri.hh"

namespace ten {

//! basic http server
class http_server : public netsock_server {
public:

    //! wrapper around an http request/response
    struct request {
        http_request &req;
        netsock &sock;
        http_response resp {404};
        bool resp_sent {false};
        std::chrono::high_resolution_clock::time_point start;

        request(http_request &req_, netsock &sock_)
            : req(req_), sock(sock_),
              start(std::chrono::steady_clock::now()) {}

        //! compose a uri from the request uri
        uri get_uri(std::string host="") {
            if (host.empty()) {
                host = req.get("Host");
                // TODO: transform to preserve passed in host
                if (boost::starts_with(req.uri, "http://")) {
                    return req.uri;
                }
            }

            if (host.empty()) {
                // just make up a host
                host = "localhost";
            }
            uri tmp;
            tmp.host = host;
            tmp.scheme = "http";
            tmp.path = req.uri;
            return tmp.compose();
        }

        //! send response to this request
        ssize_t send_response() {
            if (resp_sent) return 0;
            resp_sent = true;
            // TODO: Content-Length might be good to add to normal responses,
            //    but only if Transfer-Encoding isn't chunked?
            if (resp.status_code >= 400 && resp.status_code <= 599
                 && resp.get("Content-Length").empty()
                 && req.method != "HEAD")
            {
                resp.set("Content-Length", resp.body.size());
            }
            // HTTP/1.1 requires Date, so lets add it
            if (resp.get("Date").empty()) {
                char buf[128];
                struct tm tm;
                time_t now = std::chrono::system_clock::to_time_t(procnow());
                strftime(buf, sizeof(buf)-1, "%a, %d %b %Y %H:%M:%S GMT", gmtime_r(&now, &tm));
                resp.set("Date", buf);
            }
            if (resp.get("Connection").empty()) {
                // obey clients wishes if we have none of our own
                std::string conn = req.get("Connection");
                if (!conn.empty())
                    resp.set("Connection", conn);
                else if (req.http_version == http_1_0)
                    resp.set("Connection", "close");
            }

            auto data = resp.data();
            if (!resp.body.empty() && req.method != "HEAD") {
                data += resp.body;
            }
            ssize_t nw = sock.send(data.data(), data.size());
            return nw;
        }

        //! the ip of the host making the request
        //! might use the X-Forwarded-For header
        std::string agent_ip(bool use_xff=false) const {
            if (use_xff) {
                std::string xffs = req.get("X-Forwarded-For");
                const char *xff = xffs.c_str();
                if (xff) {
                    // pick the first addr
                    int i;
                    for (i=0; *xff && i<256 && !isdigit((unsigned char)*xff); i++, xff++) {}
                    if (*xff && i < 256) {
                        // now, find the end
                        const char *e = xff;
                        for (i = 0; *e && i<256 && (isdigit((unsigned char)*e) || *e == '.'); i++, e++) {}
                        if (i < 256 && e >= xff + 7 ) {
                            return std::string(xff, e - xff);
                        }
                    }
                }
            }
            address addr;
            if (sock.getpeername(addr)) {
                char buf[INET6_ADDRSTRLEN];
                if (addr.ntop(buf, sizeof(buf))) {
                    return buf;
                }
            }
            return "";
        }

        ~request() {
            if (!resp_sent) {
                // ensure a response is sent
                resp = http_response{404};
                send_response();
            }

            if (boost::iequals(resp.get("Connection"), "close")) {
                sock.close();
            }
        }
    };

    typedef std::function<void (request &)> callback_type;
public:
    std::function<void ()> connect_watch;
    std::function<void ()> disconnect_watch;
protected:
    struct route {
        std::string pattern;
        callback_type callback;
        int fnmatch_flags;

        route(const std::string &pattern_,
                const callback_type &callback_,
                int fnmatch_flags_=0)
            : pattern(pattern_), callback(callback_), fnmatch_flags(fnmatch_flags_) {}
    };

    std::vector<route> _routes;
    callback_type _log_func;
public:
    http_server(size_t stacksize_=default_stacksize, unsigned timeout_ms_=0)
        : netsock_server("http", stacksize_, timeout_ms_)
    {
    }

    //! add a callback for a uri with an fnmatch pattern
    void add_route(const std::string &pattern,
            const callback_type &callback,
            int fnmatch_flags = 0)
    {
        _routes.emplace_back(pattern, callback, fnmatch_flags);
    }

    //! set logging function, called after every request
    void set_log_callback(const callback_type &f) {
        _log_func = f;
    }

    void on_connection(netsock &s) {
        // TODO: tuneable buffer sizes
        buffer buf(4*1024);
        http_parser parser;

        if (connect_watch) {
            connect_watch();
        }
        // TODO: might want to enable this later after
        // we know this will be a long-lived connection
        // otherwise the overhead of the syscall hurts concurrency
        //s.s.setsockopt(IPPROTO_TCP, TCP_NODELAY, 1);

        http_request req;
        for (;;) {
            req.parser_init(&parser);
            bool got_headers = false;
            for (;;) {
                buf.reserve(4*1024);
                ssize_t nr = -1;
                if (buf.size() == 0) {
                    nr = s.recv(buf.back(), buf.available(), _timeout_ms);
                    if (nr < 0) goto done;
                    buf.commit(nr);
                }
                size_t nparse = buf.size();
                req.parse(&parser, buf.front(), nparse);
                buf.remove(nparse);
                if (req.complete) {
                    // handle request
                    handle_request(req, s);
                    break;
                }
                if (nr == 0) goto done;
                if (!got_headers && !req.method.empty()) {
                    got_headers = true;
                    if (req.get("Expect") == "100-continue") {
                        http_response cont_resp(100);
                        std::string data = cont_resp.data();
                        ssize_t nw = s.send(data.data(), data.size());
                        (void)nw;
                    }
                }
            }
        }
done:
        if (disconnect_watch) {
            disconnect_watch();
        }
    }

    void handle_request(http_request &req, netsock &s) {
        request r(req, s);
        const auto path = req.path();
        DVLOG(5) << "path: " << path;
        // not super efficient, but good enough
        // note: empty string pattern matches everything
        for (const auto &i : _routes) {
            DVLOG(5) << "matching pattern: " << i.pattern;
            if (i.pattern.empty() || fnmatch(i.pattern.c_str(), path.c_str(), i.fnmatch_flags) == 0) {
                try {
                    i.callback(r);
                } catch (std::exception &e) {
                    DVLOG(2) << "unhandled exception in route [" << i.pattern << "]: " << e.what();
                    r.resp = http_response(500, http_headers{"Connection", "close"});
                    std::string msg = e.what();
                    if (!msg.empty() && *msg.rbegin() != '\n')
                        msg += '\n';
                    r.resp.set_body(msg);
                    r.send_response();
                }
                break;
            }
        }
        if (_log_func) {
            _log_func(r);
        }
    }
};

} // end namespace ten

#endif // LIBTEN_HTTP_SERVER_HH
<commit_msg>get_uri() const<commit_after>#ifndef LIBTEN_HTTP_SERVER_HH
#define LIBTEN_HTTP_SERVER_HH

#include <fnmatch.h>
#include <boost/algorithm/string/predicate.hpp>
#include <boost/algorithm/string/compare.hpp>
#include <chrono>
#include <functional>
#include <netinet/tcp.h>

#include "ten/buffer.hh"
#include "ten/logging.hh"
#include "ten/task.hh"
#include "ten/net.hh"
#include "ten/http/http_message.hh"
#include "ten/uri.hh"

namespace ten {

//! basic http server
class http_server : public netsock_server {
public:

    //! wrapper around an http request/response
    struct request {
        http_request &req;
        netsock &sock;
        http_response resp {404};
        bool resp_sent {false};
        std::chrono::high_resolution_clock::time_point start;

        request(http_request &req_, netsock &sock_)
            : req(req_), sock(sock_),
              start(std::chrono::steady_clock::now()) {}

        //! compose a uri from the request uri
        uri get_uri(std::string host="") const {
            if (host.empty()) {
                host = req.get("Host");
                // TODO: transform to preserve passed in host
                if (boost::starts_with(req.uri, "http://")) {
                    return req.uri;
                }
            }

            if (host.empty()) {
                // just make up a host
                host = "localhost";
            }
            uri tmp;
            tmp.host = host;
            tmp.scheme = "http";
            tmp.path = req.uri;
            return tmp.compose();
        }

        //! send response to this request
        ssize_t send_response() {
            if (resp_sent) return 0;
            resp_sent = true;
            // TODO: Content-Length might be good to add to normal responses,
            //    but only if Transfer-Encoding isn't chunked?
            if (resp.status_code >= 400 && resp.status_code <= 599
                 && resp.get("Content-Length").empty()
                 && req.method != "HEAD")
            {
                resp.set("Content-Length", resp.body.size());
            }
            // HTTP/1.1 requires Date, so lets add it
            if (resp.get("Date").empty()) {
                char buf[128];
                struct tm tm;
                time_t now = std::chrono::system_clock::to_time_t(procnow());
                strftime(buf, sizeof(buf)-1, "%a, %d %b %Y %H:%M:%S GMT", gmtime_r(&now, &tm));
                resp.set("Date", buf);
            }
            if (resp.get("Connection").empty()) {
                // obey clients wishes if we have none of our own
                std::string conn = req.get("Connection");
                if (!conn.empty())
                    resp.set("Connection", conn);
                else if (req.http_version == http_1_0)
                    resp.set("Connection", "close");
            }

            auto data = resp.data();
            if (!resp.body.empty() && req.method != "HEAD") {
                data += resp.body;
            }
            ssize_t nw = sock.send(data.data(), data.size());
            return nw;
        }

        //! the ip of the host making the request
        //! might use the X-Forwarded-For header
        std::string agent_ip(bool use_xff=false) const {
            if (use_xff) {
                std::string xffs = req.get("X-Forwarded-For");
                const char *xff = xffs.c_str();
                if (xff) {
                    // pick the first addr
                    int i;
                    for (i=0; *xff && i<256 && !isdigit((unsigned char)*xff); i++, xff++) {}
                    if (*xff && i < 256) {
                        // now, find the end
                        const char *e = xff;
                        for (i = 0; *e && i<256 && (isdigit((unsigned char)*e) || *e == '.'); i++, e++) {}
                        if (i < 256 && e >= xff + 7 ) {
                            return std::string(xff, e - xff);
                        }
                    }
                }
            }
            address addr;
            if (sock.getpeername(addr)) {
                char buf[INET6_ADDRSTRLEN];
                if (addr.ntop(buf, sizeof(buf))) {
                    return buf;
                }
            }
            return "";
        }

        ~request() {
            if (!resp_sent) {
                // ensure a response is sent
                resp = http_response{404};
                send_response();
            }

            if (boost::iequals(resp.get("Connection"), "close")) {
                sock.close();
            }
        }
    };

    typedef std::function<void (request &)> callback_type;
public:
    std::function<void ()> connect_watch;
    std::function<void ()> disconnect_watch;
protected:
    struct route {
        std::string pattern;
        callback_type callback;
        int fnmatch_flags;

        route(const std::string &pattern_,
                const callback_type &callback_,
                int fnmatch_flags_=0)
            : pattern(pattern_), callback(callback_), fnmatch_flags(fnmatch_flags_) {}
    };

    std::vector<route> _routes;
    callback_type _log_func;
public:
    http_server(size_t stacksize_=default_stacksize, unsigned timeout_ms_=0)
        : netsock_server("http", stacksize_, timeout_ms_)
    {
    }

    //! add a callback for a uri with an fnmatch pattern
    void add_route(const std::string &pattern,
            const callback_type &callback,
            int fnmatch_flags = 0)
    {
        _routes.emplace_back(pattern, callback, fnmatch_flags);
    }

    //! set logging function, called after every request
    void set_log_callback(const callback_type &f) {
        _log_func = f;
    }

    void on_connection(netsock &s) {
        // TODO: tuneable buffer sizes
        buffer buf(4*1024);
        http_parser parser;

        if (connect_watch) {
            connect_watch();
        }
        // TODO: might want to enable this later after
        // we know this will be a long-lived connection
        // otherwise the overhead of the syscall hurts concurrency
        //s.s.setsockopt(IPPROTO_TCP, TCP_NODELAY, 1);

        http_request req;
        for (;;) {
            req.parser_init(&parser);
            bool got_headers = false;
            for (;;) {
                buf.reserve(4*1024);
                ssize_t nr = -1;
                if (buf.size() == 0) {
                    nr = s.recv(buf.back(), buf.available(), _timeout_ms);
                    if (nr < 0) goto done;
                    buf.commit(nr);
                }
                size_t nparse = buf.size();
                req.parse(&parser, buf.front(), nparse);
                buf.remove(nparse);
                if (req.complete) {
                    // handle request
                    handle_request(req, s);
                    break;
                }
                if (nr == 0) goto done;
                if (!got_headers && !req.method.empty()) {
                    got_headers = true;
                    if (req.get("Expect") == "100-continue") {
                        http_response cont_resp(100);
                        std::string data = cont_resp.data();
                        ssize_t nw = s.send(data.data(), data.size());
                        (void)nw;
                    }
                }
            }
        }
done:
        if (disconnect_watch) {
            disconnect_watch();
        }
    }

    void handle_request(http_request &req, netsock &s) {
        request r(req, s);
        const auto path = req.path();
        DVLOG(5) << "path: " << path;
        // not super efficient, but good enough
        // note: empty string pattern matches everything
        for (const auto &i : _routes) {
            DVLOG(5) << "matching pattern: " << i.pattern;
            if (i.pattern.empty() || fnmatch(i.pattern.c_str(), path.c_str(), i.fnmatch_flags) == 0) {
                try {
                    i.callback(r);
                } catch (std::exception &e) {
                    DVLOG(2) << "unhandled exception in route [" << i.pattern << "]: " << e.what();
                    r.resp = http_response(500, http_headers{"Connection", "close"});
                    std::string msg = e.what();
                    if (!msg.empty() && *msg.rbegin() != '\n')
                        msg += '\n';
                    r.resp.set_body(msg);
                    r.send_response();
                }
                break;
            }
        }
        if (_log_func) {
            _log_func(r);
        }
    }
};

} // end namespace ten

#endif // LIBTEN_HTTP_SERVER_HH
<|endoftext|>


<commit_before>#pragma once

#include "indexer/feature_data.hpp"

#include "base/base.hpp"
#include "base/stl_helpers.hpp"

#include <algorithm>
#include <array>
#include <cstdint>
#include <functional>
#include <initializer_list>
#include <set>
#include <string>
#include <type_traits>
#include <utility>
#include <vector>

#include <boost/optional.hpp>

namespace feature { class TypesHolder; }
class FeatureType;

#define DECLARE_CHECKER_INSTANCE(CheckerType) static CheckerType const & Instance() { \
                                              static CheckerType const inst; return inst; }

namespace ftypes
{
class BaseChecker
{
  size_t const m_level;

protected:
  std::vector<uint32_t> m_types;

  BaseChecker(size_t level = 2) : m_level(level) {}
  virtual ~BaseChecker() = default;

public:
  virtual bool IsMatched(uint32_t type) const;

  bool operator()(feature::TypesHolder const & types) const;
  bool operator()(FeatureType & ft) const;
  bool operator()(std::vector<uint32_t> const & types) const;
  bool operator()(uint32_t type) const { return IsMatched(type); }

  static uint32_t PrepareToMatch(uint32_t type, uint8_t level);

  template <typename TFn>
  void ForEachType(TFn && fn) const
  {
    std::for_each(m_types.cbegin(), m_types.cend(), std::forward<TFn>(fn));
  }
};

class IsPeakChecker : public BaseChecker
{
  IsPeakChecker();
public:
  DECLARE_CHECKER_INSTANCE(IsPeakChecker);
};

class IsATMChecker : public BaseChecker
{
  IsATMChecker();
public:
  DECLARE_CHECKER_INSTANCE(IsATMChecker);
};

class IsSpeedCamChecker : public BaseChecker
{
  IsSpeedCamChecker();
public:
  DECLARE_CHECKER_INSTANCE(IsSpeedCamChecker);
};

class IsFuelStationChecker : public BaseChecker
{
  IsFuelStationChecker();
public:
  DECLARE_CHECKER_INSTANCE(IsFuelStationChecker);
};

class IsRailwayStationChecker : public BaseChecker
{
  IsRailwayStationChecker();
public:
  DECLARE_CHECKER_INSTANCE(IsRailwayStationChecker);
};

class IsSubwayStationChecker : public BaseChecker
{
  IsSubwayStationChecker();

public:
  DECLARE_CHECKER_INSTANCE(IsSubwayStationChecker);
};

class IsAirportChecker : public BaseChecker
{
  IsAirportChecker();

public:
  DECLARE_CHECKER_INSTANCE(IsAirportChecker);
};

class IsSquareChecker : public BaseChecker
{
  IsSquareChecker();
  friend class IsStreetOrSuburbChecker;

public:
  DECLARE_CHECKER_INSTANCE(IsSquareChecker);
};

class IsSuburbChecker : public BaseChecker
{
  IsSuburbChecker();
  friend class IsStreetOrSuburbChecker;

public:
  DECLARE_CHECKER_INSTANCE(IsSuburbChecker);
};

class IsWayChecker : public BaseChecker
{
  IsWayChecker();
  friend class IsStreetOrSuburbChecker;

public:
  DECLARE_CHECKER_INSTANCE(IsWayChecker);
};

class IsStreetOrSuburbChecker : public BaseChecker
{
  IsStreetOrSuburbChecker();

public:
  DECLARE_CHECKER_INSTANCE(IsStreetOrSuburbChecker);
};

class IsAddressObjectChecker : public BaseChecker
{
  IsAddressObjectChecker();
public:
  DECLARE_CHECKER_INSTANCE(IsAddressObjectChecker);
};

class IsVillageChecker : public BaseChecker
{
  IsVillageChecker();
public:
  DECLARE_CHECKER_INSTANCE(IsVillageChecker);
};

class IsOneWayChecker : public BaseChecker
{
  IsOneWayChecker();
public:
  DECLARE_CHECKER_INSTANCE(IsOneWayChecker);
};

class IsRoundAboutChecker : public BaseChecker
{
  IsRoundAboutChecker();
public:
  DECLARE_CHECKER_INSTANCE(IsRoundAboutChecker);
};

class IsLinkChecker : public BaseChecker
{
  IsLinkChecker();
public:
  DECLARE_CHECKER_INSTANCE(IsLinkChecker);
};

class IsBuildingChecker : public BaseChecker
{
  IsBuildingChecker();
public:
  uint32_t GetMainType() const { return m_types[0]; }
  DECLARE_CHECKER_INSTANCE(IsBuildingChecker);
};

class IsPoiChecker : public BaseChecker
{
  IsPoiChecker();
public:
  static std::set<std::string> const kPoiTypes;

  DECLARE_CHECKER_INSTANCE(IsPoiChecker);
};

class WikiChecker : public BaseChecker
{
  WikiChecker();
public:
   static std::set<std::pair<std::string, std::string>> const kTypesForWiki;

   DECLARE_CHECKER_INSTANCE(WikiChecker);

   template <typename Ft>
   bool NeedFeature(Ft & feature) const
   {
     bool need = true;
     feature.ForEachType([&](uint32_t type) {
       if (need && !IsMatched(type))
         need = false;
     });
     return need;
   }
};

class IsPlaceChecker : public BaseChecker
{
  IsPlaceChecker();
public:
  DECLARE_CHECKER_INSTANCE(IsPlaceChecker);
};

class IsBridgeChecker : public BaseChecker
{
  virtual bool IsMatched(uint32_t type) const override;

  IsBridgeChecker();
public:
  DECLARE_CHECKER_INSTANCE(IsBridgeChecker);
};

class IsTunnelChecker : public BaseChecker
{
  virtual bool IsMatched(uint32_t type) const override;

  IsTunnelChecker();
public:
  DECLARE_CHECKER_INSTANCE(IsTunnelChecker);
};

class IsPopularityPlaceChecker : public BaseChecker
{
  IsPopularityPlaceChecker();
public:
  DECLARE_CHECKER_INSTANCE(IsPopularityPlaceChecker);
};

class IsHotelChecker : public BaseChecker
{
public:
  enum class Type
  {
    Hotel,
    Apartment,
    CampSite,
    Chalet,
    GuestHouse,
    Hostel,
    Motel,
    Resort,

    Count
  };

  using UnderlyingType = std::underlying_type_t<Type>;

  static_assert(base::Key(Type::Count) <= CHAR_BIT * sizeof(unsigned),
                "Too many types of hotels");

  static char const * GetHotelTypeTag(Type type);

  unsigned GetHotelTypesMask(FeatureType & ft) const;

  boost::optional<Type> GetHotelType(FeatureType & ft) const;

  DECLARE_CHECKER_INSTANCE(IsHotelChecker);
private:
  IsHotelChecker();

  std::array<std::pair<uint32_t, Type>, base::Key(Type::Count)> m_sortedTypes;
};

// WiFi is a type in classificator.txt,
// it should be checked for filling metadata in MapObject.
class IsWifiChecker : public BaseChecker
{
  IsWifiChecker();
public:
  DECLARE_CHECKER_INSTANCE(IsWifiChecker);
};

class IsEatChecker : public BaseChecker
{
public:
  enum class Type
  {
    Cafe,
    Bakery,
    FastFood,
    Restaurant,
    Bar,
    Pub,
    Biergarten,

    Count
  };

  DECLARE_CHECKER_INSTANCE(IsEatChecker);

  std::vector<uint32_t> const & GetTypes() const { return m_types; }
  Type GetType(uint32_t t) const;

private:
  IsEatChecker();

  std::array<std::pair<uint32_t, Type>, base::Key(Type::Count)> m_sortedTypes;
};

class IsCuisineChecker : public BaseChecker
{
  IsCuisineChecker();

public:
  DECLARE_CHECKER_INSTANCE(IsCuisineChecker);
};

class IsCityChecker : public BaseChecker
{
  IsCityChecker();
public:
  DECLARE_CHECKER_INSTANCE(IsCityChecker);
};

class IsPublicTransportStopChecker : public BaseChecker
{
  IsPublicTransportStopChecker();
public:
  DECLARE_CHECKER_INSTANCE(IsPublicTransportStopChecker);
};

/// Type of locality (do not change values and order - they have detalization order)
/// COUNTRY < STATE < CITY < ...
enum Type { NONE = -1, COUNTRY = 0, STATE, CITY, TOWN, VILLAGE, LOCALITY_COUNT };

class IsLocalityChecker : public BaseChecker
{
  IsLocalityChecker();
public:
  Type GetType(uint32_t t) const;
  Type GetType(feature::TypesHolder const & types) const;
  Type GetType(FeatureType & f) const;

  DECLARE_CHECKER_INSTANCE(IsLocalityChecker);
};

template <typename Types>
bool IsCityTownOrVillage(Types const & types)
{
  feature::TypesHolder h;
  for (auto const t : types)
    h.Add(t);
  auto const type = IsLocalityChecker::Instance().GetType(h);
  return type == CITY || type == TOWN || type == VILLAGE;
}

/// @name Get city radius and population.
/// @param r Radius in meters.
//@{
uint64_t GetPopulation(FeatureType & ft);
double GetRadiusByPopulation(uint64_t p);
uint64_t GetPopulationByRadius(double r);
//@}

/// Check if type conforms the path. Strings in the path can be
/// feature types like "highway", "living_street", "bridge" and so on
///  or *. * means any class.
/// The root name ("world") is ignored
bool IsTypeConformed(uint32_t type, base::StringIL const & path);

// Highway class. The order is important.
// The enum values follow from the biggest roads (Trunk) to the smallest ones (Service).
enum class HighwayClass
{
  Undefined = 0,  // There has not been any attempt of calculating HighwayClass.
  Error,          // There was an attempt of calculating HighwayClass but it was not successful.
  Trunk,
  Primary,
  Secondary,
  Tertiary,
  LivingStreet,
  Service,
  Pedestrian,
  Transported,    // Vehicles are transported by train or ferry.
  Count           // This value is used for internals only.
};

std::string DebugPrint(HighwayClass const cls);

HighwayClass GetHighwayClass(feature::TypesHolder const & types);
}  // namespace ftypes

namespace std
{
template<>
struct hash<ftypes::IsHotelChecker::Type>
{
  size_t operator()(ftypes::IsHotelChecker::Type type) const
  {
    using UnderlyingType = ftypes::IsHotelChecker::UnderlyingType;
    return hash<UnderlyingType>()(static_cast<UnderlyingType>(type));
  }
};
}  // namespace std
<commit_msg>[generator] Fixed logic error in WikiChecker::NeedFeature.<commit_after>#pragma once

#include "indexer/feature_data.hpp"

#include "base/base.hpp"
#include "base/stl_helpers.hpp"

#include <algorithm>
#include <array>
#include <cstdint>
#include <functional>
#include <initializer_list>
#include <set>
#include <string>
#include <type_traits>
#include <utility>
#include <vector>

#include <boost/optional.hpp>

namespace feature { class TypesHolder; }
class FeatureType;

#define DECLARE_CHECKER_INSTANCE(CheckerType) static CheckerType const & Instance() { \
                                              static CheckerType const inst; return inst; }

namespace ftypes
{
class BaseChecker
{
  size_t const m_level;

protected:
  std::vector<uint32_t> m_types;

  BaseChecker(size_t level = 2) : m_level(level) {}
  virtual ~BaseChecker() = default;

public:
  virtual bool IsMatched(uint32_t type) const;

  bool operator()(feature::TypesHolder const & types) const;
  bool operator()(FeatureType & ft) const;
  bool operator()(std::vector<uint32_t> const & types) const;
  bool operator()(uint32_t type) const { return IsMatched(type); }

  static uint32_t PrepareToMatch(uint32_t type, uint8_t level);

  template <typename TFn>
  void ForEachType(TFn && fn) const
  {
    std::for_each(m_types.cbegin(), m_types.cend(), std::forward<TFn>(fn));
  }
};

class IsPeakChecker : public BaseChecker
{
  IsPeakChecker();
public:
  DECLARE_CHECKER_INSTANCE(IsPeakChecker);
};

class IsATMChecker : public BaseChecker
{
  IsATMChecker();
public:
  DECLARE_CHECKER_INSTANCE(IsATMChecker);
};

class IsSpeedCamChecker : public BaseChecker
{
  IsSpeedCamChecker();
public:
  DECLARE_CHECKER_INSTANCE(IsSpeedCamChecker);
};

class IsFuelStationChecker : public BaseChecker
{
  IsFuelStationChecker();
public:
  DECLARE_CHECKER_INSTANCE(IsFuelStationChecker);
};

class IsRailwayStationChecker : public BaseChecker
{
  IsRailwayStationChecker();
public:
  DECLARE_CHECKER_INSTANCE(IsRailwayStationChecker);
};

class IsSubwayStationChecker : public BaseChecker
{
  IsSubwayStationChecker();

public:
  DECLARE_CHECKER_INSTANCE(IsSubwayStationChecker);
};

class IsAirportChecker : public BaseChecker
{
  IsAirportChecker();

public:
  DECLARE_CHECKER_INSTANCE(IsAirportChecker);
};

class IsSquareChecker : public BaseChecker
{
  IsSquareChecker();
  friend class IsStreetOrSuburbChecker;

public:
  DECLARE_CHECKER_INSTANCE(IsSquareChecker);
};

class IsSuburbChecker : public BaseChecker
{
  IsSuburbChecker();
  friend class IsStreetOrSuburbChecker;

public:
  DECLARE_CHECKER_INSTANCE(IsSuburbChecker);
};

class IsWayChecker : public BaseChecker
{
  IsWayChecker();
  friend class IsStreetOrSuburbChecker;

public:
  DECLARE_CHECKER_INSTANCE(IsWayChecker);
};

class IsStreetOrSuburbChecker : public BaseChecker
{
  IsStreetOrSuburbChecker();

public:
  DECLARE_CHECKER_INSTANCE(IsStreetOrSuburbChecker);
};

class IsAddressObjectChecker : public BaseChecker
{
  IsAddressObjectChecker();
public:
  DECLARE_CHECKER_INSTANCE(IsAddressObjectChecker);
};

class IsVillageChecker : public BaseChecker
{
  IsVillageChecker();
public:
  DECLARE_CHECKER_INSTANCE(IsVillageChecker);
};

class IsOneWayChecker : public BaseChecker
{
  IsOneWayChecker();
public:
  DECLARE_CHECKER_INSTANCE(IsOneWayChecker);
};

class IsRoundAboutChecker : public BaseChecker
{
  IsRoundAboutChecker();
public:
  DECLARE_CHECKER_INSTANCE(IsRoundAboutChecker);
};

class IsLinkChecker : public BaseChecker
{
  IsLinkChecker();
public:
  DECLARE_CHECKER_INSTANCE(IsLinkChecker);
};

class IsBuildingChecker : public BaseChecker
{
  IsBuildingChecker();
public:
  uint32_t GetMainType() const { return m_types[0]; }
  DECLARE_CHECKER_INSTANCE(IsBuildingChecker);
};

class IsPoiChecker : public BaseChecker
{
  IsPoiChecker();
public:
  static std::set<std::string> const kPoiTypes;

  DECLARE_CHECKER_INSTANCE(IsPoiChecker);
};

class WikiChecker : public BaseChecker
{
  WikiChecker();
public:
   static std::set<std::pair<std::string, std::string>> const kTypesForWiki;

   DECLARE_CHECKER_INSTANCE(WikiChecker);

   template <typename Ft>
   bool NeedFeature(Ft & feature) const
   {
     bool need = true;
     feature.ForEachType([&](uint32_t type) {
       if (need && IsMatched(type))
         need = false;
     });
     return !need;
   }
};

class IsPlaceChecker : public BaseChecker
{
  IsPlaceChecker();
public:
  DECLARE_CHECKER_INSTANCE(IsPlaceChecker);
};

class IsBridgeChecker : public BaseChecker
{
  virtual bool IsMatched(uint32_t type) const override;

  IsBridgeChecker();
public:
  DECLARE_CHECKER_INSTANCE(IsBridgeChecker);
};

class IsTunnelChecker : public BaseChecker
{
  virtual bool IsMatched(uint32_t type) const override;

  IsTunnelChecker();
public:
  DECLARE_CHECKER_INSTANCE(IsTunnelChecker);
};

class IsPopularityPlaceChecker : public BaseChecker
{
  IsPopularityPlaceChecker();
public:
  DECLARE_CHECKER_INSTANCE(IsPopularityPlaceChecker);
};

class IsHotelChecker : public BaseChecker
{
public:
  enum class Type
  {
    Hotel,
    Apartment,
    CampSite,
    Chalet,
    GuestHouse,
    Hostel,
    Motel,
    Resort,

    Count
  };

  using UnderlyingType = std::underlying_type_t<Type>;

  static_assert(base::Key(Type::Count) <= CHAR_BIT * sizeof(unsigned),
                "Too many types of hotels");

  static char const * GetHotelTypeTag(Type type);

  unsigned GetHotelTypesMask(FeatureType & ft) const;

  boost::optional<Type> GetHotelType(FeatureType & ft) const;

  DECLARE_CHECKER_INSTANCE(IsHotelChecker);
private:
  IsHotelChecker();

  std::array<std::pair<uint32_t, Type>, base::Key(Type::Count)> m_sortedTypes;
};

// WiFi is a type in classificator.txt,
// it should be checked for filling metadata in MapObject.
class IsWifiChecker : public BaseChecker
{
  IsWifiChecker();
public:
  DECLARE_CHECKER_INSTANCE(IsWifiChecker);
};

class IsEatChecker : public BaseChecker
{
public:
  enum class Type
  {
    Cafe,
    Bakery,
    FastFood,
    Restaurant,
    Bar,
    Pub,
    Biergarten,

    Count
  };

  DECLARE_CHECKER_INSTANCE(IsEatChecker);

  std::vector<uint32_t> const & GetTypes() const { return m_types; }
  Type GetType(uint32_t t) const;

private:
  IsEatChecker();

  std::array<std::pair<uint32_t, Type>, base::Key(Type::Count)> m_sortedTypes;
};

class IsCuisineChecker : public BaseChecker
{
  IsCuisineChecker();

public:
  DECLARE_CHECKER_INSTANCE(IsCuisineChecker);
};

class IsCityChecker : public BaseChecker
{
  IsCityChecker();
public:
  DECLARE_CHECKER_INSTANCE(IsCityChecker);
};

class IsPublicTransportStopChecker : public BaseChecker
{
  IsPublicTransportStopChecker();
public:
  DECLARE_CHECKER_INSTANCE(IsPublicTransportStopChecker);
};

/// Type of locality (do not change values and order - they have detalization order)
/// COUNTRY < STATE < CITY < ...
enum Type { NONE = -1, COUNTRY = 0, STATE, CITY, TOWN, VILLAGE, LOCALITY_COUNT };

class IsLocalityChecker : public BaseChecker
{
  IsLocalityChecker();
public:
  Type GetType(uint32_t t) const;
  Type GetType(feature::TypesHolder const & types) const;
  Type GetType(FeatureType & f) const;

  DECLARE_CHECKER_INSTANCE(IsLocalityChecker);
};

template <typename Types>
bool IsCityTownOrVillage(Types const & types)
{
  feature::TypesHolder h;
  for (auto const t : types)
    h.Add(t);
  auto const type = IsLocalityChecker::Instance().GetType(h);
  return type == CITY || type == TOWN || type == VILLAGE;
}

/// @name Get city radius and population.
/// @param r Radius in meters.
//@{
uint64_t GetPopulation(FeatureType & ft);
double GetRadiusByPopulation(uint64_t p);
uint64_t GetPopulationByRadius(double r);
//@}

/// Check if type conforms the path. Strings in the path can be
/// feature types like "highway", "living_street", "bridge" and so on
///  or *. * means any class.
/// The root name ("world") is ignored
bool IsTypeConformed(uint32_t type, base::StringIL const & path);

// Highway class. The order is important.
// The enum values follow from the biggest roads (Trunk) to the smallest ones (Service).
enum class HighwayClass
{
  Undefined = 0,  // There has not been any attempt of calculating HighwayClass.
  Error,          // There was an attempt of calculating HighwayClass but it was not successful.
  Trunk,
  Primary,
  Secondary,
  Tertiary,
  LivingStreet,
  Service,
  Pedestrian,
  Transported,    // Vehicles are transported by train or ferry.
  Count           // This value is used for internals only.
};

std::string DebugPrint(HighwayClass const cls);

HighwayClass GetHighwayClass(feature::TypesHolder const & types);
}  // namespace ftypes

namespace std
{
template<>
struct hash<ftypes::IsHotelChecker::Type>
{
  size_t operator()(ftypes::IsHotelChecker::Type type) const
  {
    using UnderlyingType = ftypes::IsHotelChecker::UnderlyingType;
    return hash<UnderlyingType>()(static_cast<UnderlyingType>(type));
  }
};
}  // namespace std
<|endoftext|>


<commit_before>/* -*- Mode: C++; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */
/*
 * Version: MPL 1.1 / GPLv3+ / LGPLv3+
 *
 * The contents of this file are subject to the Mozilla Public License Version
 * 1.1 (the "License"); you may not use this file except in compliance with
 * the License. You may obtain a copy of the License at
 * http://www.mozilla.org/MPL/
 *
 * Software distributed under the License is distributed on an "AS IS" basis,
 * WITHOUT WARRANTY OF ANY KIND, either express or implied. See the License
 * for the specific language governing rights and limitations under the
 * License.
 *
 * The Initial Developer of the Original Code is
 *       Caolán McNamara <caolanm@redhat.com>
 *
 * Contributor(s):
 *   Caolán McNamara <caolanm@redhat.com>
 *
 * Alternatively, the contents of this file may be used under the terms of
 * either the GNU General Public License Version 3 or later (the "GPLv3+"), or
 * the GNU Lesser General Public License Version 3 or later (the "LGPLv3+"),
 * in which case the provisions of the GPLv3+ or the LGPLv3+ are applicable
 * instead of those above.
 */
#include "precompiled_i18npool.hxx"

#include <cppunit/TestSuite.h>
#include <cppunit/TestFixture.h>
#include <cppunit/TestCase.h>
#include <cppunit/plugin/TestPlugIn.h>
#include <cppunit/extensions/HelperMacros.h>

#include <cppuhelper/compbase1.hxx>
#include <cppuhelper/bootstrap.hxx>
#include <cppuhelper/basemutex.hxx>
#include <com/sun/star/i18n/XBreakIterator.hpp>

#include <string.h>

using namespace ::com::sun::star;

class TestBreakIterator : public CppUnit::TestFixture
{
public:
    TestBreakIterator();
    ~TestBreakIterator();

    virtual void setUp();
    virtual void tearDown();

    void testLineBreaking();

    CPPUNIT_TEST_SUITE(TestBreakIterator);
    CPPUNIT_TEST(testLineBreaking);
    CPPUNIT_TEST_SUITE_END();

private:
    uno::Reference<uno::XComponentContext> m_xContext;
    uno::Reference<lang::XMultiComponentFactory> m_xFactory;
    uno::Reference<lang::XMultiServiceFactory> m_xMSF;
};

//See https://bugs.freedesktop.org/show_bug.cgi?id=31271 for motivation
void TestBreakIterator::testLineBreaking()
{
    uno::Reference< i18n::XBreakIterator > xBreak(m_xMSF->createInstance(
        ::rtl::OUString(RTL_CONSTASCII_USTRINGPARAM("com.sun.star.i18n.BreakIterator"))),uno::UNO_QUERY);
    CPPUNIT_ASSERT_MESSAGE("Expected Break Iterator", xBreak.is());

    ::rtl::OUString aTest1(RTL_CONSTASCII_USTRINGPARAM("(some text here)"));

    i18n::LineBreakHyphenationOptions aHyphOptions;
    i18n::LineBreakUserOptions aUserOptions;
    lang::Locale aLocale;

    aLocale.Language = ::rtl::OUString(RTL_CONSTASCII_USTRINGPARAM("en"));
    aLocale.Country = ::rtl::OUString(RTL_CONSTASCII_USTRINGPARAM("US"));

    {
        //Here we want the line break to leave text here) on the next line
        i18n::LineBreakResults aResult = xBreak->getLineBreak(aTest1, strlen("(some tex"), aLocale, 0, aHyphOptions, aUserOptions);
        CPPUNIT_ASSERT_MESSAGE("Expected a break at the the start of the word", aResult.breakIndex == 6);
    }

    {
        //Here we want the line break to leave "here)" on the next line
        i18n::LineBreakResults aResult = xBreak->getLineBreak(aTest1, strlen("(some text here"), aLocale, 0, aHyphOptions, aUserOptions);
        CPPUNIT_ASSERT_MESSAGE("Expected a break at the the start of the word", aResult.breakIndex == 11);
    }
}

TestBreakIterator::TestBreakIterator()
{
    m_xContext = cppu::defaultBootstrap_InitialComponentContext();
    m_xFactory = m_xContext->getServiceManager();
    m_xMSF = uno::Reference<lang::XMultiServiceFactory>(m_xFactory, uno::UNO_QUERY_THROW);
}

void TestBreakIterator::setUp()
{
}

TestBreakIterator::~TestBreakIterator()
{
}

void TestBreakIterator::tearDown()
{
}

CPPUNIT_TEST_SUITE_REGISTRATION(TestBreakIterator);

CPPUNIT_PLUGIN_IMPLEMENT();

/* vim:set shiftwidth=4 softtabstop=4 expandtab: */
<commit_msg>Try and protect the Indic grapheme/cell skipping with a unit test.<commit_after>/* -*- Mode: C++; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */
/*
 * Version: MPL 1.1 / GPLv3+ / LGPLv3+
 *
 * The contents of this file are subject to the Mozilla Public License Version
 * 1.1 (the "License"); you may not use this file except in compliance with
 * the License. You may obtain a copy of the License at
 * http://www.mozilla.org/MPL/
 *
 * Software distributed under the License is distributed on an "AS IS" basis,
 * WITHOUT WARRANTY OF ANY KIND, either express or implied. See the License
 * for the specific language governing rights and limitations under the
 * License.
 *
 * The Initial Developer of the Original Code is
 *       Caolán McNamara <caolanm@redhat.com>
 *
 * Contributor(s):
 *   Caolán McNamara <caolanm@redhat.com>
 *
 * Alternatively, the contents of this file may be used under the terms of
 * either the GNU General Public License Version 3 or later (the "GPLv3+"), or
 * the GNU Lesser General Public License Version 3 or later (the "LGPLv3+"),
 * in which case the provisions of the GPLv3+ or the LGPLv3+ are applicable
 * instead of those above.
 */
#include "precompiled_i18npool.hxx"

#include <cppunit/TestSuite.h>
#include <cppunit/TestFixture.h>
#include <cppunit/TestCase.h>
#include <cppunit/plugin/TestPlugIn.h>
#include <cppunit/extensions/HelperMacros.h>

#include <cppuhelper/compbase1.hxx>
#include <cppuhelper/bootstrap.hxx>
#include <cppuhelper/basemutex.hxx>
#include <com/sun/star/i18n/XBreakIterator.hpp>
#include <com/sun/star/i18n/CharacterIteratorMode.hpp>

#include <string.h>

using namespace ::com::sun::star;

class TestBreakIterator : public CppUnit::TestFixture
{
public:
    TestBreakIterator();
    ~TestBreakIterator();

    virtual void setUp();
    virtual void tearDown();

    void testLineBreaking();
    void testGraphemeIteration();

    CPPUNIT_TEST_SUITE(TestBreakIterator);
    CPPUNIT_TEST(testLineBreaking);
    CPPUNIT_TEST(testGraphemeIteration);
    CPPUNIT_TEST_SUITE_END();

private:
    uno::Reference<uno::XComponentContext> m_xContext;
    uno::Reference<lang::XMultiComponentFactory> m_xFactory;
    uno::Reference<lang::XMultiServiceFactory> m_xMSF;
    uno::Reference<i18n::XBreakIterator> m_xBreak;
};

//See https://bugs.freedesktop.org/show_bug.cgi?id=31271 for motivation
void TestBreakIterator::testLineBreaking()
{
    ::rtl::OUString aTest1(RTL_CONSTASCII_USTRINGPARAM("(some text here)"));

    i18n::LineBreakHyphenationOptions aHyphOptions;
    i18n::LineBreakUserOptions aUserOptions;
    lang::Locale aLocale;

    aLocale.Language = ::rtl::OUString(RTL_CONSTASCII_USTRINGPARAM("en"));
    aLocale.Country = ::rtl::OUString(RTL_CONSTASCII_USTRINGPARAM("US"));

    {
        //Here we want the line break to leave text here) on the next line
        i18n::LineBreakResults aResult = m_xBreak->getLineBreak(aTest1, strlen("(some tex"), aLocale, 0, aHyphOptions, aUserOptions);
        CPPUNIT_ASSERT_MESSAGE("Expected a break at the the start of the word", aResult.breakIndex == 6);
    }

    {
        //Here we want the line break to leave "here)" on the next line
        i18n::LineBreakResults aResult = m_xBreak->getLineBreak(aTest1, strlen("(some text here"), aLocale, 0, aHyphOptions, aUserOptions);
        CPPUNIT_ASSERT_MESSAGE("Expected a break at the the start of the word", aResult.breakIndex == 11);
    }
}

//See http://qa.openoffice.org/issues/show_bug.cgi?id=111152 for motivation
void TestBreakIterator::testGraphemeIteration()
{
    lang::Locale aLocale;
    aLocale.Language = ::rtl::OUString(RTL_CONSTASCII_USTRINGPARAM("bn"));
    aLocale.Country = ::rtl::OUString(RTL_CONSTASCII_USTRINGPARAM("IN"));

    {
        sal_Unicode BA_HALANT_LA[] = { 0x09AC, 0x09CD, 0x09AF };
        ::rtl::OUString aTest1(BA_HALANT_LA, SAL_N_ELEMENTS(BA_HALANT_LA));

        sal_Int32 nDone=0;
        sal_Int32 nPos;
        nPos = m_xBreak->nextCharacters(aTest1, 0, aLocale,
            i18n::CharacterIteratorMode::SKIPCELL, 1, nDone);
        CPPUNIT_ASSERT_MESSAGE("Should skip full grapheme", nPos == SAL_N_ELEMENTS(BA_HALANT_LA));
        nPos = m_xBreak->previousCharacters(aTest1, SAL_N_ELEMENTS(BA_HALANT_LA), aLocale,
            i18n::CharacterIteratorMode::SKIPCELL, 1, nDone);
        CPPUNIT_ASSERT_MESSAGE("Should skip full grapheme", nPos == 0);
    }

    {
        sal_Unicode HA_HALANT_NA_VOWELSIGNI[] = { 0x09B9, 0x09CD, 0x09A3, 0x09BF };
        ::rtl::OUString aTest1(HA_HALANT_NA_VOWELSIGNI, SAL_N_ELEMENTS(HA_HALANT_NA_VOWELSIGNI));

        sal_Int32 nDone=0;
        sal_Int32 nPos;
        nPos = m_xBreak->nextCharacters(aTest1, 0, aLocale,
            i18n::CharacterIteratorMode::SKIPCELL, 1, nDone);
        CPPUNIT_ASSERT_MESSAGE("Should skip full grapheme", nPos == SAL_N_ELEMENTS(HA_HALANT_NA_VOWELSIGNI));
        nPos = m_xBreak->previousCharacters(aTest1, SAL_N_ELEMENTS(HA_HALANT_NA_VOWELSIGNI), aLocale,
            i18n::CharacterIteratorMode::SKIPCELL, 1, nDone);
        CPPUNIT_ASSERT_MESSAGE("Should skip full grapheme", nPos == 0);
    }

    {
        sal_Unicode TA_HALANT_MA_HALANT_YA  [] = { 0x09A4, 0x09CD, 0x09AE, 0x09CD, 0x09AF };
        ::rtl::OUString aTest1(TA_HALANT_MA_HALANT_YA, SAL_N_ELEMENTS(TA_HALANT_MA_HALANT_YA));

        sal_Int32 nDone=0;
        sal_Int32 nPos;
        nPos = m_xBreak->nextCharacters(aTest1, 0, aLocale,
            i18n::CharacterIteratorMode::SKIPCELL, 1, nDone);
        CPPUNIT_ASSERT_MESSAGE("Should skip full grapheme", nPos == SAL_N_ELEMENTS(TA_HALANT_MA_HALANT_YA));
        nPos = m_xBreak->previousCharacters(aTest1, SAL_N_ELEMENTS(TA_HALANT_MA_HALANT_YA), aLocale,
            i18n::CharacterIteratorMode::SKIPCELL, 1, nDone);
        CPPUNIT_ASSERT_MESSAGE("Should skip full grapheme", nPos == 0);
    }
}

TestBreakIterator::TestBreakIterator()
{
    m_xContext = cppu::defaultBootstrap_InitialComponentContext();
    m_xFactory = m_xContext->getServiceManager();
    m_xMSF = uno::Reference<lang::XMultiServiceFactory>(m_xFactory, uno::UNO_QUERY_THROW);
    m_xBreak = uno::Reference< i18n::XBreakIterator >(m_xMSF->createInstance(
        ::rtl::OUString(RTL_CONSTASCII_USTRINGPARAM("com.sun.star.i18n.BreakIterator"))),
        uno::UNO_QUERY_THROW);
}

void TestBreakIterator::setUp()
{
}

TestBreakIterator::~TestBreakIterator()
{
}

void TestBreakIterator::tearDown()
{
}

CPPUNIT_TEST_SUITE_REGISTRATION(TestBreakIterator);

CPPUNIT_PLUGIN_IMPLEMENT();

/* vim:set shiftwidth=4 softtabstop=4 expandtab: */
<|endoftext|>


<commit_before>//
// Copyright (c) 2013-2014 Christoph Malek
// See LICENSE for more information.
//

#ifndef RJ_SHARED_LEVEL_MANAGER_LEVEL_MANAGER_HPP
#define RJ_SHARED_LEVEL_MANAGER_LEVEL_MANAGER_HPP

#include "level.hpp"
#include "packer_pack.hpp"
#include "packer_unpack.hpp"

#include <mlk/filesystem/filesystem.h>

#include <string>

namespace rj
{
	class level_manager
	{
		mlk::fs::file_handle m_filemgr;
		std::string m_abs_path;

	public:
		level_manager(const std::string& abs_level_path) :
			m_abs_path{mlk::fs::dir_handle{abs_level_path}.get_path()}
		{ }

		level open_level(const std::string& level_name)
		{
			if(!m_filemgr.reopen(this->make_path(level_name), std::ios::in))
				return level{};

			auto data(m_filemgr.read_all());
			level_packer<packer_mode::unpack> unpacker{data};
			return unpacker.get_level();
		}

		bool save_level(const level_packer<packer_mode::pack>& lv, const std::string& level_name)
		{
			if(!m_filemgr.reopen(this->make_path(level_name), std::ios::out | std::ios::trunc))
				return false;
			m_filemgr.write(lv.packed_data());
			return true;
		}

	private:
		std::string make_path(const std::string& level_name)
		{return m_abs_path + level_name + ".rjl";}
	};
}

#endif // RJ_SHARED_LEVEL_MANAGER_LEVEL_MANAGER_HPP
<commit_msg>level_manager: autoload<commit_after>//
// Copyright (c) 2013-2014 Christoph Malek
// See LICENSE for more information.
//

#ifndef RJ_SHARED_LEVEL_MANAGER_LEVEL_MANAGER_HPP
#define RJ_SHARED_LEVEL_MANAGER_LEVEL_MANAGER_HPP

#include "level.hpp"
#include "packer_pack.hpp"
#include "packer_unpack.hpp"

#include <mlk/filesystem/filesystem.h>

#include <map>
#include <string>

namespace rj
{
	using level_id = std::string;

	class level_manager
	{
		mlk::fs::dir_handle m_dirh;
		mlk::fs::file_handle m_filemgr;
		const std::string& m_abs_path;

		std::map<std::string, level> m_loaded_levels;

	public:
		level_manager(const std::string& abs_level_path) :
			m_dirh{abs_level_path},
			m_abs_path{m_dirh.get_path()}
		{this->init();}

		const level& open_level(const level_id& id)
		{
			this->load_to_mgr(id, this->make_path(id));
			if(m_loaded_levels.find(id) == std::end(m_loaded_levels))
				throw std::runtime_error{"error while open level"};
			return m_loaded_levels[id];
		}

		bool save_level(const level_packer<packer_mode::pack>& lv, const std::string& id)
		{
			if(!m_filemgr.reopen(this->make_path(id), std::ios::out | std::ios::trunc))
				return false;
			m_filemgr.write(lv.packed_data());
			return true;
		}

		std::size_t num_levels() const noexcept
		{return m_loaded_levels.size();}

	private:
		void init()
		{
			mlk::lout("rj::level_manager") << "loading levels recursive from directory '" << m_abs_path << "'...";
			auto content(m_dirh.get_content<true>());
			auto count(0);
			for(auto& a : content)
				if(a.type == mlk::fs::item_type::file)
				{
					this->load_to_mgr(a.name, a.path);
					++count;
				}
			mlk::lout("rj::level_manager") << "loaded " << count << " levels";
		}

		void load_to_mgr(const level_id& id, const std::string& path)
		{
			if(!m_filemgr.reopen(path, std::ios::in))
				return;

			auto data(m_filemgr.read_all());
			level_packer<packer_mode::unpack> unpacker{data};
			m_loaded_levels[id] = unpacker.get_level();
		}

		std::string make_path(const level_id& id) const noexcept
		{return m_abs_path + id + ".rjl";}
	};
}

#endif // RJ_SHARED_LEVEL_MANAGER_LEVEL_MANAGER_HPP
<|endoftext|>


<commit_before>/******************************************************************************
IrstLM: IRST Language Model Toolkit, compile LM
Copyright (C) 2006 Marcello Federico, ITC-irst Trento, Italy

This library is free software; you can redistribute it and/or
modify it under the terms of the GNU Lesser General Public
License as published by the Free Software Foundation; either
version 2.1 of the License, or (at your option) any later version.

This library is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
Lesser General Public License for more details.

You should have received a copy of the GNU Lesser General Public
License along with this library; if not, write to the Free Software
Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301 USA

******************************************************************************/

using namespace std;

#include <iostream>
#include <fstream>
#include <vector>
#include <string>
#include <stdlib.h>

#include "math.h"
#include "lmtable.h"

//----------------------------------------------------------------------
//  Special type and global variable for the BIN CLUSTERING algorithm
//
//
//----------------------------------------------------------------------

typedef struct{
  double pt;
  int idx;
  short code;
}BinEntry;

int cmpBinEntry(const void* a,const void* b){
  if (*(double *)a > *(double*)b)
    return 1;
  else if (*(double *)a < *(double*)b)
    return -1;
  else
    return 0;
}

BinEntry* bintable=NULL;

//----------------------------------------------------------------------
//  Global entry points
//----------------------------------------------------------------------

int parseWords(char *sentence, char **words, int max);

int ComputeCluster(int nc, double* cl,int N,double* Pts);

//----------------------------------------------------------------------
//  Global parameters (some are set in getArgs())
//----------------------------------------------------------------------

int       k      = 256;   // number of centers
const int MAXLEV = 11;    //maximum n-gram size

//----------------------------------------------------------------------
//  Main program
//----------------------------------------------------------------------

void usage(const char *msg = 0) {
  if (msg) { std::cerr << msg << std::endl; }
  std::cerr << "Usage: quantize-lm input-file.lm [output-file.qlm]" << std::endl;
  if (!msg) std::cerr << std::endl
    << "  quantize-lm reads a standard LM file in ARPA format and produces" << std::endl
    << "  a version of it with quantized probabilities and back-off weights"<< std::endl
    << "  that the IRST LMtoolkit can compile." << std::endl;
  }

int main(int argc, const char **argv)
{

  //Process Parameters

  if (argc < 2) { usage(); exit(1); }
  std::vector<std::string> files;
  for (int i=1; i < argc; i++) {
    std::string opt = argv[i];
    files.push_back(opt);
  }
  if (files.size() > 2) { usage("Too many arguments"); exit(1); }
  if (files.size() < 1) { usage("Please specify a LM file to read from"); exit(1); }

  std::string infile = files[0];
  if (files.size() == 1) {
    std::string::size_type p = infile.rfind('/');
    if (p != std::string::npos && ((p+1) < infile.size())) {
      files.push_back(infile.substr(p+1) + ".qlm");
    } else {
      files.push_back(infile + ".qlm");
    }
  }

  std::cout << "Reading " << infile << "..." << std::endl;

  std::fstream inp(infile.c_str());
  if (!inp.good()) {
    std::cerr << "Failed to open " << infile << "!\n";
    exit(1);
  }

  std::string outfile = files[1];
  std::ofstream out(outfile.c_str());
  std::cout << "Writing " << outfile << "..." << std::endl;

  int nPts = 0;  // actual number of points

  // *** Read ARPA FILE **

  int numNgrams[MAXLEV + 1]; /* # n-grams for each order */
  int Order,MaxOrder;
  int n;

  float logprob,logbow, logten=log(10.0);

  double* dataPts=NULL;
  double* centersP=NULL; double* centersB=NULL;

  int* mapP=NULL; int* mapB=NULL;

  int centers=k;
  streampos iposition;

  out << "qARPA\n"; //print output header

  for (int i=1;i<=MAXLEV;i++) numNgrams[i]=0;

  char line[1024];

  while (inp.getline(line,1024)){

    bool backslash = (line[0] == '\\');

    if (sscanf(line, "ngram %d=%d", &Order, &n) == 2) {
      numNgrams[Order] = n;
      MaxOrder=Order;
    }

    if (backslash && sscanf(line, "\\%d-grams", &Order) == 1) {

      out << line << "\n";
      cerr << "-- Start processing of " << Order << "-grams\n";
      assert(Order <= MAXLEV);

      int N=numNgrams[Order];
      centers=k;
      if (Order==1) centers=256; // always use 256 centers

      char* words[MAXLEV+3];
      dataPts=new double[N]; // allocate data

      iposition=inp.tellg();

      for (nPts=0;nPts<N;nPts++){
        inp.getline(line,1024);
        int howmany = parseWords(line, words, Order + 3);
        assert(howmany == Order+2 || howmany == Order+1);
        sscanf(words[0],"%f",&logprob);
        dataPts[nPts]=exp(logprob * logten);
      }

      cerr << "quantizing " << N << " probabilities\n";

      centersP=new double[centers];
      mapP=new int[N];

      ComputeCluster(centers,centersP,N,dataPts);

      assert(bintable !=NULL);
      for (int p=0;p<N;p++){
        mapP[bintable[p].idx]=bintable[p].code;
      }

      if (Order<MaxOrder){

        inp.seekg(iposition);

        for (nPts=0;nPts<N;nPts++){
          inp.getline(line,1024);
          int howmany = parseWords(line, words, Order + 3);
          if (howmany==Order+2) //backoff is written
            sscanf(words[Order+1],"%f",&logbow);
          else
            logbow=0; // backoff is implicit
          dataPts[nPts]=exp(logbow * logten);
        }

        centersB=new double[centers];
        mapB=new int[N];

        cerr << "quantizing " << N << " backoff weights\n";
        ComputeCluster(centers,centersB,N,dataPts);

        assert(bintable !=NULL);
        for (int p=0;p<N;p++){
          mapB[bintable[p].idx]=bintable[p].code;
        }

      }

      inp.seekg(iposition);

      out << centers << "\n";
      for (nPts=0;nPts<centers;nPts++){
        out << log(centersP[nPts])/logten;
        if (Order<MaxOrder) out << " " << log(centersB[nPts])/logten;
        out << "\n";
      }

      for (nPts=0;nPts<numNgrams[Order];nPts++){

        inp.getline(line,1024);

        parseWords(line, words, Order + 3);

        out << mapP[nPts];

        for (int i=1;i<=Order;i++) out << "\t" << words[i];

        if (Order < MaxOrder) out << "\t" << mapB[nPts];

        out << "\n";

      }

      if (mapP){delete [] mapP;mapP=NULL;}
      if (mapB){delete [] mapB;mapB=NULL;}

      if (centersP){delete [] centersP; centersP=NULL;}
      if (centersB){delete [] centersB; centersB=NULL;}

      delete [] dataPts;

      continue;

    }

    out << line << "\n";
  }

  cerr << "---- done\n";

  out.flush();
  inp.flush();

  out.close();
  inp.close();

}

// Compute Clusters

int ComputeCluster(int centers,double* ctrs,int N,double* dataPts){

  //cerr << "\nExecuting Clutering Algorithm:  k=" << centers<< "\n";

  if (bintable) delete [] bintable;

  bintable=new BinEntry[N];
  for (int i=0;i<N;i++){
    bintable[i].pt=dataPts[i];
    bintable[i].idx=i;
    bintable[i].code=0;
  }

  //cout << "start sort \n";
  qsort(bintable,N,sizeof(BinEntry),cmpBinEntry);

  int different=1;

  for (int i=1;i<N;i++)
    if (bintable[i].pt!=bintable[i-1].pt)
      different++;

  int interval=different/centers;
  if (interval==0) interval++;

  int* population=new int[centers];
  int* species=new int[centers];

  //cout << " Different entries=" << different
  //     << " Total Entries=" << N << " Bin Size=" << interval << "\n";

  for (int i=0;i<centers;i++){
    population[i]=species[i]=0;
    ctrs[i]=0.0;
  }

  // initial values
  bintable[0].code=0;
  population[0]=1;
  species[0]=1;

  int currcode=0;
  different=1;

  for (int i=1;i<N;i++){

    if ((bintable[i].pt!=bintable[i-1].pt)){
      different++;
      if ((different % interval) == 0)
        if ((currcode+1) < centers
            &&
            population[currcode]>0){
                currcode++;
        }
    }

      if (bintable[i].pt == bintable[i-1].pt)
        bintable[i].code=bintable[i-1].code;
      else{
        bintable[i].code=currcode;
        species[currcode]++;
      }

      population[bintable[i].code]++;

      assert(bintable[i].code < centers);

      ctrs[bintable[i].code]+=bintable[i].pt;

  }

    for (int i=0;i<centers;i++){
      if (population[i]>0)
        ctrs[i]/=(float)population[i];
      //cout << i << " population " << population[i] << " species " << species[i] <<"\n";
    }

    cout.flush();

    delete [] population;

    return 1;

}

//----------------------------------------------------------------------
//  Reading/Printing utilities
//      readPt - read a point from input stream into data storage
//              at position i.  Returns false on error or EOF.
//      printPt - prints a points to output file
//----------------------------------------------------------------------

int parseWords(char *sentence, char **words, int max)
{
  char *word;
  int i = 0;

  char *const wordSeparators = " \t\r\n";

  for (word = strtok(sentence, wordSeparators);
       i < max && word != 0;
       i++, word = strtok(0, wordSeparators))
  {
    words[i] = word;
  }
  if (i < max) {
    words[i] = 0;
  }

  return i;
}

  <commit_msg>test<commit_after>/******************************************************************************
IrstLM: IRST Language Model Toolkit, compile LM
Copyright (C) 2006 Marcello Federico, ITC-irst Trento, Italy

This library is free software; you can redistribute it and/or
modify it under the terms of the GNU Lesser General Public
License as published by the Free Software Foundation; either
version 2.1 of the License, or (at your option) any later version.

This library is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
Lesser General Public License for more details.

You should have received a copy of the GNU Lesser General Public
License along with this library; if not, write to the Free Software
Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301 USA

******************************************************************************/

using namespace std;

#include <iostream>
#include <fstream>
#include <vector>
#include <string>
#include <stdlib.h>

#include "math.h"
#include "lmtable.h"

//----------------------------------------------------------------------
//  Special type and global variable for the BIN CLUSTERING algorithm
//
//
//----------------------------------------------------------------------

typedef struct{
  double pt;
  int idx;
  short code;
}BinEntry;

int cmpBinEntry(const void* a,const void* b){
  if (*(double *)a > *(double*)b)
    return 1;
  else if (*(double *)a < *(double*)b)
    return -1;
  else
    return 0;
}

BinEntry* bintable=NULL;

//----------------------------------------------------------------------
//  Global entry points
//----------------------------------------------------------------------

int parseWords(char *sentence, char **words, int max);

int ComputeCluster(int nc, double* cl,int N,double* Pts);

//----------------------------------------------------------------------
//  Global parameters (some are set in getArgs())
//----------------------------------------------------------------------

int       k      = 256;   // number of centers
const int MAXLEV = 11;    //maximum n-gram size

//----------------------------------------------------------------------
//  Main program
//----------------------------------------------------------------------

void usage(const char *msg = 0) {
  if (msg) { std::cerr << msg << std::endl; }
  std::cerr << "Usage: quantize-lm input-file.lm [output-file.qlm]" << std::endl;
  if (!msg) std::cerr << std::endl
    << "  quantize-lm reads a standard LM file in ARPA format and produces" << std::endl
    << "  a version of it with quantized probabilities and back-off weights"<< std::endl
    << "  that the IRST LMtoolkit can compile." << std::endl;
  }

int main(int argc, const char **argv)
{

  //Process Parameters

  if (argc < 2) { usage(); exit(1); }
  std::vector<std::string> files;
  for (int i=1; i < argc; i++) {
    std::string opt = argv[i];
    files.push_back(opt);
  }
  if (files.size() > 2) { usage("Too many arguments"); exit(1); }
  if (files.size() < 1) { usage("Please specify a LM file to read from"); exit(1); }

  std::string infile = files[0];
  if (files.size() == 1) {
    std::string::size_type p = infile.rfind('/');
    if (p != std::string::npos && ((p+1) < infile.size())) {
      files.push_back(infile.substr(p+1) + ".qlm");
    } else {
      files.push_back(infile + ".qlm");
    }
  }

  std::cout << "Reading " << infile << "..." << std::endl;

  std::fstream inp(infile.c_str());
  if (!inp.good()) {
    std::cerr << "Failed to open " << infile << "!\n";
    exit(1);
  }

  std::string outfile = files[1];
  std::ofstream out(outfile.c_str());
  std::cout << "Writing " << outfile << "..." << std::endl;

  int nPts = 0;  // actual number of points

  // *** Read ARPA FILE **

  int numNgrams[MAXLEV + 1]; /* # n-grams for each order */
  int Order,MaxOrder;
  int n;

  float logprob,logbow, logten=log(10.0);

  double* dataPts=NULL;
  double* centersP=NULL; double* centersB=NULL;

  int* mapP=NULL; int* mapB=NULL;

  int centers=k;
  streampos iposition;

  out << "qARPA\n"; //print output header

  for (int i=1;i<=MAXLEV;i++) numNgrams[i]=0;

  char line[1024];

  while (inp.getline(line,1024)){

    bool backslash = (line[0] == '\\');

    if (sscanf(line, "ngram %d=%d", &Order, &n) == 2) {
      numNgrams[Order] = n;
      MaxOrder=Order;
    }

    if (backslash && sscanf(line, "\\%d-grams", &Order) == 1) {

      out << line << "\n";
      cerr << "-- Start processing of " << Order << "-grams\n";
      assert(Order <= MAXLEV);

      int N=numNgrams[Order];
      centers=k;
      if (Order==1) centers=256; // always use 256 centers

      char* words[MAXLEV+3];
      dataPts=new double[N]; // allocate data

      iposition=inp.tellg();

      for (nPts=0;nPts<N;nPts++){
        inp.getline(line,1024);
        int howmany = parseWords(line, words, Order + 3);
        assert(howmany == Order+2 || howmany == Order+1);
        sscanf(words[0],"%f",&logprob);
        dataPts[nPts]=exp(logprob * logten);
      }

      cerr << "quantizing " << N << " probabilities\n";

      centersP=new double[centers];
      mapP=new int[N];

      ComputeCluster(centers,centersP,N,dataPts);

      assert(bintable !=NULL);
      for (int p=0;p<N;p++){
        mapP[bintable[p].idx]=bintable[p].code;
      }

      if (Order<MaxOrder){

        inp.seekg(iposition);

        for (nPts=0;nPts<N;nPts++){
          inp.getline(line,1024);
          int howmany = parseWords(line, words, Order + 3);
          if (howmany==Order+2) //backoff is written
            sscanf(words[Order+1],"%f",&logbow);
          else
            logbow=0; // backoff is implicit
          dataPts[nPts]=exp(logbow * logten);
        }

        centersB=new double[centers];
        mapB=new int[N];

        cerr << "quantizing " << N << " backoff weights\n";
        ComputeCluster(centers,centersB,N,dataPts);

        assert(bintable !=NULL);
        for (int p=0;p<N;p++){
          mapB[bintable[p].idx]=bintable[p].code;
        }

      }

      inp.seekg(iposition);

      out << centers << "\n";
      for (nPts=0;nPts<centers;nPts++){
        out << log(centersP[nPts])/logten;
        if (Order<MaxOrder) out << " " << log(centersB[nPts])/logten;
        out << "\n";
      }

      for (nPts=0;nPts<numNgrams[Order];nPts++){

        inp.getline(line,1024);

        parseWords(line, words, Order + 3);

        out << mapP[nPts];

        for (int i=1;i<=Order;i++) out << "\t" << words[i];

        if (Order < MaxOrder) out << "\t" << mapB[nPts];

        out << "\n";

      }

      if (mapP){delete [] mapP;mapP=NULL;}
      if (mapB){delete [] mapB;mapB=NULL;}

      if (centersP){delete [] centersP; centersP=NULL;}
      if (centersB){delete [] centersB; centersB=NULL;}

      delete [] dataPts;

      continue;

    }

    out << line << "\n";
  }

  cerr << "---- done\n";

  out.flush();
  inp.flush();

  out.close();
  inp.close();

}

// Compute Clusters

int ComputeCluster(int centers,double* ctrs,int N,double* dataPts){

  //cerr << "\nExecuting Clutering Algorithm:  k=" << centers<< "\n";

  if (bintable) delete [] bintable;

  bintable=new BinEntry[N];
  for (int i=0;i<N;i++){
    bintable[i].pt=dataPts[i];
    bintable[i].idx=i;
    bintable[i].code=0;
  }

  //cout << "start sort \n";
  qsort(bintable,N,sizeof(BinEntry),cmpBinEntry);

  int different=1;

  for (int i=1;i<N;i++)
    if (bintable[i].pt!=bintable[i-1].pt)
      different++;

  int interval=different/centers;
  if (interval==0) interval++;

  int* population=new int[centers];
  int* species=new int[centers];

  //cout << " Different entries=" << different
  //     << " Total Entries=" << N << " Bin Size=" << interval << "\n";

  for (int i=0;i<centers;i++){
    population[i]=species[i]=0;
    ctrs[i]=0.0;
  }

  // initial values
  bintable[0].code=0;
  population[0]=1;
  species[0]=1;

  int currcode=0;
  different=1;

  for (int i=1;i<N;i++){

    if ((bintable[i].pt!=bintable[i-1].pt)){
      different++;
      if ((different % interval) == 0)
        if ((currcode+1) < centers
            &&
            population[currcode]>0){
                currcode++;
        }
    }

      if (bintable[i].pt == bintable[i-1].pt)
        bintable[i].code=bintable[i-1].code;
      else{
        bintable[i].code=currcode;
        species[currcode]++;
      }

      population[bintable[i].code]++;

      assert(bintable[i].code < centers);

      ctrs[bintable[i].code]+=bintable[i].pt;

  }

    for (int i=0;i<centers;i++){
      if (population[i]>0)
        ctrs[i]/=(float)population[i];
      //cout << i << " population " << population[i] << " species " << species[i] <<"\n";
    }

    cout.flush();

    delete [] population;

    return 1;

}

//----------------------------------------------------------------------
//  Reading/Printing utilities
//      readPt - read a point from input stream into data storage
//              at position i.  Returns false on error or EOF.
//      printPt - prints a points to output file
//----------------------------------------------------------------------

int parseWords(char *sentence, char **words, int max)
{
  char *word;
  int i = 0;

  char *const wordSeparators = " \t\r\n";

  for (word = strtok(sentence, wordSeparators);
       i < max && word != 0;
       i++, word = strtok(0, wordSeparators))
  {
    words[i] = word;
  }
  if (i < max) {
    words[i] = 0;
  }

  return i;
}

<|endoftext|>


<commit_before>// Algorithm to find the closest pair of points in a 2-d plane.
// For reference: http://en.wikipedia.org/wiki/Closest_pair_of_points_problem

#include <algorithm>
#include <cassert>
#include <cmath>
#include <cfloat>
#include <iterator>
#include <set>
#include <utility>
#include <vector>

struct Point {
    int x;
    int y;

    Point() {}
    Point(int x, int y) : x(x), y(y) {}

    bool operator==(const Point& other) const {
        return this->x == other.x && this->y == other.y;
    }
};

typedef std::pair<Point, Point> PointPair;

// Returns the Euclidean distance between 2-d points p1 and p2.
float distance(const Point& p1, const Point& p2) {
    return std::sqrt(std::pow(p2.x - p1.x, 2) + std::pow(p2.y - p1.y, 2));
}

struct sort_by_x_value {
    bool operator()(const Point& p1, const Point& p2) const {
        return p1.x < p2.x;
    }
};

struct sort_by_y_value {
    bool operator()(const Point& p1, const Point& p2) const {
        return p1.y < p2.y;
    }
};

// Comparison functor used to determine if pair1 is closer in distance than
// pair2, for use with std::min as follows.
struct is_closer_in_distance {
    bool operator()(const PointPair& pair1, const PointPair& pair2) const {
        float pair1_distance = distance(pair1.first, pair1.second);
        float pair2_distance = distance(pair2.first, pair2.second);
        return pair1_distance < pair2_distance;
    }
};

// Given a pair of iterators to a collection of unique points, returns the
// closest pair of points in range [first, last) using an O(n^2) brute-force
// approach that compares pairwise distances for all pairs of points in [first,
// last).
template <class InputIterator>
PointPair find_closest_pair_bruteforce(InputIterator first,
                                       InputIterator last) {
    float min_distance = FLT_MAX;
    PointPair closest_pair;

    InputIterator i, j;
    for (i = first; i != last; i++) {
        j = i;
        j++;
        for (; j != last; j++) {
            float dist = distance(*i, *j);
            if (dist < min_distance) {
                min_distance = dist;
                closest_pair.first = *i;
                closest_pair.second = *j;
            }
        }
    }

    return closest_pair;
}

// Given a pair of iterators to a collection of unique points sorted by x-value
// and y-value, Px [px_begin, px_end) and Py [py_begin, py_end) respectively,
// and the previously calculated closest_non_split_pair of points, with
// pairwise distance delta, returns the closest split pair of points with
// pairwise distance less than delta. Returns closest_non_split_pair if no
// split pair is found with pairwise distance less than delta.
//
// A split pair of points is a pair of points in which one point lies left of
// the vertical divide x_mid, the midpoint of Px, and the other point lies to
// the right.
template <class RandomAccessIterator>
PointPair find_closest_split_pair(RandomAccessIterator px_begin,
                                  RandomAccessIterator px_end,
                                  RandomAccessIterator py_begin,
                                  RandomAccessIterator py_end,
                                  const PointPair& closest_non_split_pair) {
    Point px_mid = *(px_begin + ((px_end - px_begin) / 2));

    float delta = distance(closest_non_split_pair.first,
                           closest_non_split_pair.second);

    // sy is the subset of points that lie at most delta to the left or right
    // of the vertical divide x_mid, sorted by y-value.
    std::vector<Point> sy;

    RandomAccessIterator py_iter = py_begin;
    for (; py_iter != py_end; py_iter++) {
        if (std::abs(py_iter->x - px_mid.x) <= delta) {
            sy.push_back(*py_iter);
        }
    }

    PointPair closest_split_pair = closest_non_split_pair;
    float min_delta = delta;

    // For each point in sy, we need only consider at most 6 of its neighbors.
    std::vector<Point>::iterator i, j;
    for (i = sy.begin(); i != sy.end(); i++) {
        for (j = i + 1; j != i + 7 && j != sy.end(); j++) {
            float delta = distance(*i, *j);
            if (delta < min_delta) {
                min_delta = delta;
                closest_split_pair.first = *i;
                closest_split_pair.second = *j;
            }
        }
    }

    return closest_split_pair;
}

// Given a pair of iterators to a collection of unique points sorted by x-value
// and y-value, Px [px_begin, px_end) and Py [py_begin, py_end) respectively,
// returns the pair of points that are closest in distance using an O(nlogn)
// divide-and-conquer algorithm.
template <class RandomAccessIterator>
PointPair find_closest_pair_rec(RandomAccessIterator px_begin,
                                RandomAccessIterator px_end,
                                RandomAccessIterator py_begin,
                                RandomAccessIterator py_end) {
    if ((px_end - px_begin < 4) || (py_end - py_begin < 4)) {
        std::set<Point, sort_by_x_value> points(px_begin, px_end);
        std::copy(py_begin, py_end, std::inserter(points, points.begin()));
        return find_closest_pair_bruteforce(points.begin(), points.end());
    }

    int middle = (px_end - px_begin) / 2;

    PointPair closest_pair_left = find_closest_pair_rec(px_begin,
                                                        px_begin + middle,
                                                        py_begin,
                                                        py_begin + middle);

    PointPair closest_pair_right = find_closest_pair_rec(px_begin + middle,
                                                         px_end,
                                                         py_begin + middle,
                                                         py_end);

    PointPair closest_pair = std::min(closest_pair_left, closest_pair_right,
                                      is_closer_in_distance());

    PointPair closest_split_pair = find_closest_split_pair(px_begin, px_end,
                                                           py_begin, py_end,
                                                           closest_pair);

    return std::min(closest_pair, closest_split_pair, is_closer_in_distance());
}

// Given a pair of iterators to a collection of unique points, returns the pair
// of points that are closest in distance using an O(nlogn) divide-and-conquer
// algorithm.
template <class InputIterator>
PointPair find_closest_pair(InputIterator first, InputIterator last) {
    std::vector<Point> px(first, last);
    std::sort(px.begin(), px.end(), sort_by_x_value());

    std::vector<Point> py(first, last);
    std::sort(py.begin(), py.end(), sort_by_y_value());

    return find_closest_pair_rec(px.begin(), px.end(), py.begin(), py.end());
}

int main(int argc, char** argv) {
    Point tmp[] = {Point(1, 0), Point(2, 6), Point(2, 9), Point(3, 1),
                   Point(4, 9), Point(6, 0), Point(8, 6)};

    std::vector<Point> points(tmp, tmp + (sizeof tmp / sizeof tmp[0]));

    PointPair expected(Point(2, 9), Point(4, 9));

    assert(find_closest_pair_bruteforce(points.begin(), points.end())
           == expected);
    assert(find_closest_pair(points.begin(), points.end()) == expected);
}
<commit_msg>Minor refactoring: add operator< to Point<commit_after>// Algorithm to find the closest pair of points in a 2-d plane.
// For reference: http://en.wikipedia.org/wiki/Closest_pair_of_points_problem

#include <algorithm>
#include <cassert>
#include <cmath>
#include <cfloat>
#include <iterator>
#include <set>
#include <utility>
#include <vector>

struct Point {
    int x;
    int y;

    Point() {}
    Point(int x, int y) : x(x), y(y) {}

    bool operator==(const Point& other) const {
        return this->x == other.x && this->y == other.y;
    }

    bool operator<(const Point& other) const {
        if (this->x == other.x) {
            return this->y < other.y;
        }
        return this->x < other.x;
    }
};

typedef std::pair<Point, Point> PointPair;

// Returns the Euclidean distance between 2-d points p1 and p2.
float distance(const Point& p1, const Point& p2) {
    return std::sqrt(std::pow(p2.x - p1.x, 2) + std::pow(p2.y - p1.y, 2));
}

struct sort_by_x_value {
    bool operator()(const Point& p1, const Point& p2) const {
        return p1.x < p2.x;
    }
};

struct sort_by_y_value {
    bool operator()(const Point& p1, const Point& p2) const {
        return p1.y < p2.y;
    }
};

// Comparison functor used to determine if pair1 is closer in distance than
// pair2, for use with std::min as follows.
struct is_closer_in_distance {
    bool operator()(const PointPair& pair1, const PointPair& pair2) const {
        float pair1_distance = distance(pair1.first, pair1.second);
        float pair2_distance = distance(pair2.first, pair2.second);
        return pair1_distance < pair2_distance;
    }
};

// Given a pair of iterators to a collection of unique points, returns the
// closest pair of points in range [first, last) using an O(n^2) brute-force
// approach that compares pairwise distances for all pairs of points in [first,
// last).
template <class InputIterator>
PointPair find_closest_pair_bruteforce(InputIterator first,
                                       InputIterator last) {
    float min_distance = FLT_MAX;
    PointPair closest_pair;

    InputIterator i, j;
    for (i = first; i != last; i++) {
        j = i;
        j++;
        for (; j != last; j++) {
            float dist = distance(*i, *j);
            if (dist < min_distance) {
                min_distance = dist;
                closest_pair.first = *i;
                closest_pair.second = *j;
            }
        }
    }

    return closest_pair;
}

// Given a pair of iterators to a collection of unique points sorted by x-value
// and y-value, Px [px_begin, px_end) and Py [py_begin, py_end) respectively,
// and the previously calculated closest_non_split_pair of points, with
// pairwise distance delta, returns the closest split pair of points with
// pairwise distance less than delta. Returns closest_non_split_pair if no
// split pair is found with pairwise distance less than delta.
//
// A split pair of points is a pair of points in which one point lies left of
// the vertical divide x_mid, the midpoint of Px, and the other point lies to
// the right.
template <class RandomAccessIterator>
PointPair find_closest_split_pair(RandomAccessIterator px_begin,
                                  RandomAccessIterator px_end,
                                  RandomAccessIterator py_begin,
                                  RandomAccessIterator py_end,
                                  const PointPair& closest_non_split_pair) {
    Point px_mid = *(px_begin + ((px_end - px_begin) / 2));

    float delta = distance(closest_non_split_pair.first,
                           closest_non_split_pair.second);

    // sy is the subset of points that lie at most delta to the left or right
    // of the vertical divide x_mid, sorted by y-value.
    std::vector<Point> sy;

    RandomAccessIterator py_iter = py_begin;
    for (; py_iter != py_end; py_iter++) {
        if (std::abs(py_iter->x - px_mid.x) <= delta) {
            sy.push_back(*py_iter);
        }
    }

    PointPair closest_split_pair = closest_non_split_pair;
    float min_delta = delta;

    // For each point in sy, we need only consider at most 6 of its neighbors.
    std::vector<Point>::iterator i, j;
    for (i = sy.begin(); i != sy.end(); i++) {
        for (j = i + 1; j != i + 7 && j != sy.end(); j++) {
            float delta = distance(*i, *j);
            if (delta < min_delta) {
                min_delta = delta;
                closest_split_pair.first = *i;
                closest_split_pair.second = *j;
            }
        }
    }

    return closest_split_pair;
}

// Given a pair of iterators to a collection of unique points sorted by x-value
// and y-value, Px [px_begin, px_end) and Py [py_begin, py_end) respectively,
// returns the pair of points that are closest in distance using an O(nlogn)
// divide-and-conquer algorithm.
template <class RandomAccessIterator>
PointPair find_closest_pair_rec(RandomAccessIterator px_begin,
                                RandomAccessIterator px_end,
                                RandomAccessIterator py_begin,
                                RandomAccessIterator py_end) {
    if ((px_end - px_begin < 4) || (py_end - py_begin < 4)) {
        std::set<Point> points(px_begin, px_end);
        std::copy(py_begin, py_end, std::inserter(points, points.begin()));
        return find_closest_pair_bruteforce(points.begin(), points.end());
    }

    int middle = (px_end - px_begin) / 2;

    PointPair closest_pair_left = find_closest_pair_rec(px_begin,
                                                        px_begin + middle,
                                                        py_begin,
                                                        py_begin + middle);

    PointPair closest_pair_right = find_closest_pair_rec(px_begin + middle,
                                                         px_end,
                                                         py_begin + middle,
                                                         py_end);

    PointPair closest_pair = std::min(closest_pair_left, closest_pair_right,
                                      is_closer_in_distance());

    PointPair closest_split_pair = find_closest_split_pair(px_begin, px_end,
                                                           py_begin, py_end,
                                                           closest_pair);

    return std::min(closest_pair, closest_split_pair, is_closer_in_distance());
}

// Given a pair of iterators to a collection of unique points, returns the pair
// of points that are closest in distance using an O(nlogn) divide-and-conquer
// algorithm.
template <class InputIterator>
PointPair find_closest_pair(InputIterator first, InputIterator last) {
    std::vector<Point> px(first, last);
    std::sort(px.begin(), px.end(), sort_by_x_value());

    std::vector<Point> py(first, last);
    std::sort(py.begin(), py.end(), sort_by_y_value());

    return find_closest_pair_rec(px.begin(), px.end(), py.begin(), py.end());
}

int main(int argc, char** argv) {
    Point tmp[] = {Point(1, 0), Point(2, 6), Point(2, 9), Point(3, 1),
                   Point(4, 9), Point(6, 0), Point(8, 6)};

    std::vector<Point> points(tmp, tmp + (sizeof tmp / sizeof tmp[0]));

    PointPair expected(Point(2, 9), Point(4, 9));

    assert(find_closest_pair_bruteforce(points.begin(), points.end())
           == expected);
    assert(find_closest_pair(points.begin(), points.end()) == expected);
}
<|endoftext|>


<commit_before>/* Copyright (c) 2008-2017 the MRtrix3 contributors
 *
 * This Source Code Form is subject to the terms of the Mozilla Public
 * License, v. 2.0. If a copy of the MPL was not distributed with this
 * file, you can obtain one at http://mozilla.org/MPL/2.0/.
 *
 * MRtrix is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty
 * of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
 *
 * For more details, see http://www.mrtrix.org/.
 */

#include <iomanip>
#include <vector>

#include "command.h"
#include "datatype.h"
#include "image.h"
#include "image_helpers.h"
#include "memory.h"
#include "algo/histogram.h"
#include "algo/loop.h"
#include "file/ofstream.h"
#include "stats.h"

using namespace MR;
using namespace App;

void usage ()
{
AUTHOR = "J-Donald Tournier (jdtournier@gmail.com)";

SYNOPSIS = "Compute images statistics";

ARGUMENTS
  + Argument ("image",
  "the input image from which statistics will be computed.")
  .type_image_in ();

OPTIONS
  + Stats::Options

  + OptionGroup ("Additional options for mrstats")
  + Option ("allvolumes", "generate statistics across all image volumes, rather than one set of statistics per image volume")
  + Option ("ignorezero", "ignore zero-valued input voxels.");

}

typedef Stats::value_type value_type;
typedef Stats::complex_type complex_type;

class Volume_loop
{ NOMEMALIGN
  public:
    Volume_loop (Image<complex_type>& in) :
        image (in),
        is_4D (in.ndim() == 4),
        status (true)
    {
      if (is_4D)
        image.index(3) = 0;
    }

    void operator++ ()
    {
      if (is_4D) {
        image.index(3)++;
      } else {
        assert (status);
        status = false;
      }
    }
    operator bool() const
    {
      if (is_4D)
        return (image.index(3) >= 0 && image.index(3) < image.size(3));
      else
        return status;
    }

  private:
    Image<complex_type>& image;
    const bool is_4D;
    bool status;
};

void run_volume (Stats::Stats& stats, Image<complex_type>& data, Image<bool>& mask)
{
  if (mask.valid()) {
    for (auto l = Loop(0,3) (data, mask); l; ++l) {
      if (mask.value())
        stats (data.value());
    }
  } else {
    for (auto l = Loop(0,3) (data); l; ++l)
      stats (data.value());
  }
}

void run ()
{

  auto header = Header::open (argument[0]);
  if (header.ndim() > 4)
    throw Exception ("mrstats is not designed to handle images greater than 4D");
  const bool is_complex = header.datatype().is_complex();
  auto data = header.get_image<complex_type>();
  const bool ignorezero = get_options("ignorezero").size();

  auto opt = get_options ("mask");
  Image<bool> mask;
  if (opt.size()) {
    mask = Image<bool>::open (opt[0][0]);
    check_dimensions (mask, header, 0, 3);
  }

  vector<std::string> fields;
  opt = get_options ("output");
  for (size_t n = 0; n < opt.size(); ++n)
    fields.push_back (opt[n][0]);

  if (App::log_level && fields.empty())
    Stats::print_header (is_complex);

  if (get_options ("allvolumes").size()) {

    Stats::Stats stats (is_complex, ignorezero);
    for (auto i = Volume_loop (data); i; ++i)
      run_volume (stats, data, mask);
    stats.print (data, fields);

  } else {

    for (auto i = Volume_loop (data); i; ++i) {
      Stats::Stats stats (is_complex, ignorezero);
      run_volume (stats, data, mask);
      stats.print (data, fields);
    }

  }
}
<commit_msg>mrstats: removed duplicate ignorezero, kept the one in stats.cpp<commit_after>/* Copyright (c) 2008-2017 the MRtrix3 contributors
 *
 * This Source Code Form is subject to the terms of the Mozilla Public
 * License, v. 2.0. If a copy of the MPL was not distributed with this
 * file, you can obtain one at http://mozilla.org/MPL/2.0/.
 *
 * MRtrix is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty
 * of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
 *
 * For more details, see http://www.mrtrix.org/.
 */

#include <iomanip>
#include <vector>

#include "command.h"
#include "datatype.h"
#include "image.h"
#include "image_helpers.h"
#include "memory.h"
#include "algo/histogram.h"
#include "algo/loop.h"
#include "file/ofstream.h"
#include "stats.h"

using namespace MR;
using namespace App;

void usage ()
{
AUTHOR = "J-Donald Tournier (jdtournier@gmail.com)";

SYNOPSIS = "Compute images statistics";

ARGUMENTS
  + Argument ("image",
  "the input image from which statistics will be computed.")
  .type_image_in ();

OPTIONS
  + Stats::Options

  + OptionGroup ("Additional options for mrstats")
  + Option ("allvolumes", "generate statistics across all image volumes, rather than one set of statistics per image volume");

}

typedef Stats::value_type value_type;
typedef Stats::complex_type complex_type;

class Volume_loop
{ NOMEMALIGN
  public:
    Volume_loop (Image<complex_type>& in) :
        image (in),
        is_4D (in.ndim() == 4),
        status (true)
    {
      if (is_4D)
        image.index(3) = 0;
    }

    void operator++ ()
    {
      if (is_4D) {
        image.index(3)++;
      } else {
        assert (status);
        status = false;
      }
    }
    operator bool() const
    {
      if (is_4D)
        return (image.index(3) >= 0 && image.index(3) < image.size(3));
      else
        return status;
    }

  private:
    Image<complex_type>& image;
    const bool is_4D;
    bool status;
};

void run_volume (Stats::Stats& stats, Image<complex_type>& data, Image<bool>& mask)
{
  if (mask.valid()) {
    for (auto l = Loop(0,3) (data, mask); l; ++l) {
      if (mask.value())
        stats (data.value());
    }
  } else {
    for (auto l = Loop(0,3) (data); l; ++l)
      stats (data.value());
  }
}

void run ()
{

  auto header = Header::open (argument[0]);
  if (header.ndim() > 4)
    throw Exception ("mrstats is not designed to handle images greater than 4D");
  const bool is_complex = header.datatype().is_complex();
  auto data = header.get_image<complex_type>();
  const bool ignorezero = get_options("ignorezero").size();

  auto opt = get_options ("mask");
  Image<bool> mask;
  if (opt.size()) {
    mask = Image<bool>::open (opt[0][0]);
    check_dimensions (mask, header, 0, 3);
  }

  vector<std::string> fields;
  opt = get_options ("output");
  for (size_t n = 0; n < opt.size(); ++n)
    fields.push_back (opt[n][0]);

  if (App::log_level && fields.empty())
    Stats::print_header (is_complex);

  if (get_options ("allvolumes").size()) {

    Stats::Stats stats (is_complex, ignorezero);
    for (auto i = Volume_loop (data); i; ++i)
      run_volume (stats, data, mask);
    stats.print (data, fields);

  } else {

    for (auto i = Volume_loop (data); i; ++i) {
      Stats::Stats stats (is_complex, ignorezero);
      run_volume (stats, data, mask);
      stats.print (data, fields);
    }

  }
}
<|endoftext|>


<commit_before><commit_msg>comment out of data<commit_after><|endoftext|>