<commit_before>/*
 * Copyright (C) 2015 ScyllaDB
 */

/*
 * This file is part of Scylla.
 *
 * Scylla is free software: you can redistribute it and/or modify
 * it under the terms of the GNU Affero General Public License as published by
 * the Free Software Foundation, either version 3 of the License, or
 * (at your option) any later version.
 *
 * Scylla is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 * GNU General Public License for more details.
 *
 * You should have received a copy of the GNU General Public License
 * along with Scylla.  If not, see <http://www.gnu.org/licenses/>.
 */

#pragma once

#include <unordered_map>
#include <exception>

#include <seastar/core/sstring.hh>
#include <seastar/core/sharded.hh>
#include <seastar/core/future.hh>

#include "database.hh"
#include "utils/UUID.hh"

class repair_exception : public std::exception {
private:
    sstring _what;
public:
    repair_exception(sstring msg) : _what(std::move(msg)) { }
    virtual const char* what() const noexcept override { return _what.c_str(); }
};

class repair_stopped_exception : public repair_exception {
public:
    repair_stopped_exception() : repair_exception("Repair stopped") { }
};

// NOTE: repair_start() can be run on any node, but starts a node-global
// operation.
// repair_start() starts the requested repair on this node. It returns an
// integer id which can be used to query the repair's status with
// repair_get_status(). The returned future<int> becomes available quickly,
// as soon as repair_get_status() can be used - it doesn't wait for the
// repair to complete.
future<int> repair_start(seastar::sharded<database>& db, sstring keyspace,
        std::unordered_map<sstring, sstring> options);

// TODO: Have repair_progress contains a percentage progress estimator
// instead of just "RUNNING".
enum class repair_status { RUNNING, SUCCESSFUL, FAILED };

// repair_get_status() returns a future because it needs to run code on a
// different CPU (cpu 0) and that might be a deferring operation.
future<repair_status> repair_get_status(seastar::sharded<database>& db, int id);

// returns a vector with the ids of the active repairs
future<std::vector<int>> get_active_repairs(seastar::sharded<database>& db);

// repair_shutdown() stops all ongoing repairs started on this node (and
// prevents any further repairs from being started). It returns a future
// saying when all repairs have stopped, and attempts to stop them as
// quickly as possible (we do not wait for repairs to finish but rather
// stop them abruptly).
future<> repair_shutdown(seastar::sharded<database>& db);

// Abort all the repairs
future<> repair_abort_all(seastar::sharded<database>& db);

enum class repair_checksum {
    legacy = 0,
    streamed = 1,
};

// The class partition_checksum calculates a 256-bit cryptographically-secure
// checksum of a set of partitions fed to it. The checksum of a partition set
// is calculated by calculating a strong hash function (SHA-256) of each
// individual partition, and then XORing the individual hashes together.
// XOR is good enough for merging strong checksums, and allows us to
// independently calculate the checksums of different subsets of the original
// set, and then combine the results into one checksum with the add() method.
// The hash of an individual partition uses both its key and value.
class partition_checksum {
private:
    std::array<uint8_t, 32> _digest; // 256 bits
private:
    static future<partition_checksum> compute_legacy(flat_mutation_reader m);
    static future<partition_checksum> compute_streamed(flat_mutation_reader m);
public:
    constexpr partition_checksum() : _digest{} { }
    explicit partition_checksum(std::array<uint8_t, 32> digest) : _digest(std::move(digest)) { }
    static future<partition_checksum> compute(flat_mutation_reader mr, repair_checksum rt);
    void add(const partition_checksum& other);
    bool operator==(const partition_checksum& other) const;
    bool operator!=(const partition_checksum& other) const { return !operator==(other); }
    friend std::ostream& operator<<(std::ostream&, const partition_checksum&);
    const std::array<uint8_t, 32>& digest() const;
};

// Calculate the checksum of the data held on all shards of a column family,
// in the given token range.
// All parameters to this function are constant references, and the caller
// must ensure they live as long as the future returned by this function is
// not resolved.
future<partition_checksum> checksum_range(seastar::sharded<database> &db,
        const sstring& keyspace, const sstring& cf,
        const ::dht::token_range& range, repair_checksum rt);

// Represent a position of a mutation_fragment read from a flat mutation
// reader. Repair nodes negotiate a small range identified by two
// repair_sync_boundary to work on in each round.
struct repair_sync_boundary {
    dht::decorated_key pk;
    position_in_partition position;
    class tri_compare {
        dht::ring_position_comparator _pk_cmp;
        position_in_partition::tri_compare _position_cmp;
    public:
        tri_compare(const schema& s) : _pk_cmp(s), _position_cmp(s) { }
        int operator()(const repair_sync_boundary& a, const repair_sync_boundary& b) const {
            int ret = _pk_cmp(a.pk, b.pk);
            if (ret == 0) {
                ret = _position_cmp(a.position, b.position);
            }
            return ret;
        }
    };
    friend std::ostream& operator<<(std::ostream& os, const repair_sync_boundary& x) {
        return os << "{ " << x.pk << "," <<  x.position << " }";
    }
};

// Hash of a repair row
class repair_hash {
public:
    uint64_t hash = 0;
    repair_hash() = default;
    explicit repair_hash(uint64_t h) : hash(h) {
    }
    void clear() {
        hash = 0;
    }
    void add(const repair_hash& other) {
        hash ^= other.hash;
    }
    bool operator==(const repair_hash& x) const {
        return x.hash == hash;
    }
    bool operator!=(const repair_hash& x) const {
        return x.hash != hash;
    }
    bool operator<(const repair_hash& x) const {
        return x.hash < hash;
    }
    friend std::ostream& operator<<(std::ostream& os, const repair_hash& x) {
        return os << x.hash;
    }
};

namespace std {
template<>
struct hash<partition_checksum> {
    size_t operator()(partition_checksum sum) const {
        size_t h = 0;
        std::copy_n(sum.digest().begin(), std::min(sizeof(size_t), sizeof(sum.digest())), reinterpret_cast<uint8_t*>(&h));
        return h;
    }
};
}
<commit_msg>repair: Introduce get_sync_boundary_response<commit_after>/*
 * Copyright (C) 2015 ScyllaDB
 */

/*
 * This file is part of Scylla.
 *
 * Scylla is free software: you can redistribute it and/or modify
 * it under the terms of the GNU Affero General Public License as published by
 * the Free Software Foundation, either version 3 of the License, or
 * (at your option) any later version.
 *
 * Scylla is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 * GNU General Public License for more details.
 *
 * You should have received a copy of the GNU General Public License
 * along with Scylla.  If not, see <http://www.gnu.org/licenses/>.
 */

#pragma once

#include <unordered_map>
#include <exception>

#include <seastar/core/sstring.hh>
#include <seastar/core/sharded.hh>
#include <seastar/core/future.hh>

#include "database.hh"
#include "utils/UUID.hh"

class repair_exception : public std::exception {
private:
    sstring _what;
public:
    repair_exception(sstring msg) : _what(std::move(msg)) { }
    virtual const char* what() const noexcept override { return _what.c_str(); }
};

class repair_stopped_exception : public repair_exception {
public:
    repair_stopped_exception() : repair_exception("Repair stopped") { }
};

// NOTE: repair_start() can be run on any node, but starts a node-global
// operation.
// repair_start() starts the requested repair on this node. It returns an
// integer id which can be used to query the repair's status with
// repair_get_status(). The returned future<int> becomes available quickly,
// as soon as repair_get_status() can be used - it doesn't wait for the
// repair to complete.
future<int> repair_start(seastar::sharded<database>& db, sstring keyspace,
        std::unordered_map<sstring, sstring> options);

// TODO: Have repair_progress contains a percentage progress estimator
// instead of just "RUNNING".
enum class repair_status { RUNNING, SUCCESSFUL, FAILED };

// repair_get_status() returns a future because it needs to run code on a
// different CPU (cpu 0) and that might be a deferring operation.
future<repair_status> repair_get_status(seastar::sharded<database>& db, int id);

// returns a vector with the ids of the active repairs
future<std::vector<int>> get_active_repairs(seastar::sharded<database>& db);

// repair_shutdown() stops all ongoing repairs started on this node (and
// prevents any further repairs from being started). It returns a future
// saying when all repairs have stopped, and attempts to stop them as
// quickly as possible (we do not wait for repairs to finish but rather
// stop them abruptly).
future<> repair_shutdown(seastar::sharded<database>& db);

// Abort all the repairs
future<> repair_abort_all(seastar::sharded<database>& db);

enum class repair_checksum {
    legacy = 0,
    streamed = 1,
};

// The class partition_checksum calculates a 256-bit cryptographically-secure
// checksum of a set of partitions fed to it. The checksum of a partition set
// is calculated by calculating a strong hash function (SHA-256) of each
// individual partition, and then XORing the individual hashes together.
// XOR is good enough for merging strong checksums, and allows us to
// independently calculate the checksums of different subsets of the original
// set, and then combine the results into one checksum with the add() method.
// The hash of an individual partition uses both its key and value.
class partition_checksum {
private:
    std::array<uint8_t, 32> _digest; // 256 bits
private:
    static future<partition_checksum> compute_legacy(flat_mutation_reader m);
    static future<partition_checksum> compute_streamed(flat_mutation_reader m);
public:
    constexpr partition_checksum() : _digest{} { }
    explicit partition_checksum(std::array<uint8_t, 32> digest) : _digest(std::move(digest)) { }
    static future<partition_checksum> compute(flat_mutation_reader mr, repair_checksum rt);
    void add(const partition_checksum& other);
    bool operator==(const partition_checksum& other) const;
    bool operator!=(const partition_checksum& other) const { return !operator==(other); }
    friend std::ostream& operator<<(std::ostream&, const partition_checksum&);
    const std::array<uint8_t, 32>& digest() const;
};

// Calculate the checksum of the data held on all shards of a column family,
// in the given token range.
// All parameters to this function are constant references, and the caller
// must ensure they live as long as the future returned by this function is
// not resolved.
future<partition_checksum> checksum_range(seastar::sharded<database> &db,
        const sstring& keyspace, const sstring& cf,
        const ::dht::token_range& range, repair_checksum rt);

// Represent a position of a mutation_fragment read from a flat mutation
// reader. Repair nodes negotiate a small range identified by two
// repair_sync_boundary to work on in each round.
struct repair_sync_boundary {
    dht::decorated_key pk;
    position_in_partition position;
    class tri_compare {
        dht::ring_position_comparator _pk_cmp;
        position_in_partition::tri_compare _position_cmp;
    public:
        tri_compare(const schema& s) : _pk_cmp(s), _position_cmp(s) { }
        int operator()(const repair_sync_boundary& a, const repair_sync_boundary& b) const {
            int ret = _pk_cmp(a.pk, b.pk);
            if (ret == 0) {
                ret = _position_cmp(a.position, b.position);
            }
            return ret;
        }
    };
    friend std::ostream& operator<<(std::ostream& os, const repair_sync_boundary& x) {
        return os << "{ " << x.pk << "," <<  x.position << " }";
    }
};

// Hash of a repair row
class repair_hash {
public:
    uint64_t hash = 0;
    repair_hash() = default;
    explicit repair_hash(uint64_t h) : hash(h) {
    }
    void clear() {
        hash = 0;
    }
    void add(const repair_hash& other) {
        hash ^= other.hash;
    }
    bool operator==(const repair_hash& x) const {
        return x.hash == hash;
    }
    bool operator!=(const repair_hash& x) const {
        return x.hash != hash;
    }
    bool operator<(const repair_hash& x) const {
        return x.hash < hash;
    }
    friend std::ostream& operator<<(std::ostream& os, const repair_hash& x) {
        return os << x.hash;
    }
};

// Return value of the REPAIR_GET_SYNC_BOUNDARY RPC verb
struct get_sync_boundary_response {
    std::optional<repair_sync_boundary> boundary;
    repair_hash row_buf_combined_csum;
    // The current size of the row buf
    uint64_t row_buf_size;
    // The number of bytes this verb read from disk
    uint64_t new_rows_size;
    // The number of rows this verb read from disk
    uint64_t new_rows_nr;
};

namespace std {
template<>
struct hash<partition_checksum> {
    size_t operator()(partition_checksum sum) const {
        size_t h = 0;
        std::copy_n(sum.digest().begin(), std::min(sizeof(size_t), sizeof(sum.digest())), reinterpret_cast<uint8_t*>(&h));
        return h;
    }
};
}
<|endoftext|>


<commit_before>#include <node.h>
#include <v8.h>
#include <iostream>

#include "pow.h"

using namespace v8;
using namespace std;

/**
 *
 */
int strToB10(const char* b2){
  int b10 = 0,
      len = strlen(b2),
      i = len-1,
      p;

  for(p = 0; i > -1; p++, i--)
    b10 += (b2[i] - 48) * pow(2, p);

  return b10;
}

/**
 *
 */
int arrToB10(int* pointer){
  int b10 = 0;

  cout << "arrToB10 pointer value: " << *pointer << endl;

  return b10;
}

/**
 * B10 JS API
 */
void B10(const v8::FunctionCallbackInfo<Value>& args){
  Isolate* isolate = Isolate::GetCurrent();
  HandleScope scope(isolate);

  if(args[0]->IsString()){
    String::Utf8Value js_str(args[0]);
    const char* str = *js_str;

    cout << "str: " << str << endl;

    //Local<Number> js_b10 = Number::New(isolate, strToB10(str));
    //args.GetReturnValue().Set(js_b10);
  }
  else{

    Local<Object>  array  = args[0]->ToObject();
    Handle<Object> buffer = array->Get(
        String::NewFromUtf8(isolate, "buffer"))->ToObject();
    unsigned int offset = array->Get(
        String::NewFromUtf8(isolate, "byteOffset"))->Uint32Value();
    int length = array->Get(
        String::NewFromUtf8(isolate, "byteLength"))->Uint32Value();

    //Local<Number> js_b10 = Number::New(isolate, arrToB10(*array));
    //args.GetReturnValue().Set(js_b10);

/*
    int var = 25;
    int* var_pointer = &var;
    cout << "var_pointer: " << var_pointer << "\n";

    int var = 25;
    int* var_pointer = &var;
    arrToB10(var_pointer);
*/

    int t;
    for(int i = 0; i < length; i++){
      t = array->Get(i)->Uint32Value();
      cout << "t: " << t << endl;
    }

/*
    Local<Object> array  = args[0]->ToObject();
    Local<Number> js_b10 = Number::New(isolate, arrToB10(*array));
    args.GetReturnValue().Set(js_b10);
*/
    //arrToB10(array);

    //int p = &array;

/*
    Local<Object> array  = args[0]->ToObject();
    Local<Number> js_b10 = Number::New(isolate, arrToB10(*array));
    args.GetReturnValue().Set(js_b10);
*/

    /*int t;
    for(int i = 0; i < length; i++){
      t = array->Get(i)->Uint32Value();
      cout << "t: " << t << "\n";
    }

    cout << "offset: " << offset << "\n";
    cout << "length: " << length << "\n";*/
  }

}

void Init(Handle<Object> exports) {
  Isolate* isolate = Isolate::GetCurrent();

  exports->Set(String::NewFromUtf8(isolate, "b10"),
      FunctionTemplate::New(isolate, B10)->GetFunction());
}

NODE_MODULE(b2, Init)
<commit_msg>Working b2.b10(string)<commit_after>#include <node.h>
#include <v8.h>
#include <iostream>

#include "pow.h"

using namespace v8;
using namespace std;

/**
 *
 */
int strToB10(const char* b2){
  int b10 = 0,
      len = strlen(b2),
      i = len-1,
      p;

  for(p = 0; i > -1; p++, i--)
    b10 += (b2[i] - 48) * pow(2, p);

  return b10;
}

/**
 *
 */
int arrToB10(int* pointer){
  int b10 = 0;

  cout << "arrToB10 pointer value: " << *pointer << endl;

  return b10;
}

/**
 * B10 JS API
 */
void B10(const v8::FunctionCallbackInfo<Value>& args){
  Isolate* isolate = Isolate::GetCurrent();
  HandleScope scope(isolate);

  if(args[0]->IsString()){
    String::Utf8Value js_str(args[0]);
    const char* str = *js_str;

    Local<Number> js_b10 = Number::New(isolate, strToB10(str));
    args.GetReturnValue().Set(js_b10);
  }
  else{

    Local<Object>  array  = args[0]->ToObject();
    Handle<Object> buffer = array->Get(
        String::NewFromUtf8(isolate, "buffer"))->ToObject();
    unsigned int offset = array->Get(
        String::NewFromUtf8(isolate, "byteOffset"))->Uint32Value();
    int length = array->Get(
        String::NewFromUtf8(isolate, "byteLength"))->Uint32Value();

    //Local<Number> js_b10 = Number::New(isolate, arrToB10(*array));
    //args.GetReturnValue().Set(js_b10);

/*
    int var = 25;
    int* var_pointer = &var;
    cout << "var_pointer: " << var_pointer << "\n";

    int var = 25;
    int* var_pointer = &var;
    arrToB10(var_pointer);
*/

    int t;
    for(int i = 0; i < length; i++){
      t = array->Get(i)->Uint32Value();
      cout << "t: " << t << endl;
    }

/*
    Local<Object> array  = args[0]->ToObject();
    Local<Number> js_b10 = Number::New(isolate, arrToB10(*array));
    args.GetReturnValue().Set(js_b10);
*/
    //arrToB10(array);

    //int p = &array;

/*
    Local<Object> array  = args[0]->ToObject();
    Local<Number> js_b10 = Number::New(isolate, arrToB10(*array));
    args.GetReturnValue().Set(js_b10);
*/

    /*int t;
    for(int i = 0; i < length; i++){
      t = array->Get(i)->Uint32Value();
      cout << "t: " << t << "\n";
    }

    cout << "offset: " << offset << "\n";
    cout << "length: " << length << "\n";*/
  }

}

void Init(Handle<Object> exports) {
  Isolate* isolate = Isolate::GetCurrent();

  exports->Set(String::NewFromUtf8(isolate, "b10"),
      FunctionTemplate::New(isolate, B10)->GetFunction());
}

NODE_MODULE(b2, Init)
<|endoftext|>


<commit_before>/**
 * OpenAL cross platform audio library
 * Copyright (C) 2011 by authors.
 * This library is free software; you can redistribute it and/or
 *  modify it under the terms of the GNU Library General Public
 *  License as published by the Free Software Foundation; either
 *  version 2 of the License, or (at your option) any later version.
 *
 * This library is distributed in the hope that it will be useful,
 *  but WITHOUT ANY WARRANTY; without even the implied warranty of
 *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
 *  Library General Public License for more details.
 *
 * You should have received a copy of the GNU Library General Public
 *  License along with this library; if not, write to the
 *  Free Software Foundation, Inc.,
 *  51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
 * Or go to http://www.gnu.org/copyleft/lgpl.html
 */

#ifdef _WIN32
#ifdef __MINGW32__
#define _WIN32_IE 0x501
#else
#define _WIN32_IE 0x400
#endif
#endif

#include "config.h"

#include <algorithm>
#include <cerrno>
#include <cstdarg>
#include <cstdlib>
#include <cstdio>
#include <cstring>
#include <mutex>
#include <string>

#ifdef HAVE_DIRENT_H
#include <dirent.h>
#endif
#ifdef HAVE_SYS_SYSCONF_H
#include <sys/sysconf.h>
#endif

#ifdef HAVE_PROC_PIDPATH
#include <libproc.h>
#endif

#ifdef __FreeBSD__
#include <sys/types.h>
#include <sys/sysctl.h>
#endif

#ifndef _WIN32
#include <unistd.h>
#elif defined(_WIN32_IE)
#include <shlobj.h>
#endif

#include "alcmain.h"
#include "almalloc.h"
#include "alfstream.h"
#include "alspan.h"
#include "alstring.h"
#include "compat.h"
#include "logging.h"
#include "strutils.h"
#include "vector.h"

#ifdef _WIN32

const PathNamePair &GetProcBinary()
{
    static PathNamePair ret;
    if(!ret.fname.empty() || !ret.path.empty())
        return ret;

    al::vector<WCHAR> fullpath(256);
    DWORD len;
    while((len=GetModuleFileNameW(nullptr, fullpath.data(), static_cast<DWORD>(fullpath.size()))) == fullpath.size())
        fullpath.resize(fullpath.size() << 1);
    if(len == 0)
    {
        ERR("Failed to get process name: error %lu\n", GetLastError());
        return ret;
    }

    fullpath.resize(len);
    if(fullpath.back() != 0)
        fullpath.push_back(0);

    auto sep = std::find(fullpath.rbegin()+1, fullpath.rend(), '\\');
    sep = std::find(fullpath.rbegin()+1, sep, '/');
    if(sep != fullpath.rend())
    {
        *sep = 0;
        ret.fname = wstr_to_utf8(&*sep + 1);
        ret.path = wstr_to_utf8(fullpath.data());
    }
    else
        ret.fname = wstr_to_utf8(fullpath.data());

    TRACE("Got binary: %s, %s\n", ret.path.c_str(), ret.fname.c_str());
    return ret;
}

void al_print(FILE *logfile, const char *fmt, ...)
{
    al::vector<char> dynmsg;
    char stcmsg[256];
    char *str{stcmsg};

    va_list args, args2;
    va_start(args, fmt);
    va_copy(args2, args);
    int msglen{std::vsnprintf(str, sizeof(stcmsg), fmt, args)};
    if UNLIKELY(msglen >= 0 && static_cast<size_t>(msglen) >= sizeof(stcmsg))
    {
        dynmsg.resize(static_cast<size_t>(msglen) + 1u);
        str = dynmsg.data();
        msglen = std::vsnprintf(str, dynmsg.size(), fmt, args2);
    }
    va_end(args2);
    va_end(args);

    std::wstring wstr{utf8_to_wstr(str)};
    fputws(wstr.c_str(), logfile);
    fflush(logfile);
}

static inline int is_slash(int c)
{ return (c == '\\' || c == '/'); }

static void DirectorySearch(const char *path, const char *ext, al::vector<std::string> *const results)
{
    std::string pathstr{path};
    pathstr += "\\*";
    pathstr += ext;
    TRACE("Searching %s\n", pathstr.c_str());

    std::wstring wpath{utf8_to_wstr(pathstr.c_str())};
    WIN32_FIND_DATAW fdata;
    HANDLE hdl{FindFirstFileW(wpath.c_str(), &fdata)};
    if(hdl == INVALID_HANDLE_VALUE) return;

    const auto base = results->size();

    do {
        results->emplace_back();
        std::string &str = results->back();
        str = path;
        str += '\\';
        str += wstr_to_utf8(fdata.cFileName);
    } while(FindNextFileW(hdl, &fdata));
    FindClose(hdl);

    const al::span<std::string> newlist{results->data()+base, results->size()-base};
    std::sort(newlist.begin(), newlist.end());
    for(const auto &name : newlist)
        TRACE(" got %s\n", name.c_str());
}

al::vector<std::string> SearchDataFiles(const char *ext, const char *subdir)
{
    static std::mutex search_lock;
    std::lock_guard<std::mutex> _{search_lock};

    /* If the path is absolute, use it directly. */
    al::vector<std::string> results;
    if(isalpha(subdir[0]) && subdir[1] == ':' && is_slash(subdir[2]))
    {
        std::string path{subdir};
        std::replace(path.begin(), path.end(), '/', '\\');
        DirectorySearch(path.c_str(), ext, &results);
        return results;
    }
    if(subdir[0] == '\\' && subdir[1] == '\\' && subdir[2] == '?' && subdir[3] == '\\')
    {
        DirectorySearch(subdir, ext, &results);
        return results;
    }

    std::string path;

    /* Search the app-local directory. */
    if(auto localpath = al::getenv(L"ALSOFT_LOCAL_PATH"))
    {
        path = wstr_to_utf8(localpath->c_str());
        if(is_slash(path.back()))
            path.pop_back();
    }
    else if(WCHAR *cwdbuf{_wgetcwd(nullptr, 0)})
    {
        path = wstr_to_utf8(cwdbuf);
        if(is_slash(path.back()))
            path.pop_back();
        free(cwdbuf);
    }
    else
        path = ".";
    std::replace(path.begin(), path.end(), '/', '\\');
    DirectorySearch(path.c_str(), ext, &results);

    /* Search the local and global data dirs. */
    static const int ids[2]{ CSIDL_APPDATA, CSIDL_COMMON_APPDATA };
    for(int id : ids)
    {
        WCHAR buffer[MAX_PATH];
        if(SHGetSpecialFolderPathW(nullptr, buffer, id, FALSE) == FALSE)
            continue;

        path = wstr_to_utf8(buffer);
        if(!is_slash(path.back()))
            path += '\\';
        path += subdir;
        std::replace(path.begin(), path.end(), '/', '\\');

        DirectorySearch(path.c_str(), ext, &results);
    }

    return results;
}

void SetRTPriority(void)
{
    bool failed = false;
    if(RTPrioLevel > 0)
        failed = !SetThreadPriority(GetCurrentThread(), THREAD_PRIORITY_TIME_CRITICAL);
    if(failed) ERR("Failed to set priority level for thread\n");
}

#else

#if defined(HAVE_PTHREAD_SETSCHEDPARAM) && !defined(__OpenBSD__)
#include <pthread.h>
#include <sched.h>
#endif

const PathNamePair &GetProcBinary()
{
    static PathNamePair ret;
    if(!ret.fname.empty() || !ret.path.empty())
        return ret;

    al::vector<char> pathname;
#ifdef __FreeBSD__
    size_t pathlen;
    int mib[4] = { CTL_KERN, KERN_PROC, KERN_PROC_PATHNAME, -1 };
    if(sysctl(mib, 4, nullptr, &pathlen, nullptr, 0) == -1)
        WARN("Failed to sysctl kern.proc.pathname: %s\n", strerror(errno));
    else
    {
        pathname.resize(pathlen + 1);
        sysctl(mib, 4, pathname.data(), &pathlen, nullptr, 0);
        pathname.resize(pathlen);
    }
#endif
#ifdef HAVE_PROC_PIDPATH
    if(pathname.empty())
    {
        char procpath[PROC_PIDPATHINFO_MAXSIZE]{};
        const pid_t pid{getpid()};
        if(proc_pidpath(pid, procpath, sizeof(procpath)) < 1)
            ERR("proc_pidpath(%d, ...) failed: %s\n", pid, strerror(errno));
        else
            pathname.insert(pathname.end(), procpath, procpath+strlen(procpath));
    }
#endif
    if(pathname.empty())
    {
        pathname.resize(256);

        const char *selfname{"/proc/self/exe"};
        ssize_t len{readlink(selfname, pathname.data(), pathname.size())};
        if(len == -1 && errno == ENOENT)
        {
            selfname = "/proc/self/file";
            len = readlink(selfname, pathname.data(), pathname.size());
        }
        if(len == -1 && errno == ENOENT)
        {
            selfname = "/proc/curproc/exe";
            len = readlink(selfname, pathname.data(), pathname.size());
        }
        if(len == -1 && errno == ENOENT)
        {
            selfname = "/proc/curproc/file";
            len = readlink(selfname, pathname.data(), pathname.size());
        }

        while(len > 0 && static_cast<size_t>(len) == pathname.size())
        {
            pathname.resize(pathname.size() << 1);
            len = readlink(selfname, pathname.data(), pathname.size());
        }
        if(len <= 0)
        {
            WARN("Failed to readlink %s: %s\n", selfname, strerror(errno));
            return ret;
        }

        pathname.resize(static_cast<size_t>(len));
    }
    while(!pathname.empty() && pathname.back() == 0)
        pathname.pop_back();

    auto sep = std::find(pathname.crbegin(), pathname.crend(), '/');
    if(sep != pathname.crend())
    {
        ret.path = std::string(pathname.cbegin(), sep.base()-1);
        ret.fname = std::string(sep.base(), pathname.cend());
    }
    else
        ret.fname = std::string(pathname.cbegin(), pathname.cend());

    TRACE("Got binary: %s, %s\n", ret.path.c_str(), ret.fname.c_str());
    return ret;
}

void al_print(FILE *logfile, const char *fmt, ...)
{
    va_list ap;

    va_start(ap, fmt);
    vfprintf(logfile, fmt, ap);
    va_end(ap);

    fflush(logfile);
}

static void DirectorySearch(const char *path, const char *ext, al::vector<std::string> *const results)
{
    TRACE("Searching %s for *%s\n", path, ext);
    DIR *dir{opendir(path)};
    if(!dir) return;

    const auto base = results->size();
    const size_t extlen{strlen(ext)};

    struct dirent *dirent;
    while((dirent=readdir(dir)) != nullptr)
    {
        if(strcmp(dirent->d_name, ".") == 0 || strcmp(dirent->d_name, "..") == 0)
            continue;

        const size_t len{strlen(dirent->d_name)};
        if(len <= extlen) continue;
        if(al::strcasecmp(dirent->d_name+len-extlen, ext) != 0)
            continue;

        results->emplace_back();
        std::string &str = results->back();
        str = path;
        if(str.back() != '/')
            str.push_back('/');
        str += dirent->d_name;
    }
    closedir(dir);

    const al::span<std::string> newlist{results->data()+base, results->size()-base};
    std::sort(newlist.begin(), newlist.end());
    for(const auto &name : newlist)
        TRACE(" got %s\n", name.c_str());
}

al::vector<std::string> SearchDataFiles(const char *ext, const char *subdir)
{
    static std::mutex search_lock;
    std::lock_guard<std::mutex> _{search_lock};

    al::vector<std::string> results;
    if(subdir[0] == '/')
    {
        DirectorySearch(subdir, ext, &results);
        return results;
    }

    /* Search the app-local directory. */
    if(auto localpath = al::getenv("ALSOFT_LOCAL_PATH"))
        DirectorySearch(localpath->c_str(), ext, &results);
    else
    {
        al::vector<char> cwdbuf(256);
        while(!getcwd(cwdbuf.data(), cwdbuf.size()))
        {
            if(errno != ERANGE)
            {
                cwdbuf.clear();
                break;
            }
            cwdbuf.resize(cwdbuf.size() << 1);
        }
        if(cwdbuf.empty())
            DirectorySearch(".", ext, &results);
        else
        {
            DirectorySearch(cwdbuf.data(), ext, &results);
            cwdbuf.clear();
        }
    }

    // Search local data dir
    if(auto datapath = al::getenv("XDG_DATA_HOME"))
    {
        std::string &path = *datapath;
        if(path.back() != '/')
            path += '/';
        path += subdir;
        DirectorySearch(path.c_str(), ext, &results);
    }
    else if(auto homepath = al::getenv("HOME"))
    {
        std::string &path = *homepath;
        if(path.back() == '/')
            path.pop_back();
        path += "/.local/share/";
        path += subdir;
        DirectorySearch(path.c_str(), ext, &results);
    }

    // Search global data dirs
    std::string datadirs{al::getenv("XDG_DATA_DIRS").value_or("/usr/local/share/:/usr/share/")};

    size_t curpos{0u};
    while(curpos < datadirs.size())
    {
        size_t nextpos{datadirs.find(':', curpos)};

        std::string path{(nextpos != std::string::npos) ?
            datadirs.substr(curpos, nextpos++ - curpos) : datadirs.substr(curpos)};
        curpos = nextpos;

        if(path.empty()) continue;
        if(path.back() != '/')
            path += '/';
        path += subdir;

        DirectorySearch(path.c_str(), ext, &results);
    }

    return results;
}

void SetRTPriority()
{
    bool failed = false;
#if defined(HAVE_PTHREAD_SETSCHEDPARAM) && !defined(__OpenBSD__)
    if(RTPrioLevel > 0)
    {
        struct sched_param param;
        /* Use the minimum real-time priority possible for now (on Linux this
         * should be 1 for SCHED_RR) */
        param.sched_priority = sched_get_priority_min(SCHED_RR);
        failed = !!pthread_setschedparam(pthread_self(), SCHED_RR, &param);
    }
#else
    /* Real-time priority not available */
    failed = (RTPrioLevel>0);
#endif
    if(failed)
        ERR("Failed to set priority level for thread\n");
}

#endif
<commit_msg>Cleanup some helper code<commit_after>/**
 * OpenAL cross platform audio library
 * Copyright (C) 2011 by authors.
 * This library is free software; you can redistribute it and/or
 *  modify it under the terms of the GNU Library General Public
 *  License as published by the Free Software Foundation; either
 *  version 2 of the License, or (at your option) any later version.
 *
 * This library is distributed in the hope that it will be useful,
 *  but WITHOUT ANY WARRANTY; without even the implied warranty of
 *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
 *  Library General Public License for more details.
 *
 * You should have received a copy of the GNU Library General Public
 *  License along with this library; if not, write to the
 *  Free Software Foundation, Inc.,
 *  51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
 * Or go to http://www.gnu.org/copyleft/lgpl.html
 */

#ifdef _WIN32
#ifdef __MINGW32__
#define _WIN32_IE 0x501
#else
#define _WIN32_IE 0x400
#endif
#endif

#include "config.h"

#include <algorithm>
#include <cerrno>
#include <cstdarg>
#include <cstdlib>
#include <cstdio>
#include <cstring>
#include <mutex>
#include <string>

#ifdef HAVE_DIRENT_H
#include <dirent.h>
#endif
#ifdef HAVE_SYS_SYSCONF_H
#include <sys/sysconf.h>
#endif

#ifdef HAVE_PROC_PIDPATH
#include <libproc.h>
#endif

#ifdef __FreeBSD__
#include <sys/types.h>
#include <sys/sysctl.h>
#endif

#ifndef _WIN32
#include <unistd.h>
#elif defined(_WIN32_IE)
#include <shlobj.h>
#endif

#include "alcmain.h"
#include "almalloc.h"
#include "alfstream.h"
#include "alspan.h"
#include "alstring.h"
#include "compat.h"
#include "logging.h"
#include "strutils.h"
#include "vector.h"

#ifdef _WIN32

const PathNamePair &GetProcBinary()
{
    static PathNamePair ret;
    if(!ret.fname.empty() || !ret.path.empty())
        return ret;

    auto fullpath = al::vector<WCHAR>(256);
    DWORD len;
    while((len=GetModuleFileNameW(nullptr, fullpath.data(), static_cast<DWORD>(fullpath.size()))) == fullpath.size())
        fullpath.resize(fullpath.size() << 1);
    if(len == 0)
    {
        ERR("Failed to get process name: error %lu\n", GetLastError());
        return ret;
    }

    fullpath.resize(len);
    if(fullpath.back() != 0)
        fullpath.push_back(0);

    auto sep = std::find(fullpath.rbegin()+1, fullpath.rend(), '\\');
    sep = std::find(fullpath.rbegin()+1, sep, '/');
    if(sep != fullpath.rend())
    {
        *sep = 0;
        ret.fname = wstr_to_utf8(&*sep + 1);
        ret.path = wstr_to_utf8(fullpath.data());
    }
    else
        ret.fname = wstr_to_utf8(fullpath.data());

    TRACE("Got binary: %s, %s\n", ret.path.c_str(), ret.fname.c_str());
    return ret;
}

void al_print(FILE *logfile, const char *fmt, ...)
{
    al::vector<char> dynmsg;
    char stcmsg[256];
    char *str{stcmsg};

    va_list args, args2;
    va_start(args, fmt);
    va_copy(args2, args);
    int msglen{std::vsnprintf(str, sizeof(stcmsg), fmt, args)};
    if UNLIKELY(msglen >= 0 && static_cast<size_t>(msglen) >= sizeof(stcmsg))
    {
        dynmsg.resize(static_cast<size_t>(msglen) + 1u);
        str = dynmsg.data();
        msglen = std::vsnprintf(str, dynmsg.size(), fmt, args2);
    }
    va_end(args2);
    va_end(args);

    std::wstring wstr{utf8_to_wstr(str)};
    fputws(wstr.c_str(), logfile);
    fflush(logfile);
}

namespace {

void DirectorySearch(const char *path, const char *ext, al::vector<std::string> *const results)
{
    std::string pathstr{path};
    pathstr += "\\*";
    pathstr += ext;
    TRACE("Searching %s\n", pathstr.c_str());

    std::wstring wpath{utf8_to_wstr(pathstr.c_str())};
    WIN32_FIND_DATAW fdata;
    HANDLE hdl{FindFirstFileW(wpath.c_str(), &fdata)};
    if(hdl == INVALID_HANDLE_VALUE) return;

    const auto base = results->size();

    do {
        results->emplace_back();
        std::string &str = results->back();
        str = path;
        str += '\\';
        str += wstr_to_utf8(fdata.cFileName);
    } while(FindNextFileW(hdl, &fdata));
    FindClose(hdl);

    const al::span<std::string> newlist{results->data()+base, results->size()-base};
    std::sort(newlist.begin(), newlist.end());
    for(const auto &name : newlist)
        TRACE(" got %s\n", name.c_str());
}

} // namespace

al::vector<std::string> SearchDataFiles(const char *ext, const char *subdir)
{
    auto is_slash = [](int c) noexcept -> int { return (c == '\\' || c == '/'); }

    static std::mutex search_lock;
    std::lock_guard<std::mutex> _{search_lock};

    /* If the path is absolute, use it directly. */
    al::vector<std::string> results;
    if(isalpha(subdir[0]) && subdir[1] == ':' && is_slash(subdir[2]))
    {
        std::string path{subdir};
        std::replace(path.begin(), path.end(), '/', '\\');
        DirectorySearch(path.c_str(), ext, &results);
        return results;
    }
    if(subdir[0] == '\\' && subdir[1] == '\\' && subdir[2] == '?' && subdir[3] == '\\')
    {
        DirectorySearch(subdir, ext, &results);
        return results;
    }

    std::string path;

    /* Search the app-local directory. */
    if(auto localpath = al::getenv(L"ALSOFT_LOCAL_PATH"))
    {
        path = wstr_to_utf8(localpath->c_str());
        if(is_slash(path.back()))
            path.pop_back();
    }
    else if(WCHAR *cwdbuf{_wgetcwd(nullptr, 0)})
    {
        path = wstr_to_utf8(cwdbuf);
        if(is_slash(path.back()))
            path.pop_back();
        free(cwdbuf);
    }
    else
        path = ".";
    std::replace(path.begin(), path.end(), '/', '\\');
    DirectorySearch(path.c_str(), ext, &results);

    /* Search the local and global data dirs. */
    static const int ids[2]{ CSIDL_APPDATA, CSIDL_COMMON_APPDATA };
    for(int id : ids)
    {
        WCHAR buffer[MAX_PATH];
        if(SHGetSpecialFolderPathW(nullptr, buffer, id, FALSE) == FALSE)
            continue;

        path = wstr_to_utf8(buffer);
        if(!is_slash(path.back()))
            path += '\\';
        path += subdir;
        std::replace(path.begin(), path.end(), '/', '\\');

        DirectorySearch(path.c_str(), ext, &results);
    }

    return results;
}

void SetRTPriority(void)
{
    if(RTPrioLevel > 0)
    {
        if(!SetThreadPriority(GetCurrentThread(), THREAD_PRIORITY_TIME_CRITICAL))
            ERR("Failed to set priority level for thread\n");
    }
}

#else

#if defined(HAVE_PTHREAD_SETSCHEDPARAM) && !defined(__OpenBSD__)
#include <pthread.h>
#include <sched.h>
#endif

const PathNamePair &GetProcBinary()
{
    static PathNamePair ret;
    if(!ret.fname.empty() || !ret.path.empty())
        return ret;

    al::vector<char> pathname;
#ifdef __FreeBSD__
    size_t pathlen;
    int mib[4] = { CTL_KERN, KERN_PROC, KERN_PROC_PATHNAME, -1 };
    if(sysctl(mib, 4, nullptr, &pathlen, nullptr, 0) == -1)
        WARN("Failed to sysctl kern.proc.pathname: %s\n", strerror(errno));
    else
    {
        pathname.resize(pathlen + 1);
        sysctl(mib, 4, pathname.data(), &pathlen, nullptr, 0);
        pathname.resize(pathlen);
    }
#endif
#ifdef HAVE_PROC_PIDPATH
    if(pathname.empty())
    {
        char procpath[PROC_PIDPATHINFO_MAXSIZE]{};
        const pid_t pid{getpid()};
        if(proc_pidpath(pid, procpath, sizeof(procpath)) < 1)
            ERR("proc_pidpath(%d, ...) failed: %s\n", pid, strerror(errno));
        else
            pathname.insert(pathname.end(), procpath, procpath+strlen(procpath));
    }
#endif
    if(pathname.empty())
    {
        static const char SelfLinkNames[][32]{
            "/proc/self/exe",
            "/proc/self/file",
            "/proc/curproc/exe",
            "/proc/curproc/file"
        };

        pathname.resize(256);

        const char *selfname{};
        ssize_t len{};
        for(const char *name : SelfLinkNames)
        {
            selfname = name;
            len = readlink(selfname, pathname.data(), pathname.size());
            if(len >= 0 || errno != ENOENT) break;
        }

        while(len > 0 && static_cast<size_t>(len) == pathname.size())
        {
            pathname.resize(pathname.size() << 1);
            len = readlink(selfname, pathname.data(), pathname.size());
        }
        if(len <= 0)
        {
            WARN("Failed to readlink %s: %s\n", selfname, strerror(errno));
            return ret;
        }

        pathname.resize(static_cast<size_t>(len));
    }
    while(!pathname.empty() && pathname.back() == 0)
        pathname.pop_back();

    auto sep = std::find(pathname.crbegin(), pathname.crend(), '/');
    if(sep != pathname.crend())
    {
        ret.path = std::string(pathname.cbegin(), sep.base()-1);
        ret.fname = std::string(sep.base(), pathname.cend());
    }
    else
        ret.fname = std::string(pathname.cbegin(), pathname.cend());

    TRACE("Got binary: %s, %s\n", ret.path.c_str(), ret.fname.c_str());
    return ret;
}

void al_print(FILE *logfile, const char *fmt, ...)
{
    va_list ap;

    va_start(ap, fmt);
    vfprintf(logfile, fmt, ap);
    va_end(ap);

    fflush(logfile);
}

namespace {

void DirectorySearch(const char *path, const char *ext, al::vector<std::string> *const results)
{
    TRACE("Searching %s for *%s\n", path, ext);
    DIR *dir{opendir(path)};
    if(!dir) return;

    const auto base = results->size();
    const size_t extlen{strlen(ext)};

    while(struct dirent *dirent{readdir(dir)})
    {
        if(strcmp(dirent->d_name, ".") == 0 || strcmp(dirent->d_name, "..") == 0)
            continue;

        const size_t len{strlen(dirent->d_name)};
        if(len <= extlen) continue;
        if(al::strcasecmp(dirent->d_name+len-extlen, ext) != 0)
            continue;

        results->emplace_back();
        std::string &str = results->back();
        str = path;
        if(str.back() != '/')
            str.push_back('/');
        str += dirent->d_name;
    }
    closedir(dir);

    const al::span<std::string> newlist{results->data()+base, results->size()-base};
    std::sort(newlist.begin(), newlist.end());
    for(const auto &name : newlist)
        TRACE(" got %s\n", name.c_str());
}

} // namespace

al::vector<std::string> SearchDataFiles(const char *ext, const char *subdir)
{
    static std::mutex search_lock;
    std::lock_guard<std::mutex> _{search_lock};

    al::vector<std::string> results;
    if(subdir[0] == '/')
    {
        DirectorySearch(subdir, ext, &results);
        return results;
    }

    /* Search the app-local directory. */
    if(auto localpath = al::getenv("ALSOFT_LOCAL_PATH"))
        DirectorySearch(localpath->c_str(), ext, &results);
    else
    {
        al::vector<char> cwdbuf(256);
        while(!getcwd(cwdbuf.data(), cwdbuf.size()))
        {
            if(errno != ERANGE)
            {
                cwdbuf.clear();
                break;
            }
            cwdbuf.resize(cwdbuf.size() << 1);
        }
        if(cwdbuf.empty())
            DirectorySearch(".", ext, &results);
        else
        {
            DirectorySearch(cwdbuf.data(), ext, &results);
            cwdbuf.clear();
        }
    }

    // Search local data dir
    if(auto datapath = al::getenv("XDG_DATA_HOME"))
    {
        std::string &path = *datapath;
        if(path.back() != '/')
            path += '/';
        path += subdir;
        DirectorySearch(path.c_str(), ext, &results);
    }
    else if(auto homepath = al::getenv("HOME"))
    {
        std::string &path = *homepath;
        if(path.back() == '/')
            path.pop_back();
        path += "/.local/share/";
        path += subdir;
        DirectorySearch(path.c_str(), ext, &results);
    }

    // Search global data dirs
    std::string datadirs{al::getenv("XDG_DATA_DIRS").value_or("/usr/local/share/:/usr/share/")};

    size_t curpos{0u};
    while(curpos < datadirs.size())
    {
        size_t nextpos{datadirs.find(':', curpos)};

        std::string path{(nextpos != std::string::npos) ?
            datadirs.substr(curpos, nextpos++ - curpos) : datadirs.substr(curpos)};
        curpos = nextpos;

        if(path.empty()) continue;
        if(path.back() != '/')
            path += '/';
        path += subdir;

        DirectorySearch(path.c_str(), ext, &results);
    }

    return results;
}

void SetRTPriority()
{
#if defined(HAVE_PTHREAD_SETSCHEDPARAM) && !defined(__OpenBSD__)
    if(RTPrioLevel > 0)
    {
        struct sched_param param{};
        /* Use the minimum real-time priority possible for now (on Linux this
         * should be 1 for SCHED_RR).
         */
        param.sched_priority = sched_get_priority_min(SCHED_RR);
        if(pthread_setschedparam(pthread_self(), SCHED_RR, &param))
            ERR("Failed to set priority level for thread\n");
    }
#else
    /* Real-time priority not available */
    if(RTPrioLevel > 0)
        ERR("Cannot set priority level for thread\n");
#endif
}

#endif
<|endoftext|>


<commit_before>/*
 * DISTRHO Plugin Framework (DPF)
 * Copyright (C) 2012-2015 Filipe Coelho <falktx@falktx.com>
 *
 * Permission to use, copy, modify, and/or distribute this software for any purpose with
 * or without fee is hereby granted, provided that the above copyright notice and this
 * permission notice appear in all copies.
 *
 * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES WITH REGARD
 * TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS. IN
 * NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL
 * DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER
 * IN AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN
 * CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
 */

#ifndef DISTRHO_LEAK_DETECTOR_HPP_INCLUDED
#define DISTRHO_LEAK_DETECTOR_HPP_INCLUDED

#include "../DistrhoUtils.hpp"

START_NAMESPACE_DISTRHO

// -----------------------------------------------------------------------
// The following code was based from juce-core LeakDetector class
// Copyright (C) 2013 Raw Material Software Ltd.

/** A good old-fashioned C macro concatenation helper.
   This combines two items (which may themselves be macros) into a single string,
   avoiding the pitfalls of the ## macro operator.
*/
#define DISTRHO_JOIN_MACRO_HELPER(a, b) a ## b
#define DISTRHO_JOIN_MACRO(item1, item2) DISTRHO_JOIN_MACRO_HELPER(item1, item2)

/** This macro lets you embed a leak-detecting object inside a class.\n
    To use it, simply declare a DISTRHO_LEAK_DETECTOR(YourClassName) inside a private section
    of the class declaration. E.g.
    \code
    class MyClass
    {
    public:
        MyClass();
        void blahBlah();

    private:
        DISTRHO_LEAK_DETECTOR(MyClass)
    };
    \endcode
*/
#define DISTRHO_LEAK_DETECTOR(ClassName)                                          \
    friend class DISTRHO_NAMESPACE::LeakedObjectDetector<ClassName>;              \
    static const char* getLeakedObjectClassName() noexcept { return #ClassName; } \
    DISTRHO_NAMESPACE::LeakedObjectDetector<ClassName> DISTRHO_JOIN_MACRO(leakDetector_, ClassName);

#define DISTRHO_DECLARE_NON_COPYABLE_WITH_LEAK_DETECTOR(ClassName) \
    DISTRHO_DECLARE_NON_COPY_CLASS(ClassName)                      \
    DISTRHO_LEAK_DETECTOR(ClassName)

//==============================================================================
/**
    Embedding an instance of this class inside another class can be used as a low-overhead
    way of detecting leaked instances.

    This class keeps an internal static count of the number of instances that are
    active, so that when the app is shutdown and the static destructors are called,
    it can check whether there are any left-over instances that may have been leaked.

    To use it, use the DISTRHO_LEAK_DETECTOR macro as a simple way to put one in your
    class declaration.
*/
template<class OwnerClass>
class LeakedObjectDetector
{
public:
    //==============================================================================
    LeakedObjectDetector() noexcept                            { ++(getCounter().numObjects); }
    LeakedObjectDetector(const LeakedObjectDetector&) noexcept { ++(getCounter().numObjects); }

    ~LeakedObjectDetector() noexcept
    {
        if (--(getCounter().numObjects) < 0)
        {
            /** If you hit this, then you've managed to delete more instances of this class than you've
                created.. That indicates that you're deleting some dangling pointers.

                Note that although this assertion will have been triggered during a destructor, it might
                not be this particular deletion that's at fault - the incorrect one may have happened
                at an earlier point in the program, and simply not been detected until now.

                Most errors like this are caused by using old-fashioned, non-RAII techniques for
                your object management. Tut, tut. Always, always use ScopedPointers, OwnedArrays,
                ReferenceCountedObjects, etc, and avoid the 'delete' operator at all costs!
            */
            d_stderr2("*** Dangling pointer deletion! Class: '%s', Count: %i", getLeakedObjectClassName(), getCounter().numObjects);
        }
    }

private:
    //==============================================================================
    class LeakCounter
    {
    public:
        LeakCounter() noexcept
            : numObjects(0) {}

        ~LeakCounter() noexcept
        {
            if (numObjects > 0)
            {
                /** If you hit this, then you've leaked one or more objects of the type specified by
                    the 'OwnerClass' template parameter - the name should have been printed by the line above.

                    If you're leaking, it's probably because you're using old-fashioned, non-RAII techniques for
                    your object management. Tut, tut. Always, always use ScopedPointers, OwnedArrays,
                    ReferenceCountedObjects, etc, and avoid the 'delete' operator at all costs!
                */
                d_stderr2("*** Leaked objects detected: %i instance(s) of class '%s'", numObjects, getLeakedObjectClassName());
            }
        }

        // this should be an atomic...
        volatile int numObjects;
    };

    static const char* getLeakedObjectClassName() noexcept
    {
        return OwnerClass::getLeakedObjectClassName();
    }

    static LeakCounter& getCounter() noexcept
    {
        static LeakCounter counter;
        return counter;
    }
};

// -----------------------------------------------------------------------

END_NAMESPACE_DISTRHO

#endif // DISTRHO_LEAK_DETECTOR_HPP_INCLUDED
<commit_msg>Don't use leak detection on release builds<commit_after>/*
 * DISTRHO Plugin Framework (DPF)
 * Copyright (C) 2012-2015 Filipe Coelho <falktx@falktx.com>
 *
 * Permission to use, copy, modify, and/or distribute this software for any purpose with
 * or without fee is hereby granted, provided that the above copyright notice and this
 * permission notice appear in all copies.
 *
 * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES WITH REGARD
 * TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS. IN
 * NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL
 * DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER
 * IN AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN
 * CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
 */

#ifndef DISTRHO_LEAK_DETECTOR_HPP_INCLUDED
#define DISTRHO_LEAK_DETECTOR_HPP_INCLUDED

#include "../DistrhoUtils.hpp"

START_NAMESPACE_DISTRHO

// -----------------------------------------------------------------------
// The following code was based from juce-core LeakDetector class
// Copyright (C) 2013 Raw Material Software Ltd.

/** A good old-fashioned C macro concatenation helper.
   This combines two items (which may themselves be macros) into a single string,
   avoiding the pitfalls of the ## macro operator.
*/
#define DISTRHO_JOIN_MACRO_HELPER(a, b) a ## b
#define DISTRHO_JOIN_MACRO(item1, item2) DISTRHO_JOIN_MACRO_HELPER(item1, item2)

#ifdef DEBUG
/** This macro lets you embed a leak-detecting object inside a class.\n
    To use it, simply declare a DISTRHO_LEAK_DETECTOR(YourClassName) inside a private section
    of the class declaration. E.g.
    \code
    class MyClass
    {
    public:
        MyClass();
        void blahBlah();

    private:
        DISTRHO_LEAK_DETECTOR(MyClass)
    };
    \endcode
*/
# define DISTRHO_LEAK_DETECTOR(ClassName)                                         \
    friend class DISTRHO_NAMESPACE::LeakedObjectDetector<ClassName>;              \
    static const char* getLeakedObjectClassName() noexcept { return #ClassName; } \
    DISTRHO_NAMESPACE::LeakedObjectDetector<ClassName> DISTRHO_JOIN_MACRO(leakDetector_, ClassName);

# define DISTRHO_DECLARE_NON_COPYABLE_WITH_LEAK_DETECTOR(ClassName) \
    DISTRHO_DECLARE_NON_COPY_CLASS(ClassName)                       \
    DISTRHO_LEAK_DETECTOR(ClassName)
#else
/** Don't use leak detection on release builds. */
# define DISTRHO_LEAK_DETECTOR(ClassName)
# define DISTRHO_DECLARE_NON_COPYABLE_WITH_LEAK_DETECTOR(ClassName) \
    DISTRHO_DECLARE_NON_COPY_CLASS(ClassName)
#endif

//==============================================================================
/**
    Embedding an instance of this class inside another class can be used as a low-overhead
    way of detecting leaked instances.

    This class keeps an internal static count of the number of instances that are
    active, so that when the app is shutdown and the static destructors are called,
    it can check whether there are any left-over instances that may have been leaked.

    To use it, use the DISTRHO_LEAK_DETECTOR macro as a simple way to put one in your
    class declaration.
*/
template<class OwnerClass>
class LeakedObjectDetector
{
public:
    //==============================================================================
    LeakedObjectDetector() noexcept                            { ++(getCounter().numObjects); }
    LeakedObjectDetector(const LeakedObjectDetector&) noexcept { ++(getCounter().numObjects); }

    ~LeakedObjectDetector() noexcept
    {
        if (--(getCounter().numObjects) < 0)
        {
            /** If you hit this, then you've managed to delete more instances of this class than you've
                created.. That indicates that you're deleting some dangling pointers.

                Note that although this assertion will have been triggered during a destructor, it might
                not be this particular deletion that's at fault - the incorrect one may have happened
                at an earlier point in the program, and simply not been detected until now.

                Most errors like this are caused by using old-fashioned, non-RAII techniques for
                your object management. Tut, tut. Always, always use ScopedPointers, OwnedArrays,
                ReferenceCountedObjects, etc, and avoid the 'delete' operator at all costs!
            */
            d_stderr2("*** Dangling pointer deletion! Class: '%s', Count: %i", getLeakedObjectClassName(), getCounter().numObjects);
        }
    }

private:
    //==============================================================================
    class LeakCounter
    {
    public:
        LeakCounter() noexcept
            : numObjects(0) {}

        ~LeakCounter() noexcept
        {
            if (numObjects > 0)
            {
                /** If you hit this, then you've leaked one or more objects of the type specified by
                    the 'OwnerClass' template parameter - the name should have been printed by the line above.

                    If you're leaking, it's probably because you're using old-fashioned, non-RAII techniques for
                    your object management. Tut, tut. Always, always use ScopedPointers, OwnedArrays,
                    ReferenceCountedObjects, etc, and avoid the 'delete' operator at all costs!
                */
                d_stderr2("*** Leaked objects detected: %i instance(s) of class '%s'", numObjects, getLeakedObjectClassName());
            }
        }

        // this should be an atomic...
        volatile int numObjects;
    };

    static const char* getLeakedObjectClassName() noexcept
    {
        return OwnerClass::getLeakedObjectClassName();
    }

    static LeakCounter& getCounter() noexcept
    {
        static LeakCounter counter;
        return counter;
    }
};

// -----------------------------------------------------------------------

END_NAMESPACE_DISTRHO

#endif // DISTRHO_LEAK_DETECTOR_HPP_INCLUDED
<|endoftext|>


<commit_before>// Copyright 2014 Google Inc. All rights reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

#include "index.h"

#include <memory>
#include <string>

#include <netinet/if_ether.h>  // ethhdr
#include <netinet/in.h>        // ntohs(), ntohl()
#include <netinet/tcp.h>       // tcphdr
#include <netinet/udp.h>       // udphdr
#include <netinet/ip.h>        // iphdr
#include <netinet/ip6.h>       // ip6_hdr

#include <leveldb/env.h>
#include <leveldb/slice.h>
#include <leveldb/status.h>
#include <leveldb/table_builder.h>

namespace leveldb {

// Augment leveldb::Slice just slightly, so we can use it inside ordered
// data structures like our in-memory trees.
bool operator<(const leveldb::Slice& a, const leveldb::Slice& b) {
  return a.compare(b) < 0;
}

}  // namespace leveldb

namespace st {

void Index::Process(const Packet& p, int64_t block_offset) {
  packets_++;
  int64_t packet_offset = block_offset + p.offset_in_block;
  const char* start = p.data.data();
  const char* limit = start + p.data.size();
  if (start + sizeof(struct ethhdr) > limit) {
    return;
  }
  auto eth = reinterpret_cast<const struct ethhdr*>(start);
  start += sizeof(struct ethhdr);
  auto type = ntohs(eth->h_proto);
  if (type == ETH_P_8021Q) {
    if (start + 4 > limit) {
      return;
    }
    type = ntohs(*reinterpret_cast<const uint16_t*>(start + 2));
    start += 4;
  }
  leveldb::Slice src_ip, dst_ip;
  uint8_t protocol = 0;
  switch (type) {
    case ETH_P_IP: {
      if (start + sizeof(struct iphdr) > limit) {
        return;
      }
      auto ip4 = reinterpret_cast<const struct iphdr*>(start);
      src_ip = leveldb::Slice(reinterpret_cast<const char*>(&ip4->saddr), 4);
      dst_ip = leveldb::Slice(reinterpret_cast<const char*>(&ip4->daddr), 4);
      size_t len = ip4->ihl;
      len *= 4;
      if (len < 20) return;
      protocol = ip4->protocol;
      start += len;
      break;
    }
    case ETH_P_IPV6: {
      if (start + sizeof(struct ip6_hdr) > limit) {
        return;
      }
      auto ip6 = reinterpret_cast<const struct ip6_hdr*>(start);
      protocol = ip6->ip6_ctlun.ip6_un1.ip6_un1_nxt;
      start += sizeof(struct ip6_hdr);
      src_ip = leveldb::Slice(reinterpret_cast<const char*>(&ip6->ip6_src), 16);
      dst_ip = leveldb::Slice(reinterpret_cast<const char*>(&ip6->ip6_dst), 16);
      bool ip6extensions = true;
      while (ip6extensions) {
        switch (protocol) {
          case IPPROTO_FRAGMENT: {
            if (start + sizeof(struct ip6_frag) > limit) {
              return;
            }
            auto ip6frag = reinterpret_cast<const struct ip6_frag*>(start);
            if (ntohs(ip6frag->ip6f_offlg) & 0xfff8) {
              // If we're not the first fragment, break out of the loop so we
              // can store the IPs we have but recognize in the protocol switch
              // later on that we don't know what this packet is.
              ip6extensions = false;
              break;
            }
            // otherwise, fall through to treating this like any other
            // extention.
          }
          case IPPROTO_HOPOPTS:
          case IPPROTO_ROUTING:
          case IPPROTO_DSTOPTS:
          case IPPROTO_MH: {
            if (start + sizeof(struct ip6_ext) > limit) {
              return;
            }
            auto ip6ext = reinterpret_cast<const struct ip6_ext*>(start);
            protocol = ip6ext->ip6e_nxt;
            start += (ip6ext->ip6e_len + 1) * 8;
            break;
          }
          default:
            ip6extensions = false;
        }
      }
      break;
    }
    default:
      return;
  }
  AddIP(src_ip, packet_offset);
  AddIP(dst_ip, packet_offset);
  AddProtocol(protocol, packet_offset);
  uint16_t src_port, dst_port;
  switch (protocol) {
    case IPPROTO_TCP: {
      if (start + sizeof(struct tcphdr) > limit) {
        return;
      }
      auto tcp = reinterpret_cast<const struct tcphdr*>(start);
      src_port = ntohs(tcp->source);
      dst_port = ntohs(tcp->dest);
      break;
    }
    case IPPROTO_UDP: {
      if (start + sizeof(struct udphdr) > limit) {
        return;
      }
      auto udp = reinterpret_cast<const struct udphdr*>(start);
      src_port = ntohs(udp->source);
      dst_port = ntohs(udp->dest);
      break;
    }
    default:
      return;
  }
  AddPort(src_port, packet_offset);
  AddPort(dst_port, packet_offset);
}

namespace {

void WriteToIndex(char first, const char* start, int size, int64_t pos,
                  leveldb::TableBuilder* ss) {
  char buf[17];
  CHECK(size <= 16);
  CHECK(pos < int64_t(1) << 32);
  buf[0] = first;
  memcpy(buf + 1, start, size);
  uint32_t pos32 = htonl(pos);
  ss->Add(leveldb::Slice(buf, size + 1),
          leveldb::Slice(reinterpret_cast<const char*>(&pos32), 4));
}

}  // namespace

Error Index::Flush() {
  leveldb::WritableFile* file;
  string filename = HiddenFile(dirname_, micros_);
  auto status = leveldb::Env::Default()->NewWritableFile(filename, &file);
  std::unique_ptr<leveldb::WritableFile> cleaner(file);
  if (!status.ok()) {
    return ERROR("could not open '" + filename + "': " + status.ToString());
  }

  leveldb::Options options;
  options.compression = leveldb::kNoCompression;
  leveldb::TableBuilder index_ss(options, file);
  for (auto iter : proto_) {
    for (auto pos : iter.second) {
      WriteToIndex(1, reinterpret_cast<const char*>(&iter.first), 1, pos,
                   &index_ss);
    }
  }
  for (auto iter : port_) {
    uint16_t port = htons(iter.first);
    for (auto pos : iter.second) {
      WriteToIndex(2, reinterpret_cast<const char*>(&port), 2, pos, &index_ss);
    }
  }
  for (auto iter : ip4_) {
    for (auto pos : iter.second) {
      WriteToIndex(4, iter.first.data(), 4, pos, &index_ss);
    }
  }
  for (auto iter : ip6_) {
    for (auto pos : iter.second) {
      WriteToIndex(6, iter.first.data(), 16, pos, &index_ss);
    }
  }
  auto finished = index_ss.Finish();
  if (!finished.ok()) {
    return ERROR("could not finish writing index table '" + filename + "': " +
                 finished.ToString());
  }
  auto closed = file->Close();
  if (!closed.ok()) {
    return ERROR("could not close index table '" + filename + "': " +
                 closed.ToString());
  }
  string unhidden = UnhiddenFile(dirname_, micros_);
  LOG(INFO) << "Wrote all index files for " << filename << ", moving to "
            << unhidden;
  RETURN_IF_ERROR(Errno(rename(filename.c_str(), unhidden.c_str())), "rename");
  LOG(V1) << "Stored " << packets_ << " with " << ip4_.size() << " IP4 "
          << ip6_.size() << " IP6 " << proto_.size() << " protos "
          << port_.size() << " ports";
  return SUCCESS;
}

void Index::AddIP(leveldb::Slice ip, int64_t pos) {
  CHECK(ip.size() == 4 || ip.size() == 16);
  auto tree = ip.size() == 4 ? &ip4_ : &ip6_;
  auto finder = tree->find(ip);
  if (finder == tree->end()) {
    ip = ip_pieces_.Store(ip);
    (*tree)[ip].push_back(pos);
  } else {
    finder->second.push_back(pos);
  }
}
void Index::AddProtocol(uint8_t proto, int64_t pos) {
  auto finder = proto_.find(proto);
  if (finder == proto_.end()) {
    proto_[proto].push_back(pos);
  } else {
    finder->second.push_back(pos);
  }
}
void Index::AddPort(uint16_t port, int64_t pos) {
  auto finder = port_.find(port);
  if (finder == port_.end()) {
    port_[port].push_back(pos);
  } else {
    finder->second.push_back(pos);
  }
}

}  // namespace st
<commit_msg>Only use MH if it's available (not in older Linux versions).<commit_after>// Copyright 2014 Google Inc. All rights reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

#include "index.h"

#include <memory>
#include <string>

#include <netinet/if_ether.h>  // ethhdr
#include <netinet/in.h>        // ntohs(), ntohl()
#include <netinet/tcp.h>       // tcphdr
#include <netinet/udp.h>       // udphdr
#include <netinet/ip.h>        // iphdr
#include <netinet/ip6.h>       // ip6_hdr

#include <leveldb/env.h>
#include <leveldb/slice.h>
#include <leveldb/status.h>
#include <leveldb/table_builder.h>

namespace leveldb {

// Augment leveldb::Slice just slightly, so we can use it inside ordered
// data structures like our in-memory trees.
bool operator<(const leveldb::Slice& a, const leveldb::Slice& b) {
  return a.compare(b) < 0;
}

}  // namespace leveldb

namespace st {

void Index::Process(const Packet& p, int64_t block_offset) {
  packets_++;
  int64_t packet_offset = block_offset + p.offset_in_block;
  const char* start = p.data.data();
  const char* limit = start + p.data.size();
  if (start + sizeof(struct ethhdr) > limit) {
    return;
  }
  auto eth = reinterpret_cast<const struct ethhdr*>(start);
  start += sizeof(struct ethhdr);
  auto type = ntohs(eth->h_proto);
  if (type == ETH_P_8021Q) {
    if (start + 4 > limit) {
      return;
    }
    type = ntohs(*reinterpret_cast<const uint16_t*>(start + 2));
    start += 4;
  }
  leveldb::Slice src_ip, dst_ip;
  uint8_t protocol = 0;
  switch (type) {
    case ETH_P_IP: {
      if (start + sizeof(struct iphdr) > limit) {
        return;
      }
      auto ip4 = reinterpret_cast<const struct iphdr*>(start);
      src_ip = leveldb::Slice(reinterpret_cast<const char*>(&ip4->saddr), 4);
      dst_ip = leveldb::Slice(reinterpret_cast<const char*>(&ip4->daddr), 4);
      size_t len = ip4->ihl;
      len *= 4;
      if (len < 20) return;
      protocol = ip4->protocol;
      start += len;
      break;
    }
    case ETH_P_IPV6: {
      if (start + sizeof(struct ip6_hdr) > limit) {
        return;
      }
      auto ip6 = reinterpret_cast<const struct ip6_hdr*>(start);
      protocol = ip6->ip6_ctlun.ip6_un1.ip6_un1_nxt;
      start += sizeof(struct ip6_hdr);
      src_ip = leveldb::Slice(reinterpret_cast<const char*>(&ip6->ip6_src), 16);
      dst_ip = leveldb::Slice(reinterpret_cast<const char*>(&ip6->ip6_dst), 16);
      bool ip6extensions = true;
      while (ip6extensions) {
        switch (protocol) {
          case IPPROTO_FRAGMENT: {
            if (start + sizeof(struct ip6_frag) > limit) {
              return;
            }
            auto ip6frag = reinterpret_cast<const struct ip6_frag*>(start);
            if (ntohs(ip6frag->ip6f_offlg) & 0xfff8) {
              // If we're not the first fragment, break out of the loop so we
              // can store the IPs we have but recognize in the protocol switch
              // later on that we don't know what this packet is.
              ip6extensions = false;
              break;
            }
            // otherwise, fall through to treating this like any other
            // extention.
          }
#ifdef IPPROTO_MH
          case IPPROTO_MH:
#endif
          case IPPROTO_HOPOPTS:
          case IPPROTO_ROUTING:
          case IPPROTO_DSTOPTS: {
            if (start + sizeof(struct ip6_ext) > limit) {
              return;
            }
            auto ip6ext = reinterpret_cast<const struct ip6_ext*>(start);
            protocol = ip6ext->ip6e_nxt;
            start += (ip6ext->ip6e_len + 1) * 8;
            break;
          }
          default:
            ip6extensions = false;
        }
      }
      break;
    }
    default:
      return;
  }
  AddIP(src_ip, packet_offset);
  AddIP(dst_ip, packet_offset);
  AddProtocol(protocol, packet_offset);
  uint16_t src_port, dst_port;
  switch (protocol) {
    case IPPROTO_TCP: {
      if (start + sizeof(struct tcphdr) > limit) {
        return;
      }
      auto tcp = reinterpret_cast<const struct tcphdr*>(start);
      src_port = ntohs(tcp->source);
      dst_port = ntohs(tcp->dest);
      break;
    }
    case IPPROTO_UDP: {
      if (start + sizeof(struct udphdr) > limit) {
        return;
      }
      auto udp = reinterpret_cast<const struct udphdr*>(start);
      src_port = ntohs(udp->source);
      dst_port = ntohs(udp->dest);
      break;
    }
    default:
      return;
  }
  AddPort(src_port, packet_offset);
  AddPort(dst_port, packet_offset);
}

namespace {

void WriteToIndex(char first, const char* start, int size, int64_t pos,
                  leveldb::TableBuilder* ss) {
  char buf[17];
  CHECK(size <= 16);
  CHECK(pos < int64_t(1) << 32);
  buf[0] = first;
  memcpy(buf + 1, start, size);
  uint32_t pos32 = htonl(pos);
  ss->Add(leveldb::Slice(buf, size + 1),
          leveldb::Slice(reinterpret_cast<const char*>(&pos32), 4));
}

}  // namespace

Error Index::Flush() {
  leveldb::WritableFile* file;
  string filename = HiddenFile(dirname_, micros_);
  auto status = leveldb::Env::Default()->NewWritableFile(filename, &file);
  std::unique_ptr<leveldb::WritableFile> cleaner(file);
  if (!status.ok()) {
    return ERROR("could not open '" + filename + "': " + status.ToString());
  }

  leveldb::Options options;
  options.compression = leveldb::kNoCompression;
  leveldb::TableBuilder index_ss(options, file);
  for (auto iter : proto_) {
    for (auto pos : iter.second) {
      WriteToIndex(1, reinterpret_cast<const char*>(&iter.first), 1, pos,
                   &index_ss);
    }
  }
  for (auto iter : port_) {
    uint16_t port = htons(iter.first);
    for (auto pos : iter.second) {
      WriteToIndex(2, reinterpret_cast<const char*>(&port), 2, pos, &index_ss);
    }
  }
  for (auto iter : ip4_) {
    for (auto pos : iter.second) {
      WriteToIndex(4, iter.first.data(), 4, pos, &index_ss);
    }
  }
  for (auto iter : ip6_) {
    for (auto pos : iter.second) {
      WriteToIndex(6, iter.first.data(), 16, pos, &index_ss);
    }
  }
  auto finished = index_ss.Finish();
  if (!finished.ok()) {
    return ERROR("could not finish writing index table '" + filename + "': " +
                 finished.ToString());
  }
  auto closed = file->Close();
  if (!closed.ok()) {
    return ERROR("could not close index table '" + filename + "': " +
                 closed.ToString());
  }
  string unhidden = UnhiddenFile(dirname_, micros_);
  LOG(INFO) << "Wrote all index files for " << filename << ", moving to "
            << unhidden;
  RETURN_IF_ERROR(Errno(rename(filename.c_str(), unhidden.c_str())), "rename");
  LOG(V1) << "Stored " << packets_ << " with " << ip4_.size() << " IP4 "
          << ip6_.size() << " IP6 " << proto_.size() << " protos "
          << port_.size() << " ports";
  return SUCCESS;
}

void Index::AddIP(leveldb::Slice ip, int64_t pos) {
  CHECK(ip.size() == 4 || ip.size() == 16);
  auto tree = ip.size() == 4 ? &ip4_ : &ip6_;
  auto finder = tree->find(ip);
  if (finder == tree->end()) {
    ip = ip_pieces_.Store(ip);
    (*tree)[ip].push_back(pos);
  } else {
    finder->second.push_back(pos);
  }
}
void Index::AddProtocol(uint8_t proto, int64_t pos) {
  auto finder = proto_.find(proto);
  if (finder == proto_.end()) {
    proto_[proto].push_back(pos);
  } else {
    finder->second.push_back(pos);
  }
}
void Index::AddPort(uint16_t port, int64_t pos) {
  auto finder = port_.find(port);
  if (finder == port_.end()) {
    port_[port].push_back(pos);
  } else {
    finder->second.push_back(pos);
  }
}

}  // namespace st
<|endoftext|>


<commit_before>/*
 * Copyright 2014 Open Connectome Project (http://openconnecto.me)
 * Written by Da Zheng (zhengda1936@gmail.com)
 *
 * This file is part of FlashMatrix.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

#include <unordered_map>

#include "io_interface.h"

#include "matrix_config.h"
#include "mem_worker_thread.h"
#include "EM_object.h"
#include "local_mem_buffer.h"

namespace fm
{

namespace detail
{

#ifdef USE_HWLOC
std::vector<int> get_cpus(int node_id)
{
	std::vector<int> io_cpus = safs::get_io_cpus();
	std::vector<int> logical_units = cpus.get_node(node_id).get_logical_units();
	std::set<int> cpu_set(logical_units.begin(), logical_units.end());
	// Remove the logical units where I/O threads run.
	for (size_t j = 0; j < io_cpus.size(); j++)
		cpu_set.erase(io_cpus[j]);
	return std::vector<int>(cpu_set.begin(), cpu_set.end());
}
#endif

mem_thread_pool::mem_thread_pool(int num_nodes, int nthreads_per_node)
{
	tot_num_tasks = 0;
	threads.resize(num_nodes);
	for (int i = 0; i < num_nodes; i++) {
		// Get the CPU cores that are in node i.
#ifdef USE_HWLOC
		std::vector<int> cpus = get_cpus(i);
#endif
		threads[i].resize(nthreads_per_node);
		for (int j = 0; j < nthreads_per_node; j++) {
			std::string name
				= std::string("mem-worker-") + itoa(i) + "-" + itoa(j);
#ifdef USE_HWLOC
			if (safs::get_io_cpus().empty())
				threads[i][j] = std::shared_ptr<pool_task_thread>(
						new pool_task_thread(i * nthreads_per_node + j, name, i));
			else
				threads[i][j] = std::shared_ptr<pool_task_thread>(
						new pool_task_thread(i * nthreads_per_node + j, name,
							cpus, i));
#else
			threads[i][j] = std::shared_ptr<pool_task_thread>(
					new pool_task_thread(i * nthreads_per_node + j, name, i));
#endif
			threads[i][j]->start();
		}
	}
	ntasks_per_node.resize(num_nodes);
}

/*
 * This method dispatches tasks in a round-robin fashion.
 */
void mem_thread_pool::process_task(int node_id, thread_task *task)
{
	if (node_id < 0)
		node_id = tot_num_tasks % get_num_nodes();

	assert((size_t) node_id < threads.size());
	size_t idx = ntasks_per_node[node_id] % threads[node_id].size();
	threads[node_id][idx]->add_task(task);
	ntasks_per_node[node_id]++;
	tot_num_tasks++;
}

void mem_thread_pool::wait4complete()
{
	for (size_t i = 0; i < threads.size(); i++) {
		size_t nthreads = threads[i].size();
		for (size_t j = 0; j < nthreads; j++)
			threads[i][j]->wait4complete();
	}

	// After all workers complete, we should try to clear the memory buffers
	// in each worker thread to reduce memory consumption.
	// We might want to keep the memory buffer for I/O on dense matrices.
	if (matrix_conf.is_keep_mem_buf())
		detail::local_mem_buffer::clear_bufs(
				detail::local_mem_buffer::MAT_PORTION);
	else
		detail::local_mem_buffer::clear_bufs();
}

size_t mem_thread_pool::get_num_pending() const
{
	size_t ret = 0;
	for (size_t i = 0; i < threads.size(); i++) {
		size_t nthreads = threads[i].size();
		for (size_t j = 0; j < nthreads; j++)
			ret += threads[i][j]->get_num_pending();
	}
	return ret;
}

static mem_thread_pool::ptr global_threads;
enum thread_pool_state {
	UNINIT,
	ACTIVE,
	INACTIVE,
};
static thread_pool_state pool_state = thread_pool_state::UNINIT;

/*
 * When we disable thread pool, we use the main thread to perform
 * computation.
 */
bool mem_thread_pool::disable_thread_pool()
{
	pool_state = thread_pool_state::INACTIVE;
	return true;
}

bool mem_thread_pool::enable_thread_pool()
{
	pool_state = thread_pool_state::ACTIVE;
	return true;
}

mem_thread_pool::ptr mem_thread_pool::get_global_mem_threads()
{
	assert(pool_state != thread_pool_state::INACTIVE);
	if (global_threads == NULL) {
		int nthreads_per_node
			= matrix_conf.get_num_DM_threads() / matrix_conf.get_num_nodes();
		assert(nthreads_per_node > 0);
		global_threads = mem_thread_pool::create(matrix_conf.get_num_nodes(),
				nthreads_per_node);
		enable_thread_pool();
	}
	return global_threads;
}

size_t mem_thread_pool::get_global_num_threads()
{
	// When we disable the thread pool, we use the main thread for computation.
	// So the number of threads is 1.
	if (pool_state == thread_pool_state::INACTIVE)
		return 1;
	else
		return get_global_mem_threads()->get_num_threads();
}

int mem_thread_pool::get_curr_thread_id()
{
	// When we disable the thread pool, we use the main thread for computation.
	// And we use 0 as the thread id of the main thread.
	if (pool_state == thread_pool_state::INACTIVE)
		return 0;
	else {
		detail::pool_task_thread *curr
			= dynamic_cast<detail::pool_task_thread *>(thread::get_curr_thread());
		assert(curr);
		return curr->get_pool_thread_id();
	}
}

void mem_thread_pool::init_global_mem_threads(int num_nodes,
		int nthreads_per_node)
{
	if (global_threads == NULL) {
		global_threads = mem_thread_pool::create(num_nodes, nthreads_per_node);
		enable_thread_pool();
	}
}

void mem_thread_pool::destroy()
{
	global_threads = NULL;
}

void io_worker_task::run()
{
	std::vector<safs::io_interface::ptr> ios;
	pthread_spin_lock(&lock);
	for (auto it = EM_objs.begin(); it != EM_objs.end(); it++) {
		std::vector<safs::io_interface::ptr> tmp = (*it)->create_ios();
		ios.insert(ios.end(), tmp.begin(), tmp.end());
	}
	pthread_spin_unlock(&lock);
	safs::io_select::ptr select = safs::create_io_select(ios);

	// The task runs until there are no tasks left in the queue.
	while (dispatch->issue_task())
		safs::wait4ios(select, max_pending_ios);
	// Test if all I/O instances have processed all requests.
	size_t num_pending = safs::wait4ios(select, 0);
	assert(num_pending == 0);

	pthread_spin_lock(&lock);
	EM_objs.clear();
	pthread_spin_unlock(&lock);

	for (size_t i = 0; i < ios.size(); i++) {
		portion_callback &cb = static_cast<portion_callback &>(
				ios[i]->get_callback());
		assert(!cb.has_callback());
	}
}

global_counter::global_counter()
{
	counts.resize(mem_thread_pool::get_global_num_threads());
	reset();
}

void global_counter::inc(size_t val)
{
	int id = mem_thread_pool::get_curr_thread_id();
	counts[id].count += val;
}

void global_counter::reset()
{
	for (size_t i = 0; i < counts.size(); i++)
		counts[i].count = 0;
}

size_t global_counter::get() const
{
	size_t tot = 0;
	for (size_t i = 0; i < counts.size(); i++)
		tot += counts[i].count;
	return tot;
}

}

}
<commit_msg>[Matrix]: remove assert in getting pool thread ID.<commit_after>/*
 * Copyright 2014 Open Connectome Project (http://openconnecto.me)
 * Written by Da Zheng (zhengda1936@gmail.com)
 *
 * This file is part of FlashMatrix.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

#include <unordered_map>

#include "io_interface.h"

#include "matrix_config.h"
#include "mem_worker_thread.h"
#include "EM_object.h"
#include "local_mem_buffer.h"

namespace fm
{

namespace detail
{

#ifdef USE_HWLOC
std::vector<int> get_cpus(int node_id)
{
	std::vector<int> io_cpus = safs::get_io_cpus();
	std::vector<int> logical_units = cpus.get_node(node_id).get_logical_units();
	std::set<int> cpu_set(logical_units.begin(), logical_units.end());
	// Remove the logical units where I/O threads run.
	for (size_t j = 0; j < io_cpus.size(); j++)
		cpu_set.erase(io_cpus[j]);
	return std::vector<int>(cpu_set.begin(), cpu_set.end());
}
#endif

mem_thread_pool::mem_thread_pool(int num_nodes, int nthreads_per_node)
{
	tot_num_tasks = 0;
	threads.resize(num_nodes);
	for (int i = 0; i < num_nodes; i++) {
		// Get the CPU cores that are in node i.
#ifdef USE_HWLOC
		std::vector<int> cpus = get_cpus(i);
#endif
		threads[i].resize(nthreads_per_node);
		for (int j = 0; j < nthreads_per_node; j++) {
			std::string name
				= std::string("mem-worker-") + itoa(i) + "-" + itoa(j);
#ifdef USE_HWLOC
			if (safs::get_io_cpus().empty())
				threads[i][j] = std::shared_ptr<pool_task_thread>(
						new pool_task_thread(i * nthreads_per_node + j, name, i));
			else
				threads[i][j] = std::shared_ptr<pool_task_thread>(
						new pool_task_thread(i * nthreads_per_node + j, name,
							cpus, i));
#else
			threads[i][j] = std::shared_ptr<pool_task_thread>(
					new pool_task_thread(i * nthreads_per_node + j, name, i));
#endif
			threads[i][j]->start();
		}
	}
	ntasks_per_node.resize(num_nodes);
}

/*
 * This method dispatches tasks in a round-robin fashion.
 */
void mem_thread_pool::process_task(int node_id, thread_task *task)
{
	if (node_id < 0)
		node_id = tot_num_tasks % get_num_nodes();

	assert((size_t) node_id < threads.size());
	size_t idx = ntasks_per_node[node_id] % threads[node_id].size();
	threads[node_id][idx]->add_task(task);
	ntasks_per_node[node_id]++;
	tot_num_tasks++;
}

void mem_thread_pool::wait4complete()
{
	for (size_t i = 0; i < threads.size(); i++) {
		size_t nthreads = threads[i].size();
		for (size_t j = 0; j < nthreads; j++)
			threads[i][j]->wait4complete();
	}

	// After all workers complete, we should try to clear the memory buffers
	// in each worker thread to reduce memory consumption.
	// We might want to keep the memory buffer for I/O on dense matrices.
	if (matrix_conf.is_keep_mem_buf())
		detail::local_mem_buffer::clear_bufs(
				detail::local_mem_buffer::MAT_PORTION);
	else
		detail::local_mem_buffer::clear_bufs();
}

size_t mem_thread_pool::get_num_pending() const
{
	size_t ret = 0;
	for (size_t i = 0; i < threads.size(); i++) {
		size_t nthreads = threads[i].size();
		for (size_t j = 0; j < nthreads; j++)
			ret += threads[i][j]->get_num_pending();
	}
	return ret;
}

static mem_thread_pool::ptr global_threads;
enum thread_pool_state {
	UNINIT,
	ACTIVE,
	INACTIVE,
};
static thread_pool_state pool_state = thread_pool_state::UNINIT;

/*
 * When we disable thread pool, we use the main thread to perform
 * computation.
 */
bool mem_thread_pool::disable_thread_pool()
{
	pool_state = thread_pool_state::INACTIVE;
	return true;
}

bool mem_thread_pool::enable_thread_pool()
{
	pool_state = thread_pool_state::ACTIVE;
	return true;
}

mem_thread_pool::ptr mem_thread_pool::get_global_mem_threads()
{
	assert(pool_state != thread_pool_state::INACTIVE);
	if (global_threads == NULL) {
		int nthreads_per_node
			= matrix_conf.get_num_DM_threads() / matrix_conf.get_num_nodes();
		assert(nthreads_per_node > 0);
		global_threads = mem_thread_pool::create(matrix_conf.get_num_nodes(),
				nthreads_per_node);
		enable_thread_pool();
	}
	return global_threads;
}

size_t mem_thread_pool::get_global_num_threads()
{
	// When we disable the thread pool, we use the main thread for computation.
	// So the number of threads is 1.
	if (pool_state == thread_pool_state::INACTIVE)
		return 1;
	else
		return get_global_mem_threads()->get_num_threads();
}

int mem_thread_pool::get_curr_thread_id()
{
	// When we disable the thread pool, we use the main thread for computation.
	// And we use 0 as the thread id of the main thread.
	if (pool_state == thread_pool_state::INACTIVE)
		return 0;
	else {
		detail::pool_task_thread *curr
			= dynamic_cast<detail::pool_task_thread *>(thread::get_curr_thread());
		if (curr)
			return curr->get_pool_thread_id();
		else
			return -1;
	}
}

void mem_thread_pool::init_global_mem_threads(int num_nodes,
		int nthreads_per_node)
{
	if (global_threads == NULL) {
		global_threads = mem_thread_pool::create(num_nodes, nthreads_per_node);
		enable_thread_pool();
	}
}

void mem_thread_pool::destroy()
{
	global_threads = NULL;
}

void io_worker_task::run()
{
	std::vector<safs::io_interface::ptr> ios;
	pthread_spin_lock(&lock);
	for (auto it = EM_objs.begin(); it != EM_objs.end(); it++) {
		std::vector<safs::io_interface::ptr> tmp = (*it)->create_ios();
		ios.insert(ios.end(), tmp.begin(), tmp.end());
	}
	pthread_spin_unlock(&lock);
	safs::io_select::ptr select = safs::create_io_select(ios);

	// The task runs until there are no tasks left in the queue.
	while (dispatch->issue_task())
		safs::wait4ios(select, max_pending_ios);
	// Test if all I/O instances have processed all requests.
	size_t num_pending = safs::wait4ios(select, 0);
	assert(num_pending == 0);

	pthread_spin_lock(&lock);
	EM_objs.clear();
	pthread_spin_unlock(&lock);

	for (size_t i = 0; i < ios.size(); i++) {
		portion_callback &cb = static_cast<portion_callback &>(
				ios[i]->get_callback());
		assert(!cb.has_callback());
	}
}

global_counter::global_counter()
{
	counts.resize(mem_thread_pool::get_global_num_threads());
	reset();
}

void global_counter::inc(size_t val)
{
	int id = mem_thread_pool::get_curr_thread_id();
	counts[id].count += val;
}

void global_counter::reset()
{
	for (size_t i = 0; i < counts.size(); i++)
		counts[i].count = 0;
}

size_t global_counter::get() const
{
	size_t tot = 0;
	for (size_t i = 0; i < counts.size(); i++)
		tot += counts[i].count;
	return tot;
}

}

}
<|endoftext|>


<commit_before>/* StreamFeel Dataset Utility/Serializer (Author: Adam Browne)
 * This utility encodes the dataset and adds new records to existing datasets.
 * the serialized data we get from the trained model is used in the actual extension. */

// text_categorizer.h was modified to include "encode / decode" to use the serialization/deserialization.

#include <algorithm>
#include <iostream>
#include <sstream>
#include <vector>
#include <fstream>
#include <string>

#include "../../../mitielib/include/mitie/text_categorizer.h"
#include "../../../mitielib/include/mitie/text_categorizer_trainer.h"

using namespace mitie;
using namespace dlib;

//an abstraction intended to keep the # of checks down (?)
struct DataInst {
	std::string file_name, lbl, type, file;
	char selection;
	int curOffset;
	std::string dir = "/Users/adambrowne/Desktop/Personal/StreamFeel/src/inject/";

	DataInst(char choice){
		selection = choice;
		if(selection == 'r'){
			curOffset = 0;
			lbl = "Relevant (y/n):\n", file = "relevance.txt";
			type = "relevance_data", file_name = dir + "default.js";
		} else {
			curOffset = 16;
			lbl = "Sentiment:\n", file = "sentiment.txt";
			type = "sentiment", file_name = dir + "sentiment.js";
		}
	}
	//reads the contents of the file as a string.
	std::string readData() {
		std::ifstream fileReader(file);
		std::stringstream buffer;
		buffer << fileReader.rdbuf();
		return buffer.str();
	}

};

/* In order to categorize a msg, we must first tokenize it.
* this helper function does that by parsing spaces. */
std::vector<std::string> tokenize_msg(std::string& input, char delim){
    std::vector<std::string> tokens;
        tokens.reserve(input.size());
        int curSpace = input.find(delim), curPos = 0;
        while(curSpace != std::string::npos){
            tokens.push_back(input.substr(curPos,curSpace-curPos));
            curPos = curSpace+1;
            curSpace = input.find(delim,curPos);
        }
        //if no space, or last region, push it in.
        tokens.push_back(input.substr(curPos));
        return tokens;
}

//split by commas
std::vector<std::string> split(std::string& raw){
	return tokenize_msg(raw,',');
}

/* Load an existing dataset from a file. */
std::vector<std::string> dataset(DataInst& inst){
	std::string raw = inst.readData();
	return split(raw);
}

text_categorizer BuildCategorizer(std::vector<std::string> const& data) {
    text_categorizer_trainer fit;
    int i = 0, invariant = data.size() - 1;
    while(i < invariant){
        std::string msg = data[i], label = data[i+1];
        std::vector<std::string> token = tokenize_msg(msg,' ');
        fit.add(token,label);
        i += 2;
    }
    fit.set_num_threads(4);
    return fit.train();
}

/* Serialize works by pulling the existing dataset, subsequently training it,
 * then serializing (encodes) the data to the vector stream and returning its' buffer */
std::vector<char> serialize(DataInst& inst) {
    text_categorizer executor;
    std::vector<std::string> data = dataset(inst);
    executor = BuildCategorizer(data);
    std::vector<char> buf;
    dlib::vectorstream strm(buf);
    executor.encode(executor,strm);
    return buf;
}

//input validation for adding records.
bool notValid(const std::string& input, char type) {
	if(type == 'r')
		return input != "y" && input != "n";
	else
		return (input != "funny" && input != "sad" && input != "happy"
			&& input != "angry" && input != "confused" && input != "curious"
			&& input != "gentlemanly" && input != "astonished" && input != "friendly");
}

void readRecord(std::string& msg, DataInst& inst, std::string& label) {
	std::cout << "Enter message:\n";
	std::cin.ignore();
	std::getline(std::cin,msg);
	std::cout << inst.lbl;
	do {
		std::cin >> label;
	} while(notValid(label,inst.selection) == true);
	//need the delimeter ',': can't confuse it.
	msg.erase(std::remove(msg.begin(), msg.end(), ','), msg.end());
}

void addRecord(DataInst& inst) {
	std::string msg, label;
	//open the dataset for appending because we're adding another record.
	std::ofstream fileWriter;
	fileWriter.open(inst.file, std::ios_base::app);
	//append the new record to the dataset.
	readRecord(msg,inst,label);
	std::string new_msg = "," + msg + "," + label;
	fileWriter << new_msg;
	std::cout << "Added " << msg << "," << label << std::endl;
}

void displayMenu() {
	std::cout << "1. Add Relevance\n";
	std::cout << "2. Add Sentiment\n";
	std::cout << "3. Update Relevance\n";
	std::cout << "4. Update Sentiment\n";
	std::cout << "5. Display Menu\n";
}

/* SegmenData breaks up the entire serialized buffer into eight different chunks
 * They're subsequently interpreted by the native client module. */
void segmentData(std::vector<char>& buffer, std::string& updated_data, DataInst& inst){
	int index = 0, origBound = buffer.size() / 16, curBound;
	curBound = origBound;
	int curOffset = inst.curOffset;
	std::string type = inst.type;
	while(index != buffer.size()){
		std::string offset = to_string(curOffset++);
		updated_data += ("\nvar " + type + offset + " = [\n" + offset + ",");
		for(; index < curBound; ++index){
			int byte = buffer[index];
			updated_data += (to_string(byte) + ",");
		}
		updated_data.pop_back();
		updated_data += "\n];";
		curBound = origBound + curBound;
		if(curBound + origBound > buffer.size())
			curBound = buffer.size();
	}
}

/* Serializes a trained categorizer model and writes its
 * raw representation to the javascript file containing it,
 * this will be deserialized by the native client module. */
void updateDataset(DataInst& inst){
	//get the trained buffer (raw representation of the model)
	std::vector<char> buffer = serialize(inst);
	std::string updated_data;
	segmentData(buffer,updated_data,inst);
	std::ofstream dataWriter(inst.file_name);
	dataWriter << updated_data;
}

//1 - Update Rel, 2 - Update Sent, 3 - Add Rel, 4 - Add Sent - 5: display menu
int main() {
	int choice; //1 - 4:
	displayMenu();
	do {
		std::cin >> choice;
		auto rel = DataInst('r'), sen = DataInst('s');
		switch(choice){
			case 1:
				addRecord(rel);
				break;
			case 2:
				addRecord(sen);
				break;
			case 3:
				updateDataset(rel);
				break;
			case 4:
				updateDataset(sen);
				break;
			case 5:
				displayMenu();
				break;
		}
	} while(choice >= 1 && choice <= 5);
}

<commit_msg>refactor sfeel util x3<commit_after>/* StreamFeel Dataset Utility/Serializer (Author: Adam Browne)
 * This utility encodes the dataset and adds new records to existing datasets.
 * the serialized data we get from the trained model is used in the actual extension. */

// text_categorizer.h was modified to include "encode / decode" to use the serialization/deserialization.

#include <algorithm>
#include <iostream>
#include <sstream>
#include <vector>
#include <fstream>
#include <string>

#include "../../../mitielib/include/mitie/text_categorizer.h"
#include "../../../mitielib/include/mitie/text_categorizer_trainer.h"

using namespace mitie;
using namespace dlib;

//an abstraction intended to keep the # of checks down (?)
struct DataInst {
	std::string file_name, lbl, type, file;
	char selection;
	int curOffset;
	std::string dir = "/Users/adambrowne/Desktop/Personal/StreamFeel/src/inject/";

	DataInst(char choice){
		selection = choice;
		if(selection == 'r'){
			curOffset = 0;
			lbl = "Relevant (y/n):\n", file = "relevance.txt";
			type = "relevance_data", file_name = dir + "default.js";
		} else {
			curOffset = 16;
			lbl = "Sentiment:\n", file = "sentiment.txt";
			type = "sentiment", file_name = dir + "sentiment.js";
		}
	}
	//reads the contents of the file as a string.
	std::string readData() {
		std::ifstream fileReader(file);
		std::stringstream buffer;
		buffer << fileReader.rdbuf();
		return buffer.str();
	}

};

/* In order to categorize a msg, we must first tokenize it.
* this helper function does that by parsing spaces. */
std::vector<std::string> tokenize_msg(std::string& input, char delim){
    std::vector<std::string> tokens;
        tokens.reserve(input.size());
        int curSpace = input.find(delim), curPos = 0;
        while(curSpace != std::string::npos){
            tokens.push_back(input.substr(curPos,curSpace-curPos));
            curPos = curSpace+1;
            curSpace = input.find(delim,curPos);
        }
        //if no space, or last region, push it in.
        tokens.push_back(input.substr(curPos));
        return tokens;
}

//split by commas
std::vector<std::string> split(std::string& raw){
	return tokenize_msg(raw,',');
}

/* Load an existing dataset from a file. */
std::vector<std::string> dataset(DataInst& inst){
	std::string raw = inst.readData();
	return split(raw);
}

text_categorizer BuildCategorizer(std::vector<std::string> const& data) {
    text_categorizer_trainer fit;
    int i = 0, invariant = data.size() - 1;
    while(i < invariant){
        std::string msg = data[i], label = data[i+1];
        std::vector<std::string> token = tokenize_msg(msg,' ');
        fit.add(token,label);
        i += 2;
    }
    fit.set_num_threads(4);
    return fit.train();
}

/* Serialize works by pulling the existing dataset, subsequently training it,
 * then serializing (encodes) the data to the vector stream and returning its' buffer */
std::vector<char> serialize(DataInst& inst) {
    text_categorizer executor;
    std::vector<std::string> data = dataset(inst);
    executor = BuildCategorizer(data);
    std::vector<char> buf;
    dlib::vectorstream strm(buf);
    executor.encode(executor,strm);
    return buf;
}

//input validation for adding records.
bool notValid(const std::string& input, char type) {
	if(type == 'r')
		return input != "y" && input != "n";
	else
		return (input != "funny" && input != "sad" && input != "happy"
			&& input != "angry" && input != "confused" && input != "curious"
			&& input != "gentlemanly" && input != "astonished" && input != "friendly");
}

std::string readLabel(char sel) {
	std::string label;
	while(notValid(label,sel) == true)
		std::cin >> label;
	return label;
}

void readRecord(std::string& msg, DataInst& inst, std::string& label) {
	std::cout << "Enter message:\n";
	std::cin.ignore();
	std::getline(std::cin,msg);
	std::cout << inst.lbl;
	label = readLabel(inst.selection);
	msg.erase(std::remove(msg.begin(), msg.end(), ','), msg.end());
}

/* We open the dataset for appending so we can add another record.
 * Input validation is performed to ensure accurate labels. */
void addRecord(DataInst& inst) {
	std::string msg, label;
	std::ofstream fileWriter;
	fileWriter.open(inst.file, std::ios_base::app);
	readRecord(msg,inst,label);
	std::string new_msg = "," + msg + "," + label;
	fileWriter << new_msg;
	std::cout << "Added " << msg << "," << label << std::endl;
}

void displayMenu() {
	std::cout << "1. Add Relevance\n";
	std::cout << "2. Add Sentiment\n";
	std::cout << "3. Update Relevance\n";
	std::cout << "4. Update Sentiment\n";
	std::cout << "5. Display Menu\n";
}

/* SegmenData breaks up the entire serialized buffer into eight different chunks
 * They're subsequently interpreted by the native client module. */
void segmentData(std::vector<char>& buffer, std::string& updated_data, DataInst& inst){
	int index = 0, origBound = buffer.size() / 16, curBound;
	curBound = origBound;
	int curOffset = inst.curOffset;
	std::string type = inst.type;
	while(index != buffer.size()){
		std::string offset = to_string(curOffset++);
		updated_data += ("\nvar " + type + offset + " = [\n" + offset + ",");
		for(; index < curBound; ++index){
			int byte = buffer[index];
			updated_data += (to_string(byte) + ",");
		}
		updated_data.pop_back();
		updated_data += "\n];";
		curBound = origBound + curBound;
		if(curBound + origBound > buffer.size())
			curBound = buffer.size();
	}
}

/* Serializes a trained categorizer model and writes its
 * raw representation to the javascript file containing it,
 * this will be deserialized by the native client module. */
void updateDataset(DataInst& inst){
	//get the trained buffer (raw representation of the model)
	std::vector<char> buffer = serialize(inst);
	std::string updated_data;
	segmentData(buffer,updated_data,inst);
	std::ofstream dataWriter(inst.file_name);
	dataWriter << updated_data;
}

//1 - Update Rel, 2 - Update Sent, 3 - Add Rel, 4 - Add Sent - 5: display menu
int main() {
	int choice; //1 - 4:
	displayMenu();
	do {
		std::cin >> choice;
		auto rel = DataInst('r'), sen = DataInst('s');
		switch(choice){
			case 1:
				addRecord(rel);
				break;
			case 2:
				addRecord(sen);
				break;
			case 3:
				updateDataset(rel);
				break;
			case 4:
				updateDataset(sen);
				break;
			case 5:
				displayMenu();
				break;
		}
	} while(choice >= 1 && choice <= 5);
}

<|endoftext|>