<commit_before>/**
 * Copyright (c) 2016 DeepCortex GmbH <legal@eventql.io>
 * Authors:
 *   - Paul Asmuth <paul@eventql.io>
 *
 * This program is free software: you can redistribute it and/or modify it under
 * the terms of the GNU Affero General Public License ("the license") as
 * published by the Free Software Foundation, either version 3 of the License,
 * or any later version.
 *
 * In accordance with Section 7(e) of the license, the licensing of the Program
 * under the license does not imply a trademark license. Therefore any rights,
 * title and interest in our trademarks remain entirely with us.
 *
 * This program is distributed in the hope that it will be useful, but WITHOUT
 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
 * FOR A PARTICULAR PURPOSE. See the license for more details.
 *
 * You can be released from the requirements of the license by purchasing a
 * commercial license. Buying such a license is mandatory as soon as you develop
 * commercial activities involving this program without disclosing the source
 * code of your own applications
 */
#include <stdlib.h>
#include <unistd.h>
#include <signal.h>
#include <poll.h>

#include "eventql/eventql.h"
#include "eventql/util/application.h"
#include "eventql/util/logging.h"
#include "eventql/util/io/TerminalOutputStream.h"
#include "eventql/util/wallclock.h"
#include "eventql/util/cli/CLI.h"
#include "eventql/util/cli/flagparser.h"
#include "eventql/util/cli/term.h"
#include "eventql/cli/console.h"
#include "eventql/cli/cli_config.h"

thread::EventLoop ev;

static bool hasSTDIN() {
  struct pollfd p = {
    .fd = STDIN_FILENO,
    .events = POLLIN | POLLRDBAND | POLLRDNORM | POLLPRI
  };

  return poll(&p, 1, 0) == 1;
}

static void printError(const String& error_string) {
  auto stderr_os = TerminalOutputStream::fromStream(OutputStream::getStderr());
  stderr_os->print(
      "ERROR:",
      { TerminalStyle::RED, TerminalStyle::UNDERSCORE });
  stderr_os->print(" " + error_string + "\n");
}

int main(int argc, const char** argv) {
  Application::init();
  Application::logToStderr("evql");

  cli::FlagParser flags;

  flags.defineFlag(
      "help",
      cli::FlagParser::T_SWITCH,
      false,
      "?",
      NULL,
      "help",
      "<help>");

  flags.defineFlag(
      "version",
      cli::FlagParser::T_SWITCH,
      false,
      "v",
      NULL,
      "print version",
      "<switch>");

  flags.defineFlag(
      "config",
      ::cli::FlagParser::T_STRING,
      false,
      "c",
      NULL,
      "path to config file",
      "<config_file>");

  flags.defineFlag(
      "config_set",
      ::cli::FlagParser::T_STRING,
      false,
      "C",
      NULL,
      "set config option",
      "<key>=<val>");

  flags.defineFlag(
      "file",
      cli::FlagParser::T_STRING,
      false,
      "f",
      NULL,
      "query file",
      "<file>");

  flags.defineFlag(
      "lang",
      cli::FlagParser::T_STRING,
      false,
      "l",
      NULL,
      "query language",
      "<lang>");

  flags.defineFlag(
      "exec",
      cli::FlagParser::T_STRING,
      false,
      "e",
      NULL,
      "query string",
      "<query_str>");

  flags.defineFlag(
      "host",
      cli::FlagParser::T_STRING,
      false,
      "h",
      NULL,
      "eventql server hostname",
      "<host>");

  flags.defineFlag(
      "port",
      cli::FlagParser::T_INTEGER,
      false,
      "p",
      NULL,
      "eventql server port",
      "<port>");

  flags.defineFlag(
      "user",
      cli::FlagParser::T_STRING,
      false,
      "u",
      NULL,
      "username",
      "<user>");

  flags.defineFlag(
      "database",
      cli::FlagParser::T_STRING,
      false,
      "d",
      NULL,
      "database",
      "<db>");

  flags.defineFlag(
      "auth_token",
      cli::FlagParser::T_STRING,
      false,
      NULL,
      NULL,
      "auth token",
      "<token>");

  flags.defineFlag(
      "history_file",
      cli::FlagParser::T_STRING,
      false,
      NULL,
      NULL,
      "history path",
      "<path>");

  flags.defineFlag(
      "history_maxlen",
      cli::FlagParser::T_INTEGER,
      false,
      NULL,
      NULL,
      "history length",
      "<number of history entries>");

  flags.defineFlag(
      "loglevel",
      cli::FlagParser::T_STRING,
      false,
      NULL,
      "INFO",
      "loglevel",
      "<level>");

  flags.defineFlag(
      "batch",
      cli::FlagParser::T_SWITCH,
      false,
      "B",
      NULL,
      "batch",
      "<batch mode>");

  flags.defineFlag(
      "quiet",
      cli::FlagParser::T_SWITCH,
      false,
      "q",
      NULL,
      "quiet",
      "<quiet>");

  flags.parseArgv(argc, argv);

  Logger::get()->setMinimumLogLevel(
      strToLogLevel(flags.getString("loglevel")));

  auto stdin_is = InputStream::getStdin();
  auto stdout_os = OutputStream::getStdout();
  auto stderr_os = OutputStream::getStderr();

  /* print help */
  if (flags.isSet("help") || flags.isSet("version")) {
    stdout_os->write(
        StringUtil::format(
            "EventQL $0 ($1)\n"
            "Copyright (c) 2016, DeepCortex GmbH. All rights reserved.\n\n",
            eventql::kVersionString,
            eventql::kBuildID));
  }

  if (flags.isSet("version")) {
    return 0;
  }

  /* print help */
  if (flags.isSet("help")) {
    stdout_os->write(
        "Usage: $ evql [OPTIONS] [query]\n"
        "       $ evql [OPTIONS] -f file\n\n"
        "   -f, --file <file>         Read query from file\n"
        "   -e, --exec <query_str>    Execute query string\n"
        "   -l, --lang <lang>         Set the query language ('sql' or 'js')\n"
        "   -D, --database <db>       Select a database\n"
        "   -h, --host <hostname>     Set the EventQL server hostname\n"
        "   -p, --port <port>         Set the EventQL server port\n"
        "   -u, --user <user>         Set the auth username\n"
        "   --password <password>     Set the auth password (if required)\n"
        "   --auth_token <token>      Set the auth token (if required)\n"
        "   -c, --config <file>       Load config from file\n"
        "   -C name=value             Define a config value on the command line\n"
        "   -B, --batch               Run in batch mode (streaming result output)\n"
        "   -q, --quiet               Be quiet (disables query progress)\n"
        "   --verbose                 Print debug output to STDERR\n"
        "   -v, --version             Display the version of this binary and exit\n"
        "   -?, --help                Display this help text and exit\n"
        "                                                       \n"
        "Examples:                                              \n"
        "   $ evql                        # start an interactive shell\n"
        "   $ evql -h localhost -p 9175   # start an interactive shell\n"
        "   $ evql < query.sql            # execute query from STDIN\n"
        "   $ evql -f query.sql           # execute query in query.sql\n"
        "   $ evql -l js -f query.js      # execute query in query.js\n"
        "   $ evql -e 'SELECT 42;'        # execute 'SELECT 42'\n"
    );
    return 0;
  }

  /* console options */
  eventql::ProcessConfigBuilder cfg_builder;
  cfg_builder.setProperty("client.timeout", "5000000");

  if (flags.isSet("config")) {
    auto config_file_path = flags.getString("config");
    auto rc = cfg_builder.loadFile(config_file_path);
    if (!rc.isSuccess()) {
      printError("error while loading config file: " + rc.message());
      return 1;
    }
  }

  for (const auto& opt : flags.getStrings("config_set")) {
    auto opt_key_end = opt.find("=");
    if (opt_key_end == String::npos) {
      printError(StringUtil::format("invalid config option: $0", opt));;
      return 1;
    }

    auto opt_key = opt.substr(0, opt_key_end);
    auto opt_value = opt.substr(opt_key_end + 1);
    cfg_builder.setProperty(opt_key, opt_value);
  }

  {
    auto status = cfg_builder.loadDefaultConfigFile("evql");
    if (!status.isSuccess()) {
      printError(status.message());
      return 1;
    }
  }

  if (flags.isSet("host")) {
    cfg_builder.setProperty("client.host", flags.getString("host"));
  }

  if (flags.isSet("port")) {
    cfg_builder.setProperty(
        "client.port",
        StringUtil::toString(flags.getInt("port")));
  }

  if (flags.isSet("database")) {
    cfg_builder.setProperty("client.database", flags.getString("database"));
  }

  if (flags.isSet("user")) {
    cfg_builder.setProperty("client.user", flags.getString("user"));
  }

  if (flags.isSet("password")) {
    cfg_builder.setProperty("client.password", flags.getString("password"));
  }

  if (flags.isSet("auth_token")) {
    cfg_builder.setProperty("client.auth_token", flags.getString("auth_token"));
  }

  if (flags.isSet("lang")) {
    cfg_builder.setProperty("client.lang", flags.getString("lang"));
  }

  if (flags.isSet("batch")) {
    cfg_builder.setProperty("client.batch", "true");
  }

  /* cli config */
  eventql::cli::CLIConfig cli_cfg(cfg_builder.getConfig());

  if (flags.getArgv().size() > 0) {
    printError(StringUtil::format(
        "invalid argument: '$0', run evql --help for help\n",
        flags.getArgv()[0]));

    return 1;
  }

  /* cli */
  eventql::cli::Console console(cli_cfg);

  {
    auto rc = console.connect();
    if (!rc.isSuccess()) {
      printError(StringUtil::format(
          "error while connecting to server: $0", rc.getMessage()));
      return 1;
    }
  }

  int rc = 0;

  auto file = cli_cfg.getFile();
  auto language = cli_cfg.getLanguage();
  if (file.isEmpty() &&
      !language.isEmpty() &&
      language.get() == eventql::cli::CLIConfig::kLanguage::JAVASCRIPT) {
    printError("missing --file flag. Set --file for javascript");
    rc = 1;
    goto exit;
  }

  if (!file.isEmpty()) {
    if (language.isEmpty()) {
      printError(
          "invalid --lang flag. Must be one of 'sql', 'js' or 'javascript'");
      rc = 1;
      goto exit;
    }

    auto query = FileUtil::read(file.get());
    switch (language.get()) {
      case eventql::cli::CLIConfig::kLanguage::SQL: {
        Status ret = console.runQuery(query.toString());
        rc = ret.isSuccess() ? 0 : 1;
        goto exit;
      }

      case eventql::cli::CLIConfig::kLanguage::JAVASCRIPT: {
        Status ret = console.runJS(query.toString());
        rc = ret.isSuccess() ? 0 : 1;
        goto exit;
      }
    }
  }

  if (flags.isSet("exec")) {
    auto ret = console.runQuery(flags.getString("exec"));
    rc = ret.isSuccess() ? 0 : 1;
    goto exit;
  }

  if (hasSTDIN() || flags.isSet("batch") || !stdout_os->isTTY()) {
    String query;
    while (stdin_is->readLine(&query)) {
      auto ret = console.runQuery(query);
      query = "";
      if (ret.isError()) {
        rc = 1;
        goto exit;
      }
    }
  } else {
    console.startInteractiveShell();
  }

exit:
  console.close();
  return rc;
}

<commit_msg>don't ignore --file flag in evql<commit_after>/**
 * Copyright (c) 2016 DeepCortex GmbH <legal@eventql.io>
 * Authors:
 *   - Paul Asmuth <paul@eventql.io>
 *
 * This program is free software: you can redistribute it and/or modify it under
 * the terms of the GNU Affero General Public License ("the license") as
 * published by the Free Software Foundation, either version 3 of the License,
 * or any later version.
 *
 * In accordance with Section 7(e) of the license, the licensing of the Program
 * under the license does not imply a trademark license. Therefore any rights,
 * title and interest in our trademarks remain entirely with us.
 *
 * This program is distributed in the hope that it will be useful, but WITHOUT
 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
 * FOR A PARTICULAR PURPOSE. See the license for more details.
 *
 * You can be released from the requirements of the license by purchasing a
 * commercial license. Buying such a license is mandatory as soon as you develop
 * commercial activities involving this program without disclosing the source
 * code of your own applications
 */
#include <stdlib.h>
#include <unistd.h>
#include <signal.h>
#include <poll.h>

#include "eventql/eventql.h"
#include "eventql/util/application.h"
#include "eventql/util/logging.h"
#include "eventql/util/io/TerminalOutputStream.h"
#include "eventql/util/wallclock.h"
#include "eventql/util/cli/CLI.h"
#include "eventql/util/cli/flagparser.h"
#include "eventql/util/cli/term.h"
#include "eventql/cli/console.h"
#include "eventql/cli/cli_config.h"

thread::EventLoop ev;

static bool hasSTDIN() {
  struct pollfd p = {
    .fd = STDIN_FILENO,
    .events = POLLIN | POLLRDBAND | POLLRDNORM | POLLPRI
  };

  return poll(&p, 1, 0) == 1;
}

static void printError(const String& error_string) {
  auto stderr_os = TerminalOutputStream::fromStream(OutputStream::getStderr());
  stderr_os->print(
      "ERROR:",
      { TerminalStyle::RED, TerminalStyle::UNDERSCORE });
  stderr_os->print(" " + error_string + "\n");
}

int main(int argc, const char** argv) {
  Application::init();
  Application::logToStderr("evql");

  cli::FlagParser flags;

  flags.defineFlag(
      "help",
      cli::FlagParser::T_SWITCH,
      false,
      "?",
      NULL,
      "help",
      "<help>");

  flags.defineFlag(
      "version",
      cli::FlagParser::T_SWITCH,
      false,
      "v",
      NULL,
      "print version",
      "<switch>");

  flags.defineFlag(
      "config",
      ::cli::FlagParser::T_STRING,
      false,
      "c",
      NULL,
      "path to config file",
      "<config_file>");

  flags.defineFlag(
      "config_set",
      ::cli::FlagParser::T_STRING,
      false,
      "C",
      NULL,
      "set config option",
      "<key>=<val>");

  flags.defineFlag(
      "file",
      cli::FlagParser::T_STRING,
      false,
      "f",
      NULL,
      "query file",
      "<file>");

  flags.defineFlag(
      "lang",
      cli::FlagParser::T_STRING,
      false,
      "l",
      NULL,
      "query language",
      "<lang>");

  flags.defineFlag(
      "exec",
      cli::FlagParser::T_STRING,
      false,
      "e",
      NULL,
      "query string",
      "<query_str>");

  flags.defineFlag(
      "host",
      cli::FlagParser::T_STRING,
      false,
      "h",
      NULL,
      "eventql server hostname",
      "<host>");

  flags.defineFlag(
      "port",
      cli::FlagParser::T_INTEGER,
      false,
      "p",
      NULL,
      "eventql server port",
      "<port>");

  flags.defineFlag(
      "user",
      cli::FlagParser::T_STRING,
      false,
      "u",
      NULL,
      "username",
      "<user>");

  flags.defineFlag(
      "database",
      cli::FlagParser::T_STRING,
      false,
      "d",
      NULL,
      "database",
      "<db>");

  flags.defineFlag(
      "auth_token",
      cli::FlagParser::T_STRING,
      false,
      NULL,
      NULL,
      "auth token",
      "<token>");

  flags.defineFlag(
      "history_file",
      cli::FlagParser::T_STRING,
      false,
      NULL,
      NULL,
      "history path",
      "<path>");

  flags.defineFlag(
      "history_maxlen",
      cli::FlagParser::T_INTEGER,
      false,
      NULL,
      NULL,
      "history length",
      "<number of history entries>");

  flags.defineFlag(
      "loglevel",
      cli::FlagParser::T_STRING,
      false,
      NULL,
      "INFO",
      "loglevel",
      "<level>");

  flags.defineFlag(
      "batch",
      cli::FlagParser::T_SWITCH,
      false,
      "B",
      NULL,
      "batch",
      "<batch mode>");

  flags.defineFlag(
      "quiet",
      cli::FlagParser::T_SWITCH,
      false,
      "q",
      NULL,
      "quiet",
      "<quiet>");

  flags.parseArgv(argc, argv);

  Logger::get()->setMinimumLogLevel(
      strToLogLevel(flags.getString("loglevel")));

  auto stdin_is = InputStream::getStdin();
  auto stdout_os = OutputStream::getStdout();
  auto stderr_os = OutputStream::getStderr();

  /* print help */
  if (flags.isSet("help") || flags.isSet("version")) {
    stdout_os->write(
        StringUtil::format(
            "EventQL $0 ($1)\n"
            "Copyright (c) 2016, DeepCortex GmbH. All rights reserved.\n\n",
            eventql::kVersionString,
            eventql::kBuildID));
  }

  if (flags.isSet("version")) {
    return 0;
  }

  /* print help */
  if (flags.isSet("help")) {
    stdout_os->write(
        "Usage: $ evql [OPTIONS] [query]\n"
        "       $ evql [OPTIONS] -f file\n\n"
        "   -f, --file <file>         Read query from file\n"
        "   -e, --exec <query_str>    Execute query string\n"
        "   -l, --lang <lang>         Set the query language ('sql' or 'js')\n"
        "   -D, --database <db>       Select a database\n"
        "   -h, --host <hostname>     Set the EventQL server hostname\n"
        "   -p, --port <port>         Set the EventQL server port\n"
        "   -u, --user <user>         Set the auth username\n"
        "   --password <password>     Set the auth password (if required)\n"
        "   --auth_token <token>      Set the auth token (if required)\n"
        "   -c, --config <file>       Load config from file\n"
        "   -C name=value             Define a config value on the command line\n"
        "   -B, --batch               Run in batch mode (streaming result output)\n"
        "   -q, --quiet               Be quiet (disables query progress)\n"
        "   --verbose                 Print debug output to STDERR\n"
        "   -v, --version             Display the version of this binary and exit\n"
        "   -?, --help                Display this help text and exit\n"
        "                                                       \n"
        "Examples:                                              \n"
        "   $ evql                        # start an interactive shell\n"
        "   $ evql -h localhost -p 9175   # start an interactive shell\n"
        "   $ evql < query.sql            # execute query from STDIN\n"
        "   $ evql -f query.sql           # execute query in query.sql\n"
        "   $ evql -l js -f query.js      # execute query in query.js\n"
        "   $ evql -e 'SELECT 42;'        # execute 'SELECT 42'\n"
    );
    return 0;
  }

  /* console options */
  eventql::ProcessConfigBuilder cfg_builder;
  cfg_builder.setProperty("client.timeout", "5000000");

  if (flags.isSet("config")) {
    auto config_file_path = flags.getString("config");
    auto rc = cfg_builder.loadFile(config_file_path);
    if (!rc.isSuccess()) {
      printError("error while loading config file: " + rc.message());
      return 1;
    }
  }

  for (const auto& opt : flags.getStrings("config_set")) {
    auto opt_key_end = opt.find("=");
    if (opt_key_end == String::npos) {
      printError(StringUtil::format("invalid config option: $0", opt));;
      return 1;
    }

    auto opt_key = opt.substr(0, opt_key_end);
    auto opt_value = opt.substr(opt_key_end + 1);
    cfg_builder.setProperty(opt_key, opt_value);
  }

  {
    auto status = cfg_builder.loadDefaultConfigFile("evql");
    if (!status.isSuccess()) {
      printError(status.message());
      return 1;
    }
  }

  if (flags.isSet("host")) {
    cfg_builder.setProperty("client.host", flags.getString("host"));
  }

  if (flags.isSet("port")) {
    cfg_builder.setProperty(
        "client.port",
        StringUtil::toString(flags.getInt("port")));
  }

  if (flags.isSet("database")) {
    cfg_builder.setProperty("client.database", flags.getString("database"));
  }

  if (flags.isSet("user")) {
    cfg_builder.setProperty("client.user", flags.getString("user"));
  }

  if (flags.isSet("password")) {
    cfg_builder.setProperty("client.password", flags.getString("password"));
  }

  if (flags.isSet("auth_token")) {
    cfg_builder.setProperty("client.auth_token", flags.getString("auth_token"));
  }

  if (flags.isSet("file")) {
    cfg_builder.setProperty("client.file", flags.getString("file"));
  }

  if (flags.isSet("lang")) {
    cfg_builder.setProperty("client.lang", flags.getString("lang"));
  }

  if (flags.isSet("batch")) {
    cfg_builder.setProperty("client.batch", "true");
  }

  /* cli config */
  eventql::cli::CLIConfig cli_cfg(cfg_builder.getConfig());

  if (flags.getArgv().size() > 0) {
    printError(StringUtil::format(
        "invalid argument: '$0', run evql --help for help\n",
        flags.getArgv()[0]));

    return 1;
  }

  /* cli */
  eventql::cli::Console console(cli_cfg);

  {
    auto rc = console.connect();
    if (!rc.isSuccess()) {
      printError(StringUtil::format(
          "error while connecting to server: $0", rc.getMessage()));
      return 1;
    }
  }

  int rc = 0;

  auto file = cli_cfg.getFile();
  auto language = cli_cfg.getLanguage();
  if (file.isEmpty() &&
      !language.isEmpty() &&
      language.get() == eventql::cli::CLIConfig::kLanguage::JAVASCRIPT) {
    printError("missing --file flag. Set --file for javascript");
    rc = 1;
    goto exit;
  }

  if (!file.isEmpty()) {
    if (language.isEmpty()) {
      printError(
          "invalid --lang flag. Must be one of 'sql', 'js' or 'javascript'");
      rc = 1;
      goto exit;
    }

    auto query = FileUtil::read(file.get());
    switch (language.get()) {
      case eventql::cli::CLIConfig::kLanguage::SQL: {
        Status ret = console.runQuery(query.toString());
        rc = ret.isSuccess() ? 0 : 1;
        goto exit;
      }

      case eventql::cli::CLIConfig::kLanguage::JAVASCRIPT: {
        Status ret = console.runJS(query.toString());
        rc = ret.isSuccess() ? 0 : 1;
        goto exit;
      }
    }
  }

  if (flags.isSet("exec")) {
    auto ret = console.runQuery(flags.getString("exec"));
    rc = ret.isSuccess() ? 0 : 1;
    goto exit;
  }

  if (hasSTDIN() || flags.isSet("batch") || !stdout_os->isTTY()) {
    String query;
    while (stdin_is->readLine(&query)) {
      auto ret = console.runQuery(query);
      query = "";
      if (ret.isError()) {
        rc = 1;
        goto exit;
      }
    }
  } else {
    console.startInteractiveShell();
  }

exit:
  console.close();
  return rc;
}

<|endoftext|>


<commit_before>// Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.

#include <vespa/eval/tensor/dense/typed_cells.h>
#include <vespa/searchlib/common/bitvector.h>
#include <vespa/searchlib/tensor/distance_functions.h>
#include <vespa/searchlib/tensor/doc_vector_access.h>
#include <vespa/searchlib/tensor/hnsw_index.h>
#include <vespa/searchlib/tensor/random_level_generator.h>
#include <vespa/searchlib/tensor/inv_log_level_generator.h>
#include <vespa/vespalib/gtest/gtest.h>
#include <vespa/vespalib/util/generationhandler.h>
#include <vespa/vespalib/data/slime/slime.h>
#include <vespa/vespalib/util/threadstackexecutor.h>
#include <vespa/vespalib/util/blockingthreadstackexecutor.h>
#include <vespa/vespalib/util/lambdatask.h>

#include <vector>
#include <sys/types.h>
#include <sys/stat.h>
#include <fcntl.h>
#include <unistd.h>
#include <stdio.h>
#include <chrono>
#include <cstdlib>

#include <vespa/log/log.h>
LOG_SETUP("stress_hnsw_mt");

using vespalib::GenerationHandler;
using vespalib::MemoryUsage;
using namespace search::tensor;
using namespace vespalib::slime;
using vespalib::Slime;
using search::BitVector;

#define NUM_DIMS 128
#define NUM_POSSIBLE_V 100000
#define NUM_POSSIBLE_DOCS 10000
#define NUM_OPS 50000

class RndGen {
private:
    std::mt19937_64 urng;
    std::uniform_real_distribution<double> uf;
public:
    RndGen() : urng(0x1234deadbeef5678uLL), uf(0.0, 1.0) {}

    double nextUniform() {
        return uf(urng);
    }
};

using ConstVectorRef = vespalib::ConstArrayRef<float>;

struct MallocPointVector {
    float v[NUM_DIMS];
    operator ConstVectorRef() const { return ConstVectorRef(v, NUM_DIMS); }
};
static MallocPointVector *aligned_alloc_pv(size_t num) {
    size_t num_bytes = num * sizeof(MallocPointVector);
    double mega_bytes = num_bytes / (1024.0*1024.0);
    fprintf(stderr, "allocate %.2f MB of vectors\n", mega_bytes);
    char *mem = (char *)malloc(num_bytes + 512);
    mem += 512;
    size_t val = (size_t)mem;
    size_t unalign = val % 512;
    mem -= unalign;
    return reinterpret_cast<MallocPointVector *>(mem);
}

void read_vector_file(MallocPointVector *p) {
    std::string data_set = "sift";
    std::string data_dir = ".";
    char *home = getenv("HOME");
    if (home) {
        data_dir = home;
        data_dir += "/" + data_set;
    }
    std::string fn = data_dir + "/" + data_set + "_base.fvecs";
    int fd = open(fn.c_str(), O_RDONLY);
    if (fd < 0) {
        perror(fn.c_str());
        exit(1);
    }
    int d;
    size_t rv;
    fprintf(stderr, "reading %u vectors from %s\n", NUM_POSSIBLE_V, fn.c_str());
    for (uint32_t i = 0; i < NUM_POSSIBLE_V; ++i) {
        rv = read(fd, &d, 4);
        ASSERT_EQ(rv, 4u);
        ASSERT_EQ(d, NUM_DIMS);
        rv = read(fd, &p[i].v, NUM_DIMS*sizeof(float));
        ASSERT_EQ(rv, sizeof(MallocPointVector));
    }
    close(fd);
    fprintf(stderr, "reading %u vectors OK\n", NUM_POSSIBLE_V);
}

class MyDocVectorStore : public DocVectorAccess {
private:
    MallocPointVector *_vectors;
public:
    MyDocVectorStore() {
        _vectors = aligned_alloc_pv(NUM_POSSIBLE_DOCS);
    }
    MyDocVectorStore& set(uint32_t docid, ConstVectorRef vec) {
        assert(docid < NUM_POSSIBLE_DOCS);
        memcpy(&_vectors[docid], vec.cbegin(), sizeof(MallocPointVector));
        return *this;
    }
    vespalib::tensor::TypedCells get_vector(uint32_t docid) const override {
        assert(docid < NUM_POSSIBLE_DOCS);
        ConstVectorRef ref(_vectors[docid]);
        return vespalib::tensor::TypedCells(ref);
    }
};

using FloatSqEuclideanDistance = SquaredEuclideanDistance<float>;
using HnswIndexUP = std::unique_ptr<HnswIndex>;

class Stressor : public ::testing::Test {
private:
    struct LoadedVectors {
        MallocPointVector *pv_storage;
        void load() {
            pv_storage = aligned_alloc_pv(size());
            read_vector_file(pv_storage);
        }
        size_t size() const { return NUM_POSSIBLE_V; }
        vespalib::ConstArrayRef<float> operator[] (size_t i) {
            return pv_storage[i];
        }
    } loaded_vectors;
public:
    BitVector::UP in_progress;
    BitVector::UP existing_ids;
    RndGen rng;
    MyDocVectorStore vectors;
    GenerationHandler gen_handler;
    HnswIndexUP index;
    vespalib::BlockingThreadStackExecutor multi_prepare_workers;
    vespalib::BlockingThreadStackExecutor write_thread;

    using PrepUP = std::unique_ptr<PrepareResult>;
    using ReadGuard = GenerationHandler::Guard;

    // union of data required by tasks
    struct TaskBase : vespalib::Executor::Task {
        Stressor &parent;
        uint32_t docid;
        ConstVectorRef vec;
        PrepUP prepare_result;
        ReadGuard read_guard;

        TaskBase(Stressor &p, uint32_t d, ConstVectorRef v, PrepUP r, ReadGuard g)
            : parent(p), docid(d), vec(v), prepare_result(std::move(r)), read_guard(g)
        {}
        TaskBase(Stressor &p, uint32_t d, ConstVectorRef v, ReadGuard g)
            : TaskBase(p, d, v, PrepUP(), g) {}
        TaskBase(Stressor &p, uint32_t d, ReadGuard g)
            : TaskBase(p, d, ConstVectorRef(), PrepUP(), g) {}
        TaskBase(Stressor &p, uint32_t d, ConstVectorRef v, PrepUP r)
            : TaskBase(p, d, v, std::move(r), ReadGuard()) {}
        TaskBase(Stressor &p, uint32_t d)
            : TaskBase(p, d, ConstVectorRef(), PrepUP(), ReadGuard()) {}
    };

    struct CompleteAddTask : TaskBase {
        using TaskBase::TaskBase;
        void run() override {
            parent.vectors.set(docid, vec);
            parent.index->complete_add_document(docid, std::move(prepare_result));
            parent.existing_ids->setBit(docid);
            parent.commit(docid);
        }
    };

    struct TwoPhaseAddTask  : TaskBase {
        using TaskBase::TaskBase;
        void run() override {
            auto v = vespalib::tensor::TypedCells(vec);
            auto up = parent.index->prepare_add_document(docid, v, read_guard);
            auto task = std::make_unique<CompleteAddTask>(parent, docid, vec, std::move(up));
            auto r = parent.write_thread.execute(std::move(task));
            if (r) {
                fprintf(stderr, "Failed posting complete add task!");
                abort();
            }
        }
    };

    struct CompleteRemoveTask : TaskBase {
        using TaskBase::TaskBase;
        void run() override {
            parent.index->remove_document(docid);
            parent.existing_ids->clearBit(docid);
            parent.commit(docid);
        }
    };

    struct TwoPhaseRemoveTask : TaskBase {
        using TaskBase::TaskBase;
        void run() override {
            auto task = std::make_unique<CompleteRemoveTask>(parent, docid);
            auto r = parent.write_thread.execute(std::move(task));
            if (r) {
                fprintf(stderr, "Failed posting complete remove task!");
                abort();
            }
        }
    };

    struct CompleteUpdateTask : TaskBase {
        using TaskBase::TaskBase;
        void run() override {
            parent.index->remove_document(docid);
            parent.vectors.set(docid, vec);
            parent.index->complete_add_document(docid, std::move(prepare_result));
            EXPECT_EQ(parent.existing_ids->testBit(docid), true);
            parent.commit(docid);
        }
    };

    struct TwoPhaseUpdateTask : TaskBase {
        using TaskBase::TaskBase;
        void run() override {
            auto v = vespalib::tensor::TypedCells(vec);
            auto up = parent.index->prepare_add_document(docid, v, read_guard);
            EXPECT_EQ(bool(up), true);
            auto task = std::make_unique<CompleteUpdateTask>(parent, docid, vec, std::move(up));
            auto r = parent.write_thread.execute(std::move(task));
            if (r) {
                fprintf(stderr, "Failed posting complete remove task!");
                abort();
            }
        }
    };

    Stressor()
        : loaded_vectors(),
          in_progress(BitVector::create(NUM_POSSIBLE_DOCS)),
          existing_ids(BitVector::create(NUM_POSSIBLE_DOCS)),
          rng(),
          vectors(),
          gen_handler(),
          index(),
          multi_prepare_workers(10, 128*1024, 50),
          write_thread(1, 128*1024, 15)
    {
        loaded_vectors.load();
    }

    ~Stressor() {}

    void init() {
        uint32_t m = 16;
        index = std::make_unique<HnswIndex>(vectors, std::make_unique<FloatSqEuclideanDistance>(),
                                            std::make_unique<InvLogLevelGenerator>(m),
                                            HnswIndex::Config(2*m, m, 200, true));
    }
    size_t get_rnd(size_t size) {
        return rng.nextUniform() * size;
    }
    void add_document(uint32_t docid) {
        size_t vec_num = get_rnd(loaded_vectors.size());
        ConstVectorRef vec = loaded_vectors[vec_num];
        auto guard = take_read_guard();
        auto task = std::make_unique<TwoPhaseAddTask>(*this, docid, vec, guard);
        auto r = multi_prepare_workers.execute(std::move(task));
        ASSERT_EQ(bool(r), false);
    }
    void remove_document(uint32_t docid) {
        auto guard = take_read_guard();
        auto task = std::make_unique<TwoPhaseRemoveTask>(*this, docid, guard);
        auto r = multi_prepare_workers.execute(std::move(task));
        ASSERT_EQ(bool(r), false);
    }
    void update_document(uint32_t docid) {
        size_t vec_num = get_rnd(loaded_vectors.size());
        ConstVectorRef vec = loaded_vectors[vec_num];
        auto guard = take_read_guard();
        auto task = std::make_unique<TwoPhaseUpdateTask>(*this, docid, vec, guard);
        auto r = multi_prepare_workers.execute(std::move(task));
        ASSERT_EQ(bool(r), false);
    }
    void commit(uint32_t docid) {
        index->transfer_hold_lists(gen_handler.getCurrentGeneration());
        gen_handler.incGeneration();
        gen_handler.updateFirstUsedGeneration();
        index->trim_hold_lists(gen_handler.getFirstUsedGeneration());
        in_progress->clearBit(docid);
        // printf("commit: %u\n", docid);
    }
    void gen_operation() {
        do {
            uint32_t docid = get_rnd(NUM_POSSIBLE_DOCS);
            if (in_progress->testBit(docid)) continue;
            in_progress->setBit(docid);
            if (existing_ids->testBit(docid)) {
                if (get_rnd(100) < 70) {
                    // printf("start remove op: %u\n", docid);
                    remove_document(docid);
                } else {
                    // printf("start update op: %u\n", docid);
                    update_document(docid);
                }
            } else {
                // printf("start add op: %u\n", docid);
                add_document(docid);
            }
            return;
        } while (true);
    }
    GenerationHandler::Guard take_read_guard() {
        return gen_handler.takeGuard();
    }
    MemoryUsage memory_usage() const {
        return index->memory_usage();
    }
};

TEST_F(Stressor, stress)
{
    init();
    for (int i = 0; i < NUM_OPS; ++i) {
        gen_operation();
        if (i % 1000 == 0) {
            fprintf(stderr, "generating operations %d / %d\n", i, NUM_OPS);
            auto r = write_thread.execute(vespalib::makeLambdaTask([&]() {
                        EXPECT_TRUE(index->check_link_symmetry());
            }));
            EXPECT_EQ(bool(r), false);
        }
    }
    fprintf(stderr, "waiting for queued operations...\n");
    multi_prepare_workers.sync();
    write_thread.sync();
    in_progress->invalidateCachedCount();
    EXPECT_EQ(in_progress->countTrueBits(), 0);
    EXPECT_TRUE(index->check_link_symmetry());
    fprintf(stderr, "all done.\n");
}

GTEST_MAIN_RUN_ALL_TESTS()
<commit_msg>minor stress-test tweaks:<commit_after>// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.

#include <vespa/eval/tensor/dense/typed_cells.h>
#include <vespa/searchlib/common/bitvector.h>
#include <vespa/searchlib/tensor/distance_functions.h>
#include <vespa/searchlib/tensor/doc_vector_access.h>
#include <vespa/searchlib/tensor/hnsw_index.h>
#include <vespa/searchlib/tensor/random_level_generator.h>
#include <vespa/searchlib/tensor/inv_log_level_generator.h>
#include <vespa/vespalib/gtest/gtest.h>
#include <vespa/vespalib/util/generationhandler.h>
#include <vespa/vespalib/util/blockingthreadstackexecutor.h>
#include <vespa/vespalib/util/lambdatask.h>

#include <vector>
#include <sys/types.h>
#include <sys/stat.h>
#include <fcntl.h>
#include <unistd.h>
#include <stdio.h>
#include <chrono>
#include <cstdlib>

#include <vespa/log/log.h>
LOG_SETUP("stress_hnsw_mt");

using vespalib::GenerationHandler;
using vespalib::MemoryUsage;
using namespace search::tensor;
using search::BitVector;

#define NUM_DIMS 128
#define NUM_POSSIBLE_V 1000000
#define NUM_POSSIBLE_DOCS 30000
#define NUM_OPS 1000000

class RndGen {
private:
    std::mt19937_64 urng;
    std::uniform_real_distribution<double> uf;
public:
    RndGen() : urng(0x1234deadbeef5678uLL), uf(0.0, 1.0) {}

    double nextUniform() {
        return uf(urng);
    }
};

using ConstVectorRef = vespalib::ConstArrayRef<float>;

struct MallocPointVector {
    float v[NUM_DIMS];
    operator ConstVectorRef() const { return ConstVectorRef(v, NUM_DIMS); }
};
static MallocPointVector *aligned_alloc_pv(size_t num) {
    size_t num_bytes = num * sizeof(MallocPointVector);
    double mega_bytes = num_bytes / (1024.0*1024.0);
    fprintf(stderr, "allocate %.2f MB of vectors\n", mega_bytes);
    char *mem = (char *)malloc(num_bytes + 512);
    mem += 512;
    size_t val = (size_t)mem;
    size_t unalign = val % 512;
    mem -= unalign;
    return reinterpret_cast<MallocPointVector *>(mem);
}

void read_vector_file(MallocPointVector *p) {
    std::string data_set = "sift";
    std::string data_dir = ".";
    char *home = getenv("HOME");
    if (home) {
        data_dir = home;
        data_dir += "/" + data_set;
    }
    std::string fn = data_dir + "/" + data_set + "_base.fvecs";
    int fd = open(fn.c_str(), O_RDONLY);
    if (fd < 0) {
        perror(fn.c_str());
        exit(1);
    }
    int d;
    size_t rv;
    fprintf(stderr, "reading %u vectors from %s\n", NUM_POSSIBLE_V, fn.c_str());
    for (uint32_t i = 0; i < NUM_POSSIBLE_V; ++i) {
        rv = read(fd, &d, 4);
        ASSERT_EQ(rv, 4u);
        ASSERT_EQ(d, NUM_DIMS);
        rv = read(fd, &p[i].v, NUM_DIMS*sizeof(float));
        ASSERT_EQ(rv, sizeof(MallocPointVector));
    }
    close(fd);
    fprintf(stderr, "reading %u vectors OK\n", NUM_POSSIBLE_V);
}

class MyDocVectorStore : public DocVectorAccess {
private:
    MallocPointVector *_vectors;
public:
    MyDocVectorStore() {
        _vectors = aligned_alloc_pv(NUM_POSSIBLE_DOCS);
    }
    MyDocVectorStore& set(uint32_t docid, ConstVectorRef vec) {
        assert(docid < NUM_POSSIBLE_DOCS);
        memcpy(&_vectors[docid], vec.cbegin(), sizeof(MallocPointVector));
        return *this;
    }
    vespalib::tensor::TypedCells get_vector(uint32_t docid) const override {
        assert(docid < NUM_POSSIBLE_DOCS);
        ConstVectorRef ref(_vectors[docid]);
        return vespalib::tensor::TypedCells(ref);
    }
};

using FloatSqEuclideanDistance = SquaredEuclideanDistance<float>;
using HnswIndexUP = std::unique_ptr<HnswIndex>;

class Stressor : public ::testing::Test {
private:
    struct LoadedVectors {
        MallocPointVector *pv_storage;
        void load() {
            pv_storage = aligned_alloc_pv(size());
            read_vector_file(pv_storage);
        }
        size_t size() const { return NUM_POSSIBLE_V; }
        vespalib::ConstArrayRef<float> operator[] (size_t i) {
            return pv_storage[i];
        }
    } loaded_vectors;
public:
    BitVector::UP in_progress;
    BitVector::UP existing_ids;
    RndGen rng;
    MyDocVectorStore vectors;
    GenerationHandler gen_handler;
    HnswIndexUP index;
    vespalib::BlockingThreadStackExecutor multi_prepare_workers;
    vespalib::BlockingThreadStackExecutor write_thread;

    using PrepUP = std::unique_ptr<PrepareResult>;
    using ReadGuard = GenerationHandler::Guard;

    // union of data required by tasks
    struct TaskBase : vespalib::Executor::Task {
        Stressor &parent;
        uint32_t docid;
        ConstVectorRef vec;
        PrepUP prepare_result;
        ReadGuard read_guard;

        TaskBase(Stressor &p, uint32_t d, ConstVectorRef v, PrepUP r, ReadGuard g)
            : parent(p), docid(d), vec(v), prepare_result(std::move(r)), read_guard(g)
        {}
        TaskBase(Stressor &p, uint32_t d, ConstVectorRef v, ReadGuard g)
            : TaskBase(p, d, v, PrepUP(), g) {}
        TaskBase(Stressor &p, uint32_t d, ReadGuard g)
            : TaskBase(p, d, ConstVectorRef(), PrepUP(), g) {}
        TaskBase(Stressor &p, uint32_t d, ConstVectorRef v, PrepUP r)
            : TaskBase(p, d, v, std::move(r), ReadGuard()) {}
        TaskBase(Stressor &p, uint32_t d)
            : TaskBase(p, d, ConstVectorRef(), PrepUP(), ReadGuard()) {}
    };

    struct CompleteAddTask : TaskBase {
        using TaskBase::TaskBase;
        void run() override {
            parent.vectors.set(docid, vec);
            parent.index->complete_add_document(docid, std::move(prepare_result));
            parent.existing_ids->setBit(docid);
            parent.commit(docid);
        }
    };

    struct TwoPhaseAddTask  : TaskBase {
        using TaskBase::TaskBase;
        void run() override {
            auto v = vespalib::tensor::TypedCells(vec);
            auto up = parent.index->prepare_add_document(docid, v, read_guard);
            auto task = std::make_unique<CompleteAddTask>(parent, docid, vec, std::move(up));
            auto r = parent.write_thread.execute(std::move(task));
            if (r) {
                fprintf(stderr, "Failed posting complete add task!");
                abort();
            }
        }
    };

    struct CompleteRemoveTask : TaskBase {
        using TaskBase::TaskBase;
        void run() override {
            parent.index->remove_document(docid);
            parent.existing_ids->clearBit(docid);
            parent.commit(docid);
        }
    };

    struct TwoPhaseRemoveTask : TaskBase {
        using TaskBase::TaskBase;
        void run() override {
            auto task = std::make_unique<CompleteRemoveTask>(parent, docid);
            auto r = parent.write_thread.execute(std::move(task));
            if (r) {
                fprintf(stderr, "Failed posting complete remove task!");
                abort();
            }
        }
    };

    struct CompleteUpdateTask : TaskBase {
        using TaskBase::TaskBase;
        void run() override {
            parent.index->remove_document(docid);
            parent.vectors.set(docid, vec);
            parent.index->complete_add_document(docid, std::move(prepare_result));
            EXPECT_EQ(parent.existing_ids->testBit(docid), true);
            parent.commit(docid);
        }
    };

    struct TwoPhaseUpdateTask : TaskBase {
        using TaskBase::TaskBase;
        void run() override {
            auto v = vespalib::tensor::TypedCells(vec);
            auto up = parent.index->prepare_add_document(docid, v, read_guard);
            EXPECT_EQ(bool(up), true);
            auto task = std::make_unique<CompleteUpdateTask>(parent, docid, vec, std::move(up));
            auto r = parent.write_thread.execute(std::move(task));
            if (r) {
                fprintf(stderr, "Failed posting complete remove task!");
                abort();
            }
        }
    };

    Stressor()
        : loaded_vectors(),
          in_progress(BitVector::create(NUM_POSSIBLE_DOCS)),
          existing_ids(BitVector::create(NUM_POSSIBLE_DOCS)),
          rng(),
          vectors(),
          gen_handler(),
          index(),
          multi_prepare_workers(10, 128*1024, 50),
          write_thread(1, 128*1024, 15)
    {
        loaded_vectors.load();
    }

    ~Stressor() {}

    void init() {
        uint32_t m = 16;
        index = std::make_unique<HnswIndex>(vectors, std::make_unique<FloatSqEuclideanDistance>(),
                                            std::make_unique<InvLogLevelGenerator>(m),
                                            HnswIndex::Config(2*m, m, 200, true));
    }
    size_t get_rnd(size_t size) {
        return rng.nextUniform() * size;
    }
    void add_document(uint32_t docid) {
        size_t vec_num = get_rnd(loaded_vectors.size());
        ConstVectorRef vec = loaded_vectors[vec_num];
        auto guard = take_read_guard();
        auto task = std::make_unique<TwoPhaseAddTask>(*this, docid, vec, guard);
        auto r = multi_prepare_workers.execute(std::move(task));
        ASSERT_EQ(bool(r), false);
    }
    void remove_document(uint32_t docid) {
        auto guard = take_read_guard();
        auto task = std::make_unique<TwoPhaseRemoveTask>(*this, docid, guard);
        auto r = multi_prepare_workers.execute(std::move(task));
        ASSERT_EQ(bool(r), false);
    }
    void update_document(uint32_t docid) {
        size_t vec_num = get_rnd(loaded_vectors.size());
        ConstVectorRef vec = loaded_vectors[vec_num];
        auto guard = take_read_guard();
        auto task = std::make_unique<TwoPhaseUpdateTask>(*this, docid, vec, guard);
        auto r = multi_prepare_workers.execute(std::move(task));
        ASSERT_EQ(bool(r), false);
    }
    void commit(uint32_t docid) {
        index->transfer_hold_lists(gen_handler.getCurrentGeneration());
        gen_handler.incGeneration();
        gen_handler.updateFirstUsedGeneration();
        index->trim_hold_lists(gen_handler.getFirstUsedGeneration());
        in_progress->clearBit(docid);
        // printf("commit: %u\n", docid);
    }
    void gen_operation() {
        do {
            uint32_t docid = get_rnd(NUM_POSSIBLE_DOCS);
            if (in_progress->testBit(docid)) continue;
            in_progress->setBit(docid);
            if (existing_ids->testBit(docid)) {
                if (get_rnd(100) < 70) {
                    // printf("start remove op: %u\n", docid);
                    remove_document(docid);
                } else {
                    // printf("start update op: %u\n", docid);
                    update_document(docid);
                }
            } else {
                // printf("start add op: %u\n", docid);
                add_document(docid);
            }
            return;
        } while (true);
    }
    GenerationHandler::Guard take_read_guard() {
        return gen_handler.takeGuard();
    }
    MemoryUsage memory_usage() const {
        return index->memory_usage();
    }
};

TEST_F(Stressor, stress)
{
    init();
    for (int i = 0; i < NUM_OPS; ++i) {
        gen_operation();
        if (i % 1000 == 0) {
            fprintf(stderr, "generating operations %d / %d\n", i, NUM_OPS);
            auto r = write_thread.execute(vespalib::makeLambdaTask([&]() {
                        EXPECT_TRUE(index->check_link_symmetry());
            }));
            EXPECT_EQ(bool(r), false);
        }
    }
    fprintf(stderr, "waiting for queued operations...\n");
    multi_prepare_workers.sync();
    write_thread.sync();
    in_progress->invalidateCachedCount();
    EXPECT_EQ(in_progress->countTrueBits(), 0);
    EXPECT_TRUE(index->check_link_symmetry());
    fprintf(stderr, "all done.\n");
}

GTEST_MAIN_RUN_ALL_TESTS()
<|endoftext|>


<commit_before>/******************************************************************************
 * Copyright 2017 The Apollo Authors. All Rights Reserved.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 * http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 *****************************************************************************/

#include "modules/common/util/down_sampler.h"

#include "modules/common/math/vec2d.h"

#include <vector>

#include "gtest/gtest.h"

namespace apollo {
namespace common {
namespace util {

TEST(DownSamplerTest, DownSampleByAngle) {
    using ::apollo::common::math::Vec2d;

    std::vector<Vec2d> points;
    points.emplace_back(-405.778, -134.969);
    points.emplace_back(-403.919, -127.696);
    points.emplace_back(-400.635, -115.407);
    points.emplace_back(-397.997, -105.291);
    points.emplace_back(-395.801, -96.8637);
    points.emplace_back(-392.889, -86.1015);
    points.emplace_back(-388.054, -67.9935);
    points.emplace_back(-385.994, -60.1831);
    points.emplace_back(-378.213, -30.2776);
    points.emplace_back(-376.702, -24.5804);
    points.emplace_back(-373.825, -13.3855);
    points.emplace_back(-367.583, 10.4028);
    points.emplace_back(-363.025, 27.4212);

    std::vector<int> sampled_indices;

    EXPECT_TRUE(DownSampleByAngle(points, 0.1, &sampled_indices));
    EXPECT_EQ(2, sampled_indices.size());
    EXPECT_EQ(0, sampled_indices[0]);
    EXPECT_EQ(12, sampled_indices[1]);
}

}  // namespace util
}  // namespace common
}  // namespace apollo

<commit_msg>fix unit test points_downsampler_test compile issue<commit_after>/******************************************************************************
 * Copyright 2017 The Apollo Authors. All Rights Reserved.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 * http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 *****************************************************************************/

#include "modules/common/util/points_downsampler.h"

#include "modules/common/math/vec2d.h"

#include <vector>

#include "gtest/gtest.h"

namespace apollo {
namespace common {
namespace util {

TEST(DownSamplerTest, DownSampleByAngle) {
  using ::apollo::common::math::Vec2d;

  std::vector<Vec2d> points;
  points.emplace_back(-405.778, -134.969);
  points.emplace_back(-403.919, -127.696);
  points.emplace_back(-400.635, -115.407);
  points.emplace_back(-397.997, -105.291);
  points.emplace_back(-395.801, -96.8637);
  points.emplace_back(-392.889, -86.1015);
  points.emplace_back(-388.054, -67.9935);
  points.emplace_back(-385.994, -60.1831);
  points.emplace_back(-378.213, -30.2776);
  points.emplace_back(-376.702, -24.5804);
  points.emplace_back(-373.825, -13.3855);
  points.emplace_back(-367.583, 10.4028);
  points.emplace_back(-363.025, 27.4212);

  std::vector<int> sampled_indices;

  EXPECT_TRUE(DownSampleByAngle(points, 0.1, &sampled_indices));
  EXPECT_EQ(2, sampled_indices.size());
  EXPECT_EQ(0, sampled_indices[0]);
  EXPECT_EQ(12, sampled_indices[1]);
}

}  // namespace util
}  // namespace common
}  // namespace apollo
<|endoftext|>


<commit_before>/******************************************************************************
 * This file is part of the Mula project
 * Copyright (c) 2011 Laszlo Papp <lpapp@kde.org>
 *
 * This library is free software; you can redistribute it and/or
 * modify it under the terms of the GNU Lesser General Public
 * License as published by the Free Software Foundation; either
 * version 2.1 of the License, or (at your option) any later version.
 *
 * This library is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
 * Lesser General Public License for more details.
 *
 * You should have received a copy of the GNU Lesser General Public
 * License along with this library; if not, write to the Free Software
 * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301  USA
 */

#include "dictionarycache.h"

using namespace MulaPluginStarDict;

class DictionaryCache::Private
{
    public:
        Private()
            : chunk(0)
            , byteArray(0)
            , stamp(0)
            , count(0)
        {
        }

        ~Private()
        {
        }

        int chunk;
        QByteArray byteArray;
        int stamp;
        int count;
};

DictionaryCache::DictionaryCache()
    : d( new Private )
{
}

DictionaryCache::DictionaryCache(int chunk, QByteArray byteArray, int stamp, int count)
    : d( new Private )
{
    d->chunk = chunk;
    d->byteArray = byteArray;
    d->stamp = stamp;
    d->count = count;
}

DictionaryCache::~DictionaryCache()
{
}

void
DictionaryCache::setChunk(int chunk)
{
    d->chunk = chunk;
}

int
DictionaryCache::chunk() const
{
    return d->chunk;
}

void
DictionaryCache::setByteArray(QByteArray byteArray)
{
    d->byteArray = byteArray;
}

QByteArray&
DictionaryCache::byteArray() const
{
    return d->byteArray;
}

void setStamp(int stamp)
{
}

int
DictionaryCache::stamp() const
{
    return d->stamp;
}

void
DictionaryCache::setCount(int count)
{
    d->count = count;
}

int
DictionaryCache::count() const
{
    return d->count;
}

<commit_msg>Fix the stamp setting getter since the method body was empty<commit_after>/******************************************************************************
 * This file is part of the Mula project
 * Copyright (c) 2011 Laszlo Papp <lpapp@kde.org>
 *
 * This library is free software; you can redistribute it and/or
 * modify it under the terms of the GNU Lesser General Public
 * License as published by the Free Software Foundation; either
 * version 2.1 of the License, or (at your option) any later version.
 *
 * This library is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
 * Lesser General Public License for more details.
 *
 * You should have received a copy of the GNU Lesser General Public
 * License along with this library; if not, write to the Free Software
 * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301  USA
 */

#include "dictionarycache.h"

using namespace MulaPluginStarDict;

class DictionaryCache::Private
{
    public:
        Private()
            : chunk(0)
            , byteArray(0)
            , stamp(0)
            , count(0)
        {
        }

        ~Private()
        {
        }

        int chunk;
        QByteArray byteArray;
        int stamp;
        int count;
};

DictionaryCache::DictionaryCache()
    : d( new Private )
{
}

DictionaryCache::DictionaryCache(int chunk, QByteArray byteArray, int stamp, int count)
    : d( new Private )
{
    d->chunk = chunk;
    d->byteArray = byteArray;
    d->stamp = stamp;
    d->count = count;
}

DictionaryCache::~DictionaryCache()
{
}

void
DictionaryCache::setChunk(int chunk)
{
    d->chunk = chunk;
}

int
DictionaryCache::chunk() const
{
    return d->chunk;
}

void
DictionaryCache::setByteArray(QByteArray byteArray)
{
    d->byteArray = byteArray;
}

QByteArray&
DictionaryCache::byteArray() const
{
    return d->byteArray;
}

void
DictionaryCache::setStamp(int stamp)
{
    d->stamp = stamp;
}

int
DictionaryCache::stamp() const
{
    return d->stamp;
}

void
DictionaryCache::setCount(int count)
{
    d->count = count;
}

int
DictionaryCache::count() const
{
    return d->count;
}

<|endoftext|>


<commit_before>#include <nix/file/FileFS.hpp>
#include <nix/hdf5/BlockHDF5.hpp>
#include <nix/hdf5/SectionHDF5.hpp>
#include <nix/util/util.hpp>

using namespace std;
using namespace nix;

namespace nix {
namespace file {

    // Format definition
#define FILE_VERSION std::vector<int>{1, 0, 0}
#define FILE_FORMAT  std::string("nix")

    static unsigned int map_file_mode(FileMode mode) {
        switch (mode) {
            case FileMode::ReadWrite:
                return ios::in | ios::out;

            case FileMode::ReadOnly:
                return ios::out;

            case FileMode::Overwrite:
                return ios::trunc;

            default:
                return ios::out;
        }
    }

    FileFS::FileFS(const string &name, FileMode mode) {
        boost::filesystem::path p(name.c_str());
        if (!boost::filesystem::exists(p)) {
            mode = FileMode::Overwrite;
        }

        this->file_mode = map_file_mode(mode);
        this->root_path = p;
        this->open_or_create();
        setCreatedAt();
        setUpdatedAt();

        if (!checkHeader()) {
            throw std::runtime_error("Invalid file header: either file format or file version not correct");
        }
    }

    void FileFS::open_or_create() {
        //TODO respect the file mode
        boost::filesystem::path data("data");
        boost::filesystem::path metadata("metadata");

        this->data_path = this->root_path / data;
        this->metadata_path = this->root_path / metadata;
        if (!boost::filesystem::exists(this->root_path)) {
            boost::filesystem::create_directory(this->root_path);
            boost::filesystem::create_directories(this->data_path);
            boost::filesystem::create_directories(this->metadata_path);
        } else {
            if (!boost::filesystem::exists(this->data_path)) {
                boost::filesystem::create_directories(this->data_path);
            }
            if (!boost::filesystem::exists(this->metadata_path)) {
                boost::filesystem::create_directories(this->metadata_path);
            }
        }
        this->attributes = AttributesFS(this->location());
    }

    ndsize_t FileFS::blockCount() const {
        ndsize_t count = 0;
        boost::filesystem::directory_iterator end;
        if (boost::filesystem::exists(this->data_path) && boost::filesystem::is_directory(this->data_path)) {
            boost::filesystem::directory_iterator di(this->data_path);
            while (di != end) {
                count ++;
            }
        }
        return count;
    }

    bool FileFS::hasBlock(const std::string &name_or_id) const  {
        return false;
    }

    std::shared_ptr<base::IBlock> FileFS::getBlock(const std::string &name_or_id) const {
        shared_ptr<nix::hdf5::BlockHDF5> block;
        /*
                    boost::optional<Group> group = data.findGroupByNameOrAttribute("entity_id", name_or_id);
                    if (group)
                        block = make_shared<BlockHDF5>(file(), *group);
        */
        return block;
    }

    std::shared_ptr<base::IBlock> FileFS::getBlock(ndsize_t index) const {
        shared_ptr<nix::hdf5::BlockHDF5> block;
        return block;
    }

    std::shared_ptr<base::IBlock> FileFS::createBlock(const std::string &name, const std::string &type) {
        shared_ptr<nix::hdf5::BlockHDF5> block;
        return block;
    }

    bool FileFS::deleteBlock(const std::string &name_or_id) {
        return false;
    }

    //--------------------------------------------------
    // Methods concerning sections
    //--------------------------------------------------

    bool FileFS::hasSection(const std::string &name_or_id) const {
        return false;
    }

    std::shared_ptr<base::ISection> FileFS::getSection(const std::string &name_or_id) const {
        shared_ptr<nix::hdf5::SectionHDF5> block;
        return block;
    }

    std::shared_ptr<base::ISection> FileFS::getSection(ndsize_t index) const {
        shared_ptr<nix::hdf5::SectionHDF5> block;
        return block;
    }

    ndsize_t FileFS::sectionCount() const {
        ndsize_t count = 0;
        boost::filesystem::directory_iterator end;
        if (boost::filesystem::exists(this->metadata_path) && boost::filesystem::is_directory(this->metadata_path)) {
            boost::filesystem::directory_iterator di(this->metadata_path);
            while (di != end) {
                count ++;
            }
        }
        return count;
    }

    std::shared_ptr<base::ISection> FileFS::createSection(const std::string &name, const std::string &type) {
        shared_ptr<hdf5::SectionHDF5> block;
        return block;
    }

    bool FileFS::deleteSection(const std::string &name_or_id) {
        return  false;
    }

    //--------------------------------------------------
    // Methods for file attribute access.
    //--------------------------------------------------

    std::vector<int> FileFS::version() const {
        vector<int> version;
        return  version;
    }

    std::string FileFS::format() const {
        return "nix";
    }

    std::string FileFS::location() const {
        return boost::filesystem::absolute(this->root_path).string();
    }

    time_t FileFS::createdAt() const {
        return util::strToTime("00:00.00");
    }

    time_t FileFS::updatedAt() const {
        return util::strToTime("00:00.00");
    }

    void FileFS::setUpdatedAt(){
        if (!this->attributes.has("updated_at")) {
            time_t t = time(NULL);
            this->attributes.set("updated_at", util::timeToStr(t));
        }
    }

    void FileFS::forceUpdatedAt() {
        time_t t = time(NULL);
        this->attributes.set("updated_at", util::timeToStr(t));
    }

    void FileFS::setCreatedAt() {
        if (!this->attributes.has("created_at")) {
            time_t t = time(NULL);
            this->attributes.set("created_at", util::timeToStr(t));
        }
    }

    void FileFS::forceCreatedAt(time_t t) {
        this->attributes.set("created_at", util::timeToStr(t));
    }

    void FileFS::close() {}

    bool FileFS::isOpen() const {
        return false;
    }

    bool FileFS::operator==(const FileFS &other) const {
        return false;
    }

    bool FileFS::operator!=(const FileFS &other) const {
        return false;
    }

    FileFS::~FileFS() {

    }

bool FileFS::checkHeader() {
    bool check = true;
    vector<int> version;
    string str;
    // check format
    if (this->attributes.has("format")) {
        this->attributes.get("format", str);
        if (str != FILE_FORMAT) {
            check = false;
        }
    } else {
        this->attributes.set("format", FILE_FORMAT);
    }
    // check version
    if (this->attributes.has("version")) {
        this->attributes.get("version", version);
        if (version != FILE_VERSION) {
            check = false;
        }
    } else {
        this->attributes.set("version", FILE_VERSION);
    }
    return check;
}

} // namespace file
} // namespace nix<commit_msg>[file backend/FileFS] indentation<commit_after>#include <nix/file/FileFS.hpp>
#include <nix/hdf5/BlockHDF5.hpp>
#include <nix/hdf5/SectionHDF5.hpp>
#include <nix/util/util.hpp>

using namespace std;
using namespace nix;

namespace nix {
namespace file {

    // Format definition
#define FILE_VERSION std::vector<int>{1, 0, 0}
#define FILE_FORMAT  std::string("nix")

static unsigned int map_file_mode(FileMode mode) {
    switch (mode) {
        case FileMode::ReadWrite:
            return ios::in | ios::out;

        case FileMode::ReadOnly:
            return ios::out;

        case FileMode::Overwrite:
            return ios::trunc;

        default:
            return ios::out;
    }
}

FileFS::FileFS(const string &name, FileMode mode) {
    boost::filesystem::path p(name.c_str());
    if (!boost::filesystem::exists(p)) {
        mode = FileMode::Overwrite;
    }

    this->file_mode = map_file_mode(mode);
    this->root_path = p;
    this->open_or_create();
    setCreatedAt();
    setUpdatedAt();

    if (!checkHeader()) {
        throw std::runtime_error("Invalid file header: either file format or file version not correct");
    }
}

void FileFS::open_or_create() {
    //TODO respect the file mode
    boost::filesystem::path data("data");
    boost::filesystem::path metadata("metadata");

    this->data_path = this->root_path / data;
    this->metadata_path = this->root_path / metadata;
    if (!boost::filesystem::exists(this->root_path)) {
        boost::filesystem::create_directory(this->root_path);
        boost::filesystem::create_directories(this->data_path);
        boost::filesystem::create_directories(this->metadata_path);
    } else {
        if (!boost::filesystem::exists(this->data_path)) {
            boost::filesystem::create_directories(this->data_path);
        }
        if (!boost::filesystem::exists(this->metadata_path)) {
            boost::filesystem::create_directories(this->metadata_path);
        }
    }
    this->attributes = AttributesFS(this->location());
}

ndsize_t FileFS::blockCount() const {
    ndsize_t count = 0;
    boost::filesystem::directory_iterator end;
    if (boost::filesystem::exists(this->data_path) && boost::filesystem::is_directory(this->data_path)) {
        boost::filesystem::directory_iterator di(this->data_path);
        while (di != end) {
            count ++;
        }
    }
    return count;
}

bool FileFS::hasBlock(const std::string &name_or_id) const  {
    return false;
}

std::shared_ptr<base::IBlock> FileFS::getBlock(const std::string &name_or_id) const {
    shared_ptr<nix::hdf5::BlockHDF5> block;
    /*
    boost::optional<Group> group = data.findGroupByNameOrAttribute("entity_id", name_or_id);
    if (group)
       block = make_shared<BlockHDF5>(file(), *group);
    */
    return block;
}

std::shared_ptr<base::IBlock> FileFS::getBlock(ndsize_t index) const {
    shared_ptr<nix::hdf5::BlockHDF5> block;
    return block;
}

std::shared_ptr<base::IBlock> FileFS::createBlock(const std::string &name, const std::string &type) {
    shared_ptr<nix::hdf5::BlockHDF5> block;
    return block;
}

bool FileFS::deleteBlock(const std::string &name_or_id) {
    return false;
}

//--------------------------------------------------
// Methods concerning sections
//--------------------------------------------------

bool FileFS::hasSection(const std::string &name_or_id) const {
    return false;
}

std::shared_ptr<base::ISection> FileFS::getSection(const std::string &name_or_id) const {
    shared_ptr<nix::hdf5::SectionHDF5> block;
    return block;
}

std::shared_ptr<base::ISection> FileFS::getSection(ndsize_t index) const {
    shared_ptr<nix::hdf5::SectionHDF5> block;
    return block;
}

ndsize_t FileFS::sectionCount() const {
    ndsize_t count = 0;
    boost::filesystem::directory_iterator end;
    if (boost::filesystem::exists(this->metadata_path) && boost::filesystem::is_directory(this->metadata_path)) {
        boost::filesystem::directory_iterator di(this->metadata_path);
        while (di != end) {
            count ++;
        }
    }
    return count;
}

std::shared_ptr<base::ISection> FileFS::createSection(const std::string &name, const std::string &type) {
    shared_ptr<hdf5::SectionHDF5> block;
    return block;
}

bool FileFS::deleteSection(const std::string &name_or_id) {
    return  false;
}

//--------------------------------------------------
// Methods for file attribute access.
//--------------------------------------------------

std::vector<int> FileFS::version() const {
    vector<int> version;
    return  version;
}

std::string FileFS::format() const {
    return "nix";
}

std::string FileFS::location() const {
    return boost::filesystem::absolute(this->root_path).string();
}

time_t FileFS::createdAt() const {
    return util::strToTime("00:00.00");
}

time_t FileFS::updatedAt() const {
    return util::strToTime("00:00.00");
}

void FileFS::setUpdatedAt(){
    if (!this->attributes.has("updated_at")) {
        time_t t = time(NULL);
        this->attributes.set("updated_at", util::timeToStr(t));
    }
}

void FileFS::forceUpdatedAt() {
    time_t t = time(NULL);
    this->attributes.set("updated_at", util::timeToStr(t));
}

void FileFS::setCreatedAt() {
    if (!this->attributes.has("created_at")) {
        time_t t = time(NULL);
        this->attributes.set("created_at", util::timeToStr(t));
    }
}

void FileFS::forceCreatedAt(time_t t) {
    this->attributes.set("created_at", util::timeToStr(t));
}

void FileFS::close() {}

bool FileFS::isOpen() const {
    return false;
}

bool FileFS::operator==(const FileFS &other) const {
    return false;
}

bool FileFS::operator!=(const FileFS &other) const {
    return false;
}

FileFS::~FileFS() {

}

bool FileFS::checkHeader() {
    bool check = true;
    vector<int> version;
    string str;
    // check format
    if (this->attributes.has("format")) {
        this->attributes.get("format", str);
        if (str != FILE_FORMAT) {
            check = false;
        }
    } else {
        this->attributes.set("format", FILE_FORMAT);
    }
    // check version
    if (this->attributes.has("version")) {
        this->attributes.get("version", version);
        if (version != FILE_VERSION) {
            check = false;
        }
    } else {
        this->attributes.set("version", FILE_VERSION);
    }
    return check;
}

} // namespace file
} // namespace nix<|endoftext|>


<commit_before>#include "filtergraph.h"

#include "camerafilter.h"
#include "kvazaarfilter.h"
#include "rgb32toyuv.h"
#include "openhevcfilter.h"
#include "yuvtorgb32.h"
#include "framedsourcefilter.h"
#include "rtpsinkfilter.h"
#include "displayfilter.h"

#include "audiocapturefilter.h"
#include "audiooutputdevice.h"
#include "audiooutput.h"
#include "opusencoderfilter.h"
#include "opusdecoderfilter.h"
#include "speexaecfilter.h"

#include "common.h"

FilterGraph::FilterGraph(StatisticsInterface* stats):
  peers_(),
  videoSend_(),
  audioSend_(),
  selfView_(NULL),
  stats_(stats),
  format_()
{
  Q_ASSERT(stats);
  // TODO negotiate these values with all included filters
  format_.setSampleRate(48000);
  format_.setChannelCount(2);
  format_.setSampleSize(16);
  format_.setSampleType(QAudioFormat::SignedInt);
  format_.setByteOrder(QAudioFormat::LittleEndian);
  format_.setCodec("audio/pcm");
}

void FilterGraph::init(VideoWidget* selfView, QSize resolution)
{
  resolution_ = resolution;
  selfView_ = selfView;
  frameRate_ = 30;

  initSelfView(selfView, resolution);
}

void FilterGraph::initSelfView(VideoWidget *selfView, QSize resolution)
{
  if(videoSend_.size() > 0)
  {
    destroyFilters(videoSend_);
  }

  // Sending video graph
  videoSend_.push_back(new CameraFilter("", stats_, resolution));

  if(selfView)
  {
    // connect selfview to camera
    DisplayFilter* selfviewFilter = new DisplayFilter("Self_", stats_, selfView, 1111);
    selfviewFilter->setProperties(true);
    videoSend_.push_back(selfviewFilter);
    videoSend_.at(0)->addOutConnection(videoSend_.back());
    videoSend_.back()->start();
  }
}

void FilterGraph::initVideoSend(QSize resolution)
{
  if(videoSend_.size() != 2)
  {
    if(videoSend_.size() > 2)
    {
      destroyFilters(videoSend_);
    }
    initSelfView(selfView_, resolution_);
  }

  videoSend_.push_back(new RGB32toYUV("", stats_));
  videoSend_.at(0)->addOutConnection(videoSend_.back()); // attach to camera
  videoSend_.back()->start();

  KvazaarFilter* kvz = new KvazaarFilter("", stats_);
  kvz->init(resolution, frameRate_, 1);
  videoSend_.push_back(kvz);
  videoSend_.at(videoSend_.size() - 2)->addOutConnection(videoSend_.back());
  videoSend_.back()->start();
}

void FilterGraph::initAudioSend()
{
  // Do this before adding participants, otherwise AEC filter wont get attached

  AudioCaptureFilter* capture = new AudioCaptureFilter("", stats_);
  capture->initializeAudio(format_);
  audioSend_.push_back(capture);

  OpusEncoderFilter *encoder = new OpusEncoderFilter("", stats_);
  encoder->init(format_);
  audioSend_.push_back(encoder);
  audioSend_.at(audioSend_.size() - 2)->addOutConnection(audioSend_.back());
  audioSend_.back()->start();
}

void FilterGraph::checkParticipant(int16_t id)
{
  Q_ASSERT(stats_);
  Q_ASSERT(id > -1);

  qDebug() << "Checking participant with id:" << id;

  if(peers_.size() > (unsigned int)id)
  {
    if(peers_.at(id) != NULL)
    {
      qDebug() << "Filter graph: Peer exists";
      return;
    }
    else
    {
      qDebug() << "Filter graph: Replacing old participant with id:" << id;
      peers_.at(id) = new Peer;
    }
  }
  else
  {
    while(peers_.size() < (unsigned int)id)
    {
      peers_.push_back(0);
    }
    peers_.push_back(new Peer);

    qDebug() << "Filter graph: Adding participant to end";
  }

  peers_.at(id)->output = 0;
  peers_.at(id)->audioFramedSource = 0;
  peers_.at(id)->videoFramedSource = 0;
  peers_.at(id)->audioSink = 0;
  peers_.at(id)->videoSink = 0;
}

void FilterGraph::sendVideoto(int16_t id, Filter *videoFramedSource)
{
  Q_ASSERT(id > -1);
  Q_ASSERT(videoFramedSource);

  qDebug() << "Adding send video for peer:" << id;

  // make sure we are generating video
  if(videoSend_.size() < 3)
  {
    initVideoSend(resolution_);
  }

  // add participant if necessary
  checkParticipant(id);

  if(peers_.at(id)->videoFramedSource)
  {
    qWarning() << "Warning: We are already sending video to participant.";
    return;
  }

  peers_.at(id)->videoFramedSource = videoFramedSource;

  videoSend_.back()->addOutConnection(videoFramedSource);
}

void FilterGraph::receiveVideoFrom(int16_t id, Filter *videoSink, VideoWidget *view)
{
  Q_ASSERT(id > -1);
  Q_ASSERT(videoSink);

  // add participant if necessary
  checkParticipant(id);

  if(peers_.at(id)->videoSink)
  {
    qWarning() << "Warning: We are receiving video from this participant:" << id;
    return;
  }

  peers_.at(id)->videoSink = videoSink;

  OpenHEVCFilter* decoder =  new OpenHEVCFilter(QString::number(id) + "_", stats_);
  decoder->init();
  peers_.at(id)->videoReceive.push_back(decoder);
  peers_.at(id)->videoSink->addOutConnection(peers_.at(id)->videoReceive.back());
  peers_.at(id)->videoReceive.back()->start();

  peers_.at(id)->videoReceive.push_back(new YUVtoRGB32(QString::number(id) + "_", stats_));
  peers_.at(id)->videoReceive.at(peers_.at(id)->videoReceive.size()-2)
      ->addOutConnection(peers_.at(id)->videoReceive.back());
  peers_.at(id)->videoReceive.back()->start();

  peers_.at(id)->videoReceive.push_back(new DisplayFilter(QString::number(id) + "_", stats_, view, id));
  peers_.at(id)->videoReceive.at(peers_.at(id)->videoReceive.size()-2)
      ->addOutConnection(peers_.at(id)->videoReceive.back());
  peers_.at(id)->videoReceive.back()->start();
}

void FilterGraph::sendAudioTo(int16_t id, Filter* audioFramedSource)
{
  Q_ASSERT(id > -1);
  Q_ASSERT(audioFramedSource);

  // just in case it is wanted later. AEC filter has to be attached
  if(audioSend_.size() == 0)
  {
    initAudioSend();
  }

  // add participant if necessary
  checkParticipant(id);

  if(peers_.at(id)->audioFramedSource)
  {
    qWarning() << "Warning: We are sending audio from to participant:" << id;
    return;
  }

  peers_.at(id)->audioFramedSource = audioFramedSource;

  audioSend_.back()->addOutConnection(audioFramedSource);
}

void FilterGraph::receiveAudioFrom(int16_t id, Filter* audioSink)
{
  Q_ASSERT(id > -1);
  Q_ASSERT(audioSink);

  // just in case it is wanted later. AEC filter has to be attached
  if(AEC && audioSend_.size() == 0)
  {
    initAudioSend();
  }

  // add participant if necessary
  checkParticipant(id);

  if(peers_.at(id)->audioSink)
  {
    qWarning() << "Warning: We are receiving video from this participant:" << id;
    return;
  }
  peers_.at(id)->audioSink = audioSink;

  OpusDecoderFilter *decoder = new OpusDecoderFilter(QString::number(id) + "_", stats_);
  decoder->init(format_);

  peers_.at(id)->audioReceive.push_back(decoder);
  peers_.at(id)->audioSink->addOutConnection(peers_.at(id)->audioReceive.back());
  peers_.at(id)->audioReceive.back()->start();

  if(audioSend_.size() > 0 && AEC)
  {
    peers_.at(id)->audioReceive.push_back(new SpeexAECFilter(QString::number(id) + "_", stats_, format_));
    audioSend_.at(0)->addOutConnection(peers_.at(id)->audioReceive.back());
    peers_.at(id)->audioReceive.at(peers_.at(id)->audioReceive.size()-2)
        ->addOutConnection(peers_.at(id)->audioReceive.back());
    peers_.at(id)->audioReceive.back()->start();
  }
  else
  {
    qWarning() << "WARNING: Did not attach echo cancellation";
  }

  peers_.at(id)->output = new AudioOutput(stats_, id);
  peers_.at(id)->output->initializeAudio(format_);
  AudioOutputDevice* outputModule = peers_.at(id)->output->getOutputModule();

  outputModule->init(peers_.at(id)->audioReceive.back());
}

void FilterGraph::uninit()
{
  for(Peer* p : peers_)
  {
    if(p != NULL)
    {
      destroyPeer(p);
      p = 0;
    }
  }
  peers_.clear();

  destroyFilters(videoSend_);
  destroyFilters(audioSend_);
}

void changeState(Filter* f, bool state)
{
  if(state)
  {
    f->emptyBuffer();
    f->start();
  }
  else
  {
    f->stop();
    f->emptyBuffer();

    while(f->isRunning())
    {
      qSleep(1);
    }
  }
}

void FilterGraph::mic(bool state)
{

  if(audioSend_.size() > 0)
  {
    if(!state)
    {
      qDebug() << "Stopping microphone";
      audioSend_.at(0)->stop();
    }
    else
    {
      qDebug() << "Starting microphone";
      audioSend_.at(0)->start();
    }
  }
}

void FilterGraph::camera(bool state)
{
  if(videoSend_.size() > 0)
  {
    if(!state)
    {
      qDebug() << "Stopping camera";
      videoSend_.at(0)->stop();
    }
    else
    {
      qDebug() << "Starting camera";
      videoSend_.at(0)->start();
    }
  }
}

void FilterGraph::running(bool state)
{
  for(Filter* f : videoSend_)
  {
    changeState(f, state);
  }
  for(Filter* f : audioSend_)
  {
    changeState(f, state);
  }

  for(Peer* p : peers_)
  {
    if(p != NULL)
    {
      if(p->audioFramedSource)
      {
        changeState(p->audioFramedSource, state);
      }
      if(p->videoFramedSource)
      {
        changeState(p->videoFramedSource, state);
      }
      for(Filter* f : p->audioReceive)
      {
        changeState(f, state);
      }
      for(Filter* f : p->videoReceive)
      {
        changeState(f, state);
      }
    }
  }
}

void FilterGraph::destroyFilters(std::vector<Filter*>& filters)
{
  qDebug() << "Destroying filter graph with" << filters.size() << "filters.";
  for( Filter *f : filters )
  {
    changeState(f, false);
    delete f;
  }

  filters.clear();
}

void FilterGraph::destroyPeer(Peer* peer)
{
  if(peer->audioFramedSource)
  {
    audioSend_.back()->removeOutConnection(peer->audioFramedSource);
    //peer->audioFramedSource is destroyed by RTPStreamer
    peer->audioFramedSource = 0;

    if(AEC)
    {
      audioSend_.at(0)->removeOutConnection(peer->audioReceive.back());
    }
  }
  if(peer->videoFramedSource)
  {
    videoSend_.back()->removeOutConnection(peer->videoFramedSource);
    //peer->videoFramedSource is destroyed by RTPStreamer
    peer->videoFramedSource = 0;
  }

  if(peer->audioSink)
  {
    peer->audioSink->removeOutConnection(peer->audioReceive.front());
  }

  if(peer->videoSink)
  {
    peer->videoSink->removeOutConnection(peer->videoReceive.front());
  }

  destroyFilters(peer->audioReceive);
  destroyFilters(peer->videoReceive);

  if(peer->output)
  {
    delete peer->output;
    peer->output = 0;
  }
  delete peer;
}

void FilterGraph::removeParticipant(int16_t id)
{
  qDebug() << "Removing peer index:" << id << "/" << peers_.size();
  Q_ASSERT(id < peers_.size());
  if(peers_.at(id) != NULL)
    destroyPeer(peers_.at(id));
  peers_.at(id) = NULL;
}

void FilterGraph::print()
{
  QString audioDotFile = "digraph AudioGraph {\r\n";

  for(auto f : audioSend_)
  {
    audioDotFile += f->printOutputs();
  }

  for(unsigned int i = 0; i <peers_.size(); ++i)
  {
    if(peers_.at(i) != NULL)
    {
      for(auto f : peers_.at(i)->audioReceive)
      {
        audioDotFile += f->printOutputs();
      }

      audioDotFile += peers_.at(i)->audioSink->printOutputs();
      audioDotFile += peers_.at(i)->audioFramedSource->printOutputs();
    }
  }
  audioDotFile += "}";

  qDebug() << audioDotFile;

  QString videoDotFile = "digraph VideoGraph {\r\n";

  for(auto f : videoSend_)
  {
    videoDotFile += f->printOutputs();
  }

  for(unsigned int i = 0; i <peers_.size(); ++i)
  {
    if(peers_.at(i) != NULL)
    {
      for(auto f : peers_.at(i)->videoReceive)
      {
        videoDotFile += f->printOutputs();
      }

      videoDotFile += peers_.at(i)->videoSink->printOutputs();
      videoDotFile += peers_.at(i)->videoFramedSource->printOutputs();
    }
  }
  videoDotFile += "}";
  qDebug() << videoDotFile;

  QString aFilename="audiograph.dot";
  QFile aFile( aFilename );
  if ( aFile.open(QIODevice::WriteOnly) )
  {
      QTextStream stream( &aFile );
      stream << audioDotFile << endl;
  }

  QString vFilename="videograph.dot";
  QFile vFile( vFilename );
  if ( vFile.open(QIODevice::WriteOnly) )
  {
      QTextStream stream( &vFile );
      stream << videoDotFile << endl;
  }
}
<commit_msg>Destruction of sending filtergraphs if the last participant leaves.<commit_after>#include "filtergraph.h"

#include "camerafilter.h"
#include "kvazaarfilter.h"
#include "rgb32toyuv.h"
#include "openhevcfilter.h"
#include "yuvtorgb32.h"
#include "framedsourcefilter.h"
#include "rtpsinkfilter.h"
#include "displayfilter.h"

#include "audiocapturefilter.h"
#include "audiooutputdevice.h"
#include "audiooutput.h"
#include "opusencoderfilter.h"
#include "opusdecoderfilter.h"
#include "speexaecfilter.h"

#include "common.h"

FilterGraph::FilterGraph(StatisticsInterface* stats):
  peers_(),
  videoSend_(),
  audioSend_(),
  selfView_(NULL),
  stats_(stats),
  format_()
{
  Q_ASSERT(stats);
  // TODO negotiate these values with all included filters
  format_.setSampleRate(48000);
  format_.setChannelCount(2);
  format_.setSampleSize(16);
  format_.setSampleType(QAudioFormat::SignedInt);
  format_.setByteOrder(QAudioFormat::LittleEndian);
  format_.setCodec("audio/pcm");
}

void FilterGraph::init(VideoWidget* selfView, QSize resolution)
{
  resolution_ = resolution;
  selfView_ = selfView;
  frameRate_ = 30;

  initSelfView(selfView, resolution);
}

void FilterGraph::initSelfView(VideoWidget *selfView, QSize resolution)
{
  if(videoSend_.size() > 0)
  {
    destroyFilters(videoSend_);
  }

  // Sending video graph
  videoSend_.push_back(new CameraFilter("", stats_, resolution));

  if(selfView)
  {
    // connect selfview to camera
    DisplayFilter* selfviewFilter = new DisplayFilter("Self_", stats_, selfView, 1111);
    selfviewFilter->setProperties(true);
    videoSend_.push_back(selfviewFilter);
    videoSend_.at(0)->addOutConnection(videoSend_.back());
    videoSend_.back()->start();
  }
}

void FilterGraph::initVideoSend(QSize resolution)
{
  if(videoSend_.size() != 2)
  {
    if(videoSend_.size() > 2)
    {
      destroyFilters(videoSend_);
    }
    initSelfView(selfView_, resolution_);
  }

  videoSend_.push_back(new RGB32toYUV("", stats_));
  videoSend_.at(0)->addOutConnection(videoSend_.back()); // attach to camera
  videoSend_.back()->start();

  KvazaarFilter* kvz = new KvazaarFilter("", stats_);
  kvz->init(resolution, frameRate_, 1);
  videoSend_.push_back(kvz);
  videoSend_.at(videoSend_.size() - 2)->addOutConnection(videoSend_.back());
  videoSend_.back()->start();
}

void FilterGraph::initAudioSend()
{
  // Do this before adding participants, otherwise AEC filter wont get attached

  AudioCaptureFilter* capture = new AudioCaptureFilter("", stats_);
  capture->initializeAudio(format_);
  audioSend_.push_back(capture);

  OpusEncoderFilter *encoder = new OpusEncoderFilter("", stats_);
  encoder->init(format_);
  audioSend_.push_back(encoder);
  audioSend_.at(audioSend_.size() - 2)->addOutConnection(audioSend_.back());
  audioSend_.back()->start();
}

void FilterGraph::checkParticipant(int16_t id)
{
  Q_ASSERT(stats_);
  Q_ASSERT(id > -1);

  qDebug() << "Checking participant with id:" << id;

  if(peers_.size() > (unsigned int)id)
  {
    if(peers_.at(id) != NULL)
    {
      qDebug() << "Filter graph: Peer exists";
      return;
    }
    else
    {
      qDebug() << "Filter graph: Replacing old participant with id:" << id;
      peers_.at(id) = new Peer;
    }
  }
  else
  {
    while(peers_.size() < (unsigned int)id)
    {
      peers_.push_back(0);
    }
    peers_.push_back(new Peer);

    qDebug() << "Filter graph: Adding participant to end";
  }

  peers_.at(id)->output = 0;
  peers_.at(id)->audioFramedSource = 0;
  peers_.at(id)->videoFramedSource = 0;
  peers_.at(id)->audioSink = 0;
  peers_.at(id)->videoSink = 0;
}

void FilterGraph::sendVideoto(int16_t id, Filter *videoFramedSource)
{
  Q_ASSERT(id > -1);
  Q_ASSERT(videoFramedSource);

  qDebug() << "Adding send video for peer:" << id;

  // make sure we are generating video
  if(videoSend_.size() < 3)
  {
    initVideoSend(resolution_);
  }

  // add participant if necessary
  checkParticipant(id);

  if(peers_.at(id)->videoFramedSource)
  {
    qWarning() << "Warning: We are already sending video to participant.";
    return;
  }

  peers_.at(id)->videoFramedSource = videoFramedSource;

  videoSend_.back()->addOutConnection(videoFramedSource);
}

void FilterGraph::receiveVideoFrom(int16_t id, Filter *videoSink, VideoWidget *view)
{
  Q_ASSERT(id > -1);
  Q_ASSERT(videoSink);

  // add participant if necessary
  checkParticipant(id);

  if(peers_.at(id)->videoSink)
  {
    qWarning() << "Warning: We are receiving video from this participant:" << id;
    return;
  }

  peers_.at(id)->videoSink = videoSink;

  OpenHEVCFilter* decoder =  new OpenHEVCFilter(QString::number(id) + "_", stats_);
  decoder->init();
  peers_.at(id)->videoReceive.push_back(decoder);
  peers_.at(id)->videoSink->addOutConnection(peers_.at(id)->videoReceive.back());
  peers_.at(id)->videoReceive.back()->start();

  peers_.at(id)->videoReceive.push_back(new YUVtoRGB32(QString::number(id) + "_", stats_));
  peers_.at(id)->videoReceive.at(peers_.at(id)->videoReceive.size()-2)
      ->addOutConnection(peers_.at(id)->videoReceive.back());
  peers_.at(id)->videoReceive.back()->start();

  peers_.at(id)->videoReceive.push_back(new DisplayFilter(QString::number(id) + "_", stats_, view, id));
  peers_.at(id)->videoReceive.at(peers_.at(id)->videoReceive.size()-2)
      ->addOutConnection(peers_.at(id)->videoReceive.back());
  peers_.at(id)->videoReceive.back()->start();
}

void FilterGraph::sendAudioTo(int16_t id, Filter* audioFramedSource)
{
  Q_ASSERT(id > -1);
  Q_ASSERT(audioFramedSource);

  // just in case it is wanted later. AEC filter has to be attached
  if(audioSend_.size() == 0)
  {
    initAudioSend();
  }

  // add participant if necessary
  checkParticipant(id);

  if(peers_.at(id)->audioFramedSource)
  {
    qWarning() << "Warning: We are sending audio from to participant:" << id;
    return;
  }

  peers_.at(id)->audioFramedSource = audioFramedSource;

  audioSend_.back()->addOutConnection(audioFramedSource);
}

void FilterGraph::receiveAudioFrom(int16_t id, Filter* audioSink)
{
  Q_ASSERT(id > -1);
  Q_ASSERT(audioSink);

  // just in case it is wanted later. AEC filter has to be attached
  if(AEC && audioSend_.size() == 0)
  {
    initAudioSend();
  }

  // add participant if necessary
  checkParticipant(id);

  if(peers_.at(id)->audioSink)
  {
    qWarning() << "Warning: We are receiving video from this participant:" << id;
    return;
  }
  peers_.at(id)->audioSink = audioSink;

  OpusDecoderFilter *decoder = new OpusDecoderFilter(QString::number(id) + "_", stats_);
  decoder->init(format_);

  peers_.at(id)->audioReceive.push_back(decoder);
  peers_.at(id)->audioSink->addOutConnection(peers_.at(id)->audioReceive.back());
  peers_.at(id)->audioReceive.back()->start();

  if(audioSend_.size() > 0 && AEC)
  {
    peers_.at(id)->audioReceive.push_back(new SpeexAECFilter(QString::number(id) + "_", stats_, format_));
    audioSend_.at(0)->addOutConnection(peers_.at(id)->audioReceive.back());
    peers_.at(id)->audioReceive.at(peers_.at(id)->audioReceive.size()-2)
        ->addOutConnection(peers_.at(id)->audioReceive.back());
    peers_.at(id)->audioReceive.back()->start();
  }
  else
  {
    qWarning() << "WARNING: Did not attach echo cancellation";
  }

  peers_.at(id)->output = new AudioOutput(stats_, id);
  peers_.at(id)->output->initializeAudio(format_);
  AudioOutputDevice* outputModule = peers_.at(id)->output->getOutputModule();

  outputModule->init(peers_.at(id)->audioReceive.back());
}

void FilterGraph::uninit()
{
  for(Peer* p : peers_)
  {
    if(p != NULL)
    {
      destroyPeer(p);
      p = 0;
    }
  }
  peers_.clear();

  destroyFilters(videoSend_);
  destroyFilters(audioSend_);
}

void changeState(Filter* f, bool state)
{
  if(state)
  {
    f->emptyBuffer();
    f->start();
  }
  else
  {
    f->stop();
    f->emptyBuffer();

    while(f->isRunning())
    {
      qSleep(1);
    }
  }
}

void FilterGraph::mic(bool state)
{

  if(audioSend_.size() > 0)
  {
    if(!state)
    {
      qDebug() << "Stopping microphone";
      audioSend_.at(0)->stop();
    }
    else
    {
      qDebug() << "Starting microphone";
      audioSend_.at(0)->start();
    }
  }
}

void FilterGraph::camera(bool state)
{
  if(videoSend_.size() > 0)
  {
    if(!state)
    {
      qDebug() << "Stopping camera";
      videoSend_.at(0)->stop();
    }
    else
    {
      qDebug() << "Starting camera";
      videoSend_.at(0)->start();
    }
  }
}

void FilterGraph::running(bool state)
{
  for(Filter* f : videoSend_)
  {
    changeState(f, state);
  }
  for(Filter* f : audioSend_)
  {
    changeState(f, state);
  }

  for(Peer* p : peers_)
  {
    if(p != NULL)
    {
      if(p->audioFramedSource)
      {
        changeState(p->audioFramedSource, state);
      }
      if(p->videoFramedSource)
      {
        changeState(p->videoFramedSource, state);
      }
      for(Filter* f : p->audioReceive)
      {
        changeState(f, state);
      }
      for(Filter* f : p->videoReceive)
      {
        changeState(f, state);
      }
    }
  }
}

void FilterGraph::destroyFilters(std::vector<Filter*>& filters)
{
  qDebug() << "Destroying filter graph with" << filters.size() << "filters.";
  for( Filter *f : filters )
  {
    changeState(f, false);
    delete f;
  }

  filters.clear();
}

void FilterGraph::destroyPeer(Peer* peer)
{
  if(peer->audioFramedSource)
  {
    audioSend_.back()->removeOutConnection(peer->audioFramedSource);
    //peer->audioFramedSource is destroyed by RTPStreamer
    peer->audioFramedSource = 0;

    if(AEC)
    {
      audioSend_.at(0)->removeOutConnection(peer->audioReceive.back());
    }
  }
  if(peer->videoFramedSource)
  {
    videoSend_.back()->removeOutConnection(peer->videoFramedSource);
    //peer->videoFramedSource is destroyed by RTPStreamer
    peer->videoFramedSource = 0;
  }

  if(peer->audioSink)
  {
    peer->audioSink->removeOutConnection(peer->audioReceive.front());
  }

  if(peer->videoSink)
  {
    peer->videoSink->removeOutConnection(peer->videoReceive.front());
  }

  destroyFilters(peer->audioReceive);
  destroyFilters(peer->videoReceive);

  if(peer->output)
  {
    delete peer->output;
    peer->output = 0;
  }
  delete peer;
}

void FilterGraph::removeParticipant(int16_t id)
{
  qDebug() << "Removing peer index:" << id << "/" << peers_.size();
  Q_ASSERT(id < peers_.size());
  if(peers_.at(id) != NULL)
    destroyPeer(peers_.at(id));
  peers_.at(id) = NULL;

  bool peerPresent = false;
  for(auto peer : peers_)
  {
    if (peer != NULL)
      peerPresent = true;
  }

  if(!peerPresent)
  {
    destroyFilters(videoSend_);
    destroyFilters(audioSend_);
  }
}

void FilterGraph::print()
{
  QString audioDotFile = "digraph AudioGraph {\r\n";

  for(auto f : audioSend_)
  {
    audioDotFile += f->printOutputs();
  }

  for(unsigned int i = 0; i <peers_.size(); ++i)
  {
    if(peers_.at(i) != NULL)
    {
      for(auto f : peers_.at(i)->audioReceive)
      {
        audioDotFile += f->printOutputs();
      }

      audioDotFile += peers_.at(i)->audioSink->printOutputs();
      audioDotFile += peers_.at(i)->audioFramedSource->printOutputs();
    }
  }
  audioDotFile += "}";

  qDebug() << audioDotFile;

  QString videoDotFile = "digraph VideoGraph {\r\n";

  for(auto f : videoSend_)
  {
    videoDotFile += f->printOutputs();
  }

  for(unsigned int i = 0; i <peers_.size(); ++i)
  {
    if(peers_.at(i) != NULL)
    {
      for(auto f : peers_.at(i)->videoReceive)
      {
        videoDotFile += f->printOutputs();
      }

      videoDotFile += peers_.at(i)->videoSink->printOutputs();
      videoDotFile += peers_.at(i)->videoFramedSource->printOutputs();
    }
  }
  videoDotFile += "}";
  qDebug() << videoDotFile;

  QString aFilename="audiograph.dot";
  QFile aFile( aFilename );
  if ( aFile.open(QIODevice::WriteOnly) )
  {
      QTextStream stream( &aFile );
      stream << audioDotFile << endl;
  }

  QString vFilename="videograph.dot";
  QFile vFile( vFilename );
  if ( vFile.open(QIODevice::WriteOnly) )
  {
      QTextStream stream( &vFile );
      stream << videoDotFile << endl;
  }
}
<|endoftext|>