<commit_before>#include <clang-c/Index.h>
#include <iostream>
#include <unordered_map>
#include <unordered_set>
#include <string.h>
#include <sys/utsname.h>
#include <sys/stat.h>

#include "ast.h"
#include "bpftrace.h"
#include "clang_parser.h"
#include "types.h"
#include "utils.h"
#include "headers.h"

namespace bpftrace {

std::unordered_map<std::string, CXCursor> indirect_structs;
std::unordered_set<std::string> unvisited_indirect_structs;

static std::string get_clang_string(CXString string)
{
  std::string str = clang_getCString(string);
  clang_disposeString(string);
  return str;
}

static void remove_struct_prefix(std::string &str)
{
  if (strncmp(str.c_str(), "struct ", 7) == 0)
    str.erase(0, 7);
}

static CXCursor get_indirect_field_parent_struct(CXCursor c)
{
  CXCursor parent = clang_getCursorSemanticParent(c);

  while (!clang_Cursor_isNull(parent) && indirect_structs.count(get_clang_string(clang_getTypeSpelling(clang_getCanonicalType(clang_getCursorType(parent))))) > 0) {
    parent = clang_getCursorSemanticParent(parent);
  }

  return parent;
}

static std::string get_parent_struct_name(CXCursor c)
{
  CXCursor parent = get_indirect_field_parent_struct(c);

  return get_clang_string(clang_getCursorSpelling(parent));
}

static int get_indirect_field_offset(CXCursor c)
{
  int offset = 0;
  CXCursor parent = get_indirect_field_parent_struct(c);
  auto ident = get_clang_string(clang_getCursorSpelling(c));
  offset = clang_Type_getOffsetOf(clang_getCursorType(parent), ident.c_str()) / 8;

  return offset;
}

// NOTE(mmarchini): as suggested in http://clang-developers.42468.n3.nabble.com/Extracting-macro-information-using-libclang-the-C-Interface-to-Clang-td4042648.html#message4042666
static bool translateMacro(CXCursor cursor, std::string &name, std::string &value)
{
  CXToken* tokens    = nullptr;
  unsigned numTokens = 0;
  CXTranslationUnit transUnit = clang_Cursor_getTranslationUnit(cursor);
  CXSourceRange srcRange  = clang_getCursorExtent(cursor);
  clang_tokenize(transUnit, srcRange, &tokens, &numTokens);
  for (unsigned n=0; n<numTokens; n++)
  {
    auto tokenText = clang_getTokenSpelling(transUnit, tokens[n]);
    if (n == 0)
    {
      value.clear();
      name = clang_getCString(tokenText);
      if (name[0] == '_')
        break;
    }
    else
    {
      CXTokenKind tokenKind = clang_getTokenKind(tokens[n]);
      if (tokenKind != CXToken_Comment)
      {
        const char* text = clang_getCString(tokenText);
        if (text)
          value += text;
      }
    }
  }
  clang_disposeTokens(transUnit, tokens, numTokens);
  return value.length() != 0;
}

static SizedType get_sized_type(CXType clang_type)
{
  auto size = clang_Type_getSizeOf(clang_type);
  auto typestr = get_clang_string(clang_getTypeSpelling(clang_type));
  remove_struct_prefix(typestr);

  switch (clang_type.kind)
  {
    case CXType_Bool:
    case CXType_Char_S:
    case CXType_Char_U:
    case CXType_SChar:
    case CXType_UChar:
    case CXType_Short:
    case CXType_UShort:
    case CXType_Int:
    case CXType_UInt:
    case CXType_Long:
    case CXType_ULong:
    case CXType_LongLong:
    case CXType_ULongLong:
      return SizedType(Type::integer, size);
    case CXType_Record:
      return SizedType(Type::cast, size, typestr);
    case CXType_Pointer:
    {
      auto pointee_type = clang_getPointeeType(clang_type);
      SizedType type;
      if (pointee_type.kind == CXType_Record)
      {
        auto pointee_typestr = get_clang_string(clang_getTypeSpelling(pointee_type));
        remove_struct_prefix(pointee_typestr);
        type = SizedType(Type::cast, sizeof(uintptr_t), pointee_typestr);
      }
      else
      {
        type = SizedType(Type::integer, sizeof(uintptr_t));
      }
      auto pointee_size = clang_Type_getSizeOf(pointee_type);
      type.is_pointer = true;
      type.pointee_size = pointee_size;
      return type;
    }
    case CXType_ConstantArray:
    {
      auto elem_type = clang_getArrayElementType(clang_type);
      auto size = clang_getArraySize(clang_type);
      if (elem_type.kind == CXType_Char_S || elem_type.kind == CXType_Char_U)
      {
        return SizedType(Type::string, size);
      }
      // TODO add support for arrays
      return SizedType(Type::none, 0);
    }
    default:
      return SizedType(Type::none, 0);
  }
}

static bool is_dir(const std::string& path)
{
  struct stat buf;

  if (::stat(path.c_str(), &buf) < 0)
    return false;

  return S_ISDIR(buf.st_mode);
}

// get_kernel_dirs returns {ksrc, kobj} - directories for pristine and
// generated kernel sources.
//
// When the kernel was built in its source tree ksrc == kobj, however when
// the kernel was build in a different directory than its source, ksrc != kobj.
//
// A notable example is Debian, which places pristine kernel headers in
//
//	/lib/modules/`uname -r`/source/
//
// and generated kernel headers in
//
//	/lib/modules/`uname -r`/build/
//
// {"", ""} is returned if no trace of kernel headers was found at all.
// Both ksrc and kobj are guaranteed to be != "", if at least some trace of kernel sources was found.
static std::tuple<std::string, std::string> get_kernel_dirs(const struct utsname& utsname)
{
#ifdef KERNEL_HEADERS_DIR
  return {KERNEL_HEADERS_DIR, KERNEL_HEADERS_DIR};
#endif

  const char *kpath_env = ::getenv("BPFTRACE_KERNEL_SOURCE");
  if (kpath_env)
    return std::make_tuple(kpath_env, kpath_env);

  std::string kdir = std::string("/lib/modules/") + utsname.release;
  auto ksrc = kdir + "/source";
  auto kobj = kdir + "/build";

  // if one of source/ or build/ is not present - try to use the other one for both.
  if (!is_dir(ksrc)) {
	  ksrc = "";
  }
  if (!is_dir(kobj)) {
	  kobj = "";
  }
  if (ksrc == "" && kobj == "") {
	  return std::make_tuple("", "");
  }
  if (ksrc == "") {
	  ksrc = kobj;
  }
  else if (kobj == "") {
	  kobj = ksrc;
  }

  return std::make_tuple(ksrc, kobj);
}

void ClangParser::parse(ast::Program *program, BPFtrace &bpftrace)
{
  auto input = program->c_definitions;
  if (input.size() == 0)
    return; // We occasionally get crashes in libclang otherwise

  CXUnsavedFile unsaved_files[] =
  {
    {
      .Filename = "definitions.h",
      .Contents = input.c_str(),
      .Length = input.size(),
    },
    {
      .Filename = "/bpftrace/include/__stddef_max_align_t.h",
      .Contents = __stddef_max_align_t_h,
      .Length = __stddef_max_align_t_h_len,
    },
    {
      .Filename = "/bpftrace/include/float.h",
      .Contents = float_h,
      .Length = float_h_len,
    },
    {
      .Filename = "/bpftrace/include/limits.h",
      .Contents = limits_h,
      .Length = limits_h_len,
    },
    {
      .Filename = "/bpftrace/include/stdarg.h",
      .Contents = stdarg_h,
      .Length = stdarg_h_len,
    },
    {
      .Filename = "/bpftrace/include/stddef.h",
      .Contents = stddef_h,
      .Length = stddef_h_len,
    },
    {
      .Filename = "/bpftrace/include/stdint.h",
      .Contents = stdint_h,
      .Length = stdint_h_len,
    },
  };

  std::vector<std::string> kflags;
  struct utsname utsname;
  uname(&utsname);
  // auto [ksrc, kobj] = get_kernel_dirs(utsname);	XXX fails with LLVM5
  std::string ksrc, kobj;
  auto kdirs = get_kernel_dirs(utsname);
  ksrc = std::get<0>(kdirs);
  kobj = std::get<1>(kdirs);

  if (ksrc != "")
    kflags = get_kernel_cflags(utsname.machine, ksrc, kobj);

  std::vector<const char *> args =
  {
    "-I", "/bpftrace/include",
  };
  for (auto &flag : kflags)
  {
    args.push_back(flag.c_str());
  }

  CXIndex index = clang_createIndex(1, 1);
  CXTranslationUnit translation_unit;
  CXErrorCode error = clang_parseTranslationUnit2(
      index,
      "definitions.h",
      &args[0], args.size(),
      unsaved_files, sizeof(unsaved_files)/sizeof(CXUnsavedFile),
      CXTranslationUnit_DetailedPreprocessingRecord,
      &translation_unit);
  if (error)
  {
    std::cerr << "Clang error while parsing C definitions: " << error << std::endl;
    std::cerr << "Input (" << input.size() << "): " << input << std::endl;
  }

  indirect_structs.clear();
  unvisited_indirect_structs.clear();

  CXCursor cursor = clang_getTranslationUnitCursor(translation_unit);

  bool iterate = true;

  do {
    clang_visitChildren(
        cursor,
        [](CXCursor c, CXCursor parent, CXClientData client_data)
        {
          if (clang_getCursorKind(c) == CXCursor_MacroDefinition)
          {
            std::string macro_name;
            std::string macro_value;
            if (translateMacro(c, macro_name, macro_value)) {
              auto &macros = static_cast<BPFtrace*>(client_data)->macros_;
              macros[macro_name] = macro_value;
            }
            return CXChildVisit_Recurse;
          }

          if (clang_getCursorKind(parent) != CXCursor_StructDecl &&
              clang_getCursorKind(parent) != CXCursor_UnionDecl)
            return CXChildVisit_Recurse;

          auto ptype = clang_getCanonicalType(clang_getCursorType(parent));
          auto ptypestr = get_clang_string(clang_getTypeSpelling(ptype));
          auto ptypesize = clang_Type_getSizeOf(ptype);

          if (clang_getCursorKind(c) == CXCursor_StructDecl ||
              clang_getCursorKind(c) == CXCursor_UnionDecl) {
            auto struct_name = get_clang_string(clang_getTypeSpelling(clang_getCanonicalType(clang_getCursorType(c))));
            indirect_structs[struct_name] = c;
            unvisited_indirect_structs.insert(struct_name);

            return CXChildVisit_Continue;
          }

          if (clang_getCursorKind(c) == CXCursor_FieldDecl)
          {
            auto &structs = static_cast<BPFtrace*>(client_data)->structs_;
            auto struct_name = get_parent_struct_name(c);
            auto ident = get_clang_string(clang_getCursorSpelling(c));
            auto offset = clang_Cursor_getOffsetOfField(c) / 8;
            auto type = clang_getCanonicalType(clang_getCursorType(c));
            auto typestr = get_clang_string(clang_getTypeSpelling(type));

            if (indirect_structs.count(typestr))
              indirect_structs.erase(typestr);

            if(indirect_structs.count(ptypestr))
              offset = get_indirect_field_offset(c);

            if (struct_name == "")
              struct_name = ptypestr;
            remove_struct_prefix(struct_name);

            structs[struct_name].fields[ident].offset = offset;
            structs[struct_name].fields[ident].type = get_sized_type(type);
            structs[struct_name].size = ptypesize;
          }

          return CXChildVisit_Recurse;
        },
        &bpftrace);
    if (unvisited_indirect_structs.size()) {
      cursor = indirect_structs[*unvisited_indirect_structs.begin()];
      unvisited_indirect_structs.erase(unvisited_indirect_structs.begin());
    } else {
      iterate = false;
    }
  } while (iterate);

  clang_disposeTranslationUnit(translation_unit);
  clang_disposeIndex(index);
}

} // namespace bpftrace
<commit_msg>[clang_parser] fix struct definition from headers<commit_after>#include <clang-c/Index.h>
#include <iostream>
#include <unordered_map>
#include <unordered_set>
#include <string.h>
#include <sys/utsname.h>
#include <sys/stat.h>

#include "ast.h"
#include "bpftrace.h"
#include "clang_parser.h"
#include "types.h"
#include "utils.h"
#include "headers.h"

namespace bpftrace {

std::unordered_map<std::string, CXCursor> indirect_structs;
std::unordered_set<std::string> unvisited_indirect_structs;

static std::string get_clang_string(CXString string)
{
  std::string str = clang_getCString(string);
  clang_disposeString(string);
  return str;
}

static void remove_struct_prefix(std::string &str)
{
  if (strncmp(str.c_str(), "struct ", 7) == 0)
    str.erase(0, 7);
}

static CXCursor get_indirect_field_parent_struct(CXCursor c)
{
  CXCursor parent = clang_getCursorSemanticParent(c);

  while (!clang_Cursor_isNull(parent) && indirect_structs.count(get_clang_string(clang_getTypeSpelling(clang_getCanonicalType(clang_getCursorType(parent))))) > 0) {
    parent = clang_getCursorSemanticParent(parent);
  }

  return parent;
}

static std::string get_parent_struct_name(CXCursor c)
{
  CXCursor parent = get_indirect_field_parent_struct(c);

  if (clang_getCursorKind(parent) != CXCursor_StructDecl &&
      clang_getCursorKind(parent) != CXCursor_UnionDecl)
    return "";

  return get_clang_string(clang_getCursorSpelling(parent));
}

static int get_indirect_field_offset(CXCursor c)
{
  int offset = 0;
  CXCursor parent = get_indirect_field_parent_struct(c);
  auto ident = get_clang_string(clang_getCursorSpelling(c));
  offset = clang_Type_getOffsetOf(clang_getCursorType(parent), ident.c_str()) / 8;

  return offset;
}

// NOTE(mmarchini): as suggested in http://clang-developers.42468.n3.nabble.com/Extracting-macro-information-using-libclang-the-C-Interface-to-Clang-td4042648.html#message4042666
static bool translateMacro(CXCursor cursor, std::string &name, std::string &value)
{
  CXToken* tokens    = nullptr;
  unsigned numTokens = 0;
  CXTranslationUnit transUnit = clang_Cursor_getTranslationUnit(cursor);
  CXSourceRange srcRange  = clang_getCursorExtent(cursor);
  clang_tokenize(transUnit, srcRange, &tokens, &numTokens);
  for (unsigned n=0; n<numTokens; n++)
  {
    auto tokenText = clang_getTokenSpelling(transUnit, tokens[n]);
    if (n == 0)
    {
      value.clear();
      name = clang_getCString(tokenText);
      if (name[0] == '_')
        break;
    }
    else
    {
      CXTokenKind tokenKind = clang_getTokenKind(tokens[n]);
      if (tokenKind != CXToken_Comment)
      {
        const char* text = clang_getCString(tokenText);
        if (text)
          value += text;
      }
    }
  }
  clang_disposeTokens(transUnit, tokens, numTokens);
  return value.length() != 0;
}

static SizedType get_sized_type(CXType clang_type)
{
  auto size = clang_Type_getSizeOf(clang_type);
  auto typestr = get_clang_string(clang_getTypeSpelling(clang_type));
  remove_struct_prefix(typestr);

  switch (clang_type.kind)
  {
    case CXType_Bool:
    case CXType_Char_S:
    case CXType_Char_U:
    case CXType_SChar:
    case CXType_UChar:
    case CXType_Short:
    case CXType_UShort:
    case CXType_Int:
    case CXType_UInt:
    case CXType_Long:
    case CXType_ULong:
    case CXType_LongLong:
    case CXType_ULongLong:
      return SizedType(Type::integer, size);
    case CXType_Record:
      return SizedType(Type::cast, size, typestr);
    case CXType_Pointer:
    {
      auto pointee_type = clang_getPointeeType(clang_type);
      SizedType type;
      if (pointee_type.kind == CXType_Record)
      {
        auto pointee_typestr = get_clang_string(clang_getTypeSpelling(pointee_type));
        remove_struct_prefix(pointee_typestr);
        type = SizedType(Type::cast, sizeof(uintptr_t), pointee_typestr);
      }
      else
      {
        type = SizedType(Type::integer, sizeof(uintptr_t));
      }
      auto pointee_size = clang_Type_getSizeOf(pointee_type);
      type.is_pointer = true;
      type.pointee_size = pointee_size;
      return type;
    }
    case CXType_ConstantArray:
    {
      auto elem_type = clang_getArrayElementType(clang_type);
      auto size = clang_getArraySize(clang_type);
      if (elem_type.kind == CXType_Char_S || elem_type.kind == CXType_Char_U)
      {
        return SizedType(Type::string, size);
      }
      // TODO add support for arrays
      return SizedType(Type::none, 0);
    }
    default:
      return SizedType(Type::none, 0);
  }
}

static bool is_dir(const std::string& path)
{
  struct stat buf;

  if (::stat(path.c_str(), &buf) < 0)
    return false;

  return S_ISDIR(buf.st_mode);
}

// get_kernel_dirs returns {ksrc, kobj} - directories for pristine and
// generated kernel sources.
//
// When the kernel was built in its source tree ksrc == kobj, however when
// the kernel was build in a different directory than its source, ksrc != kobj.
//
// A notable example is Debian, which places pristine kernel headers in
//
//	/lib/modules/`uname -r`/source/
//
// and generated kernel headers in
//
//	/lib/modules/`uname -r`/build/
//
// {"", ""} is returned if no trace of kernel headers was found at all.
// Both ksrc and kobj are guaranteed to be != "", if at least some trace of kernel sources was found.
static std::tuple<std::string, std::string> get_kernel_dirs(const struct utsname& utsname)
{
#ifdef KERNEL_HEADERS_DIR
  return {KERNEL_HEADERS_DIR, KERNEL_HEADERS_DIR};
#endif

  const char *kpath_env = ::getenv("BPFTRACE_KERNEL_SOURCE");
  if (kpath_env)
    return std::make_tuple(kpath_env, kpath_env);

  std::string kdir = std::string("/lib/modules/") + utsname.release;
  auto ksrc = kdir + "/source";
  auto kobj = kdir + "/build";

  // if one of source/ or build/ is not present - try to use the other one for both.
  if (!is_dir(ksrc)) {
	  ksrc = "";
  }
  if (!is_dir(kobj)) {
	  kobj = "";
  }
  if (ksrc == "" && kobj == "") {
	  return std::make_tuple("", "");
  }
  if (ksrc == "") {
	  ksrc = kobj;
  }
  else if (kobj == "") {
	  kobj = ksrc;
  }

  return std::make_tuple(ksrc, kobj);
}

void ClangParser::parse(ast::Program *program, BPFtrace &bpftrace)
{
  auto input = program->c_definitions;
  if (input.size() == 0)
    return; // We occasionally get crashes in libclang otherwise

  CXUnsavedFile unsaved_files[] =
  {
    {
      .Filename = "definitions.h",
      .Contents = input.c_str(),
      .Length = input.size(),
    },
    {
      .Filename = "/bpftrace/include/__stddef_max_align_t.h",
      .Contents = __stddef_max_align_t_h,
      .Length = __stddef_max_align_t_h_len,
    },
    {
      .Filename = "/bpftrace/include/float.h",
      .Contents = float_h,
      .Length = float_h_len,
    },
    {
      .Filename = "/bpftrace/include/limits.h",
      .Contents = limits_h,
      .Length = limits_h_len,
    },
    {
      .Filename = "/bpftrace/include/stdarg.h",
      .Contents = stdarg_h,
      .Length = stdarg_h_len,
    },
    {
      .Filename = "/bpftrace/include/stddef.h",
      .Contents = stddef_h,
      .Length = stddef_h_len,
    },
    {
      .Filename = "/bpftrace/include/stdint.h",
      .Contents = stdint_h,
      .Length = stdint_h_len,
    },
  };

  std::vector<std::string> kflags;
  struct utsname utsname;
  uname(&utsname);
  // auto [ksrc, kobj] = get_kernel_dirs(utsname);	XXX fails with LLVM5
  std::string ksrc, kobj;
  auto kdirs = get_kernel_dirs(utsname);
  ksrc = std::get<0>(kdirs);
  kobj = std::get<1>(kdirs);

  if (ksrc != "")
    kflags = get_kernel_cflags(utsname.machine, ksrc, kobj);

  std::vector<const char *> args =
  {
    "-I", "/bpftrace/include",
  };
  for (auto &flag : kflags)
  {
    args.push_back(flag.c_str());
  }

  CXIndex index = clang_createIndex(1, 1);
  CXTranslationUnit translation_unit;
  CXErrorCode error = clang_parseTranslationUnit2(
      index,
      "definitions.h",
      &args[0], args.size(),
      unsaved_files, sizeof(unsaved_files)/sizeof(CXUnsavedFile),
      CXTranslationUnit_DetailedPreprocessingRecord,
      &translation_unit);
  if (error)
  {
    std::cerr << "Clang error while parsing C definitions: " << error << std::endl;
    std::cerr << "Input (" << input.size() << "): " << input << std::endl;
  }

  indirect_structs.clear();
  unvisited_indirect_structs.clear();

  CXCursor cursor = clang_getTranslationUnitCursor(translation_unit);

  bool iterate = true;

  do {
    clang_visitChildren(
        cursor,
        [](CXCursor c, CXCursor parent, CXClientData client_data)
        {
          if (clang_getCursorKind(c) == CXCursor_MacroDefinition)
          {
            std::string macro_name;
            std::string macro_value;
            if (translateMacro(c, macro_name, macro_value)) {
              auto &macros = static_cast<BPFtrace*>(client_data)->macros_;
              macros[macro_name] = macro_value;
            }
            return CXChildVisit_Recurse;
          }

          if (clang_getCursorKind(parent) != CXCursor_StructDecl &&
              clang_getCursorKind(parent) != CXCursor_UnionDecl)
            return CXChildVisit_Recurse;

          auto ptype = clang_getCanonicalType(clang_getCursorType(parent));
          auto ptypestr = get_clang_string(clang_getTypeSpelling(ptype));
          auto ptypesize = clang_Type_getSizeOf(ptype);

          if (clang_getCursorKind(c) == CXCursor_StructDecl ||
              clang_getCursorKind(c) == CXCursor_UnionDecl) {
            auto struct_name = get_clang_string(clang_getTypeSpelling(clang_getCanonicalType(clang_getCursorType(c))));
            indirect_structs[struct_name] = c;
            unvisited_indirect_structs.insert(struct_name);

            return CXChildVisit_Continue;
          }

          if (clang_getCursorKind(c) == CXCursor_FieldDecl)
          {
            auto &structs = static_cast<BPFtrace*>(client_data)->structs_;
            auto struct_name = get_parent_struct_name(c);
            auto ident = get_clang_string(clang_getCursorSpelling(c));
            auto offset = clang_Cursor_getOffsetOfField(c) / 8;
            auto type = clang_getCanonicalType(clang_getCursorType(c));
            auto typestr = get_clang_string(clang_getTypeSpelling(type));

            if (indirect_structs.count(typestr))
              indirect_structs.erase(typestr);

            if(indirect_structs.count(ptypestr))
              offset = get_indirect_field_offset(c);

            if (struct_name == "")
              struct_name = ptypestr;
            remove_struct_prefix(struct_name);

            structs[struct_name].fields[ident].offset = offset;
            structs[struct_name].fields[ident].type = get_sized_type(type);
            structs[struct_name].size = ptypesize;
          }

          return CXChildVisit_Recurse;
        },
        &bpftrace);
    if (unvisited_indirect_structs.size()) {
      cursor = indirect_structs[*unvisited_indirect_structs.begin()];
      unvisited_indirect_structs.erase(unvisited_indirect_structs.begin());
    } else {
      iterate = false;
    }
  } while (iterate);

  clang_disposeTranslationUnit(translation_unit);
  clang_disposeIndex(index);
}

} // namespace bpftrace
<|endoftext|>


<commit_before>#include "defs.hh"
#include "define_short.hh"

#include "coldata1.hh"
#include "compare1.hh"
#include "connection1.hh"
#include "const_string1.hh"
#include "convert1.hh"
#include "datetime1.hh"
#include "field_names1.hh"
#include "field_types1.hh"
#include "fields1.hh"
#include "manip1.hh"
#include "null1.hh"
#include "query1.hh"
#include "resiter1.hh"
#include "result1.hh"
#include "row1.hh"
#include "set1.hh"
#include "sql_query1.hh"
#include "sql_string1.hh"
#include "stream2string1.hh"
#include "tiny_int1.hh"
#include "type_info1.hh"
#include "vallist1.hh"

#include "coldata2.hh"
#include "compare2.hh"
#include "connection2.hh"
#include "const_string2.hh"
#include "convert2.hh"
#include "datetime2.hh"
#include "field_names2.hh"
#include "field_types2.hh"
#include "fields2.hh"
#include "manip2.hh"
#include "null2.hh"
#include "query2.hh"
#include "result2.hh"
#include "row2.hh"
#include "set2.hh"
#include "sql_query2.hh"
#include "sql_string2.hh"
#include "stream2string2.hh"
#include "tiny_int2.hh"
#include "type_info2.hh"
#include "vallist2.hh"

#include "coldata3.hh"
#include "coldata4.hh"
#include "compare3.hh"
#include "connection3.hh"
#include "const_string3.hh"
#include "convert3.hh"
#include "datetime3.hh"
#include "field_names3.hh"
#include "field_types3.hh"
#include "manip3.hh"
#include "null3.hh"
#include "query3.hh"
#include "result3.hh"
#include "row3.hh"
#include "set3.hh"
#include "sql_query3.hh"
#include "sql_string3.hh"
#include "stream2string3.hh"
#include "tiny_int3.hh"
#include "type_info3.hh"
#include "vallist3.hh"

#ifdef MYSQL_NO_SHORT_NAMES
#include "undef_short"
#endif

#include "exceptions.hh"
<commit_msg>Added winsock.h conditionally on Windows platforms.<commit_after>#ifdef __WIN32__
#include <winsock.h>
#endif

#include "defs.hh"
#include "define_short.hh"

#include "coldata1.hh"
#include "compare1.hh"
#include "connection1.hh"
#include "const_string1.hh"
#include "convert1.hh"
#include "datetime1.hh"
#include "field_names1.hh"
#include "field_types1.hh"
#include "fields1.hh"
#include "manip1.hh"
#include "null1.hh"
#include "query1.hh"
#include "resiter1.hh"
#include "result1.hh"
#include "row1.hh"
#include "set1.hh"
#include "sql_query1.hh"
#include "sql_string1.hh"
#include "stream2string1.hh"
#include "tiny_int1.hh"
#include "type_info1.hh"
#include "vallist1.hh"

#include "coldata2.hh"
#include "compare2.hh"
#include "connection2.hh"
#include "const_string2.hh"
#include "convert2.hh"
#include "datetime2.hh"
#include "field_names2.hh"
#include "field_types2.hh"
#include "fields2.hh"
#include "manip2.hh"
#include "null2.hh"
#include "query2.hh"
#include "result2.hh"
#include "row2.hh"
#include "set2.hh"
#include "sql_query2.hh"
#include "sql_string2.hh"
#include "stream2string2.hh"
#include "tiny_int2.hh"
#include "type_info2.hh"
#include "vallist2.hh"

#include "coldata3.hh"
#include "coldata4.hh"
#include "compare3.hh"
#include "connection3.hh"
#include "const_string3.hh"
#include "convert3.hh"
#include "datetime3.hh"
#include "field_names3.hh"
#include "field_types3.hh"
#include "manip3.hh"
#include "null3.hh"
#include "query3.hh"
#include "result3.hh"
#include "row3.hh"
#include "set3.hh"
#include "sql_query3.hh"
#include "sql_string3.hh"
#include "stream2string3.hh"
#include "tiny_int3.hh"
#include "type_info3.hh"
#include "vallist3.hh"

#ifdef MYSQL_NO_SHORT_NAMES
#include "undef_short"
#endif

#include "exceptions.hh"
<|endoftext|>


<commit_before>#include <iostream>
#include <string>
#include "../tcp_api.cpp"

int main(int argc, char* argv[]){
  std::string ip_addr = argv[1];
  std::string filepath = argv[2];

  file_send(ip_addr, filepath);

  return 0;
}
<commit_msg>small test<commit_after>#include <iostream>
#include <string>
#include "../tcp_api.cpp"
#include "./netlinkUser.c"

int main(int argc, char* argv[]){
  std::string ip_addr = argv[1];
  std::string filepath = argv[2];

  file_send(ip_addr, filepath);

  captains_log("this is a test");

  return 0;
}
<|endoftext|>


<commit_before>/*
 * Copyright (c) 2016 dresden elektronik ingenieurtechnik gmbh.
 * All rights reserved.
 *
 * The software in this package is published under the terms of the BSD
 * style license a copy of which has been included with this distribution in
 * the LICENSE.txt file.
 *
 */

#include <QApplication>
#include <QDesktopServices>
#include <QFile>
#include <QDir>
#include <QString>
#include <QProcess>
#include "de_web_plugin.h"
#include "de_web_plugin_private.h"

#define FW_IDLE_TIMEOUT (10 * 1000)
#define FW_IDLE_TIMEOUT_LONG (240 * 1000)
#define FW_WAIT_USER_TIMEOUT (60 * 1000)

/*! Inits the firmware update manager.
 */
void DeRestPluginPrivate::initFirmwareUpdate()
{
    fwProcess = 0;
    fwUpdateState = FW_Idle;

    Q_ASSERT(apsCtrl);
    apsCtrl->setParameter(deCONZ::ParamFirmwareUpdateActive, 0);

    fwUpdateTimer = new QTimer(this);
    fwUpdateTimer->setInterval(1000);
    fwUpdateTimer->setSingleShot(true);
    connect(fwUpdateTimer, SIGNAL(timeout()),
            this, SLOT(firmwareUpdateTimerFired()));
    fwUpdateTimer->start(FW_IDLE_TIMEOUT);
}

/*! Starts the actual firmware update process.
 */
void DeRestPluginPrivate::updateFirmware()
{
    if (gwFirmwareNeedUpdate)
    {
        gwFirmwareNeedUpdate = false;
    }

    Q_ASSERT(apsCtrl);
    if (apsCtrl->getParameter(deCONZ::ParamFirmwareUpdateActive) == 0 ||
        apsCtrl->getParameter(deCONZ::ParamDeviceConnected) == 1)
    {
        DBG_Printf(DBG_INFO, "GW firmware update conditions not met, abort\n");
        fwUpdateState = FW_Idle;
        fwUpdateTimer->start(FW_IDLE_TIMEOUT);
        return;
    }

    QString gcfFlasherBin = qApp->applicationDirPath() + "/GCFFlasher";
#ifdef Q_OS_WIN
    gcfFlasherBin.append(".exe");
    QString bin = gcfFlasherBin;
#else
    QString bin = "sudo";
    fwProcessArgs.prepend(gcfFlasherBin);
#endif

    if (!fwProcess)
    {
        fwProcess = new QProcess(this);
    }

    fwProcessArgs << "-f" << fwUpdateFile;

    fwUpdateState = FW_UpdateWaitFinished;
    fwUpdateTimer->start(1000);

    fwProcess->start(bin, fwProcessArgs);
}

/*! Observes the firmware update process.
 */
void DeRestPluginPrivate::updateFirmwareWaitFinished()
{
    if (fwProcess)
    {
        if (fwProcess->bytesAvailable())
        {
            QByteArray data = fwProcess->readAllStandardOutput();
            DBG_Printf(DBG_INFO, "%s", qPrintable(data));
        }

        if (fwProcess->state() == QProcess::Starting)
        {
            DBG_Printf(DBG_INFO, "GW firmware update starting ..\n");
        }
        else if (fwProcess->state() == QProcess::Running)
        {
            DBG_Printf(DBG_INFO, "GW firmware update running ..\n");
        }
        else if (fwProcess->state() == QProcess::NotRunning)
        {
            if (fwProcess->exitStatus() == QProcess::NormalExit)
            {
                DBG_Printf(DBG_INFO, "GW firmware update exit code %d\n", fwProcess->exitCode());
            }
            else if (fwProcess->exitStatus() == QProcess::CrashExit)
            {
                DBG_Printf(DBG_INFO, "GW firmware update crashed %s\n", qPrintable(fwProcess->errorString()));
            }

            fwProcess->deleteLater();
            fwProcess = 0;
        }
    }

    // done
    if (fwProcess == 0)
    {
        apsCtrl->setParameter(deCONZ::ParamFirmwareUpdateActive, 0);
        fwUpdateState = FW_Idle;
        fwUpdateTimer->start(FW_IDLE_TIMEOUT);
    }
    else // recheck
    {
        fwUpdateTimer->start(1000);
    }
}

/*! Starts the device disconnect so that the serial port is released.
 */
void DeRestPluginPrivate::updateFirmwareDisconnectDevice()
{
    Q_ASSERT(apsCtrl);
    if (apsCtrl->getParameter(deCONZ::ParamFirmwareUpdateActive) == 0)
    {
        if (apsCtrl->getParameter(deCONZ::ParamDeviceConnected) == 1)
        {
            DBG_Printf(DBG_INFO, "GW firmware disconnect device before update\n");
        }
        apsCtrl->setParameter(deCONZ::ParamFirmwareUpdateActive, 1);
    }

    if (apsCtrl->getParameter(deCONZ::ParamDeviceConnected) == 1)
    {
        fwUpdateTimer->start(100); // recheck
    }
    else
    {
        DBG_Printf(DBG_INFO, "GW firmware start update (device not connected)\n");
        fwUpdateState = FW_Update;
        fwUpdateTimer->start(0);
    }
}

/*! Starts the firmware update.
 */
bool DeRestPluginPrivate::startUpdateFirmware()
{
    if (fwUpdateState == FW_WaitUserConfirm)
    {
        fwUpdateState = FW_DisconnectDevice;
        fwUpdateTimer->start(100);
        return true;
    }

    return false;
}

/*! Delayed trigger to update the firmware.
 */
void DeRestPluginPrivate::firmwareUpdateTimerFired()
{
    if (fwUpdateState == FW_Idle)
    {
        if (gwFirmwareNeedUpdate)
        {
            gwFirmwareNeedUpdate = false;
            updateEtag(gwConfigEtag);
        }
        fwUpdateState = FW_CheckDevices;
        fwUpdateTimer->start(0);
    }
    else if (fwUpdateState == FW_CheckDevices)
    {
        checkFirmwareDevices();
    }
    else if (fwUpdateState == FW_CheckVersion)
    {
        queryFirmwareVersion();
    }
    else if (fwUpdateState == FW_DisconnectDevice)
    {
        updateFirmwareDisconnectDevice();
    }
    else if (fwUpdateState == FW_Update)
    {
        updateFirmware();
    }
    else if (fwUpdateState == FW_UpdateWaitFinished)
    {
        updateFirmwareWaitFinished();
    }
    else // also handle FW_WaitUserConfirm state timeout
    {
        fwUpdateState = FW_Idle;
        fwUpdateTimer->start(FW_IDLE_TIMEOUT);
    }
}

/*! Lazy query of firmware version.
    Because the device might not be connected at first optaining the
    firmware version must be delayed.

    If the firmware is older then the min required firmware for the platform
    and a proper firmware update file exists, the API will announce that a
    firmware update is available.
 */
void DeRestPluginPrivate::queryFirmwareVersion()
{
    Q_ASSERT(apsCtrl);
    if (!apsCtrl)
    {
        return;
    }

    { // check for GCFFlasher binary
        QString gcfFlasherBin = qApp->applicationDirPath() + "/GCFFlasher";
#ifdef Q_OS_WIN
        gcfFlasherBin.append(".exe");
#endif

        if (!QFile::exists(gcfFlasherBin))
        {
            DBG_Printf(DBG_INFO, "GW update firmware failed, %s doesn't exist\n", qPrintable(gcfFlasherBin));
            fwUpdateState = FW_Idle;
            fwUpdateTimer->start(FW_IDLE_TIMEOUT_LONG);
            return;
        }
    }

    // does the update file exist?
    if (fwUpdateFile.isEmpty())
    {
        QString fileName;
        fileName.sprintf("deCONZ_Rpi_0x%08x.bin.GCF", GW_MIN_RPI_FW_VERSION);

        // search in different locations
        std::vector<QString> paths;
        paths.push_back(deCONZ::getStorageLocation(deCONZ::ApplicationsDataLocation) + QLatin1String("/firmware/"));
        paths.push_back(deCONZ::getStorageLocation(deCONZ::HomeLocation) + QLatin1String("/raspbee_firmware/"));
#ifdef Q_OS_OSX
        QDir dir(qApp->applicationDirPath());
        dir.cdUp();
        dir.cd("Resources");
        paths.push_back(dir.path() + "/");
#endif

        std::vector<QString>::const_iterator i = paths.begin();
        std::vector<QString>::const_iterator end = paths.end();
        for (; i != end; ++i)
        {
            if (QFile::exists(*i + fileName))
            {
                fwUpdateFile = *i + fileName;
                DBG_Printf(DBG_INFO, "GW update firmware found: %s\n", qPrintable(fwUpdateFile));
                break;
            }
        }
    }

    if (fwUpdateFile.isEmpty())
    {
        DBG_Printf(DBG_ERROR, "GW update firmware not found: %s\n", qPrintable(fwUpdateFile));
        fwUpdateState = FW_Idle;
        fwUpdateTimer->start(FW_IDLE_TIMEOUT);
        return;
    }

    uint8_t devConnected = apsCtrl->getParameter(deCONZ::ParamDeviceConnected);
    uint32_t fwVersion = apsCtrl->getParameter(deCONZ::ParamFirmwareVersion);

    Q_ASSERT(!gwFirmwareNeedUpdate);

    if (devConnected == 0 || fwVersion == 0)
    {
        // if even after 60 seconds no firmware was detected
        // ASSUME that a device is present and reachable but might not have firmware installed
        if (getUptime() >= 60)
        {
            QString str;
            str.sprintf("0x%08x", GW_MIN_RPI_FW_VERSION);

            gwFirmwareVersion = "0x00000000"; // unknown
            gwFirmwareVersionUpdate = str;
            gwConfig["fwversion"] = gwFirmwareVersion;
            gwFirmwareNeedUpdate = true;
            updateEtag(gwConfigEtag);

            fwUpdateState = FW_WaitUserConfirm;
            fwUpdateTimer->start(FW_WAIT_USER_TIMEOUT);
            return;
        }
    }
    else if (devConnected)
    {
        QString str;
        str.sprintf("0x%08x", fwVersion);

        if (gwFirmwareVersion != str)
        {
            gwFirmwareVersion = str;
            gwConfig["fwversion"] = str;
            updateEtag(gwConfigEtag);
        }

        DBG_Printf(DBG_INFO, "GW firmware version: %s\n", qPrintable(gwFirmwareVersion));

        // if the device is detected check that the firmware version is >= min version
        if ((fwVersion & FW_PLATFORM_MASK) == FW_PLATFORM_RPI)
        {
            if (fwVersion < GW_MIN_RPI_FW_VERSION)
            {
                gwFirmwareVersionUpdate.sprintf("0x%08x", GW_MIN_RPI_FW_VERSION);
                gwFirmwareNeedUpdate = true;
                updateEtag(gwConfigEtag);

                DBG_Printf(DBG_INFO, "GW firmware version shall be updated to: 0x%08x\n", GW_MIN_RPI_FW_VERSION);
                fwUpdateState = FW_WaitUserConfirm;
                fwUpdateTimer->start(FW_WAIT_USER_TIMEOUT);
                return;
            }
            else
            {
                DBG_Printf(DBG_INFO, "GW firmware version is up to date: 0x%08x\n", fwVersion);
                fwUpdateState = FW_Idle;
                fwUpdateTimer->start(FW_IDLE_TIMEOUT_LONG);
                return;
            }
        }

        if (!gwFirmwareVersionUpdate.isEmpty())
        {
            gwFirmwareVersionUpdate.clear();
            updateEtag(gwConfigEtag);
        }
    }

    fwUpdateState = FW_Idle;
    fwUpdateTimer->start(FW_IDLE_TIMEOUT);
}

/*! Checks if devices for firmware update are present.
 */
void DeRestPluginPrivate::checkFirmwareDevices()
{
    deCONZ::DeviceEnumerator devEnumerator;

    fwProcessArgs.clear();

    devEnumerator.listSerialPorts();
    const std::vector<deCONZ::DeviceEntry> &availPorts = devEnumerator.getList();

    std::vector<deCONZ::DeviceEntry>::const_iterator i = availPorts.begin();
    std::vector<deCONZ::DeviceEntry>::const_iterator end = availPorts.end();

    int raspBeeCount = 0;
    int usbDongleCount = 0;

    for (; i != end; ++i)
    {
        if (i->friendlyName.contains(QLatin1String("ConBee")))
        {
            usbDongleCount++;
        }
        else if (i->path.contains(QLatin1String("ttyAMA0")))
        {
            raspBeeCount = 1;
        }
    }

    if (usbDongleCount > 1)
    {
        DBG_Printf(DBG_INFO, "GW firmware update too many USB devices connected, abort\n");
    }
    else if (usbDongleCount == 1)
    {
        DBG_Printf(DBG_INFO, "GW firmware update select USB device\n");
        fwProcessArgs << "-d" << "0";
    }
    else if (raspBeeCount > 0 && usbDongleCount == 0)
    {
        DBG_Printf(DBG_INFO, "GW firmware update select /dev/ttyAMA0 device\n");
        fwProcessArgs << "/dev/ttyAMA0";
    }

    if (!fwProcessArgs.isEmpty())
    {
        fwUpdateState = FW_CheckVersion;
        fwUpdateTimer->start(0);
        return;
    }

    fwUpdateState = FW_Idle;
    fwUpdateTimer->start(FW_IDLE_TIMEOUT);
}
<commit_msg>Use correct RaspBee device paths for firmware update<commit_after>/*
 * Copyright (c) 2016 dresden elektronik ingenieurtechnik gmbh.
 * All rights reserved.
 *
 * The software in this package is published under the terms of the BSD
 * style license a copy of which has been included with this distribution in
 * the LICENSE.txt file.
 *
 */

#include <QApplication>
#include <QDesktopServices>
#include <QFile>
#include <QDir>
#include <QString>
#include <QProcess>
#include "de_web_plugin.h"
#include "de_web_plugin_private.h"

#define FW_IDLE_TIMEOUT (10 * 1000)
#define FW_IDLE_TIMEOUT_LONG (240 * 1000)
#define FW_WAIT_USER_TIMEOUT (60 * 1000)

/*! Inits the firmware update manager.
 */
void DeRestPluginPrivate::initFirmwareUpdate()
{
    fwProcess = 0;
    fwUpdateState = FW_Idle;

    Q_ASSERT(apsCtrl);
    apsCtrl->setParameter(deCONZ::ParamFirmwareUpdateActive, 0);

    fwUpdateTimer = new QTimer(this);
    fwUpdateTimer->setInterval(1000);
    fwUpdateTimer->setSingleShot(true);
    connect(fwUpdateTimer, SIGNAL(timeout()),
            this, SLOT(firmwareUpdateTimerFired()));
    fwUpdateTimer->start(FW_IDLE_TIMEOUT);
}

/*! Starts the actual firmware update process.
 */
void DeRestPluginPrivate::updateFirmware()
{
    if (gwFirmwareNeedUpdate)
    {
        gwFirmwareNeedUpdate = false;
    }

    Q_ASSERT(apsCtrl);
    if (apsCtrl->getParameter(deCONZ::ParamFirmwareUpdateActive) == 0 ||
        apsCtrl->getParameter(deCONZ::ParamDeviceConnected) == 1)
    {
        DBG_Printf(DBG_INFO, "GW firmware update conditions not met, abort\n");
        fwUpdateState = FW_Idle;
        fwUpdateTimer->start(FW_IDLE_TIMEOUT);
        return;
    }

    QString gcfFlasherBin = qApp->applicationDirPath() + "/GCFFlasher";
#ifdef Q_OS_WIN
    gcfFlasherBin.append(".exe");
    QString bin = gcfFlasherBin;
#else
    QString bin = "sudo";
    fwProcessArgs.prepend(gcfFlasherBin);
#endif

    if (!fwProcess)
    {
        fwProcess = new QProcess(this);
    }

    fwProcessArgs << "-f" << fwUpdateFile;

    fwUpdateState = FW_UpdateWaitFinished;
    fwUpdateTimer->start(1000);

    fwProcess->start(bin, fwProcessArgs);
}

/*! Observes the firmware update process.
 */
void DeRestPluginPrivate::updateFirmwareWaitFinished()
{
    if (fwProcess)
    {
        if (fwProcess->bytesAvailable())
        {
            QByteArray data = fwProcess->readAllStandardOutput();
            DBG_Printf(DBG_INFO, "%s", qPrintable(data));
        }

        if (fwProcess->state() == QProcess::Starting)
        {
            DBG_Printf(DBG_INFO, "GW firmware update starting ..\n");
        }
        else if (fwProcess->state() == QProcess::Running)
        {
            DBG_Printf(DBG_INFO, "GW firmware update running ..\n");
        }
        else if (fwProcess->state() == QProcess::NotRunning)
        {
            if (fwProcess->exitStatus() == QProcess::NormalExit)
            {
                DBG_Printf(DBG_INFO, "GW firmware update exit code %d\n", fwProcess->exitCode());
            }
            else if (fwProcess->exitStatus() == QProcess::CrashExit)
            {
                DBG_Printf(DBG_INFO, "GW firmware update crashed %s\n", qPrintable(fwProcess->errorString()));
            }

            fwProcess->deleteLater();
            fwProcess = 0;
        }
    }

    // done
    if (fwProcess == 0)
    {
        apsCtrl->setParameter(deCONZ::ParamFirmwareUpdateActive, 0);
        fwUpdateState = FW_Idle;
        fwUpdateTimer->start(FW_IDLE_TIMEOUT);
    }
    else // recheck
    {
        fwUpdateTimer->start(1000);
    }
}

/*! Starts the device disconnect so that the serial port is released.
 */
void DeRestPluginPrivate::updateFirmwareDisconnectDevice()
{
    Q_ASSERT(apsCtrl);
    if (apsCtrl->getParameter(deCONZ::ParamFirmwareUpdateActive) == 0)
    {
        if (apsCtrl->getParameter(deCONZ::ParamDeviceConnected) == 1)
        {
            DBG_Printf(DBG_INFO, "GW firmware disconnect device before update\n");
        }
        apsCtrl->setParameter(deCONZ::ParamFirmwareUpdateActive, 1);
    }

    if (apsCtrl->getParameter(deCONZ::ParamDeviceConnected) == 1)
    {
        fwUpdateTimer->start(100); // recheck
    }
    else
    {
        DBG_Printf(DBG_INFO, "GW firmware start update (device not connected)\n");
        fwUpdateState = FW_Update;
        fwUpdateTimer->start(0);
    }
}

/*! Starts the firmware update.
 */
bool DeRestPluginPrivate::startUpdateFirmware()
{
    if (fwUpdateState == FW_WaitUserConfirm)
    {
        fwUpdateState = FW_DisconnectDevice;
        fwUpdateTimer->start(100);
        return true;
    }

    return false;
}

/*! Delayed trigger to update the firmware.
 */
void DeRestPluginPrivate::firmwareUpdateTimerFired()
{
    if (fwUpdateState == FW_Idle)
    {
        if (gwFirmwareNeedUpdate)
        {
            gwFirmwareNeedUpdate = false;
            updateEtag(gwConfigEtag);
        }
        fwUpdateState = FW_CheckDevices;
        fwUpdateTimer->start(0);
    }
    else if (fwUpdateState == FW_CheckDevices)
    {
        checkFirmwareDevices();
    }
    else if (fwUpdateState == FW_CheckVersion)
    {
        queryFirmwareVersion();
    }
    else if (fwUpdateState == FW_DisconnectDevice)
    {
        updateFirmwareDisconnectDevice();
    }
    else if (fwUpdateState == FW_Update)
    {
        updateFirmware();
    }
    else if (fwUpdateState == FW_UpdateWaitFinished)
    {
        updateFirmwareWaitFinished();
    }
    else // also handle FW_WaitUserConfirm state timeout
    {
        fwUpdateState = FW_Idle;
        fwUpdateTimer->start(FW_IDLE_TIMEOUT);
    }
}

/*! Lazy query of firmware version.
    Because the device might not be connected at first optaining the
    firmware version must be delayed.

    If the firmware is older then the min required firmware for the platform
    and a proper firmware update file exists, the API will announce that a
    firmware update is available.
 */
void DeRestPluginPrivate::queryFirmwareVersion()
{
    Q_ASSERT(apsCtrl);
    if (!apsCtrl)
    {
        return;
    }

    { // check for GCFFlasher binary
        QString gcfFlasherBin = qApp->applicationDirPath() + "/GCFFlasher";
#ifdef Q_OS_WIN
        gcfFlasherBin.append(".exe");
#endif

        if (!QFile::exists(gcfFlasherBin))
        {
            DBG_Printf(DBG_INFO, "GW update firmware failed, %s doesn't exist\n", qPrintable(gcfFlasherBin));
            fwUpdateState = FW_Idle;
            fwUpdateTimer->start(FW_IDLE_TIMEOUT_LONG);
            return;
        }
    }

    // does the update file exist?
    if (fwUpdateFile.isEmpty())
    {
        QString fileName;
        fileName.sprintf("deCONZ_Rpi_0x%08x.bin.GCF", GW_MIN_RPI_FW_VERSION);

        // search in different locations
        std::vector<QString> paths;
        paths.push_back(deCONZ::getStorageLocation(deCONZ::ApplicationsDataLocation) + QLatin1String("/firmware/"));
        paths.push_back(deCONZ::getStorageLocation(deCONZ::HomeLocation) + QLatin1String("/raspbee_firmware/"));
#ifdef Q_OS_OSX
        QDir dir(qApp->applicationDirPath());
        dir.cdUp();
        dir.cd("Resources");
        paths.push_back(dir.path() + "/");
#endif

        std::vector<QString>::const_iterator i = paths.begin();
        std::vector<QString>::const_iterator end = paths.end();
        for (; i != end; ++i)
        {
            if (QFile::exists(*i + fileName))
            {
                fwUpdateFile = *i + fileName;
                DBG_Printf(DBG_INFO, "GW update firmware found: %s\n", qPrintable(fwUpdateFile));
                break;
            }
        }
    }

    if (fwUpdateFile.isEmpty())
    {
        DBG_Printf(DBG_ERROR, "GW update firmware not found: %s\n", qPrintable(fwUpdateFile));
        fwUpdateState = FW_Idle;
        fwUpdateTimer->start(FW_IDLE_TIMEOUT);
        return;
    }

    uint8_t devConnected = apsCtrl->getParameter(deCONZ::ParamDeviceConnected);
    uint32_t fwVersion = apsCtrl->getParameter(deCONZ::ParamFirmwareVersion);

    Q_ASSERT(!gwFirmwareNeedUpdate);

    if (devConnected == 0 || fwVersion == 0)
    {
        // if even after 60 seconds no firmware was detected
        // ASSUME that a device is present and reachable but might not have firmware installed
        if (getUptime() >= 60)
        {
            QString str;
            str.sprintf("0x%08x", GW_MIN_RPI_FW_VERSION);

            gwFirmwareVersion = "0x00000000"; // unknown
            gwFirmwareVersionUpdate = str;
            gwConfig["fwversion"] = gwFirmwareVersion;
            gwFirmwareNeedUpdate = true;
            updateEtag(gwConfigEtag);

            fwUpdateState = FW_WaitUserConfirm;
            fwUpdateTimer->start(FW_WAIT_USER_TIMEOUT);
            return;
        }
    }
    else if (devConnected)
    {
        QString str;
        str.sprintf("0x%08x", fwVersion);

        if (gwFirmwareVersion != str)
        {
            gwFirmwareVersion = str;
            gwConfig["fwversion"] = str;
            updateEtag(gwConfigEtag);
        }

        DBG_Printf(DBG_INFO, "GW firmware version: %s\n", qPrintable(gwFirmwareVersion));

        // if the device is detected check that the firmware version is >= min version
        if ((fwVersion & FW_PLATFORM_MASK) == FW_PLATFORM_RPI)
        {
            if (fwVersion < GW_MIN_RPI_FW_VERSION)
            {
                gwFirmwareVersionUpdate.sprintf("0x%08x", GW_MIN_RPI_FW_VERSION);
                gwFirmwareNeedUpdate = true;
                updateEtag(gwConfigEtag);

                DBG_Printf(DBG_INFO, "GW firmware version shall be updated to: 0x%08x\n", GW_MIN_RPI_FW_VERSION);
                fwUpdateState = FW_WaitUserConfirm;
                fwUpdateTimer->start(FW_WAIT_USER_TIMEOUT);
                return;
            }
            else
            {
                DBG_Printf(DBG_INFO, "GW firmware version is up to date: 0x%08x\n", fwVersion);
                fwUpdateState = FW_Idle;
                fwUpdateTimer->start(FW_IDLE_TIMEOUT_LONG);
                return;
            }
        }

        if (!gwFirmwareVersionUpdate.isEmpty())
        {
            gwFirmwareVersionUpdate.clear();
            updateEtag(gwConfigEtag);
        }
    }

    fwUpdateState = FW_Idle;
    fwUpdateTimer->start(FW_IDLE_TIMEOUT);
}

/*! Checks if devices for firmware update are present.
 */
void DeRestPluginPrivate::checkFirmwareDevices()
{
    deCONZ::DeviceEnumerator devEnumerator;

    fwProcessArgs.clear();

    devEnumerator.listSerialPorts();
    const std::vector<deCONZ::DeviceEntry> &availPorts = devEnumerator.getList();

    std::vector<deCONZ::DeviceEntry>::const_iterator i = availPorts.begin();
    std::vector<deCONZ::DeviceEntry>::const_iterator end = availPorts.end();

    int raspBeeCount = 0;
    int usbDongleCount = 0;

    for (; i != end; ++i)
    {
        if (i->friendlyName.contains(QLatin1String("ConBee")))
        {
            usbDongleCount++;
        }
        else if (i->friendlyName.contains(QLatin1String("RaspBee")))
        {
            raspBeeCount = 1;
        }
    }

    if (usbDongleCount > 1)
    {
        DBG_Printf(DBG_INFO, "GW firmware update too many USB devices connected, abort\n");
    }
    else if (usbDongleCount == 1)
    {
        DBG_Printf(DBG_INFO, "GW firmware update select USB device\n");
        fwProcessArgs << "-d" << "0";
    }
    else if (raspBeeCount > 0 && usbDongleCount == 0)
    {
        DBG_Printf(DBG_INFO, "GW firmware update select /dev/ttyAMA0 device\n");
        fwProcessArgs << i->path;
    }

    if (!fwProcessArgs.isEmpty())
    {
        fwUpdateState = FW_CheckVersion;
        fwUpdateTimer->start(0);
        return;
    }

    fwUpdateState = FW_Idle;
    fwUpdateTimer->start(FW_IDLE_TIMEOUT);
}
<|endoftext|>


<commit_before>/**
 * Copyright (c) 2015 - The CM Authors <legal@clickmatcher.com>
 *   All Rights Reserved.
 *
 * This file is CONFIDENTIAL -- Distribution or duplication of this material or
 * the information contained herein is strictly forbidden unless prior written
 * permission is obtained.
 */
#define BOOST_NO_CXX11_NUMERIC_LIMITS 1
#include <stdlib.h>
#include <unistd.h>
#include <signal.h>
#include "fnord-base/io/filerepository.h"
#include "fnord-base/io/fileutil.h"
#include "fnord-base/application.h"
#include "fnord-base/logging.h"
#include "fnord-base/random.h"
#include "fnord-base/thread/eventloop.h"
#include "fnord-base/thread/threadpool.h"
#include "fnord-base/wallclock.h"
#include "fnord-rpc/ServerGroup.h"
#include "fnord-rpc/RPC.h"
#include "fnord-rpc/RPCClient.h"
#include "fnord-base/cli/flagparser.h"
#include "fnord-json/json.h"
#include "fnord-json/jsonrpc.h"
#include "fnord-http/httprouter.h"
#include "fnord-http/httpserver.h"
#include "fnord-feeds/FeedService.h"
#include "fnord-feeds/RemoteFeedFactory.h"
#include "fnord-feeds/RemoteFeedReader.h"
#include "fnord-base/stats/statsdagent.h"
#include "fnord-fts/fts.h"
#include "fnord-fts/fts_common.h"
#include "fnord-mdb/MDB.h"
#include "CustomerNamespace.h"
#include "FeatureSchema.h"
#include "FeatureIndexWriter.h"
#include "IndexRequest.h"
#include "IndexBuild.h"

using namespace cm;
using namespace fnord;

std::atomic<bool> cm_indexbuild_shutdown;

void quit(int n) {
  cm_indexbuild_shutdown = true;
}

int main(int argc, const char** argv) {
  Application::init();
  Application::logToStderr();

  /* shutdown hook */
  cm_indexbuild_shutdown = false;
  struct sigaction sa;
  memset(&sa, 0, sizeof(struct sigaction));
  sa.sa_handler = quit;
  sigaction(SIGTERM, &sa, NULL);
  sigaction(SIGQUIT, &sa, NULL);
  sigaction(SIGINT, &sa, NULL);

  cli::FlagParser flags;

  flags.defineFlag(
      "index",
      cli::FlagParser::T_STRING,
      true,
      NULL,
      NULL,
      "index dir",
      "<path>");

  flags.defineFlag(
      "statsd_addr",
      fnord::cli::FlagParser::T_STRING,
      false,
      NULL,
      "127.0.0.1:8192",
      "Statsd addr",
      "<addr>");

  flags.defineFlag(
      "batch_size",
      fnord::cli::FlagParser::T_INTEGER,
      false,
      NULL,
      "2048",
      "batch_size",
      "<num>");

  flags.defineFlag(
      "buffer_size",
      fnord::cli::FlagParser::T_INTEGER,
      false,
      NULL,
      "8192",
      "buffer_size",
      "<num>");

  flags.defineFlag(
      "db_commit_size",
      fnord::cli::FlagParser::T_INTEGER,
      false,
      NULL,
      "1024",
      "db_commit_size",
      "<num>");

  flags.defineFlag(
      "db_commit_interval",
      fnord::cli::FlagParser::T_INTEGER,
      false,
      NULL,
      "5",
      "db_commit_interval",
      "<num>");

  flags.defineFlag(
      "dbsize",
      fnord::cli::FlagParser::T_INTEGER,
      false,
      NULL,
      "128",
      "max sessiondb size",
      "<MB>");

  flags.defineFlag(
      "loglevel",
      fnord::cli::FlagParser::T_STRING,
      false,
      NULL,
      "INFO",
      "loglevel",
      "<level>");

  flags.parseArgv(argc, argv);

  Logger::get()->setMinimumLogLevel(
      strToLogLevel(flags.getString("loglevel")));

  /* set up feature schema */
  FeatureSchema feature_schema;
  feature_schema.registerFeature("shop_id", 1, 1);
  feature_schema.registerFeature("category1", 2, 1);
  feature_schema.registerFeature("category2", 3, 1);
  feature_schema.registerFeature("category3", 4, 1);
  feature_schema.registerFeature("title~de", 5, 2);

  /* start event loop */
  fnord::thread::EventLoop ev;

  auto evloop_thread = std::thread([&ev] {
    ev.run();
  });

  /* set up rpc client */
  HTTPRPCClient rpc_client(&ev);

  /* start stats reporting */
  fnord::stats::StatsdAgent statsd_agent(
      fnord::net::InetAddr::resolve(flags.getString("statsd_addr")),
      10 * fnord::kMicrosPerSecond);

  statsd_agent.start();

  /* set up indexbuild */
  size_t batch_size = flags.getInt("batch_size");
  size_t buffer_size = flags.getInt("buffer_size");
  size_t db_commit_size = flags.getInt("db_commit_size");
  size_t db_commit_interval = flags.getInt("db_commit_interval");

  fnord::logInfo(
      "cm.indexbuild",
      "Starting cm-indexbuild with:\n    index=$0\n    batch_size=$1\n" \
      "    buffer_size=$2\n    db_commit_size=$3\n    db_commit_interval=$4\n"
      "    max_dbsize=$5MB",
      flags.getString("index"),
      batch_size,
      buffer_size,
      db_commit_size,
      db_commit_interval,
      flags.getInt("dbsize"));

  /* stats */
  fnord::stats::Counter<uint64_t> stat_documents_indexed_total_;
  fnord::stats::Counter<uint64_t> stat_documents_indexed_success_;
  fnord::stats::Counter<uint64_t> stat_documents_indexed_error_;

  exportStat(
      "/cm-indexbuild/global/documents_indexed_total",
      &stat_documents_indexed_total_,
      fnord::stats::ExportMode::EXPORT_DELTA);

  exportStat(
      "/cm-indexbuild/global/documents_indexed_success",
      &stat_documents_indexed_success_,
      fnord::stats::ExportMode::EXPORT_DELTA);

  exportStat(
      "/cm-indexbuild/global/documents_indexed_error",
      &stat_documents_indexed_error_,
      fnord::stats::ExportMode::EXPORT_DELTA);

  /* open featuredb db */
  auto featuredb_path = FileUtil::joinPaths(flags.getString("index"), "db");
  FileUtil::mkdir_p(featuredb_path);
  auto featuredb = mdb::MDB::open(featuredb_path);
  featuredb->setMaxSize(1000000 * flags.getInt("dbsize"));
  cm::FeatureIndexWriter feature_index_writer(&feature_schema);

  /* open full index */
  auto fullindex_path = FileUtil::joinPaths(flags.getString("index"), "docs");
  FileUtil::mkdir_p(fullindex_path);
  cm::FullIndex full_index(fullindex_path);

  /* open lucene index */
  /*
  auto index_path = FileUtil::joinPaths(
      cmdata_path,
      StringUtil::format("index/$0/fts", cmcustomer));
  FileUtil::mkdir_p(index_path);

  auto index_writer =
      fts::newLucene<fts::IndexWriter>(
          fts::FSDirectory::open(StringUtil::convertUTF8To16(index_path)),
          fts::newLucene<fts::StandardAnalyzer>(
              fts::LuceneVersion::LUCENE_CURRENT),
          true,
          fts::IndexWriter::MaxFieldLengthLIMITED);
  */

  cm::IndexBuild index_build(&feature_index_writer, &full_index);

  /* set up input feed reader */
  feeds::RemoteFeedReader feed_reader(&rpc_client);
  feed_reader.setMaxSpread(3600 * 24 * 30 * kMicrosPerSecond);

  HashMap<String, URI> input_feeds;
  auto cmcustomer = "dawanda";
  input_feeds.emplace(
      StringUtil::format(
          "$0.index_requests.feedserver01.nue01.production.fnrd.net",
          cmcustomer),
      URI("http://s01.nue01.production.fnrd.net:7001/rpc"));
  input_feeds.emplace(
      StringUtil::format(
          "$0.index_requests.feedserver02.nue01.production.fnrd.net",
          cmcustomer),
      URI("http://s02.nue01.production.fnrd.net:7001/rpc"));

  /* resume from last offset */
  auto txn = featuredb->startTransaction(true);
  try {
    for (const auto& input_feed : input_feeds) {
      uint64_t offset = 0;

      auto last_offset = txn->get(
          StringUtil::format("__indexfeed_offset~$0", input_feed.first));

      if (!last_offset.isEmpty()) {
        offset = std::stoul(last_offset.get().toString());
      }

      fnord::logInfo(
          "cm.indexbuild",
          "Adding source feed:\n    feed=$0\n    url=$1\n    offset: $2",
          input_feed.first,
          input_feed.second.toString(),
          offset);

      feed_reader.addSourceFeed(
          input_feed.second,
          input_feed.first,
          offset,
          batch_size,
          buffer_size);
    }

    txn->abort();
  } catch (...) {
    txn->abort();
    throw;
  }

  fnord::logInfo("cm.indexbuild", "Resuming IndexBuild...");
  // index document
/*
  auto doc = fts::newLucene<fts::Document>();
  doc->add(
      fts::newLucene<fts::Field>(
          L"keywords",
          L"my fnordy document",
          fts::Field::STORE_NO,
          fts::Field::INDEX_ANALYZED));

  index_writer->addDocument(doc);
*/

  //index_writer->commit();
  //index_writer->close();

  // simple search
  /*
  auto index_reader = fts::IndexReader::open(
      fts::FSDirectory::open(StringUtil::convertUTF8To16(index_path)),
      true);

  auto searcher = fts::newLucene<fts::IndexSearcher>(index_reader);

  auto analyzer = fts::newLucene<fts::StandardAnalyzer>(
      fts::LuceneVersion::LUCENE_CURRENT);

  auto query_parser = fts::newLucene<fts::QueryParser>(
      fts::LuceneVersion::LUCENE_CURRENT,
      L"keywords",
      analyzer);

  auto collector = fts::TopScoreDocCollector::create(
      500,
      false);

  auto query = query_parser->parse(L"fnordy");

  searcher->search(query, collector);
  fnord::iputs("found $0 documents", collector->getTotalHits());
  */

  DateTime last_iter;
  uint64_t rate_limit_micros = db_commit_interval * kMicrosPerSecond;

  for (;;) {
    last_iter = WallClock::now();
    feed_reader.fillBuffers();
    int i = 0;
    for (; i < db_commit_size; ++i) {
      auto entry = feed_reader.fetchNextEntry();

      if (entry.isEmpty()) {
        break;
      }

      try {
        stat_documents_indexed_total_.incr(1);
        fnord::logTrace("cm.indexbuild", "Indexing: $0", entry.get().data);
        auto index_req = json::fromJSON<cm::IndexRequest>(entry.get().data);
        index_build.updateDocument(index_req);
        stat_documents_indexed_success_.incr(1);
      } catch (const std::exception& e) {
        stat_documents_indexed_error_.incr(1);
        fnord::logError(
            "cm.indexbuild",
            e,
            "error while indexing document: $0",
            entry.get().data);
      }
    }

    auto stream_offsets = feed_reader.streamOffsets();
    String stream_offsets_str;

    index_build.commit();

    auto txn = featuredb->startTransaction();

    for (const auto& soff : stream_offsets) {
      txn->update(
          StringUtil::format("__indexfeed_offset~$0", soff.first),
          StringUtil::toString(soff.second));

      stream_offsets_str +=
          StringUtil::format("\n    offset[$0]=$1", soff.first, soff.second);
    }

    fnord::logInfo(
        "cm.indexbuild",
        "IndexBuild comitting...$0",
        stream_offsets_str);

    txn->commit();

    if (cm_indexbuild_shutdown.load() == true) {
      break;
    }

    auto etime = WallClock::now().unixMicros() - last_iter.unixMicros();
    if (i < 1 && etime < rate_limit_micros) {
      usleep(rate_limit_micros - etime);
    }
  }

  ev.shutdown();
  //evloop_thread.join();

  fnord::logInfo("cm.indexbuild", "IndexBuild exiting...");
  exit(0); // FIXPAUL

  return 0;
}

<commit_msg>move all the feed subscription code into a separate method in cm-indexbuild<commit_after>/**
 * Copyright (c) 2015 - The CM Authors <legal@clickmatcher.com>
 *   All Rights Reserved.
 *
 * This file is CONFIDENTIAL -- Distribution or duplication of this material or
 * the information contained herein is strictly forbidden unless prior written
 * permission is obtained.
 */
#define BOOST_NO_CXX11_NUMERIC_LIMITS 1
#include <stdlib.h>
#include <unistd.h>
#include <signal.h>
#include "fnord-base/io/filerepository.h"
#include "fnord-base/io/fileutil.h"
#include "fnord-base/application.h"
#include "fnord-base/logging.h"
#include "fnord-base/random.h"
#include "fnord-base/thread/eventloop.h"
#include "fnord-base/thread/threadpool.h"
#include "fnord-base/wallclock.h"
#include "fnord-rpc/ServerGroup.h"
#include "fnord-rpc/RPC.h"
#include "fnord-rpc/RPCClient.h"
#include "fnord-base/cli/flagparser.h"
#include "fnord-json/json.h"
#include "fnord-json/jsonrpc.h"
#include "fnord-http/httprouter.h"
#include "fnord-http/httpserver.h"
#include "fnord-feeds/FeedService.h"
#include "fnord-feeds/RemoteFeedFactory.h"
#include "fnord-feeds/RemoteFeedReader.h"
#include "fnord-base/stats/statsdagent.h"
#include "fnord-fts/fts.h"
#include "fnord-fts/fts_common.h"
#include "fnord-mdb/MDB.h"
#include "CustomerNamespace.h"
#include "FeatureSchema.h"
#include "FeatureIndexWriter.h"
#include "IndexRequest.h"
#include "IndexBuild.h"

using namespace cm;
using namespace fnord;

std::atomic<bool> cm_indexbuild_shutdown;

/* stats */
fnord::stats::Counter<uint64_t> stat_documents_indexed_total_;
fnord::stats::Counter<uint64_t> stat_documents_indexed_success_;
fnord::stats::Counter<uint64_t> stat_documents_indexed_error_;

void quit(int n) {
  cm_indexbuild_shutdown = true;
}

void buildIndexFromFeed(
    IndexBuild* index_build,
    RefPtr<mdb::MDB> featuredb,
    const cli::FlagParser& flags) {
  size_t batch_size = flags.getInt("batch_size");
  size_t buffer_size = flags.getInt("buffer_size");
  size_t db_commit_size = flags.getInt("db_commit_size");
  size_t db_commit_interval = flags.getInt("db_commit_interval");

  /* start event loop */
  fnord::thread::EventLoop ev;

  auto evloop_thread = std::thread([&ev] {
    ev.run();
  });

  /* set up rpc client */
  HTTPRPCClient rpc_client(&ev);

  feeds::RemoteFeedReader feed_reader(&rpc_client);
  feed_reader.setMaxSpread(3600 * 24 * 30 * kMicrosPerSecond);

  HashMap<String, URI> input_feeds;
  auto cmcustomer = "dawanda";
  input_feeds.emplace(
      StringUtil::format(
          "$0.index_requests.feedserver01.nue01.production.fnrd.net",
          cmcustomer),
      URI("http://s01.nue01.production.fnrd.net:7001/rpc"));
  input_feeds.emplace(
      StringUtil::format(
          "$0.index_requests.feedserver02.nue01.production.fnrd.net",
          cmcustomer),
      URI("http://s02.nue01.production.fnrd.net:7001/rpc"));

  /* resume from last offset */
  auto txn = featuredb->startTransaction(true);
  try {
    for (const auto& input_feed : input_feeds) {
      uint64_t offset = 0;

      auto last_offset = txn->get(
          StringUtil::format("__indexfeed_offset~$0", input_feed.first));

      if (!last_offset.isEmpty()) {
        offset = std::stoul(last_offset.get().toString());
      }

      fnord::logInfo(
          "cm.indexbuild",
          "Adding source feed:\n    feed=$0\n    url=$1\n    offset: $2",
          input_feed.first,
          input_feed.second.toString(),
          offset);

      feed_reader.addSourceFeed(
          input_feed.second,
          input_feed.first,
          offset,
          batch_size,
          buffer_size);
    }

    txn->abort();
  } catch (...) {
    txn->abort();
    throw;
  }

  fnord::logInfo("cm.indexbuild", "Resuming IndexBuild...");

  DateTime last_iter;
  uint64_t rate_limit_micros = db_commit_interval * kMicrosPerSecond;

  for (;;) {
    last_iter = WallClock::now();
    feed_reader.fillBuffers();
    int i = 0;
    for (; i < db_commit_size; ++i) {
      auto entry = feed_reader.fetchNextEntry();

      if (entry.isEmpty()) {
        break;
      }

      try {
        stat_documents_indexed_total_.incr(1);
        fnord::logTrace("cm.indexbuild", "Indexing: $0", entry.get().data);
        auto index_req = json::fromJSON<cm::IndexRequest>(entry.get().data);
        index_build->updateDocument(index_req);
        stat_documents_indexed_success_.incr(1);
      } catch (const std::exception& e) {
        stat_documents_indexed_error_.incr(1);
        fnord::logError(
            "cm.indexbuild",
            e,
            "error while indexing document: $0",
            entry.get().data);
      }
    }

    auto stream_offsets = feed_reader.streamOffsets();
    String stream_offsets_str;

    index_build->commit();

    auto txn = featuredb->startTransaction();

    for (const auto& soff : stream_offsets) {
      txn->update(
          StringUtil::format("__indexfeed_offset~$0", soff.first),
          StringUtil::toString(soff.second));

      stream_offsets_str +=
          StringUtil::format("\n    offset[$0]=$1", soff.first, soff.second);
    }

    fnord::logInfo(
        "cm.indexbuild",
        "IndexBuild comitting...$0",
        stream_offsets_str);

    txn->commit();

    if (cm_indexbuild_shutdown.load() == true) {
      break;
    }

    auto etime = WallClock::now().unixMicros() - last_iter.unixMicros();
    if (i < 1 && etime < rate_limit_micros) {
      usleep(rate_limit_micros - etime);
    }
  }

  ev.shutdown();
  evloop_thread.join();
}

int main(int argc, const char** argv) {
  Application::init();
  Application::logToStderr();

  /* shutdown hook */
  cm_indexbuild_shutdown = false;
  struct sigaction sa;
  memset(&sa, 0, sizeof(struct sigaction));
  sa.sa_handler = quit;
  sigaction(SIGTERM, &sa, NULL);
  sigaction(SIGQUIT, &sa, NULL);
  sigaction(SIGINT, &sa, NULL);

  cli::FlagParser flags;

  flags.defineFlag(
      "index",
      cli::FlagParser::T_STRING,
      true,
      NULL,
      NULL,
      "index dir",
      "<path>");

  flags.defineFlag(
      "statsd_addr",
      fnord::cli::FlagParser::T_STRING,
      false,
      NULL,
      "127.0.0.1:8192",
      "Statsd addr",
      "<addr>");

  flags.defineFlag(
      "batch_size",
      fnord::cli::FlagParser::T_INTEGER,
      false,
      NULL,
      "2048",
      "batch_size",
      "<num>");

  flags.defineFlag(
      "buffer_size",
      fnord::cli::FlagParser::T_INTEGER,
      false,
      NULL,
      "8192",
      "buffer_size",
      "<num>");

  flags.defineFlag(
      "db_commit_size",
      fnord::cli::FlagParser::T_INTEGER,
      false,
      NULL,
      "1024",
      "db_commit_size",
      "<num>");

  flags.defineFlag(
      "db_commit_interval",
      fnord::cli::FlagParser::T_INTEGER,
      false,
      NULL,
      "5",
      "db_commit_interval",
      "<num>");

  flags.defineFlag(
      "dbsize",
      fnord::cli::FlagParser::T_INTEGER,
      false,
      NULL,
      "128",
      "max sessiondb size",
      "<MB>");

  flags.defineFlag(
      "loglevel",
      fnord::cli::FlagParser::T_STRING,
      false,
      NULL,
      "INFO",
      "loglevel",
      "<level>");

  flags.parseArgv(argc, argv);

  Logger::get()->setMinimumLogLevel(
      strToLogLevel(flags.getString("loglevel")));

  /* set up feature schema */
  FeatureSchema feature_schema;
  feature_schema.registerFeature("shop_id", 1, 1);
  feature_schema.registerFeature("category1", 2, 1);
  feature_schema.registerFeature("category2", 3, 1);
  feature_schema.registerFeature("category3", 4, 1);
  feature_schema.registerFeature("title~de", 5, 2);

  /* start stats reporting */
  fnord::stats::StatsdAgent statsd_agent(
      fnord::net::InetAddr::resolve(flags.getString("statsd_addr")),
      10 * fnord::kMicrosPerSecond);

  statsd_agent.start();

  /* set up indexbuild */
  size_t batch_size = flags.getInt("batch_size");
  size_t buffer_size = flags.getInt("buffer_size");
  size_t db_commit_size = flags.getInt("db_commit_size");
  size_t db_commit_interval = flags.getInt("db_commit_interval");

  fnord::logInfo(
      "cm.indexbuild",
      "Starting cm-indexbuild with:\n    index=$0\n    batch_size=$1\n" \
      "    buffer_size=$2\n    db_commit_size=$3\n    db_commit_interval=$4\n"
      "    max_dbsize=$5MB",
      flags.getString("index"),
      batch_size,
      buffer_size,
      db_commit_size,
      db_commit_interval,
      flags.getInt("dbsize"));

  exportStat(
      "/cm-indexbuild/global/documents_indexed_total",
      &stat_documents_indexed_total_,
      fnord::stats::ExportMode::EXPORT_DELTA);

  exportStat(
      "/cm-indexbuild/global/documents_indexed_success",
      &stat_documents_indexed_success_,
      fnord::stats::ExportMode::EXPORT_DELTA);

  exportStat(
      "/cm-indexbuild/global/documents_indexed_error",
      &stat_documents_indexed_error_,
      fnord::stats::ExportMode::EXPORT_DELTA);

  /* open featuredb db */
  auto featuredb_path = FileUtil::joinPaths(flags.getString("index"), "db");
  FileUtil::mkdir_p(featuredb_path);
  auto featuredb = mdb::MDB::open(featuredb_path);
  featuredb->setMaxSize(1000000 * flags.getInt("dbsize"));
  cm::FeatureIndexWriter feature_index_writer(&feature_schema);

  /* open full index */
  auto fullindex_path = FileUtil::joinPaths(flags.getString("index"), "docs");
  FileUtil::mkdir_p(fullindex_path);
  cm::FullIndex full_index(fullindex_path);

  cm::IndexBuild index_build(&feature_index_writer, &full_index);

  buildIndexFromFeed(&index_build, featuredb, flags);

  fnord::logInfo("cm.indexbuild", "IndexBuild exiting...");
  return 0;
}

/* open lucene index */
  /*
  auto index_path = FileUtil::joinPaths(
      cmdata_path,
      StringUtil::format("index/$0/fts", cmcustomer));
  FileUtil::mkdir_p(index_path);

  auto index_writer =
      fts::newLucene<fts::IndexWriter>(
          fts::FSDirectory::open(StringUtil::convertUTF8To16(index_path)),
          fts::newLucene<fts::StandardAnalyzer>(
              fts::LuceneVersion::LUCENE_CURRENT),
          true,
          fts::IndexWriter::MaxFieldLengthLIMITED);
  */
  // index document
/*
  auto doc = fts::newLucene<fts::Document>();
  doc->add(
      fts::newLucene<fts::Field>(
          L"keywords",
          L"my fnordy document",
          fts::Field::STORE_NO,
          fts::Field::INDEX_ANALYZED));

  index_writer->addDocument(doc);
*/

  //index_writer->commit();
  //index_writer->close();

  // simple search
  /*
  auto index_reader = fts::IndexReader::open(
      fts::FSDirectory::open(StringUtil::convertUTF8To16(index_path)),
      true);

  auto searcher = fts::newLucene<fts::IndexSearcher>(index_reader);

  auto analyzer = fts::newLucene<fts::StandardAnalyzer>(
      fts::LuceneVersion::LUCENE_CURRENT);

  auto query_parser = fts::newLucene<fts::QueryParser>(
      fts::LuceneVersion::LUCENE_CURRENT,
      L"keywords",
      analyzer);

  auto collector = fts::TopScoreDocCollector::create(
      500,
      false);

  auto query = query_parser->parse(L"fnordy");

  searcher->search(query, collector);
  fnord::iputs("found $0 documents", collector->getTotalHits());
  */

<|endoftext|>


<commit_before>/*
 * Copyright 2010-2016, Tarantool AUTHORS, please see AUTHORS file.
 *
 * Redistribution and use in source and binary forms, with or
 * without modification, are permitted provided that the following
 * conditions are met:
 *
 * 1. Redistributions of source code must retain the above
 *    copyright notice, this list of conditions and the
 *    following disclaimer.
 *
 * 2. Redistributions in binary form must reproduce the above
 *    copyright notice, this list of conditions and the following
 *    disclaimer in the documentation and/or other materials
 *    provided with the distribution.
 *
 * THIS SOFTWARE IS PROVIDED BY <COPYRIGHT HOLDER> ``AS IS'' AND
 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
 * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
 * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL
 * <COPYRIGHT HOLDER> OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT,
 * INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR
 * BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
 * LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF
 * THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
 * SUCH DAMAGE.
 */
#include "memtx_space.h"
#include "space.h"
#include "iproto_constants.h"
#include "txn.h"
#include "tuple.h"
#include "xrow.h"
#include "memtx_hash.h"
#include "memtx_tree.h"
#include "memtx_rtree.h"
#include "memtx_bitset.h"
#include "port.h"

/**
 * A version of space_replace for a space which has
 * no indexes (is not yet fully built).
 */
struct tuple *
memtx_replace_no_keys(struct space *space,
		      struct tuple * /* old_tuple */,
		      struct tuple * /* new_tuple */,
		      enum dup_replace_mode /* mode */)
{
	Index *index = index_find_xc(space, 0);
	assert(index == NULL); /* not reached. */
	(void) index;
	return NULL;
}

MemtxSpace::MemtxSpace(Engine *e)
	: Handler(e)
{
	replace = memtx_replace_no_keys;
}

static inline enum dup_replace_mode
dup_replace_mode(uint32_t op)
{
	return op == IPROTO_INSERT ? DUP_INSERT : DUP_REPLACE_OR_INSERT;
}

/**
 * Do the plumbing necessary for correct statement-level
 * and transaction rollback.
 */
static inline void
memtx_txn_add_undo(struct txn *txn, struct tuple *old_tuple,
		   struct tuple *new_tuple)
{
	/*
	 * Remember the old tuple only if we replaced it
	 * successfully, to not remove a tuple inserted by
	 * another transaction in rollback().
	 */
	struct txn_stmt *stmt = txn_current_stmt(txn);
	assert(stmt->space);
	stmt->old_tuple = old_tuple;
	stmt->new_tuple = new_tuple;
}

void
MemtxSpace::applyInitialJoinRow(struct space *space, struct request *request)
{
	assert(request->type == IPROTO_INSERT);
	struct tuple *new_tuple = tuple_new_xc(space->format, request->tuple,
					       request->tuple_end);
	/* GC the new tuple if there is an exception below. */
	TupleRef ref(new_tuple);
	if (!rlist_empty(&space->on_replace)) {
		/*
		 * Emulate transactions for system spaces with triggers
		 */
		assert(in_txn() == NULL);
		request->header->server_id = 0;
		struct txn *txn = txn_begin_stmt(space);
		try {
			struct tuple *old_tuple = this->replace(space, NULL,
				new_tuple, DUP_INSERT);
			memtx_txn_add_undo(txn, old_tuple, new_tuple);
			txn_commit_stmt(txn, request);
		} catch (Exception *e) {
			say_error("rollback: %s", e->errmsg);
			txn_rollback_stmt();
			throw;
		}
	} else {
		this->replace(space, NULL, new_tuple, DUP_INSERT);
	}
	/** The new tuple is referenced by the primary key. */
}

struct tuple *
MemtxSpace::executeReplace(struct txn *txn, struct space *space,
			   struct request *request)
{
	struct tuple *new_tuple = tuple_new_xc(space->format, request->tuple,
					       request->tuple_end);
	/* GC the new tuple if there is an exception below. */
	TupleRef ref(new_tuple);
	enum dup_replace_mode mode = dup_replace_mode(request->type);
	struct tuple *old_tuple = this->replace(space, NULL, new_tuple, mode);
	memtx_txn_add_undo(txn, old_tuple, new_tuple);
	/** The new tuple is referenced by the primary key. */
	return new_tuple;
}

struct tuple *
MemtxSpace::executeDelete(struct txn *txn, struct space *space,
			  struct request *request)
{
	/* Try to find the tuple by unique key. */
	Index *pk = index_find_unique(space, request->index_id);
	const char *key = request->key;
	uint32_t part_count = mp_decode_array(&key);
	if (primary_key_validate(pk->key_def, key, part_count))
		diag_raise();
	struct tuple *old_tuple = pk->findByKey(key, part_count);
	if (old_tuple == NULL)
		return NULL;

	this->replace(space, old_tuple, NULL, DUP_REPLACE_OR_INSERT);
	memtx_txn_add_undo(txn, old_tuple, NULL);
	return old_tuple;
}

struct tuple *
MemtxSpace::executeUpdate(struct txn *txn, struct space *space,
			  struct request *request)
{
	/* Try to find the tuple by unique key. */
	Index *pk = index_find_unique(space, request->index_id);
	const char *key = request->key;
	uint32_t part_count = mp_decode_array(&key);
	if (primary_key_validate(pk->key_def, key, part_count))
		diag_raise();
	struct tuple *old_tuple = pk->findByKey(key, part_count);

	if (old_tuple == NULL)
		return NULL;

	/* Update the tuple; legacy, request ops are in request->tuple */
	struct tuple *new_tuple = tuple_update(space->format,
					       region_aligned_alloc_xc_cb,
					       &fiber()->gc,
					       old_tuple, request->tuple,
					       request->tuple_end,
					       request->index_base, NULL);
	TupleRef ref(new_tuple);
	this->replace(space, old_tuple, new_tuple, DUP_REPLACE);
	memtx_txn_add_undo(txn, old_tuple, new_tuple);
	return new_tuple;
}

void
MemtxSpace::executeUpsert(struct txn *txn, struct space *space,
			  struct request *request)
{
	Index *pk = index_find_unique(space, request->index_id);

	/* Check tuple fields */
	if (tuple_validate_raw(space->format, request->tuple))
		diag_raise();

	struct key_def *key_def = pk->key_def;
	uint32_t part_count = pk->key_def->part_count;
	/*
	 * Extract the primary key from tuple.
	 * Allocate enough memory to store the key.
	 */
	const char *key = tuple_extract_key_raw(request->tuple,
						request->tuple_end,
						key_def, NULL);
	if (key == NULL)
		diag_raise();
	/* Cut array header */
	mp_decode_array(&key);

	/* Try to find the tuple by primary key. */
	struct tuple *old_tuple = pk->findByKey(key, part_count);

	if (old_tuple == NULL) {
		/**
		 * Old tuple was not found. In a "true"
		 * non-reading-write engine, this is known only
		 * after commit. Thus any error that can happen
		 * at this point is ignored. Emulate this by
		 * suppressing the error. It's logged and ignored.
		 *
		 * Taking into account that:
		 * 1) Default tuple fields are already fully checked
		 *    at the beginning of the function
		 * 2) Space with unique secondary indexes does not support
		 *    upsert and we can't get duplicate error
		 *
		 * Thus we could get only OOM error, but according to
		 *   https://github.com/tarantool/tarantool/issues/1156
		 *   we should not suppress it
		 *
		 * So we have nothing to catch and suppress!
		 */
		if (tuple_update_check_ops(region_aligned_alloc_xc_cb, &fiber()->gc,
				       request->ops, request->ops_end,
				       request->index_base)) {
			diag_raise();
		}
		struct tuple *new_tuple = tuple_new_xc(space->format,
						       request->tuple,
						       request->tuple_end);
		TupleRef ref(new_tuple); /* useless, for unified approach */
		old_tuple = replace(space, NULL, new_tuple, DUP_INSERT);
		memtx_txn_add_undo(txn, old_tuple, new_tuple);
	} else {
		/**
		 * Update the tuple.
		 * tuple_upsert throws on totally wrong tuple ops,
		 * but ignores ops that not suitable for the tuple
		 */
		struct tuple *new_tuple;
		new_tuple = tuple_upsert(space->format,
					 region_aligned_alloc_xc_cb,
					 &fiber()->gc, old_tuple,
					 request->ops, request->ops_end,
					 request->index_base);
		TupleRef ref(new_tuple);

		/**
		 * Ignore and log all client exceptions,
		 * note that OutOfMemory is not catched.
		 */
		try {
			replace(space, old_tuple, new_tuple, DUP_REPLACE);
			memtx_txn_add_undo(txn, old_tuple, new_tuple);
		} catch (ClientError *e) {
			say_error("UPSERT failed:");
			e->log();
		}
	}
	/* Return nothing: UPSERT does not return data. */
}

Index *
MemtxSpace::createIndex(struct space *space, struct key_def *key_def_arg)
{
	(void) space;
	switch (key_def_arg->type) {
	case HASH:
		return new MemtxHash(key_def_arg);
	case TREE:
		return new MemtxTree(key_def_arg);
	case RTREE:
		return new MemtxRTree(key_def_arg);
	case BITSET:
		return new MemtxBitset(key_def_arg);
	default:
		unreachable();
		return NULL;
	}
}

void
MemtxSpace::dropIndex(Index *index)
{
	if (index->key_def->iid != 0)
		return; /* nothing to do for secondary keys */
	/*
	 * Delete all tuples in the old space if dropping the
	 * primary key.
	 */
	struct iterator *it = ((MemtxIndex*) index)->position();
	index->initIterator(it, ITER_ALL, NULL, 0);
	struct tuple *tuple;
	while ((tuple = it->next(it)))
		tuple_unref(tuple);
}

void
MemtxSpace::prepareAlterSpace(struct space *old_space, struct space *new_space)
{
	(void)new_space;
	MemtxSpace *handler = (MemtxSpace *) old_space->handler;
	replace = handler->replace;
}

void
MemtxSpace::executeSelect(struct txn *, struct space *space,
			  uint32_t index_id, uint32_t iterator,
			  uint32_t offset, uint32_t limit,
			  const char *key, const char * /* key_end */,
			  struct port *port)
{
	MemtxIndex *index = (MemtxIndex *) index_find_xc(space, index_id);

	ERROR_INJECT_EXCEPTION(ERRINJ_TESTING);

	uint32_t found = 0;
	if (iterator >= iterator_type_MAX)
		tnt_raise(IllegalParams, "Invalid iterator type");
	enum iterator_type type = (enum iterator_type) iterator;

	uint32_t part_count = key ? mp_decode_array(&key) : 0;
	if (key_validate(index->key_def, type, key, part_count))
		diag_raise();

	struct iterator *it = index->position();
	index->initIterator(it, type, key, part_count);

	struct tuple *tuple;
	while ((tuple = it->next(it)) != NULL) {
		if (offset > 0) {
			offset--;
			continue;
		}
		if (limit == found++)
			break;
		port_add_tuple(port, tuple);
	}
}
<commit_msg>memtx: improve comments in upsert implementation.<commit_after>/*
 * Copyright 2010-2016, Tarantool AUTHORS, please see AUTHORS file.
 *
 * Redistribution and use in source and binary forms, with or
 * without modification, are permitted provided that the following
 * conditions are met:
 *
 * 1. Redistributions of source code must retain the above
 *    copyright notice, this list of conditions and the
 *    following disclaimer.
 *
 * 2. Redistributions in binary form must reproduce the above
 *    copyright notice, this list of conditions and the following
 *    disclaimer in the documentation and/or other materials
 *    provided with the distribution.
 *
 * THIS SOFTWARE IS PROVIDED BY <COPYRIGHT HOLDER> ``AS IS'' AND
 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
 * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
 * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL
 * <COPYRIGHT HOLDER> OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT,
 * INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR
 * BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
 * LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF
 * THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
 * SUCH DAMAGE.
 */
#include "memtx_space.h"
#include "space.h"
#include "iproto_constants.h"
#include "txn.h"
#include "tuple.h"
#include "xrow.h"
#include "memtx_hash.h"
#include "memtx_tree.h"
#include "memtx_rtree.h"
#include "memtx_bitset.h"
#include "port.h"

/**
 * A version of space_replace for a space which has
 * no indexes (is not yet fully built).
 */
struct tuple *
memtx_replace_no_keys(struct space *space,
		      struct tuple * /* old_tuple */,
		      struct tuple * /* new_tuple */,
		      enum dup_replace_mode /* mode */)
{
	Index *index = index_find_xc(space, 0);
	assert(index == NULL); /* not reached. */
	(void) index;
	return NULL;
}

MemtxSpace::MemtxSpace(Engine *e)
	: Handler(e)
{
	replace = memtx_replace_no_keys;
}

static inline enum dup_replace_mode
dup_replace_mode(uint32_t op)
{
	return op == IPROTO_INSERT ? DUP_INSERT : DUP_REPLACE_OR_INSERT;
}

/**
 * Do the plumbing necessary for correct statement-level
 * and transaction rollback.
 */
static inline void
memtx_txn_add_undo(struct txn *txn, struct tuple *old_tuple,
		   struct tuple *new_tuple)
{
	/*
	 * Remember the old tuple only if we replaced it
	 * successfully, to not remove a tuple inserted by
	 * another transaction in rollback().
	 */
	struct txn_stmt *stmt = txn_current_stmt(txn);
	assert(stmt->space);
	stmt->old_tuple = old_tuple;
	stmt->new_tuple = new_tuple;
}

void
MemtxSpace::applyInitialJoinRow(struct space *space, struct request *request)
{
	assert(request->type == IPROTO_INSERT);
	struct tuple *new_tuple = tuple_new_xc(space->format, request->tuple,
					       request->tuple_end);
	/* GC the new tuple if there is an exception below. */
	TupleRef ref(new_tuple);
	if (!rlist_empty(&space->on_replace)) {
		/*
		 * Emulate transactions for system spaces with triggers
		 */
		assert(in_txn() == NULL);
		request->header->server_id = 0;
		struct txn *txn = txn_begin_stmt(space);
		try {
			struct tuple *old_tuple = this->replace(space, NULL,
				new_tuple, DUP_INSERT);
			memtx_txn_add_undo(txn, old_tuple, new_tuple);
			txn_commit_stmt(txn, request);
		} catch (Exception *e) {
			say_error("rollback: %s", e->errmsg);
			txn_rollback_stmt();
			throw;
		}
	} else {
		this->replace(space, NULL, new_tuple, DUP_INSERT);
	}
	/** The new tuple is referenced by the primary key. */
}

struct tuple *
MemtxSpace::executeReplace(struct txn *txn, struct space *space,
			   struct request *request)
{
	struct tuple *new_tuple = tuple_new_xc(space->format, request->tuple,
					       request->tuple_end);
	/* GC the new tuple if there is an exception below. */
	TupleRef ref(new_tuple);
	enum dup_replace_mode mode = dup_replace_mode(request->type);
	struct tuple *old_tuple = this->replace(space, NULL, new_tuple, mode);
	memtx_txn_add_undo(txn, old_tuple, new_tuple);
	/** The new tuple is referenced by the primary key. */
	return new_tuple;
}

struct tuple *
MemtxSpace::executeDelete(struct txn *txn, struct space *space,
			  struct request *request)
{
	/* Try to find the tuple by unique key. */
	Index *pk = index_find_unique(space, request->index_id);
	const char *key = request->key;
	uint32_t part_count = mp_decode_array(&key);
	if (primary_key_validate(pk->key_def, key, part_count))
		diag_raise();
	struct tuple *old_tuple = pk->findByKey(key, part_count);
	if (old_tuple == NULL)
		return NULL;

	this->replace(space, old_tuple, NULL, DUP_REPLACE_OR_INSERT);
	memtx_txn_add_undo(txn, old_tuple, NULL);
	return old_tuple;
}

struct tuple *
MemtxSpace::executeUpdate(struct txn *txn, struct space *space,
			  struct request *request)
{
	/* Try to find the tuple by unique key. */
	Index *pk = index_find_unique(space, request->index_id);
	const char *key = request->key;
	uint32_t part_count = mp_decode_array(&key);
	if (primary_key_validate(pk->key_def, key, part_count))
		diag_raise();
	struct tuple *old_tuple = pk->findByKey(key, part_count);

	if (old_tuple == NULL)
		return NULL;

	/* Update the tuple; legacy, request ops are in request->tuple */
	struct tuple *new_tuple = tuple_update(space->format,
					       region_aligned_alloc_xc_cb,
					       &fiber()->gc,
					       old_tuple, request->tuple,
					       request->tuple_end,
					       request->index_base, NULL);
	TupleRef ref(new_tuple);
	this->replace(space, old_tuple, new_tuple, DUP_REPLACE);
	memtx_txn_add_undo(txn, old_tuple, new_tuple);
	return new_tuple;
}

void
MemtxSpace::executeUpsert(struct txn *txn, struct space *space,
			  struct request *request)
{
	Index *pk = index_find_unique(space, request->index_id);

	/*
	 * Check all tuple fields: we should produce an error on
	 * malformed tuple even if upsert turns into an update.
	*/
	if (tuple_validate_raw(space->format, request->tuple))
		diag_raise();

	struct key_def *key_def = pk->key_def;
	uint32_t part_count = pk->key_def->part_count;
	/* Extract the primary key from tuple. */
	const char *key = tuple_extract_key_raw(request->tuple,
						request->tuple_end,
						key_def, NULL);
	if (key == NULL)
		diag_raise();
	/* Cut array header */
	mp_decode_array(&key);

	/* Try to find the tuple by primary key. */
	struct tuple *old_tuple = pk->findByKey(key, part_count);

	if (old_tuple == NULL) {
		/**
		 * Old tuple was not found. A write optimized
		 * engine may only know this after commit, so
		 * some errors which happen on this branch would
		 * only make it to the server log in it.
		 * To provide identical semantics, we should not throw
		 * anything. However, considering the kind of
		 * error which may occur, throwing it won't break
		 * cross-engine compatibility:
		 * - update ops are checked before commit
		 * - OOM may happen at any time
		 * - duplicate key has to be checked by
		 *   write-optimized engine before commit, so if
		 *   we get it here, it's also OK to throw it
		 * @sa https://github.com/tarantool/tarantool/issues/1156
		 */
		if (tuple_update_check_ops(region_aligned_alloc_xc_cb, &fiber()->gc,
				       request->ops, request->ops_end,
				       request->index_base)) {
			diag_raise();
		}
		struct tuple *new_tuple = tuple_new_xc(space->format,
						       request->tuple,
						       request->tuple_end);
		TupleRef ref(new_tuple); /* useless, for unified approach */
		old_tuple = replace(space, NULL, new_tuple, DUP_INSERT);
		memtx_txn_add_undo(txn, old_tuple, new_tuple);
	} else {
		/*
		 * Update the tuple.
		 * tuple_upsert() throws on totally wrong tuple ops,
		 * but ignores ops that not suitable for the tuple
		 */
		struct tuple *new_tuple;
		new_tuple = tuple_upsert(space->format,
					 region_aligned_alloc_xc_cb,
					 &fiber()->gc, old_tuple,
					 request->ops, request->ops_end,
					 request->index_base);
		TupleRef ref(new_tuple);

		/*
		 * Ignore and log all client exceptions,
		 * note that OutOfMemory is not caught.
		 */
		try {
			replace(space, old_tuple, new_tuple, DUP_REPLACE);
			memtx_txn_add_undo(txn, old_tuple, new_tuple);
		} catch (ClientError *e) {
			say_error("UPSERT failed:");
			e->log();
		}
	}
	/* Return nothing: UPSERT does not return data. */
}

Index *
MemtxSpace::createIndex(struct space *space, struct key_def *key_def_arg)
{
	(void) space;
	switch (key_def_arg->type) {
	case HASH:
		return new MemtxHash(key_def_arg);
	case TREE:
		return new MemtxTree(key_def_arg);
	case RTREE:
		return new MemtxRTree(key_def_arg);
	case BITSET:
		return new MemtxBitset(key_def_arg);
	default:
		unreachable();
		return NULL;
	}
}

void
MemtxSpace::dropIndex(Index *index)
{
	if (index->key_def->iid != 0)
		return; /* nothing to do for secondary keys */
	/*
	 * Delete all tuples in the old space if dropping the
	 * primary key.
	 */
	struct iterator *it = ((MemtxIndex*) index)->position();
	index->initIterator(it, ITER_ALL, NULL, 0);
	struct tuple *tuple;
	while ((tuple = it->next(it)))
		tuple_unref(tuple);
}

void
MemtxSpace::prepareAlterSpace(struct space *old_space, struct space *new_space)
{
	(void)new_space;
	MemtxSpace *handler = (MemtxSpace *) old_space->handler;
	replace = handler->replace;
}

void
MemtxSpace::executeSelect(struct txn *, struct space *space,
			  uint32_t index_id, uint32_t iterator,
			  uint32_t offset, uint32_t limit,
			  const char *key, const char * /* key_end */,
			  struct port *port)
{
	MemtxIndex *index = (MemtxIndex *) index_find_xc(space, index_id);

	ERROR_INJECT_EXCEPTION(ERRINJ_TESTING);

	uint32_t found = 0;
	if (iterator >= iterator_type_MAX)
		tnt_raise(IllegalParams, "Invalid iterator type");
	enum iterator_type type = (enum iterator_type) iterator;

	uint32_t part_count = key ? mp_decode_array(&key) : 0;
	if (key_validate(index->key_def, type, key, part_count))
		diag_raise();

	struct iterator *it = index->position();
	index->initIterator(it, type, key, part_count);

	struct tuple *tuple;
	while ((tuple = it->next(it)) != NULL) {
		if (offset > 0) {
			offset--;
			continue;
		}
		if (limit == found++)
			break;
		port_add_tuple(port, tuple);
	}
}
<|endoftext|>


<commit_before>/*
 * Implementation of bubble_builder.h
 */

#include <vector>
#include <utility>
#include <set>
#include <cstring>

#include "bubble_builder.h"

using std::vector;
using std::pair;
using std::set;

BubbleBuilder::BubbleBuilder(size_t len) {
    kmerLen = len;
}

Bubble BubbleBuilder::build(string& startKmer, ColorSet colors) {
    Bubble bubble = Bubble();
    // get the first color in ColorSet
    set<Color*>::iterator colorIt = colors.getBeginIterator();

    // find the next kmer that occurs in all of the colors
    // loop until there is an endKmer, or all colors have been tried
    /// @todo this loop could be optimized...
    string endKmer = "";
    while(strcmp(endKmer.c_str(), "") == 0 && colorIt != colors.getEndIterator()) {
        endKmer = findEndKmer(startKmer, *colorIt++, colors);
    }

    vector<Path> paths;
    // extend the path from kmer to endKmer for each color in colors
    for(set<Color*>::iterator it = colors.getBeginIterator(); it != colors.getEndIterator(); ++it) {
        Path path = this->extendPath(startKmer, endKmer, *it);
        paths.push_back(path);
    }

    return bubble;
}

/**
 * Helper function to get the neighbors in color of kmers.
 * @param kmers the kmers to get the neighbors from
 * @return a vector of type string with all of the neighbors of the kmers
 */
vector<string> getNeighbors(vector<string> kmers, const Color* color) {
    vector<string> neighbors;
    for(string kmer : kmers) {
        for(string neighbor : color->getSuffixNeighbors(kmer)) {
            neighbors.push_back(neighbor);
        }
    }

    return neighbors;
}

string BubbleBuilder::findEndKmer(string& startKmer, const Color* color, const ColorSet colors) {
    vector<string> neighbors({startKmer});

    while(true) {
        neighbors = getNeighbors(neighbors, color);
        if(neighbors.size() == 0) { // there are no neighbors to check, so break out of the loop
            break;
        }

        for(string neighbor : neighbors) {
            if(colors.allContainsKmer(neighbor)) {
                return neighbor;
            }
        }
    }

    // there is no kmer in color that is present in all colors
    return "";
}

Path BubbleBuilder::extendPath(string& startKmer, string& endKmer, const Color* color) {
    string currentKmer = startKmer;
    Path path = Path(color, kmerLen);

    while(currentKmer != endKmer) {
        //path.addIndex(currentIdx);
        vector<string> suffixNeighbors = color->getSuffixNeighbors(currentKmer);
        for(string neighbor : suffixNeighbors) {
            // extend the possible paths for each of the neighbors
        }

        break; // remove when the logic is actually implemented
    }

    return path;
}
<commit_msg>Created a recursive extend path... Not sure if it will work at all.<commit_after>/*
 * Implementation of bubble_builder.h
 */

#include <vector>
#include <utility>
#include <set>
#include <cstring>
#include <assert.h>

#include "bubble_builder.h"

using std::vector;
using std::pair;
using std::set;

BubbleBuilder::BubbleBuilder(size_t len) {
    kmerLen = len;
}

Bubble BubbleBuilder::build(string& startKmer, ColorSet colors) {
    Bubble bubble = Bubble();
    // get the first color in ColorSet
    set<Color*>::iterator colorIt = colors.getBeginIterator();

    // find the next kmer that occurs in all of the colors
    // loop until there is an endKmer, or all colors have been tried
    /// @todo this loop could be optimized...
    string endKmer = "";
    while(strcmp(endKmer.c_str(), "") == 0 && colorIt != colors.getEndIterator()) {
        endKmer = findEndKmer(startKmer, *colorIt++, colors);
    }

    // assert that an endKmer exists for this bubble
    assert(strcmp(endKmer.c_str(), "") != 0);

    vector<Path> paths;
    // extend the path from kmer to endKmer for each color in colors
    for(set<Color*>::iterator it = colors.getBeginIterator(); it != colors.getEndIterator(); ++it) {
        Path path = this->extendPath(startKmer, endKmer, *it);
        paths.push_back(path);
    }

    return bubble;
}

/**
 * Helper function to get the neighbors in color of kmers.
 * @param kmers the kmers to get the neighbors from
 * @return a vector of type string with all of the neighbors of the kmers
 */
vector<string> getNeighbors(vector<string> kmers, const Color* color) {
    vector<string> neighbors;
    for(string kmer : kmers) {
        for(string neighbor : color->getSuffixNeighbors(kmer)) {
            neighbors.push_back(neighbor);
        }
    }

    return neighbors;
}

string BubbleBuilder::findEndKmer(string& startKmer, const Color* color, const ColorSet colors) {
    vector<string> neighbors({startKmer});

    while(true) {
        neighbors = getNeighbors(neighbors, color);
        if(neighbors.size() == 0) { // there are no neighbors to check, so break out of the loop
            break;
        }

        for(string neighbor : neighbors) {
            if(colors.allContainsKmer(neighbor)) {
                return neighbor;
            }
        }
    }

    // there is no kmer in color that is present in all colors
    return "";
}

set<string> getNeighbors(set<string> kmers, const Color* color) {
    set<string> neighbors;
    for(string kmer : kmers) {
        for(string neighbor : color->getSuffixNeighbors(kmer)) {
            neighbors.insert(neighbor);
        }
    }

    return neighbors;
}

Path recursiveExtend(string currentKmer, string endKmer, Path path, const Color* color) {
    // the base case is reached when the currentKmer is the same as the endKmer
    if(strcmp(currentKmer.c_str(), endKmer.c_str()) == 0) {
        return path;
    }

    // call recursiveExtend on each of the neighbors of currentKmer
    for(string neighbor : getNeighbors(color->getSuffixNeighbors(currentKmer), color)) {
        Path newPath = Path(path);
        newPath.append(neighbor.substr(1, neighbor.length() - 1));
        recursiveExtend(neighbor, endKmer, newPath, color);
    }
}

Path BubbleBuilder::extendPath(string& startKmer, string& endKmer, const Color* color) {
    set<string> currentKmers({startKmer});
    map<string, string> neighborMap;

    Path path = Path(color, kmerLen);
    path.append(startKmer);

    return recursiveExtend(startKmer, endKmer, path, color);
}
<|endoftext|>