<commit_before>//===- TGLexer.cpp - Lexer for TableGen -----------------------------------===//
//
//                     The LLVM Compiler Infrastructure
//
// This file is distributed under the University of Illinois Open Source
// License. See LICENSE.TXT for details.
//
//===----------------------------------------------------------------------===//
//
// Implement the Lexer for TableGen.
//
//===----------------------------------------------------------------------===//

#include "TGLexer.h"
#include "llvm/TableGen/Error.h"
#include "llvm/Support/SourceMgr.h"
#include "llvm/Support/MemoryBuffer.h"
#include "llvm/Config/config.h"
#include "llvm/ADT/StringSwitch.h"
#include "llvm/ADT/Twine.h"
#include <cctype>
#include <cstdio>
#include <cstdlib>
#include <cstring>
#include <cerrno>
using namespace llvm;

TGLexer::TGLexer(SourceMgr &SM) : SrcMgr(SM) {
  CurBuffer = 0;
  CurBuf = SrcMgr.getMemoryBuffer(CurBuffer);
  CurPtr = CurBuf->getBufferStart();
  TokStart = 0;
}

SMLoc TGLexer::getLoc() const {
  return SMLoc::getFromPointer(TokStart);
}

/// ReturnError - Set the error to the specified string at the specified
/// location.  This is defined to always return tgtok::Error.
tgtok::TokKind TGLexer::ReturnError(const char *Loc, const Twine &Msg) {
  PrintError(Loc, Msg);
  return tgtok::Error;
}

int TGLexer::getNextChar() {
  char CurChar = *CurPtr++;
  switch (CurChar) {
  default:
    return (unsigned char)CurChar;
  case 0: {
    // A nul character in the stream is either the end of the current buffer or
    // a random nul in the file.  Disambiguate that here.
    if (CurPtr-1 != CurBuf->getBufferEnd())
      return 0;  // Just whitespace.

    // If this is the end of an included file, pop the parent file off the
    // include stack.
    SMLoc ParentIncludeLoc = SrcMgr.getParentIncludeLoc(CurBuffer);
    if (ParentIncludeLoc != SMLoc()) {
      CurBuffer = SrcMgr.FindBufferContainingLoc(ParentIncludeLoc);
      CurBuf = SrcMgr.getMemoryBuffer(CurBuffer);
      CurPtr = ParentIncludeLoc.getPointer();
      return getNextChar();
    }

    // Otherwise, return end of file.
    --CurPtr;  // Another call to lex will return EOF again.
    return EOF;
  }
  case '\n':
  case '\r':
    // Handle the newline character by ignoring it and incrementing the line
    // count.  However, be careful about 'dos style' files with \n\r in them.
    // Only treat a \n\r or \r\n as a single line.
    if ((*CurPtr == '\n' || (*CurPtr == '\r')) &&
        *CurPtr != CurChar)
      ++CurPtr;  // Eat the two char newline sequence.
    return '\n';
  }
}

tgtok::TokKind TGLexer::LexToken() {
  TokStart = CurPtr;
  // This always consumes at least one character.
  int CurChar = getNextChar();

  switch (CurChar) {
  default:
    // Handle letters: [a-zA-Z_#]
    if (isalpha(CurChar) || CurChar == '_' || CurChar == '#')
      return LexIdentifier();

    // Unknown character, emit an error.
    return ReturnError(TokStart, "Unexpected character");
  case EOF: return tgtok::Eof;
  case ':': return tgtok::colon;
  case ';': return tgtok::semi;
  case '.': return tgtok::period;
  case ',': return tgtok::comma;
  case '<': return tgtok::less;
  case '>': return tgtok::greater;
  case ']': return tgtok::r_square;
  case '{': return tgtok::l_brace;
  case '}': return tgtok::r_brace;
  case '(': return tgtok::l_paren;
  case ')': return tgtok::r_paren;
  case '=': return tgtok::equal;
  case '?': return tgtok::question;

  case 0:
  case ' ':
  case '\t':
  case '\n':
  case '\r':
    // Ignore whitespace.
    return LexToken();
  case '/':
    // If this is the start of a // comment, skip until the end of the line or
    // the end of the buffer.
    if (*CurPtr == '/')
      SkipBCPLComment();
    else if (*CurPtr == '*') {
      if (SkipCComment())
        return tgtok::Error;
    } else // Otherwise, this is an error.
      return ReturnError(TokStart, "Unexpected character");
    return LexToken();
  case '-': case '+':
  case '0': case '1': case '2': case '3': case '4': case '5': case '6':
  case '7': case '8': case '9':
    return LexNumber();
  case '"': return LexString();
  case '$': return LexVarName();
  case '[': return LexBracket();
  case '!': return LexExclaim();
  }
}

/// LexString - Lex "[^"]*"
tgtok::TokKind TGLexer::LexString() {
  const char *StrStart = CurPtr;

  CurStrVal = "";

  while (*CurPtr != '"') {
    // If we hit the end of the buffer, report an error.
    if (*CurPtr == 0 && CurPtr == CurBuf->getBufferEnd())
      return ReturnError(StrStart, "End of file in string literal");

    if (*CurPtr == '\n' || *CurPtr == '\r')
      return ReturnError(StrStart, "End of line in string literal");

    if (*CurPtr != '\\') {
      CurStrVal += *CurPtr++;
      continue;
    }

    ++CurPtr;

    switch (*CurPtr) {
    case '\\': case '\'': case '"':
      // These turn into their literal character.
      CurStrVal += *CurPtr++;
      break;
    case 't':
      CurStrVal += '\t';
      ++CurPtr;
      break;
    case 'n':
      CurStrVal += '\n';
      ++CurPtr;
      break;

    case '\n':
    case '\r':
      return ReturnError(CurPtr, "escaped newlines not supported in tblgen");

    // If we hit the end of the buffer, report an error.
    case '\0':
      if (CurPtr == CurBuf->getBufferEnd())
        return ReturnError(StrStart, "End of file in string literal");
      // FALL THROUGH
    default:
      return ReturnError(CurPtr, "invalid escape in string literal");
    }
  }

  ++CurPtr;
  return tgtok::StrVal;
}

tgtok::TokKind TGLexer::LexVarName() {
  if (!isalpha(CurPtr[0]) && CurPtr[0] != '_')
    return ReturnError(TokStart, "Invalid variable name");

  // Otherwise, we're ok, consume the rest of the characters.
  const char *VarNameStart = CurPtr++;

  while (isalpha(*CurPtr) || isdigit(*CurPtr) || *CurPtr == '_')
    ++CurPtr;

  CurStrVal.assign(VarNameStart, CurPtr);
  return tgtok::VarName;
}

tgtok::TokKind TGLexer::LexIdentifier() {
  // The first letter is [a-zA-Z_#].
  const char *IdentStart = TokStart;

  // Match the rest of the identifier regex: [0-9a-zA-Z_#]*
  while (isalpha(*CurPtr) || isdigit(*CurPtr) || *CurPtr == '_' ||
         *CurPtr == '#')
    ++CurPtr;

  // Check to see if this identifier is a keyword.
  unsigned Len = CurPtr-IdentStart;

  if (Len == 3 && !memcmp(IdentStart, "int", 3)) return tgtok::Int;
  if (Len == 3 && !memcmp(IdentStart, "bit", 3)) return tgtok::Bit;
  if (Len == 4 && !memcmp(IdentStart, "bits", 4)) return tgtok::Bits;
  if (Len == 6 && !memcmp(IdentStart, "string", 6)) return tgtok::String;
  if (Len == 4 && !memcmp(IdentStart, "list", 4)) return tgtok::List;
  if (Len == 4 && !memcmp(IdentStart, "code", 4)) return tgtok::Code;
  if (Len == 3 && !memcmp(IdentStart, "dag", 3)) return tgtok::Dag;

  if (Len == 5 && !memcmp(IdentStart, "class", 5)) return tgtok::Class;
  if (Len == 3 && !memcmp(IdentStart, "def", 3)) return tgtok::Def;
  if (Len == 8 && !memcmp(IdentStart, "multidef", 3)) return tgtok::MultiDef;
  if (Len == 4 && !memcmp(IdentStart, "defm", 4)) return tgtok::Defm;
  if (Len == 10 && !memcmp(IdentStart, "multiclass", 10))
    return tgtok::MultiClass;
  if (Len == 5 && !memcmp(IdentStart, "field", 5)) return tgtok::Field;
  if (Len == 3 && !memcmp(IdentStart, "let", 3)) return tgtok::Let;
  if (Len == 2 && !memcmp(IdentStart, "in", 2)) return tgtok::In;

  if (Len == 7 && !memcmp(IdentStart, "include", 7)) {
    if (LexInclude()) return tgtok::Error;
    return Lex();
  }

  CurStrVal.assign(IdentStart, CurPtr);
  return tgtok::Id;
}

/// LexInclude - We just read the "include" token.  Get the string token that
/// comes next and enter the include.
bool TGLexer::LexInclude() {
  // The token after the include must be a string.
  tgtok::TokKind Tok = LexToken();
  if (Tok == tgtok::Error) return true;
  if (Tok != tgtok::StrVal) {
    PrintError(getLoc(), "Expected filename after include");
    return true;
  }

  // Get the string.
  std::string Filename = CurStrVal;
  std::string IncludedFile;

  CurBuffer = SrcMgr.AddIncludeFile(Filename, SMLoc::getFromPointer(CurPtr),
                                    IncludedFile);
  if (CurBuffer == -1) {
    PrintError(getLoc(), "Could not find include file '" + Filename + "'");
    return true;
  }

  Dependencies.push_back(IncludedFile);
  // Save the line number and lex buffer of the includer.
  CurBuf = SrcMgr.getMemoryBuffer(CurBuffer);
  CurPtr = CurBuf->getBufferStart();
  return false;
}

void TGLexer::SkipBCPLComment() {
  ++CurPtr;  // skip the second slash.
  while (1) {
    switch (*CurPtr) {
    case '\n':
    case '\r':
      return;  // Newline is end of comment.
    case 0:
      // If this is the end of the buffer, end the comment.
      if (CurPtr == CurBuf->getBufferEnd())
        return;
      break;
    }
    // Otherwise, skip the character.
    ++CurPtr;
  }
}

/// SkipCComment - This skips C-style /**/ comments.  The only difference from C
/// is that we allow nesting.
bool TGLexer::SkipCComment() {
  ++CurPtr;  // skip the star.
  unsigned CommentDepth = 1;

  while (1) {
    int CurChar = getNextChar();
    switch (CurChar) {
    case EOF:
      PrintError(TokStart, "Unterminated comment!");
      return true;
    case '*':
      // End of the comment?
      if (CurPtr[0] != '/') break;

      ++CurPtr;   // End the */.
      if (--CommentDepth == 0)
        return false;
      break;
    case '/':
      // Start of a nested comment?
      if (CurPtr[0] != '*') break;
      ++CurPtr;
      ++CommentDepth;
      break;
    }
  }
}

/// LexNumber - Lex:
///    [-+]?[0-9]+
///    0x[0-9a-fA-F]+
///    0b[01]+
tgtok::TokKind TGLexer::LexNumber() {
  if (CurPtr[-1] == '0') {
    if (CurPtr[0] == 'x') {
      ++CurPtr;
      const char *NumStart = CurPtr;
      while (isxdigit(CurPtr[0]))
        ++CurPtr;

      // Requires at least one hex digit.
      if (CurPtr == NumStart)
        return ReturnError(TokStart, "Invalid hexadecimal number");

      errno = 0;
      CurIntVal = strtoll(NumStart, 0, 16);
      if (errno == EINVAL)
        return ReturnError(TokStart, "Invalid hexadecimal number");
      if (errno == ERANGE) {
        errno = 0;
        CurIntVal = (int64_t)strtoull(NumStart, 0, 16);
        if (errno == EINVAL)
          return ReturnError(TokStart, "Invalid hexadecimal number");
        if (errno == ERANGE)
          return ReturnError(TokStart, "Hexadecimal number out of range");
      }
      return tgtok::IntVal;
    } else if (CurPtr[0] == 'b') {
      ++CurPtr;
      const char *NumStart = CurPtr;
      while (CurPtr[0] == '0' || CurPtr[0] == '1')
        ++CurPtr;

      // Requires at least one binary digit.
      if (CurPtr == NumStart)
        return ReturnError(CurPtr-2, "Invalid binary number");
      CurIntVal = strtoll(NumStart, 0, 2);
      return tgtok::IntVal;
    }
  }

  // Check for a sign without a digit.
  if (!isdigit(CurPtr[0])) {
    if (CurPtr[-1] == '-')
      return tgtok::minus;
    else if (CurPtr[-1] == '+')
      return tgtok::plus;
  }

  while (isdigit(CurPtr[0]))
    ++CurPtr;
  CurIntVal = strtoll(TokStart, 0, 10);
  return tgtok::IntVal;
}

/// LexBracket - We just read '['.  If this is a code block, return it,
/// otherwise return the bracket.  Match: '[' and '[{ ( [^}]+ | }[^]] )* }]'
tgtok::TokKind TGLexer::LexBracket() {
  if (CurPtr[0] != '{')
    return tgtok::l_square;
  ++CurPtr;
  const char *CodeStart = CurPtr;
  while (1) {
    int Char = getNextChar();
    if (Char == EOF) break;

    if (Char != '}') continue;

    Char = getNextChar();
    if (Char == EOF) break;
    if (Char == ']') {
      CurStrVal.assign(CodeStart, CurPtr-2);
      return tgtok::CodeFragment;
    }
  }

  return ReturnError(CodeStart-2, "Unterminated Code Block");
}

/// LexExclaim - Lex '!' and '![a-zA-Z]+'.
tgtok::TokKind TGLexer::LexExclaim() {
  if (!isalpha(*CurPtr))
    return ReturnError(CurPtr - 1, "Invalid \"!operator\"");

  const char *Start = CurPtr++;
  while (isalpha(*CurPtr))
    ++CurPtr;

  // Check to see which operator this is.
  tgtok::TokKind Kind =
    StringSwitch<tgtok::TokKind>(StringRef(Start, CurPtr - Start))
    .Case("eq", tgtok::XEq)
    .Case("if", tgtok::XIf)
    .Case("head", tgtok::XHead)
    .Case("tail", tgtok::XTail)
    .Case("con", tgtok::XConcat)
    .Case("shl", tgtok::XSHL)
    .Case("sra", tgtok::XSRA)
    .Case("srl", tgtok::XSRL)
    .Case("cast", tgtok::XCast)
    .Case("empty", tgtok::XEmpty)
    .Case("subst", tgtok::XSubst)
    .Case("foreach", tgtok::XForEach)
    .Case("strconcat", tgtok::XStrConcat)
    .Default(tgtok::Error);

  return Kind != tgtok::Error ? Kind : ReturnError(Start-1, "Unknown operator");
}

<commit_msg>Fix Typo<commit_after>//===- TGLexer.cpp - Lexer for TableGen -----------------------------------===//
//
//                     The LLVM Compiler Infrastructure
//
// This file is distributed under the University of Illinois Open Source
// License. See LICENSE.TXT for details.
//
//===----------------------------------------------------------------------===//
//
// Implement the Lexer for TableGen.
//
//===----------------------------------------------------------------------===//

#include "TGLexer.h"
#include "llvm/TableGen/Error.h"
#include "llvm/Support/SourceMgr.h"
#include "llvm/Support/MemoryBuffer.h"
#include "llvm/Config/config.h"
#include "llvm/ADT/StringSwitch.h"
#include "llvm/ADT/Twine.h"
#include <cctype>
#include <cstdio>
#include <cstdlib>
#include <cstring>
#include <cerrno>
using namespace llvm;

TGLexer::TGLexer(SourceMgr &SM) : SrcMgr(SM) {
  CurBuffer = 0;
  CurBuf = SrcMgr.getMemoryBuffer(CurBuffer);
  CurPtr = CurBuf->getBufferStart();
  TokStart = 0;
}

SMLoc TGLexer::getLoc() const {
  return SMLoc::getFromPointer(TokStart);
}

/// ReturnError - Set the error to the specified string at the specified
/// location.  This is defined to always return tgtok::Error.
tgtok::TokKind TGLexer::ReturnError(const char *Loc, const Twine &Msg) {
  PrintError(Loc, Msg);
  return tgtok::Error;
}

int TGLexer::getNextChar() {
  char CurChar = *CurPtr++;
  switch (CurChar) {
  default:
    return (unsigned char)CurChar;
  case 0: {
    // A nul character in the stream is either the end of the current buffer or
    // a random nul in the file.  Disambiguate that here.
    if (CurPtr-1 != CurBuf->getBufferEnd())
      return 0;  // Just whitespace.

    // If this is the end of an included file, pop the parent file off the
    // include stack.
    SMLoc ParentIncludeLoc = SrcMgr.getParentIncludeLoc(CurBuffer);
    if (ParentIncludeLoc != SMLoc()) {
      CurBuffer = SrcMgr.FindBufferContainingLoc(ParentIncludeLoc);
      CurBuf = SrcMgr.getMemoryBuffer(CurBuffer);
      CurPtr = ParentIncludeLoc.getPointer();
      return getNextChar();
    }

    // Otherwise, return end of file.
    --CurPtr;  // Another call to lex will return EOF again.
    return EOF;
  }
  case '\n':
  case '\r':
    // Handle the newline character by ignoring it and incrementing the line
    // count.  However, be careful about 'dos style' files with \n\r in them.
    // Only treat a \n\r or \r\n as a single line.
    if ((*CurPtr == '\n' || (*CurPtr == '\r')) &&
        *CurPtr != CurChar)
      ++CurPtr;  // Eat the two char newline sequence.
    return '\n';
  }
}

tgtok::TokKind TGLexer::LexToken() {
  TokStart = CurPtr;
  // This always consumes at least one character.
  int CurChar = getNextChar();

  switch (CurChar) {
  default:
    // Handle letters: [a-zA-Z_#]
    if (isalpha(CurChar) || CurChar == '_' || CurChar == '#')
      return LexIdentifier();

    // Unknown character, emit an error.
    return ReturnError(TokStart, "Unexpected character");
  case EOF: return tgtok::Eof;
  case ':': return tgtok::colon;
  case ';': return tgtok::semi;
  case '.': return tgtok::period;
  case ',': return tgtok::comma;
  case '<': return tgtok::less;
  case '>': return tgtok::greater;
  case ']': return tgtok::r_square;
  case '{': return tgtok::l_brace;
  case '}': return tgtok::r_brace;
  case '(': return tgtok::l_paren;
  case ')': return tgtok::r_paren;
  case '=': return tgtok::equal;
  case '?': return tgtok::question;

  case 0:
  case ' ':
  case '\t':
  case '\n':
  case '\r':
    // Ignore whitespace.
    return LexToken();
  case '/':
    // If this is the start of a // comment, skip until the end of the line or
    // the end of the buffer.
    if (*CurPtr == '/')
      SkipBCPLComment();
    else if (*CurPtr == '*') {
      if (SkipCComment())
        return tgtok::Error;
    } else // Otherwise, this is an error.
      return ReturnError(TokStart, "Unexpected character");
    return LexToken();
  case '-': case '+':
  case '0': case '1': case '2': case '3': case '4': case '5': case '6':
  case '7': case '8': case '9':
    return LexNumber();
  case '"': return LexString();
  case '$': return LexVarName();
  case '[': return LexBracket();
  case '!': return LexExclaim();
  }
}

/// LexString - Lex "[^"]*"
tgtok::TokKind TGLexer::LexString() {
  const char *StrStart = CurPtr;

  CurStrVal = "";

  while (*CurPtr != '"') {
    // If we hit the end of the buffer, report an error.
    if (*CurPtr == 0 && CurPtr == CurBuf->getBufferEnd())
      return ReturnError(StrStart, "End of file in string literal");

    if (*CurPtr == '\n' || *CurPtr == '\r')
      return ReturnError(StrStart, "End of line in string literal");

    if (*CurPtr != '\\') {
      CurStrVal += *CurPtr++;
      continue;
    }

    ++CurPtr;

    switch (*CurPtr) {
    case '\\': case '\'': case '"':
      // These turn into their literal character.
      CurStrVal += *CurPtr++;
      break;
    case 't':
      CurStrVal += '\t';
      ++CurPtr;
      break;
    case 'n':
      CurStrVal += '\n';
      ++CurPtr;
      break;

    case '\n':
    case '\r':
      return ReturnError(CurPtr, "escaped newlines not supported in tblgen");

    // If we hit the end of the buffer, report an error.
    case '\0':
      if (CurPtr == CurBuf->getBufferEnd())
        return ReturnError(StrStart, "End of file in string literal");
      // FALL THROUGH
    default:
      return ReturnError(CurPtr, "invalid escape in string literal");
    }
  }

  ++CurPtr;
  return tgtok::StrVal;
}

tgtok::TokKind TGLexer::LexVarName() {
  if (!isalpha(CurPtr[0]) && CurPtr[0] != '_')
    return ReturnError(TokStart, "Invalid variable name");

  // Otherwise, we're ok, consume the rest of the characters.
  const char *VarNameStart = CurPtr++;

  while (isalpha(*CurPtr) || isdigit(*CurPtr) || *CurPtr == '_')
    ++CurPtr;

  CurStrVal.assign(VarNameStart, CurPtr);
  return tgtok::VarName;
}

tgtok::TokKind TGLexer::LexIdentifier() {
  // The first letter is [a-zA-Z_#].
  const char *IdentStart = TokStart;

  // Match the rest of the identifier regex: [0-9a-zA-Z_#]*
  while (isalpha(*CurPtr) || isdigit(*CurPtr) || *CurPtr == '_' ||
         *CurPtr == '#')
    ++CurPtr;

  // Check to see if this identifier is a keyword.
  unsigned Len = CurPtr-IdentStart;

  if (Len == 3 && !memcmp(IdentStart, "int", 3)) return tgtok::Int;
  if (Len == 3 && !memcmp(IdentStart, "bit", 3)) return tgtok::Bit;
  if (Len == 4 && !memcmp(IdentStart, "bits", 4)) return tgtok::Bits;
  if (Len == 6 && !memcmp(IdentStart, "string", 6)) return tgtok::String;
  if (Len == 4 && !memcmp(IdentStart, "list", 4)) return tgtok::List;
  if (Len == 4 && !memcmp(IdentStart, "code", 4)) return tgtok::Code;
  if (Len == 3 && !memcmp(IdentStart, "dag", 3)) return tgtok::Dag;

  if (Len == 5 && !memcmp(IdentStart, "class", 5)) return tgtok::Class;
  if (Len == 3 && !memcmp(IdentStart, "def", 3)) return tgtok::Def;
  if (Len == 8 && !memcmp(IdentStart, "multidef", 8)) return tgtok::MultiDef;
  if (Len == 4 && !memcmp(IdentStart, "defm", 4)) return tgtok::Defm;
  if (Len == 10 && !memcmp(IdentStart, "multiclass", 10))
    return tgtok::MultiClass;
  if (Len == 5 && !memcmp(IdentStart, "field", 5)) return tgtok::Field;
  if (Len == 3 && !memcmp(IdentStart, "let", 3)) return tgtok::Let;
  if (Len == 2 && !memcmp(IdentStart, "in", 2)) return tgtok::In;

  if (Len == 7 && !memcmp(IdentStart, "include", 7)) {
    if (LexInclude()) return tgtok::Error;
    return Lex();
  }

  CurStrVal.assign(IdentStart, CurPtr);
  return tgtok::Id;
}

/// LexInclude - We just read the "include" token.  Get the string token that
/// comes next and enter the include.
bool TGLexer::LexInclude() {
  // The token after the include must be a string.
  tgtok::TokKind Tok = LexToken();
  if (Tok == tgtok::Error) return true;
  if (Tok != tgtok::StrVal) {
    PrintError(getLoc(), "Expected filename after include");
    return true;
  }

  // Get the string.
  std::string Filename = CurStrVal;
  std::string IncludedFile;

  CurBuffer = SrcMgr.AddIncludeFile(Filename, SMLoc::getFromPointer(CurPtr),
                                    IncludedFile);
  if (CurBuffer == -1) {
    PrintError(getLoc(), "Could not find include file '" + Filename + "'");
    return true;
  }

  Dependencies.push_back(IncludedFile);
  // Save the line number and lex buffer of the includer.
  CurBuf = SrcMgr.getMemoryBuffer(CurBuffer);
  CurPtr = CurBuf->getBufferStart();
  return false;
}

void TGLexer::SkipBCPLComment() {
  ++CurPtr;  // skip the second slash.
  while (1) {
    switch (*CurPtr) {
    case '\n':
    case '\r':
      return;  // Newline is end of comment.
    case 0:
      // If this is the end of the buffer, end the comment.
      if (CurPtr == CurBuf->getBufferEnd())
        return;
      break;
    }
    // Otherwise, skip the character.
    ++CurPtr;
  }
}

/// SkipCComment - This skips C-style /**/ comments.  The only difference from C
/// is that we allow nesting.
bool TGLexer::SkipCComment() {
  ++CurPtr;  // skip the star.
  unsigned CommentDepth = 1;

  while (1) {
    int CurChar = getNextChar();
    switch (CurChar) {
    case EOF:
      PrintError(TokStart, "Unterminated comment!");
      return true;
    case '*':
      // End of the comment?
      if (CurPtr[0] != '/') break;

      ++CurPtr;   // End the */.
      if (--CommentDepth == 0)
        return false;
      break;
    case '/':
      // Start of a nested comment?
      if (CurPtr[0] != '*') break;
      ++CurPtr;
      ++CommentDepth;
      break;
    }
  }
}

/// LexNumber - Lex:
///    [-+]?[0-9]+
///    0x[0-9a-fA-F]+
///    0b[01]+
tgtok::TokKind TGLexer::LexNumber() {
  if (CurPtr[-1] == '0') {
    if (CurPtr[0] == 'x') {
      ++CurPtr;
      const char *NumStart = CurPtr;
      while (isxdigit(CurPtr[0]))
        ++CurPtr;

      // Requires at least one hex digit.
      if (CurPtr == NumStart)
        return ReturnError(TokStart, "Invalid hexadecimal number");

      errno = 0;
      CurIntVal = strtoll(NumStart, 0, 16);
      if (errno == EINVAL)
        return ReturnError(TokStart, "Invalid hexadecimal number");
      if (errno == ERANGE) {
        errno = 0;
        CurIntVal = (int64_t)strtoull(NumStart, 0, 16);
        if (errno == EINVAL)
          return ReturnError(TokStart, "Invalid hexadecimal number");
        if (errno == ERANGE)
          return ReturnError(TokStart, "Hexadecimal number out of range");
      }
      return tgtok::IntVal;
    } else if (CurPtr[0] == 'b') {
      ++CurPtr;
      const char *NumStart = CurPtr;
      while (CurPtr[0] == '0' || CurPtr[0] == '1')
        ++CurPtr;

      // Requires at least one binary digit.
      if (CurPtr == NumStart)
        return ReturnError(CurPtr-2, "Invalid binary number");
      CurIntVal = strtoll(NumStart, 0, 2);
      return tgtok::IntVal;
    }
  }

  // Check for a sign without a digit.
  if (!isdigit(CurPtr[0])) {
    if (CurPtr[-1] == '-')
      return tgtok::minus;
    else if (CurPtr[-1] == '+')
      return tgtok::plus;
  }

  while (isdigit(CurPtr[0]))
    ++CurPtr;
  CurIntVal = strtoll(TokStart, 0, 10);
  return tgtok::IntVal;
}

/// LexBracket - We just read '['.  If this is a code block, return it,
/// otherwise return the bracket.  Match: '[' and '[{ ( [^}]+ | }[^]] )* }]'
tgtok::TokKind TGLexer::LexBracket() {
  if (CurPtr[0] != '{')
    return tgtok::l_square;
  ++CurPtr;
  const char *CodeStart = CurPtr;
  while (1) {
    int Char = getNextChar();
    if (Char == EOF) break;

    if (Char != '}') continue;

    Char = getNextChar();
    if (Char == EOF) break;
    if (Char == ']') {
      CurStrVal.assign(CodeStart, CurPtr-2);
      return tgtok::CodeFragment;
    }
  }

  return ReturnError(CodeStart-2, "Unterminated Code Block");
}

/// LexExclaim - Lex '!' and '![a-zA-Z]+'.
tgtok::TokKind TGLexer::LexExclaim() {
  if (!isalpha(*CurPtr))
    return ReturnError(CurPtr - 1, "Invalid \"!operator\"");

  const char *Start = CurPtr++;
  while (isalpha(*CurPtr))
    ++CurPtr;

  // Check to see which operator this is.
  tgtok::TokKind Kind =
    StringSwitch<tgtok::TokKind>(StringRef(Start, CurPtr - Start))
    .Case("eq", tgtok::XEq)
    .Case("if", tgtok::XIf)
    .Case("head", tgtok::XHead)
    .Case("tail", tgtok::XTail)
    .Case("con", tgtok::XConcat)
    .Case("shl", tgtok::XSHL)
    .Case("sra", tgtok::XSRA)
    .Case("srl", tgtok::XSRL)
    .Case("cast", tgtok::XCast)
    .Case("empty", tgtok::XEmpty)
    .Case("subst", tgtok::XSubst)
    .Case("foreach", tgtok::XForEach)
    .Case("strconcat", tgtok::XStrConcat)
    .Default(tgtok::Error);

  return Kind != tgtok::Error ? Kind : ReturnError(Start-1, "Unknown operator");
}

<|endoftext|>


<commit_before>#include "gvki/UnderlyingCaller.h"
#include <iostream>
#include <cstdlib>

#ifndef MACRO_LIB
#include <dlfcn.h>
#endif

using namespace gvki;

#ifdef MACRO_LIB
#define SET_FCN_PTR(functionName) functionName ## U = &(::functionName);
#else
#include <unistd.h>
#define _SET_FCN_PTR(functionName, symbol) dlerror(); /* clear any old errors */ \
                       functionName ## U = reinterpret_cast<functionName ## Ty>(::dlsym(RTLD_NEXT, #symbol)); \
                       errorMsg = dlerror(); \
                       if ( errorMsg != NULL) { \
                         std::cerr << "Failed to dlsym(\"" #symbol  "\"): " << errorMsg << std::endl; \
                         _exit(255); \
                       } \
                       if ( functionName ## U == NULL) { \
                         std::cerr << "Function pointer for \"" #functionName "\" cannot be NULL" << std::endl; \
                         _exit(255); \
                       }

#ifdef __APPLE__
// Under OSX the OpenCL symbols are prefixed with an underscore [1] claims that
// dlsym() is supposed to handle that for us but testing by others seemed to
// suggest this wasn't the case.
//
// [1] https://developer.apple.com/library/mac/documentation/Darwin/Reference/ManPages/man3/dlsym.3.html
#define SET_FCN_PTR(functionName) _SET_FCN_PTR(functionName, _ ## functionName)
#else
#define SET_FCN_PTR(functionName) _SET_FCN_PTR(functionName, functionName)
#endif

#endif

UnderlyingCaller::UnderlyingCaller()
{
    const char* errorMsg = 0;
    SET_FCN_PTR(clCreateBuffer)
    SET_FCN_PTR(clCreateSubBuffer)
#pragma GCC diagnostic push
#pragma GCC diagnostic ignored "-Wdeprecated-declarations"
    // In OpenCL 1.2 these are deprecated. Don't warn about this.
    SET_FCN_PTR(clCreateImage2D)
    SET_FCN_PTR(clCreateImage3D)
#pragma GCC diagnostic pop
    SET_FCN_PTR(clCreateSampler)
    SET_FCN_PTR(clCreateProgramWithSource)
    SET_FCN_PTR(clBuildProgram)
    SET_FCN_PTR(clCreateKernel)
    SET_FCN_PTR(clSetKernelArg)
    SET_FCN_PTR(clEnqueueNDRangeKernel)
};
<commit_msg>Revert "Under OSX add an underscore prefix for the function name being passed"<commit_after>#include "gvki/UnderlyingCaller.h"
#include <iostream>
#include <cstdlib>

#ifndef MACRO_LIB
#include <dlfcn.h>
#endif

using namespace gvki;

#ifdef MACRO_LIB
#define SET_FCN_PTR(F) F ## U = &(::F);
#else
#include <unistd.h>
#define SET_FCN_PTR(F) dlerror(); /* clear any old errors */ \
                       F ## U = reinterpret_cast<F ## Ty>(::dlsym(RTLD_NEXT, #F)); \
                       errorMsg = dlerror(); \
                       if ( errorMsg != NULL) { \
                         std::cerr << "Failed to dlsym(\"" #F  "\"): " << errorMsg << std::endl; \
                         _exit(255); \
                       } \
                       if ( F ## U == NULL) { \
                         std::cerr << "Function pointer for \"" #F "\" cannot be NULL" << std::endl; \
                         _exit(255); \
                       }
#endif

UnderlyingCaller::UnderlyingCaller()
{
    const char* errorMsg = 0;
    SET_FCN_PTR(clCreateBuffer)
    SET_FCN_PTR(clCreateSubBuffer)
#pragma GCC diagnostic push
#pragma GCC diagnostic ignored "-Wdeprecated-declarations"
    // In OpenCL 1.2 these are deprecated. Don't warn about this.
    SET_FCN_PTR(clCreateImage2D)
    SET_FCN_PTR(clCreateImage3D)
#pragma GCC diagnostic pop
    SET_FCN_PTR(clCreateSampler)
    SET_FCN_PTR(clCreateProgramWithSource)
    SET_FCN_PTR(clBuildProgram)
    SET_FCN_PTR(clCreateKernel)
    SET_FCN_PTR(clSetKernelArg)
    SET_FCN_PTR(clEnqueueNDRangeKernel)
};
<|endoftext|>


<commit_before>// Copyright (c) 2021 ASMlover. All rights reserved.
//
//  _____         _             _
// |_   _|_ _  __| |_ __   ___ | | ___
//   | |/ _` |/ _` | '_ \ / _ \| |/ _ \
//   | | (_| | (_| | |_) | (_) | |  __/
//   |_|\__,_|\__,_| .__/ \___/|_|\___|
//                 |_|
//
// Redistribution and use in source and binary forms, with or without
// modification, are permitted provided that the following conditions
// are met:
//
//  * Redistributions of source code must retain the above copyright
//    notice, this list ofconditions and the following disclaimer.
//
//  * Redistributions in binary form must reproduce the above copyright
//    notice, this list of conditions and the following disclaimer in
//    the documentation and/or other materialsprovided with the
//    distribution.
//
// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS
// FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE
// COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT,
// INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING,
// BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
// LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
// CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
// LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN
// ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
// POSSIBILITY OF SUCH DAMAGE.
#include "lexer.hh"

namespace tadpole {

Token Lexer::next_token() {
  skip_whitespace();

  begpos_ = curpos_;
  if (is_tail())
    return make_token(TokenKind::TK_EOF);

  char c = advance();
  if (is_alpha(c))
    return make_identifier();
  if (is_digit(c))
    return make_numeric();

  // TODO:
  return make_error(from_fmt("unexpected character `%c`", c));
}

void Lexer::skip_whitespace() {
  // TODO:
}

Token Lexer::make_token(TokenKind kind) {
  return Token(kind, gen_literal(begpos_, curpos_), lineno_);
}

Token Lexer::make_token(TokenKind kind, const str_t& literal) {
  return Token(kind, literal, lineno_);
}

Token Lexer::make_error(const str_t& message) {
  return Token(TokenKind::TK_ERR, message, lineno_);
}

Token Lexer::make_identifier() {
  while (is_alnum(peek()))
    advance();

  str_t literal = gen_literal(begpos_, curpos_);
  return make_token(get_keyword_kind(literal), literal);
}

Token Lexer::make_numeric() {
  while (is_digit(peek()))
    advance();
  if (peek() == '.' && is_digit(peek(1))) {
    advance();
    while (is_digit(peek()))
      advance();
  }
  return make_token(TokenKind::TK_NUMERIC);
}

Token Lexer::make_string() {
#define _MAKE_CHAR(x, y) case x: c = y; advance(); break

  str_t literal;
  while (!is_tail() && peek() != '"') {
    char c = peek();
    switch (c) {
    case '\n': ++lineno_; break;
    case '\\':
      switch (peek()) {
      _MAKE_CHAR('"', '"');
      _MAKE_CHAR('\\', '\\');
      _MAKE_CHAR('%', '%');
      _MAKE_CHAR('0', '\0');
      _MAKE_CHAR('a', '\a');
      _MAKE_CHAR('b', '\b');
      _MAKE_CHAR('f', '\f');
      _MAKE_CHAR('n', '\n');
      _MAKE_CHAR('r', '\r');
      _MAKE_CHAR('t', '\t');
      _MAKE_CHAR('v', '\v');
      }
      break;
    }
    literal.push_back(c);
    advance();
  }
#undef _MAKE_CHAR

  if (is_tail())
    return make_error("unterminated string");

  advance();
  return make_token(TokenKind::TK_STRING, literal);
}

}
<commit_msg>:construction: chore(whitespace): implementation of skipping whitespace<commit_after>// Copyright (c) 2021 ASMlover. All rights reserved.
//
//  _____         _             _
// |_   _|_ _  __| |_ __   ___ | | ___
//   | |/ _` |/ _` | '_ \ / _ \| |/ _ \
//   | | (_| | (_| | |_) | (_) | |  __/
//   |_|\__,_|\__,_| .__/ \___/|_|\___|
//                 |_|
//
// Redistribution and use in source and binary forms, with or without
// modification, are permitted provided that the following conditions
// are met:
//
//  * Redistributions of source code must retain the above copyright
//    notice, this list ofconditions and the following disclaimer.
//
//  * Redistributions in binary form must reproduce the above copyright
//    notice, this list of conditions and the following disclaimer in
//    the documentation and/or other materialsprovided with the
//    distribution.
//
// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS
// FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE
// COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT,
// INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING,
// BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
// LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
// CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
// LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN
// ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
// POSSIBILITY OF SUCH DAMAGE.
#include "lexer.hh"

namespace tadpole {

Token Lexer::next_token() {
  skip_whitespace();

  begpos_ = curpos_;
  if (is_tail())
    return make_token(TokenKind::TK_EOF);

  char c = advance();
  if (is_alpha(c))
    return make_identifier();
  if (is_digit(c))
    return make_numeric();

  // TODO:
  return make_error(from_fmt("unexpected character `%c`", c));
}

void Lexer::skip_whitespace() {
  for (;;) {
    char c = peek();
    switch (c) {
    case ' ': case '\r': case '\t': advance(); break;
    case '\n': ++lineno_; advance(); break;
    case '/':
      if (peek(1) == '/') {
        while (peek() != '\n')
          advance();
      }
      else {
        return;
      }
      break;
    default: return;
    }
  }
}

Token Lexer::make_token(TokenKind kind) {
  return Token(kind, gen_literal(begpos_, curpos_), lineno_);
}

Token Lexer::make_token(TokenKind kind, const str_t& literal) {
  return Token(kind, literal, lineno_);
}

Token Lexer::make_error(const str_t& message) {
  return Token(TokenKind::TK_ERR, message, lineno_);
}

Token Lexer::make_identifier() {
  while (is_alnum(peek()))
    advance();

  str_t literal = gen_literal(begpos_, curpos_);
  return make_token(get_keyword_kind(literal), literal);
}

Token Lexer::make_numeric() {
  while (is_digit(peek()))
    advance();
  if (peek() == '.' && is_digit(peek(1))) {
    advance();
    while (is_digit(peek()))
      advance();
  }
  return make_token(TokenKind::TK_NUMERIC);
}

Token Lexer::make_string() {
#define _MAKE_CHAR(x, y) case x: c = y; advance(); break

  str_t literal;
  while (!is_tail() && peek() != '"') {
    char c = peek();
    switch (c) {
    case '\n': ++lineno_; break;
    case '\\':
      switch (peek()) {
      _MAKE_CHAR('"', '"');
      _MAKE_CHAR('\\', '\\');
      _MAKE_CHAR('%', '%');
      _MAKE_CHAR('0', '\0');
      _MAKE_CHAR('a', '\a');
      _MAKE_CHAR('b', '\b');
      _MAKE_CHAR('f', '\f');
      _MAKE_CHAR('n', '\n');
      _MAKE_CHAR('r', '\r');
      _MAKE_CHAR('t', '\t');
      _MAKE_CHAR('v', '\v');
      }
      break;
    }
    literal.push_back(c);
    advance();
  }
#undef _MAKE_CHAR

  if (is_tail())
    return make_error("unterminated string");

  advance();
  return make_token(TokenKind::TK_STRING, literal);
}

}
<|endoftext|>


<commit_before>#include "catch.hpp"

#include <SmurffCpp/Types.h>

#include <Utils/Error.h>

#include <SmurffCpp/Utils/MatrixUtils.h>
#include <SmurffCpp/Utils/Distribution.h>

namespace smurff {

namespace mu = smurff::matrix_utils;

TEST_CASE( "mvnormal/covar" ) {
  init_bmrng(1234);

  const int num_samples = 1000;
  Vector mean = mu::make_dense({1, 10} , { 1., 2., 3., 4., 5., 6., 7., 8., 9., 10.});
  Matrix covar = Matrix::Identity(10,10);

  auto randomMatrix = MvNormal(covar, mean, num_samples);

  // check mean
  REQUIRE(mu::equals_vector(randomMatrix.rowwise().sum(), num_samples * mean, num_samples/10));

  // check variance
  Matrix centered = (randomMatrix.rowwise() - mean);
  Matrix squared = centered.array().square();
  Vector var = squared.colwise().sum() / num_samples;
  REQUIRE(mu::equals_vector(var, covar.diagonal(), 0.1));

}

TEST_CASE( "mvnormal/prec" ) {
  init_bmrng(1234);

  const int num_samples = 1000;
  Vector mean = mu::make_dense({1, 10} , { 1., 2., 3., 4., 5., 6., 7., 8., 9., 10.});
  Matrix covar = Matrix::Identity(10,10);

  auto randomMatrix = MvNormal_prec(covar, mean, num_samples);

  // check mean
  REQUIRE(mu::equals_vector(randomMatrix.rowwise().sum(), num_samples * mean, num_samples/10));

  // check variance
  Matrix centered = (randomMatrix.rowwise() - mean);
  Matrix squared = centered.array().square();
  Vector var = squared.colwise().sum() / num_samples;
  REQUIRE(mu::equals_vector(var, covar.diagonal(), 0.1));

}

TEST_CASE( "CondNormalWishart" ) {
  init_bmrng(1234);
  Vector mean = matrix_utils::make_dense({1, 3} , { 1., 2., 3. });
  Matrix T = Matrix::Identity(3,3);
  int kappa = 2;
  int nu = 4;
  Matrix U(mu::make_dense({4, 3}, {1., 2., 3., 4., 5., 6., 7., 8., 9., 10., 11., 12.}));

  auto N = U.cols();
  auto NS = U.transpose() * U;
  auto NU = U.colwise().sum();

  REQUIRE(NS.rows() == NS.cols());
  REQUIRE(NS.cols() == 3);
  REQUIRE(NU.cols() == 3);

// should be the same
  auto p2 = CondNormalWishart(U, mean, kappa, T, nu);
  auto p1 = CondNormalWishart(N, NS, NU, mean, kappa, T, nu);
}

}<commit_msg>FIX: N = U.rows()<commit_after>#include "catch.hpp"

#include <SmurffCpp/Types.h>

#include <Utils/Error.h>

#include <SmurffCpp/Utils/MatrixUtils.h>
#include <SmurffCpp/Utils/Distribution.h>

namespace smurff {

namespace mu = smurff::matrix_utils;

TEST_CASE( "mvnormal/covar" ) {
  init_bmrng(1234);

  const int num_samples = 1000;
  Vector mean = mu::make_dense({1, 10} , { 1., 2., 3., 4., 5., 6., 7., 8., 9., 10.});
  Matrix covar = Matrix::Identity(10,10);

  auto randomMatrix = MvNormal(covar, mean, num_samples);

  // check mean
  REQUIRE(mu::equals_vector(randomMatrix.rowwise().sum(), num_samples * mean, num_samples/10));

  // check variance
  Matrix centered = (randomMatrix.rowwise() - mean);
  Matrix squared = centered.array().square();
  Vector var = squared.colwise().sum() / num_samples;
  REQUIRE(mu::equals_vector(var, covar.diagonal(), 0.1));

}

TEST_CASE( "mvnormal/prec" ) {
  init_bmrng(1234);

  const int num_samples = 1000;
  Vector mean = mu::make_dense({1, 10} , { 1., 2., 3., 4., 5., 6., 7., 8., 9., 10.});
  Matrix covar = Matrix::Identity(10,10);

  auto randomMatrix = MvNormal_prec(covar, mean, num_samples);

  // check mean
  REQUIRE(mu::equals_vector(randomMatrix.rowwise().sum(), num_samples * mean, num_samples/10));

  // check variance
  Matrix centered = (randomMatrix.rowwise() - mean);
  Matrix squared = centered.array().square();
  Vector var = squared.colwise().sum() / num_samples;
  REQUIRE(mu::equals_vector(var, covar.diagonal(), 0.1));

}

TEST_CASE( "CondNormalWishart" ) {
  init_bmrng(1234);
  Vector mean = matrix_utils::make_dense({1, 3} , { 1., 2., 3. });
  Matrix T = Matrix::Identity(3,3);
  int kappa = 2;
  int nu = 4;
  Matrix U(mu::make_dense({4, 3}, {1., 2., 3., 4., 5., 6., 7., 8., 9., 10., 11., 12.}));

  auto N = U.rows();
  auto NS = U.transpose() * U;
  auto NU = U.colwise().sum();

  REQUIRE(NS.rows() == NS.cols());
  REQUIRE(NS.cols() == 3);
  REQUIRE(NU.cols() == 3);

// should be the same
  auto p2 = CondNormalWishart(U, mean, kappa, T, nu);
  auto p1 = CondNormalWishart(N, NS, NU, mean, kappa, T, nu);
}

}<|endoftext|>


<commit_before>/* -*- Mode: C++; tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*- */
#ifndef SQLITE_STRATEGIES_H
#define SQLITE_STRATEGIES_H 1

#include <cstdlib>
#include <vector>
#include <algorithm>

#include "common.hh"
#include "queueditem.hh"
#include "sqlite-pst.hh"

class EventuallyPersistentEngine;

typedef enum {
    select_all,
    delete_vbucket
} vb_statement_type;

/**
 * Base class for all Sqlite strategies.
 */
class SqliteStrategy {
public:

    SqliteStrategy(const char * const fn,
                   const char * const finit,
                   const char * const pfinit,
                   size_t shards);

    virtual ~SqliteStrategy();

    sqlite3 *open();
    void close();

    size_t getNumOfDbShards() {
        return shardCount;
    }

    virtual uint16_t getDbShardId(const QueuedItem &qi) {
        return getDbShardIdForKey(qi.getKey());
    }

    virtual const std::vector<Statements *> &allStatements() = 0;

    virtual Statements *getStatements(uint16_t vbid, uint16_t vbver,
                                      const std::string &key) = 0;

    PreparedStatement *getInsVBucketStateST() {
        return ins_vb_stmt;
    }

    PreparedStatement *getClearVBucketStateST() {
        return clear_vb_stmt;
    }

    PreparedStatement *getGetVBucketStateST() {
        return sel_vb_stmt;
    }

    PreparedStatement *getClearStatsST() {
        return clear_stats_stmt;
    }

    PreparedStatement *getInsStatST() {
        return ins_stat_stmt;
    }

    virtual bool hasEfficientVBLoad() { return false; }

    virtual bool hasEfficientVBDeletion() { return false; }

    virtual std::vector<PreparedStatement*> getVBStatements(uint16_t vb, vb_statement_type vbst) {
        (void)vb;
        (void)vbst;
        std::vector<PreparedStatement*> rv;
        return rv;
    }

    virtual void closeVBStatements(std::vector<PreparedStatement*> &psts) {
        (void)psts;
    }

    virtual void destroyTables() = 0;

    virtual void optimizeWrites(std::vector<queued_item> &items) {
        (void)items;
    }

    void execute(const char * const query);

    static void disableSchemaCheck() {
        shouldCheckSchemaVersion = false;
    }

protected:

    void doFile(const char * const filename);

    uint16_t getDbShardIdForKey(const std::string &key) {
        assert(shardCount > 0);
        int h=5381;
        int i=0;
        const char *str = key.c_str();

        for(i=0; str[i] != 0x00; i++) {
            h = ((h << 5) + h) ^ str[i];
        }
        return std::abs(h) % (int)shardCount;
    }

    virtual void initDB() {
        doFile(initFile);
    }

    void checkSchemaVersion();
    void initMetaTables();

    virtual void initTables() = 0;
    virtual void initStatements() = 0;
    virtual void initMetaStatements();
    virtual void destroyStatements() {};

    void destroyMetaStatements();

    static bool shouldCheckSchemaVersion;

    sqlite3            *db;
    const char * const  filename;
    const char * const  initFile;
    const char * const  postInitFile;
    size_t              shardCount;

    PreparedStatement *ins_vb_stmt;
    PreparedStatement *clear_vb_stmt;
    PreparedStatement *sel_vb_stmt;

    PreparedStatement *clear_stats_stmt;
    PreparedStatement *ins_stat_stmt;

    uint16_t            schema_version;

private:
    DISALLOW_COPY_AND_ASSIGN(SqliteStrategy);
};

//
// ----------------------------------------------------------------------
// Concrete Strategies
// ----------------------------------------------------------------------
//

/**
 * Strategy for a single table kv store in a single DB.
 */
class SingleTableSqliteStrategy : public SqliteStrategy {
public:

    /**
     * Constructor.
     *
     * @param fn the filename of the DB
     * @param finit an init script to run as soon as the DB opens
     * @param pfinit an init script to run after initializing all schema
     * @param shards the number of shards
     */
    SingleTableSqliteStrategy(const char * const fn,
                              const char * const finit,
                              const char * const pfinit,
                              size_t shards = 1) :
        SqliteStrategy(fn, finit, pfinit, shards), statements() {
        assert(filename);
    }

    virtual ~SingleTableSqliteStrategy() { }

    const std::vector<Statements *> &allStatements() {
        return statements;
    }

    Statements *getStatements(uint16_t vbid, uint16_t vbver,
                              const std::string &key) {
        (void)vbid;
        (void)vbver;
        return statements.at(getDbShardIdForKey(key));
    }

    void destroyStatements();
    virtual void destroyTables();

    void optimizeWrites(std::vector<queued_item> &items) {
        // Sort all the queued items for each db shard by their row ids
        CompareQueuedItemsByRowId cq;
        std::sort(items.begin(), items.end(), cq);
    }

protected:
    std::vector<Statements *> statements;

    virtual void initTables();
    virtual void initStatements();

private:
    DISALLOW_COPY_AND_ASSIGN(SingleTableSqliteStrategy);
};

//
// ----------------------------------------------------------------------
// Multi DB strategy
// ----------------------------------------------------------------------
//

/**
 * A specialization of SqliteStrategy that allows multiple data
 * shards with a single kv table each.
 */
class MultiDBSingleTableSqliteStrategy : public SingleTableSqliteStrategy {
public:

    /**
     * Constructor.
     *
     * @param fn same as SqliteStrategy
     * @param sp the shard pattern
     * @param finit same as SqliteStrategy
     * @param pfinit same as SqliteStrategy
     * @param n number of DB shards to create
     */
    MultiDBSingleTableSqliteStrategy(const char * const fn,
                                     const char * const sp,
                                     const char * const finit,
                                     const char * const pfinit,
                                     int n):
        SingleTableSqliteStrategy(fn, finit, pfinit, n),
        shardpattern(sp), numTables(n) {
        assert(shardpattern);
    }

    void initDB(void);
    void initTables(void);
    void initStatements(void);
    void destroyTables(void);

private:
    const char * const shardpattern;
    int numTables;
};

//
// ----------------------------------------------------------------------
// Table Per Vbucket
// ----------------------------------------------------------------------
//

/**
 * Strategy for a table per vbucket store in a single DB.
 */
class MultiTableSqliteStrategy : public SqliteStrategy {
public:

    /**
     * Constructor.
     *
     * @param fn the filename of the DB
     * @param finit an init script to run as soon as the DB opens
     * @param pfinit an init script to run after initializing all schema
     * @param nv the maxinum number of vbuckets
     * @param shards the number of data shards
     */
    MultiTableSqliteStrategy(const char * const fn,
                             const char * const finit,
                             const char * const pfinit,
                             int nv,
                             int shards=1)
        : SqliteStrategy(fn, finit, pfinit, shards),
          nvbuckets(nv), statements() {

        assert(filename);
    }

    virtual ~MultiTableSqliteStrategy() { }

    const std::vector<Statements *> &allStatements() {
        return statements;
    }

    virtual Statements *getStatements(uint16_t vbid, uint16_t vbver,
                                      const std::string &key) {
        (void)vbver;
        (void)key;
        assert(static_cast<size_t>(vbid) < statements.size());
        return statements.at(vbid);
    }

    virtual void destroyStatements();
    virtual void destroyTables();

    bool hasEfficientVBLoad() { return true; }

    bool hasEfficientVBDeletion() { return true; }

    virtual std::vector<PreparedStatement*> getVBStatements(uint16_t vb, vb_statement_type vbst) {
        std::vector<PreparedStatement*> rv;
        assert(static_cast<size_t>(vb) < statements.size());
        switch (vbst) {
        case select_all:
            rv.push_back(statements.at(vb)->all());
            break;
        case delete_vbucket:
            rv.push_back(statements.at(vb)->del_vb());
            break;
        default:
            break;
        }
        return rv;
    }

    void closeVBStatements(std::vector<PreparedStatement*> &psts) {
        std::for_each(psts.begin(), psts.end(),
                      std::mem_fun(&PreparedStatement::reset));
    }

    void optimizeWrites(std::vector<queued_item> &items) {
        // Sort all the queued items for each db shard by its vbucket
        // ID and then its row ids
        CompareQueuedItemsByVBAndRowId cq;
        std::sort(items.begin(), items.end(), cq);
    }

protected:
    size_t nvbuckets;
    std::vector<Statements *> statements;

    virtual void initTables();
    virtual void initStatements();

private:
    DISALLOW_COPY_AND_ASSIGN(MultiTableSqliteStrategy);
};

//
// ----------------------------------------------------------------------
// Multiple Shards, Table Per Vbucket
// ----------------------------------------------------------------------
//

/**
 * Strategy for a table per vbucket store in multiple shards.
 */
class ShardedMultiTableSqliteStrategy : public MultiTableSqliteStrategy {
public:

    /**
     * Constructor.
     *
     * @param fn the filename of the DB
     * @param sp the shard pattern
     * @param finit an init script to run as soon as the DB opens
     * @param pfinit an init script to run after initializing all schema
     * @param nv the maxinum number of vbuckets
     * @param n the number of data shards
     */
    ShardedMultiTableSqliteStrategy(const char * const fn,
                                    const char * const sp,
                                    const char * const finit,
                                    const char * const pfinit,
                                    int nv,
                                    int n)
        : MultiTableSqliteStrategy(fn, finit, pfinit, nv, n),
          shardpattern(sp),
          statementsPerShard() {

        assert(filename);
    }

    virtual ~ShardedMultiTableSqliteStrategy() { }

    Statements *getStatements(uint16_t vbid, uint16_t vbver,
                              const std::string &key);

    void destroyStatements();
    void destroyTables();

    std::vector<PreparedStatement*> getVBStatements(uint16_t vb, vb_statement_type vbst);

protected:
    const char * const        shardpattern;
    // statementsPerShard[vbucket][shard]
    std::vector<std::vector<Statements*> > statementsPerShard;

    void initDB();
    void initTables();
    void initStatements();

private:
    DISALLOW_COPY_AND_ASSIGN(ShardedMultiTableSqliteStrategy);
};

//
// ----------------------------------------------------------------------
// Sharded by VBucket
// ----------------------------------------------------------------------
//

/**
 * Strategy for a table per vbucket where each vbucket exists in only
 * one shard.
 */
class ShardedByVBucketSqliteStrategy : public MultiTableSqliteStrategy {
public:

    /**
     * Constructor.
     *
     * @param fn the filename of the DB
     * @param sp the shard pattern for generating shard files
     * @param finit an init script to run as soon as the DB opens
     * @param pfinit an init script to run after initializing all schema
     * @param nv the maxinum number of vbuckets
     * @param n the number of data shards
     */
    ShardedByVBucketSqliteStrategy(const char * const fn,
                                   const char * const sp,
                                   const char * const finit,
                                   const char * const pfinit,
                                   int nv,
                                   int n)
        : MultiTableSqliteStrategy(fn, finit, pfinit, nv, n),
          shardpattern(sp) {

        assert(filename);
    }

    uint16_t getDbShardId(const QueuedItem &qi) {
        return getShardForVBucket(qi.getVBucketId());
    }

    virtual ~ShardedByVBucketSqliteStrategy() { }

    void destroyTables();

protected:
    const char * const shardpattern;

    size_t getShardForVBucket(uint16_t vb) {
        size_t rv = (static_cast<size_t>(vb) % shardCount);
        assert(rv < shardCount);
        return rv;
    }

    void initDB();
    void initTables();
    void initStatements();

private:
    DISALLOW_COPY_AND_ASSIGN(ShardedByVBucketSqliteStrategy);
};

#endif /* SQLITE_STRATEGIES_H */
<commit_msg>MB-3753 Provide specific SQL stmts even for single table strategy<commit_after>/* -*- Mode: C++; tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*- */
#ifndef SQLITE_STRATEGIES_H
#define SQLITE_STRATEGIES_H 1

#include <cstdlib>
#include <vector>
#include <algorithm>

#include "common.hh"
#include "queueditem.hh"
#include "sqlite-pst.hh"

class EventuallyPersistentEngine;

typedef enum {
    select_all,
    delete_vbucket
} vb_statement_type;

/**
 * Base class for all Sqlite strategies.
 */
class SqliteStrategy {
public:

    SqliteStrategy(const char * const fn,
                   const char * const finit,
                   const char * const pfinit,
                   size_t shards);

    virtual ~SqliteStrategy();

    sqlite3 *open();
    void close();

    size_t getNumOfDbShards() {
        return shardCount;
    }

    virtual uint16_t getDbShardId(const QueuedItem &qi) {
        return getDbShardIdForKey(qi.getKey());
    }

    virtual const std::vector<Statements *> &allStatements() = 0;

    virtual Statements *getStatements(uint16_t vbid, uint16_t vbver,
                                      const std::string &key) = 0;

    PreparedStatement *getInsVBucketStateST() {
        return ins_vb_stmt;
    }

    PreparedStatement *getClearVBucketStateST() {
        return clear_vb_stmt;
    }

    PreparedStatement *getGetVBucketStateST() {
        return sel_vb_stmt;
    }

    PreparedStatement *getClearStatsST() {
        return clear_stats_stmt;
    }

    PreparedStatement *getInsStatST() {
        return ins_stat_stmt;
    }

    virtual bool hasEfficientVBLoad() { return false; }

    virtual bool hasEfficientVBDeletion() { return false; }

    virtual std::vector<PreparedStatement*> getVBStatements(uint16_t vb, vb_statement_type vbst) {
        (void)vb;
        (void)vbst;
        std::vector<PreparedStatement*> rv;
        return rv;
    }

    virtual void closeVBStatements(std::vector<PreparedStatement*> &psts) {
        (void)psts;
    }

    virtual void destroyTables() = 0;

    virtual void optimizeWrites(std::vector<queued_item> &items) {
        (void)items;
    }

    void execute(const char * const query);

    static void disableSchemaCheck() {
        shouldCheckSchemaVersion = false;
    }

protected:

    void doFile(const char * const filename);

    uint16_t getDbShardIdForKey(const std::string &key) {
        assert(shardCount > 0);
        int h=5381;
        int i=0;
        const char *str = key.c_str();

        for(i=0; str[i] != 0x00; i++) {
            h = ((h << 5) + h) ^ str[i];
        }
        return std::abs(h) % (int)shardCount;
    }

    virtual void initDB() {
        doFile(initFile);
    }

    void checkSchemaVersion();
    void initMetaTables();

    virtual void initTables() = 0;
    virtual void initStatements() = 0;
    virtual void initMetaStatements();
    virtual void destroyStatements() {};

    void destroyMetaStatements();

    static bool shouldCheckSchemaVersion;

    sqlite3            *db;
    const char * const  filename;
    const char * const  initFile;
    const char * const  postInitFile;
    size_t              shardCount;

    PreparedStatement *ins_vb_stmt;
    PreparedStatement *clear_vb_stmt;
    PreparedStatement *sel_vb_stmt;

    PreparedStatement *clear_stats_stmt;
    PreparedStatement *ins_stat_stmt;

    uint16_t            schema_version;

private:
    DISALLOW_COPY_AND_ASSIGN(SqliteStrategy);
};

//
// ----------------------------------------------------------------------
// Concrete Strategies
// ----------------------------------------------------------------------
//

/**
 * Strategy for a single table kv store in a single DB.
 */
class SingleTableSqliteStrategy : public SqliteStrategy {
public:

    /**
     * Constructor.
     *
     * @param fn the filename of the DB
     * @param finit an init script to run as soon as the DB opens
     * @param pfinit an init script to run after initializing all schema
     * @param shards the number of shards
     */
    SingleTableSqliteStrategy(const char * const fn,
                              const char * const finit,
                              const char * const pfinit,
                              size_t shards = 1) :
        SqliteStrategy(fn, finit, pfinit, shards), statements() {
        assert(filename);
    }

    virtual ~SingleTableSqliteStrategy() { }

    const std::vector<Statements *> &allStatements() {
        return statements;
    }

    Statements *getStatements(uint16_t vbid, uint16_t vbver,
                              const std::string &key) {
        (void)vbid;
        (void)vbver;
        return statements.at(getDbShardIdForKey(key));
    }

    void destroyStatements();
    virtual void destroyTables();

    void optimizeWrites(std::vector<queued_item> &items) {
        // Sort all the queued items for each db shard by their row ids
        CompareQueuedItemsByRowId cq;
        std::sort(items.begin(), items.end(), cq);
    }

    virtual std::vector<PreparedStatement*> getVBStatements(uint16_t vb, vb_statement_type vbst) {
        (void)vb;
        std::vector<PreparedStatement*> rv;
        std::vector<Statements*>::iterator it;
        for (it = statements.begin(); it != statements.end(); ++it) {
            switch (vbst) {
            case select_all:
                rv.push_back((*it)->all());
                break;
            case delete_vbucket:
                rv.push_back((*it)->del_vb());
                break;
            default:
                break;
            }
        }

        return rv;
    }

    virtual void closeVBStatements(std::vector<PreparedStatement*> &psts) {
        std::for_each(psts.begin(), psts.end(),
                      std::mem_fun(&PreparedStatement::reset));
    }

protected:
    std::vector<Statements *> statements;

    virtual void initTables();
    virtual void initStatements();

private:
    DISALLOW_COPY_AND_ASSIGN(SingleTableSqliteStrategy);
};

//
// ----------------------------------------------------------------------
// Multi DB strategy
// ----------------------------------------------------------------------
//

/**
 * A specialization of SqliteStrategy that allows multiple data
 * shards with a single kv table each.
 */
class MultiDBSingleTableSqliteStrategy : public SingleTableSqliteStrategy {
public:

    /**
     * Constructor.
     *
     * @param fn same as SqliteStrategy
     * @param sp the shard pattern
     * @param finit same as SqliteStrategy
     * @param pfinit same as SqliteStrategy
     * @param n number of DB shards to create
     */
    MultiDBSingleTableSqliteStrategy(const char * const fn,
                                     const char * const sp,
                                     const char * const finit,
                                     const char * const pfinit,
                                     int n):
        SingleTableSqliteStrategy(fn, finit, pfinit, n),
        shardpattern(sp), numTables(n) {
        assert(shardpattern);
    }

    void initDB(void);
    void initTables(void);
    void initStatements(void);
    void destroyTables(void);

private:
    const char * const shardpattern;
    int numTables;
};

//
// ----------------------------------------------------------------------
// Table Per Vbucket
// ----------------------------------------------------------------------
//

/**
 * Strategy for a table per vbucket store in a single DB.
 */
class MultiTableSqliteStrategy : public SqliteStrategy {
public:

    /**
     * Constructor.
     *
     * @param fn the filename of the DB
     * @param finit an init script to run as soon as the DB opens
     * @param pfinit an init script to run after initializing all schema
     * @param nv the maxinum number of vbuckets
     * @param shards the number of data shards
     */
    MultiTableSqliteStrategy(const char * const fn,
                             const char * const finit,
                             const char * const pfinit,
                             int nv,
                             int shards=1)
        : SqliteStrategy(fn, finit, pfinit, shards),
          nvbuckets(nv), statements() {

        assert(filename);
    }

    virtual ~MultiTableSqliteStrategy() { }

    const std::vector<Statements *> &allStatements() {
        return statements;
    }

    virtual Statements *getStatements(uint16_t vbid, uint16_t vbver,
                                      const std::string &key) {
        (void)vbver;
        (void)key;
        assert(static_cast<size_t>(vbid) < statements.size());
        return statements.at(vbid);
    }

    virtual void destroyStatements();
    virtual void destroyTables();

    bool hasEfficientVBLoad() { return true; }

    bool hasEfficientVBDeletion() { return true; }

    virtual std::vector<PreparedStatement*> getVBStatements(uint16_t vb, vb_statement_type vbst) {
        std::vector<PreparedStatement*> rv;
        assert(static_cast<size_t>(vb) < statements.size());
        switch (vbst) {
        case select_all:
            rv.push_back(statements.at(vb)->all());
            break;
        case delete_vbucket:
            rv.push_back(statements.at(vb)->del_vb());
            break;
        default:
            break;
        }
        return rv;
    }

    void closeVBStatements(std::vector<PreparedStatement*> &psts) {
        std::for_each(psts.begin(), psts.end(),
                      std::mem_fun(&PreparedStatement::reset));
    }

    void optimizeWrites(std::vector<queued_item> &items) {
        // Sort all the queued items for each db shard by its vbucket
        // ID and then its row ids
        CompareQueuedItemsByVBAndRowId cq;
        std::sort(items.begin(), items.end(), cq);
    }

protected:
    size_t nvbuckets;
    std::vector<Statements *> statements;

    virtual void initTables();
    virtual void initStatements();

private:
    DISALLOW_COPY_AND_ASSIGN(MultiTableSqliteStrategy);
};

//
// ----------------------------------------------------------------------
// Multiple Shards, Table Per Vbucket
// ----------------------------------------------------------------------
//

/**
 * Strategy for a table per vbucket store in multiple shards.
 */
class ShardedMultiTableSqliteStrategy : public MultiTableSqliteStrategy {
public:

    /**
     * Constructor.
     *
     * @param fn the filename of the DB
     * @param sp the shard pattern
     * @param finit an init script to run as soon as the DB opens
     * @param pfinit an init script to run after initializing all schema
     * @param nv the maxinum number of vbuckets
     * @param n the number of data shards
     */
    ShardedMultiTableSqliteStrategy(const char * const fn,
                                    const char * const sp,
                                    const char * const finit,
                                    const char * const pfinit,
                                    int nv,
                                    int n)
        : MultiTableSqliteStrategy(fn, finit, pfinit, nv, n),
          shardpattern(sp),
          statementsPerShard() {

        assert(filename);
    }

    virtual ~ShardedMultiTableSqliteStrategy() { }

    Statements *getStatements(uint16_t vbid, uint16_t vbver,
                              const std::string &key);

    void destroyStatements();
    void destroyTables();

    std::vector<PreparedStatement*> getVBStatements(uint16_t vb, vb_statement_type vbst);

protected:
    const char * const        shardpattern;
    // statementsPerShard[vbucket][shard]
    std::vector<std::vector<Statements*> > statementsPerShard;

    void initDB();
    void initTables();
    void initStatements();

private:
    DISALLOW_COPY_AND_ASSIGN(ShardedMultiTableSqliteStrategy);
};

//
// ----------------------------------------------------------------------
// Sharded by VBucket
// ----------------------------------------------------------------------
//

/**
 * Strategy for a table per vbucket where each vbucket exists in only
 * one shard.
 */
class ShardedByVBucketSqliteStrategy : public MultiTableSqliteStrategy {
public:

    /**
     * Constructor.
     *
     * @param fn the filename of the DB
     * @param sp the shard pattern for generating shard files
     * @param finit an init script to run as soon as the DB opens
     * @param pfinit an init script to run after initializing all schema
     * @param nv the maxinum number of vbuckets
     * @param n the number of data shards
     */
    ShardedByVBucketSqliteStrategy(const char * const fn,
                                   const char * const sp,
                                   const char * const finit,
                                   const char * const pfinit,
                                   int nv,
                                   int n)
        : MultiTableSqliteStrategy(fn, finit, pfinit, nv, n),
          shardpattern(sp) {

        assert(filename);
    }

    uint16_t getDbShardId(const QueuedItem &qi) {
        return getShardForVBucket(qi.getVBucketId());
    }

    virtual ~ShardedByVBucketSqliteStrategy() { }

    void destroyTables();

protected:
    const char * const shardpattern;

    size_t getShardForVBucket(uint16_t vb) {
        size_t rv = (static_cast<size_t>(vb) % shardCount);
        assert(rv < shardCount);
        return rv;
    }

    void initDB();
    void initTables();
    void initStatements();

private:
    DISALLOW_COPY_AND_ASSIGN(ShardedByVBucketSqliteStrategy);
};

#endif /* SQLITE_STRATEGIES_H */
<|endoftext|>


<commit_before>#include "core/argparse.h"

#include <iostream>

#include <string>
#include <sstream>
#include <functional>
#include <vector>
#include <map>
#include <memory>

#include "core/assert.h"
#include "core/stringutils.h"
#include "core/stringmerger.h"

namespace argparse
{
  /////////////////////////////////////////////////////////////////////////////////////////
  // Name

  bool Name::IsOptional(const std::string& arg)
  {
    if(arg.empty())
    {
      return false;
    }
    return arg[0] == '-';
  }

  std::string Name::OptionalName(const std::string& name)
  {
    ASSERTX(IsOptional(name), name);
    ASSERTX(!name.empty(), name);
    return name.substr(1);
  }

  Name::Name()
      : is_optional(false)
  { }

  Name::Name(const char* str)
      : is_optional(IsOptional(str))
      , names( Split(str, ',') )
  {
    if(is_optional)
    {
      names[0] = OptionalName(names[0]);
    }

    for(auto& n: names)
    {
      n = Trim(n);
    }

    #ifdef _DEBUG
    AssertValid();
    #endif}
  }

  Name Name::Parse(const std::string& n)
  {
    return { n.c_str() };
  }

  Name Name::Optional(std::initializer_list<std::string> names)
  {
    return { true, names };
  }

  Name Name::Positional(std::string& name)
  {
    return {false, {name}};
  }

  Name::Name(bool o, const std::vector<std::string>& n)
      : is_optional(o)
      , names(n)
  {
    #ifdef _DEBUG
    AssertValid();
    #endif
  }

  #ifdef _DEBUG
  void Name::AssertValid()
  {
    if(is_optional)
    {
      for(unsigned int i=0; i<names.size(); i+=1)
      {
        ASSERTX(IsOptional(names[i])==false, i, names[i]);
      }
    }
    else
    {
      ASSERTX(names.size()==1, names.size());
    }

    // todo: assert that name only contains valid characters...
  }
  #endif

  /////////////////////////////////////////////////////////////////////////////////////////
  // ConsoleOutput

  void ConsoleOutput::OnError(const std::string& err)
  {
    std::cerr << err << "\n";
  }

  void ConsoleOutput::OnInfo(const std::string& info)
  {
    std::cout << info << "\n";
  }

  /////////////////////////////////////////////////////////////////////////////////////////
  // Running

  bool Running::HasMore() const
  {
    return next_index < arguments.size();
  }

  std::string Running::Read()
  {
    if(next_index < arguments.size())
    {
      const auto ret = arguments[next_index];
      next_index += 1;
      return ret;
    }
    else
    {
      return "";
    }
  }

  std::string Running::Peek(size_t advance) const
  {
    ASSERTX(advance > 0, advance);
    const auto suggested_index = next_index + advance - 1;
    if(suggested_index < arguments.size())
    {
      return arguments[suggested_index];
    }
    else
    {
      return "";
    }
  }

  /////////////////////////////////////////////////////////////////////////////////////////
  // Extra

  Extra& Extra::MetaVar(const std::string& m)
  {
    arg->meta_var = m;
    return *this;
  }

  Extra& Extra::Help(const std::string& h)
  {
    arg->help = h;
    return *this;
  }

  /////////////////////////////////////////////////////////////////////////////////////////
  // SimpleParser

  template<>
  ParseResult SimpleParser<std::string>(std::string* target, const std::string&, const std::string& value, Output*)
  {
    *target = value;
    return ParseResult::Ok;
  }

  /////////////////////////////////////////////////////////////////////////////////////////
  // Parser

  Parser::Parser(const std::string& n) : display_name(n)
  {
    // todo: add help command
  }

  Extra Parser::AddArgument(const Name& name, std::shared_ptr<Arg> arg)
  {
    arg->name = name;

    if(name.is_optional)
    {
      ASSERT(!name.names.empty());
      for(auto n: name.names)
      {
        ASSERTX(optional_arguments.find(n)==optional_arguments.end(), n);
        optional_arguments[n] = arg;
      }
    }
    else
    {
      positional_arguments.push_back(arg);
    }

    return Extra { arg };
  }

  ParseResult Parser::Parse(int argc, char* argv[]) const
  {
    auto args = std::vector<std::string>{};
    for (int i = 1; i < argc; i += 1)
    {
      args.push_back(argv[i]);
    }
    return Parse(argv[0], args);
  }

  ParseResult Parser::Parse(const std::string& program_name, const std::vector<std::string>& args) const
  {
    auto console = ConsoleOutput {};
    auto running = Running { program_name, args, output ? output : &console };

    size_t next_positional_index = 0;

    while(running.HasMore())
    {
      const auto is_optional = Name::IsOptional(running.Peek());
      const auto has_more_positionals = next_positional_index < positional_arguments.size();

      std::string arg_name;
      std::shared_ptr<Arg> arg = nullptr;

      if(has_more_positionals)
      {
        // if it is a valid optional, parse it, otherwise send to optional
        if(is_optional)
        {
          auto found = optional_arguments.find(Name::OptionalName(running.Peek()));
          if(found != optional_arguments.end())
          {
            arg = found->second;
          }
        }

        if(arg == nullptr)
        {
          arg = positional_arguments[next_positional_index];
          arg_name = arg->name.names[0];
          next_positional_index += 1;
        }
      }
      else
      {
        // no more positionals, must be a optional
        const auto optional_name = running.Read();
        if(!Name::IsOptional(optional_name))
        {
          running.output->OnError(Str() << "Got " << optional_name << " but all positionals are consumed");
          return ParseResult::Failed;
        }

        auto found = optional_arguments.find(Name::OptionalName(optional_name));
        if(found == optional_arguments.end())
        {
          running.output->OnError(Str() << "Not a valid argument: " << optional_name);
          return ParseResult::Failed;
        }
        arg_name = optional_name;
        arg = found->second;
      }

      auto r = arg->Parse(arg_name, &running);
      if( r != ParseResult::Ok)
      {
        return r;
      }
    }

    const auto has_more_positionals = next_positional_index < positional_arguments.size();

    std::vector<std::string> missing_positionals;
    for(size_t i=next_positional_index; i<positional_arguments.size(); i+=1)
    {
      missing_positionals.push_back(positional_arguments[i]->name.names[0]);
    }

    if(!missing_positionals.empty())
    {
      running.output->OnError( Str() << "Positionals not consumed: " << StringMerger::EnglishAnd().Generate(missing_positionals));
      return ParseResult::Failed;
    }

    return ParseResult::Ok;
  }
}
<commit_msg>removed warnings<commit_after>#include "core/argparse.h"

#include <iostream>

#include <string>
#include <sstream>
#include <functional>
#include <vector>
#include <map>
#include <memory>

#include "core/assert.h"
#include "core/stringutils.h"
#include "core/stringmerger.h"

namespace argparse
{
  /////////////////////////////////////////////////////////////////////////////////////////
  // Name

  bool Name::IsOptional(const std::string& arg)
  {
    if(arg.empty())
    {
      return false;
    }
    return arg[0] == '-';
  }

  std::string Name::OptionalName(const std::string& name)
  {
    ASSERTX(IsOptional(name), name);
    ASSERTX(!name.empty(), name);
    return name.substr(1);
  }

  Name::Name()
      : is_optional(false)
  { }

  Name::Name(const char* str)
      : is_optional(IsOptional(str))
      , names( Split(str, ',') )
  {
    if(is_optional)
    {
      names[0] = OptionalName(names[0]);
    }

    for(auto& n: names)
    {
      n = Trim(n);
    }

    #ifdef _DEBUG
    AssertValid();
    #endif
  }

  Name Name::Parse(const std::string& n)
  {
    return { n.c_str() };
  }

  Name Name::Optional(std::initializer_list<std::string> names)
  {
    return { true, names };
  }

  Name Name::Positional(std::string& name)
  {
    return {false, {name}};
  }

  Name::Name(bool o, const std::vector<std::string>& n)
      : is_optional(o)
      , names(n)
  {
    #ifdef _DEBUG
    AssertValid();
    #endif
  }

  #ifdef _DEBUG
  void Name::AssertValid()
  {
    if(is_optional)
    {
      for(unsigned int i=0; i<names.size(); i+=1)
      {
        ASSERTX(IsOptional(names[i])==false, i, names[i]);
      }
    }
    else
    {
      ASSERTX(names.size()==1, names.size());
    }

    // todo: assert that name only contains valid characters...
  }
  #endif

  /////////////////////////////////////////////////////////////////////////////////////////
  // ConsoleOutput

  void ConsoleOutput::OnError(const std::string& err)
  {
    std::cerr << err << "\n";
  }

  void ConsoleOutput::OnInfo(const std::string& info)
  {
    std::cout << info << "\n";
  }

  /////////////////////////////////////////////////////////////////////////////////////////
  // Running

  bool Running::HasMore() const
  {
    return next_index < arguments.size();
  }

  std::string Running::Read()
  {
    if(next_index < arguments.size())
    {
      const auto ret = arguments[next_index];
      next_index += 1;
      return ret;
    }
    else
    {
      return "";
    }
  }

  std::string Running::Peek(size_t advance) const
  {
    ASSERTX(advance > 0, advance);
    const auto suggested_index = next_index + advance - 1;
    if(suggested_index < arguments.size())
    {
      return arguments[suggested_index];
    }
    else
    {
      return "";
    }
  }

  /////////////////////////////////////////////////////////////////////////////////////////
  // Extra

  Extra& Extra::MetaVar(const std::string& m)
  {
    arg->meta_var = m;
    return *this;
  }

  Extra& Extra::Help(const std::string& h)
  {
    arg->help = h;
    return *this;
  }

  /////////////////////////////////////////////////////////////////////////////////////////
  // SimpleParser

  template<>
  ParseResult SimpleParser<std::string>(std::string* target, const std::string&, const std::string& value, Output*)
  {
    *target = value;
    return ParseResult::Ok;
  }

  /////////////////////////////////////////////////////////////////////////////////////////
  // Parser

  Parser::Parser(const std::string& n) : display_name(n)
  {
    // todo: add help command
  }

  Extra Parser::AddArgument(const Name& name, std::shared_ptr<Arg> arg)
  {
    arg->name = name;

    if(name.is_optional)
    {
      ASSERT(!name.names.empty());
      for(auto n: name.names)
      {
        ASSERTX(optional_arguments.find(n)==optional_arguments.end(), n);
        optional_arguments[n] = arg;
      }
    }
    else
    {
      positional_arguments.push_back(arg);
    }

    return Extra { arg };
  }

  ParseResult Parser::Parse(int argc, char* argv[]) const
  {
    auto args = std::vector<std::string>{};
    for (int i = 1; i < argc; i += 1)
    {
      args.push_back(argv[i]);
    }
    return Parse(argv[0], args);
  }

  ParseResult Parser::Parse(const std::string& program_name, const std::vector<std::string>& args) const
  {
    auto console = ConsoleOutput {};
    auto running = Running { program_name, args, output ? output : &console };

    size_t next_positional_index = 0;

    while(running.HasMore())
    {
      const auto is_optional = Name::IsOptional(running.Peek());
      const auto has_more_positionals = next_positional_index < positional_arguments.size();

      std::string arg_name;
      std::shared_ptr<Arg> arg = nullptr;

      if(has_more_positionals)
      {
        // if it is a valid optional, parse it, otherwise send to optional
        if(is_optional)
        {
          auto found = optional_arguments.find(Name::OptionalName(running.Peek()));
          if(found != optional_arguments.end())
          {
            arg = found->second;
          }
        }

        if(arg == nullptr)
        {
          arg = positional_arguments[next_positional_index];
          arg_name = arg->name.names[0];
          next_positional_index += 1;
        }
      }
      else
      {
        // no more positionals, must be a optional
        const auto optional_name = running.Read();
        if(!Name::IsOptional(optional_name))
        {
          running.output->OnError(Str() << "Got " << optional_name << " but all positionals are consumed");
          return ParseResult::Failed;
        }

        auto found = optional_arguments.find(Name::OptionalName(optional_name));
        if(found == optional_arguments.end())
        {
          running.output->OnError(Str() << "Not a valid argument: " << optional_name);
          return ParseResult::Failed;
        }
        arg_name = optional_name;
        arg = found->second;
      }

      auto r = arg->Parse(arg_name, &running);
      if( r != ParseResult::Ok)
      {
        return r;
      }
    }

    std::vector<std::string> missing_positionals;
    for(size_t i=next_positional_index; i<positional_arguments.size(); i+=1)
    {
      missing_positionals.push_back(positional_arguments[i]->name.names[0]);
    }

    if(!missing_positionals.empty())
    {
      running.output->OnError( Str() << "Positionals not consumed: " << StringMerger::EnglishAnd().Generate(missing_positionals));
      return ParseResult::Failed;
    }

    return ParseResult::Ok;
  }
}
<|endoftext|>