Evaluates a reverse polish notation token list
3426
private static Requirement evaluateReversePolishNotation(Iterator<Tokenizer.Token> iterator, Semver.SemverType type) {
        try {
            Tokenizer.Token token = iterator.next();

            if (token.type == Tokenizer.TokenType.VERSION) {
                if ("*".equals(token.value) || (type == Semver.SemverType.NPM && "latest".equals(token.value))) {
                    // Special case for "*" and "latest" in NPM
                    return new Requirement(new Range("0.0.0", Range.RangeOperator.GTE), null, null, null);
                }
                Semver version = new Semver(token.value, type);
                if (version.getMinor() != null && version.getPatch() != null) {
                    Range range = new Range(version, Range.RangeOperator.EQ);
                    return new Requirement(range, null, null, null);
                } else {
                    // If we have a version with a wildcard char (like 1.2.x, 1.2.* or 1.2), we need a tilde requirement
                    return tildeRequirement(version.getValue(), type);
                }
            } else if (token.type == Tokenizer.TokenType.HYPHEN) {
                Tokenizer.Token token3 = iterator.next(); // Note that token3 is before token2!
                Tokenizer.Token token2 = iterator.next();
                return hyphenRequirement(token2.value, token3.value, type);
            } else if (token.type.isUnary()) {
                Tokenizer.Token token2 = iterator.next();

                Range.RangeOperator rangeOp;
                switch (token.type) {
                    case EQ:
                        rangeOp = Range.RangeOperator.EQ;
                        break;
                    case LT:
                        rangeOp = Range.RangeOperator.LT;
                        break;
                    case LTE:
                        rangeOp = Range.RangeOperator.LTE;
                        break;
                    case GT:
                        rangeOp = Range.RangeOperator.GT;
                        break;
                    case GTE:
                        rangeOp = Range.RangeOperator.GTE;
                        break;
                    case TILDE:
                        return tildeRequirement(token2.value, type);
                    case CARET:
                        return caretRequirement(token2.value, type);
                    default:
                        throw new SemverException("Invalid requirement");
                }

                Range range = new Range(token2.value, rangeOp);
                return new Requirement(range, null, null, null);
            } else {
                Requirement req1 = evaluateReversePolishNotation(iterator, type);
                Requirement req2 = evaluateReversePolishNotation(iterator, type);

                RequirementOperator requirementOp;
                switch (token.type) {
                    case OR:
                        requirementOp = RequirementOperator.OR;
                        break;
                    case AND:
                        requirementOp = RequirementOperator.AND;
                        break;
                    default:
                        throw new SemverException("Invalid requirement");
                }

                return new Requirement(null, req1, requirementOp, req2);
            }
        } catch (NoSuchElementException e) {
            throw new SemverException("Invalid requirement");
        }
    }
509172
def __parse(self, tokenlist):
        """"""
        #empty stack
        #iterate over symbollist
        tokenlist = [x for x in tokenlist]
        if not isinstance(tokenlist, list):
            raise TypeError("Expected list, got %s" % tokenlist.__class__.__name__)
        LOG.debug("get_trees: checking list: " + str(tokenlist))
        stack = [(0, Extended_S)]
        while True:
            state = stack[-1][0]
            if len(tokenlist):#FIXME: tokenlist with one element is reported as false
                token = tokenlist[0]
            else:
                token = EndSymbol()
            newdic = self.__parsertable.insert(state, token)
            action = newdic["action"]
            if action == "Fail":
                return False
            elif action == "Accept":
                return True
            if action == "Reduce":
                reductionrule = newdic["rule"]
                #TODO extract len(right side) of the rule and insert left side
                for rsymbol in reversed(reductionrule.rightside):
                    state, symbol = stack.pop() # TODO: check
                state = stack[-1][0]
                state = self.__parsertable.goto(state,reductionrule.leftside[0])
                stack.append((state, reductionrule.leftside[0]))
            elif action == "Shift":
                stack.append((newdic['dest'], tokenlist.pop(0)))
            else:
                raise ValueError("Unknown action")
        return False
218224
function process_raw(token) {
	var token = escapeRegex(token);
	return {
		type: "raw",
		piece: token + "/",
		last: token + "/?"
	};
}
593603
def _subtoken_ids_to_tokens(self, subtokens):
    """"""
    escaped_tokens = "".join([
        self.subtoken_list[s] for s in subtokens
        if s < len(self.subtoken_list)])
    escaped_tokens = escaped_tokens.split("_")

    # All tokens in the vocabulary list have been escaped (see _escape_token())
    # so each token must be unescaped when decoding.
    ret = []
    for token in escaped_tokens:
      if token:
        ret.append(_unescape_token(token))
    return ret
872843
def _process_underscores(self, tokens):
        ""
        groups = [[str(''.join(el))] if b else list(el)
                  for (b,el) in itertools.groupby(tokens, lambda k: k=='_')]
        flattened = [el for group in groups for el in group]
        processed = []
        for token in flattened:
            if token == '_':  continue
            if token.startswith('_'):
                token = str(token[1:])
            if token.endswith('_'):
                token = str(token[:-1])
            processed.append(token)
        return processed
46386
function tokenizer(input) {
  const topLevelTokens = getTopLevelTokens();

  const length = input.length;

  const output = [];

  let cursor = 0;
  let lastBreak = 0;

  while (cursor < length) {
    const slice = input.slice(cursor);
    const found = matchPattern(topLevelTokens, slice, false);

    if (found && input[cursor - 1] === '\\') {
      const text = input.slice(lastBreak, cursor - 1);
      if (text) {
        output.push(new Text(text));
      }

      const escapedText = found[1][0];
      output.push(new Text(escapedText));

      cursor += escapedText.length;
      lastBreak = cursor;
    } else if (found) {
      const [Tok, matches] = found;

      const text = input.slice(lastBreak, cursor);
      if (text) {
        output.push(new Text(text));
      }
      output.push(new Tok(...matches));

      cursor += matches[0].length;
      lastBreak = cursor;
    } else {
      cursor += 1;
    }
  }
  const text = input.slice(lastBreak, cursor);
  if (text) {
    output.push(new Text(text));
  }

  // if there are more closes than opens
  // intelligently remove extra ones
  return removeExtraCloses(output);
}
766981
private String getPatternCode(final List<Token> tokens) {
        final StringBuilder sb = new StringBuilder();
        sb.append(tokens.size());
        for (final Token token : tokens) {
            sb.append(token.getType().ordinal());
        }
        return sb.toString();
    }
536248
def encodingStrings( reverse = false )
      @encodingStrings  = [ {"&" => "&amp;" },  {"<" => "&lt;"} , {">" => "&gt;"}, {"'" => "&apos;"}, {"\"" => "&quot;" }  ] if @encodingStrings.nil?
      if block_given?
         if reverse
            @encodingStrings.reverse_each{ |s| s.each{|k,v| yield v,k } }
         else
            @encodingStrings.each{ |s| s.each{|k,v| yield k,v }  }
         end
      else
         @encodingStrings
      end
   end
308802
def pformat_tokens(self, tokens):
        
        result = []
        for token_value in self.iter_token_values(tokens):
            char = self.token2ascii(token_value)
            if token_value > 0xff:
                result.append("\t$%04x -> %s" % (token_value, repr(char)))
            else:
                result.append("\t  $%02x -> %s" % (token_value, repr(char)))

        return result
766321
def token_offsets(tokens):
    """"""
    end_offset = 0
    previous_end_row = 0
    previous_end_column = 0
    for t in tokens:
        token_type = t[0]
        token_string = t[1]
        (start_row, start_column) = t[2]
        (end_row, end_column) = t[3]

        # Account for the whitespace between tokens.
        end_offset += start_column
        if previous_end_row == start_row:
            end_offset -= previous_end_column

        # Record the start offset of the token.
        start_offset = end_offset

        # Account for the length of the token itself.
        end_offset += len(token_string)

        yield (token_type,
               token_string,
               start_offset,
               end_offset)

        previous_end_row = end_row
        previous_end_column = end_column
411213
def extract_tokens(representation, separators=SEPARATOR_CHARACTERS):
    
    buff = ""
    elements = []
    last_index = 0
    last_token = None

    for index, c in enumerate(representation):
        # if separator character is found, push
        # the content of the buffer in the elements list
        if c in separators:
            if buff:
                # If the last found token is invalid,
                # raise and InvalidTokenError
                if not valid_token(buff):
                    raise InvalidTokenError(
                        "Duration representation {0} contains "
                        "an invalid token: {1}".format(representation, buff)
                    )

                # If buffer content is a separator word, for example
                # "and", just ignore it
                if not buff.strip() in SEPARATOR_TOKENS:
                    elements.append(buff)

            # Anyway, reset buffer and last token marker
            # to their zero value
            buff = ""
            last_token = None
        else:
            token = compute_char_token(c)
            if (token is not None and last_token is not None and token != last_token):
                elements.append(buff)
                buff = c
            else:
                buff += c

            last_token = token

    # push the content left in representation
    # in the elements list
    elements.append(buff)

    return list(zip(elements[::2], elements[1::2]))
998302
def get_tokens_unprocessed(self, text, stack=('root',)):
        """"""
        self.content_type = None
        return RegexLexer.get_tokens_unprocessed(self, text, stack)
568636
def list_tokens(opts):
    '''
    
    '''
    ret = []
    for (dirpath, dirnames, filenames) in salt.utils.path.os_walk(opts['token_dir']):
        for token in filenames:
            ret.append(token)
    return ret
189836
def tokentype(self, s):
        """"""
        a = s[0] if len(s)>0 else ""
        b = s[1] if len(s)>1 else ""

        if a.isdigit() or (a in ["+","-"] and b.isdigit()):
            return self.parse_number(s)
        elif a == '"': return self.parse_string(s)
        elif a == ':': return self.parse_colon(s)
        elif a == ';': return self.parse_semicolon(s)
        else:
            return self.parse_word(s)
372469
def _strip(self, tokens, criteria, prop, rev=False):
        """"""
        num = len(tokens)
        res = []
        for i, token in enumerate(reversed(tokens) if rev else tokens):
            if criteria(token) and num > i + 1:
                res.insert(0, tokens.pop()) if rev else res.append(tokens.pop(0))
            else:
                break
        if res:
            self[prop] = self._clean(' '.join(res))
        return tokens
991167
def reverseComplement(seq):
    '''
    
    '''
    revComp = ''
    complement = {'A':'T', 'C':'G', 'G':'C', 'T':'A', 'N':'N', '$':'$'}
    for c in reversed(seq):
        revComp += complement[c]
    return revComp
666058
def token_list_width(tokenlist):
    
    ZeroWidthEscape = Token.ZeroWidthEscape
    return sum(get_cwidth(c) for item in tokenlist for c in item[1] if item[0] != ZeroWidthEscape)