Finds tokens within the split statement name.
778472
private void findTokens() {
        mergedTokens = new TokenList();
        variable = null;
        state = TokenizerState.START;
        index = -1;

        while (index < rawTokens.length) {
            switch (state) {
            case START:
                next(TokenizerState.OUTSIDE_VARIABLE_CONTEXT);
                break;
            case OUTSIDE_VARIABLE_CONTEXT:
                handleTokenOutsideOfVariableContext();
                break;
            case INSIDE_VARIABLE_CONTEXT:
                handleTokenInsideOfVariableContext();
                break;
            default:
                break;
            }
        }
    }
558382
def _find_end_token(self, end_token, operation):
        
        ibuffer, icache = operation.state
        length = operation.buffered[ibuffer][2]
        result = None

        for i in range(len(operation.buffered) - ibuffer - 1):
            linenum, statement, charlength = operation.buffered[i + ibuffer + 1]
            length += charlength
            
            if end_token in statement.lower():
                #We already have the absolute char index for the start of the
                #instance; we just need to update the end.
                result = (i + ibuffer + 1, length)
                break        

        #If we didn't find a terminating statement, the full length of the operation
        #is a good estimate for the extent of the instance.
        if result is None:
            result = (None, length)

        return result
554018
def split(s):
    
    return map(lambda w: filter(lambda c: c != '\x00', w), lexsplit(s))
1273589
def get_perl_segments(self, tokens):
        
        pl_segments = []
        has_perl_tags = False
        pos = 0

        for typ, start, end in tokens:

            # Capture any leading text
            if start != pos:
                pl_seg = PPPUnalteredSegment(self, pos, start-1)
                pl_segments.append(pl_seg)

            if typ == "incl":
                # Got an `include ...

                # Extract the path and actual end position
                end, incl_path = self.parse_include(start)

                # create a reference that captures the location where the include occurred
                incl_ref = segment_map.IncludeRef(start, end, self.path, self.incl_ref)

                # Recurse and extract perl segments from included file
                incl_file_pp = FilePreprocessor(self.env, incl_path, self.search_paths, incl_ref)
                incl_tokens = incl_file_pp.tokenize()
                incl_pl_segments, incl_has_pl_tags = incl_file_pp.get_perl_segments(incl_tokens)

                pl_segments.extend(incl_pl_segments)
                has_perl_tags = has_perl_tags or incl_has_pl_tags

            else:
                # Got a Perl tag <% ... %>
                if self.text[start+2] == "=":
                    pl_seg = PPPMacroSegment(self, start, end)
                else:
                    pl_seg = PPPPerlSegment(self, start, end)
                pl_segments.append(pl_seg)
                has_perl_tags = True

            pos = end+1

        # Capture any trailing text
        text_len = len(self.text)
        if text_len > pos:
            pl_seg = PPPUnalteredSegment(self, pos, text_len-1)
            pl_segments.append(pl_seg)

        return (pl_segments, has_perl_tags)
1256165
def _parse_multi_options(options, split_token=','):
    
    """
    if options:
        return [o.strip() for o in options.split(split_token) if o.strip()]
    else:
        return options
226491
def parse_tokens(data)
      tokens = {}
      data.split("\n").each do |t|
        tokens.merge!({ t.split('=').first.downcase.to_sym => t.split('=').last })
      end
      return tokens
    end
1203383
def split_sequence(seq, n):
    """"""
    tokens = []
    while seq:
        tokens.append(seq[:n])
        seq = seq[n:]
    return tokens
1307659
public Collection<Text> split(char separator) {
        int pointer = pos;
        int limit = pos + len;
        int mark = pointer;

        ArrayList<Text> tokens = new ArrayList<Text>();
        while (pointer < limit) {
            if (chars[pointer] == separator) {
                tokens.add(new Text(chars, mark, pointer - mark));
                mark = pointer + 1;
            }
            pointer++;
        }
        
        tokens.add(new Text(chars, mark, limit - mark));        
        return tokens;
    }
444686
def get_tokens(line: str) -> Iterator[str]:
    
    for token in line.rstrip().split():
        if len(token) > 0:
            yield token
1287959
private void checkTokenized() {
        if (tokens == null) {
            if (chars == null) {
                // still call tokenize as subclass may do some work
                final List<String> split = tokenize(null, 0, 0);
                tokens = split.toArray(new String[split.size()]);
            } else {
                final List<String> split = tokenize(chars, 0, chars.length);
                tokens = split.toArray(new String[split.size()]);
            }
        }
    }
1039436
def parse(contents, tokens=None):
    """"""
    # Shortcut for users who are interested in tokens
    if tokens is None:
        tokens = [t for t in tokenize(contents)]

    token_index, body = _ast_worker(tokens, len(tokens), 0, None)

    assert token_index == len(tokens)
    assert body.arguments == []

    return ToplevelBody(statements=body.statements)
162706
function parse(tokens, basePath, iterSuffix) {
  const len = tokens.length;
  const body = [];

  let i = 0;
  while (i < len) {
    const token = tokens[i];
    if (subroutines[token.tokenType]) {
      const rest = tokens.slice(i + 1);
      const [
        branches,
        tokensHandled,
        shouldBreak,
      ] = subroutines[token.tokenType](token, rest, basePath, iterSuffix);

      body.push(...branches);

      i += tokensHandled;

      if (shouldBreak) {
        break;
      }
    } else {
      i += 1;
    }
  }

  return [body, i + 1];
}
565724
public function find()
    {
        foreach ($this->file->tokens as $i => $t) {
            if (TokenUtils::isTokenType($t, array(T_NEW))) {
                // +2 to consume whitespace
                $c = self::findClassInNextTokens($this->file->tokens, $i + 2);
                if ($c) {
                    yield $c;
                }
            } elseif (TokenUtils::isTokenType($t, array(T_IMPLEMENTS, T_EXTENDS))) {
                // +2 to consume whitespace
                $c = self::findClassInNextTokens($this->file->tokens, $i + 2);
                if ($c) {
                    yield $c;
                }
                $j = $c->to + 1;
                while ($this->file->tokens[$j] === ',') {
                    // +2 to consume comma and whitespace
                    $c = self::findClassInNextTokens($this->file->tokens, $j + 2);
                    if ($c) {
                        yield $c;
                        $j = $c->to;
                    }
                    $j++;
                }
            } elseif (TokenUtils::isTokenType($t, array(T_USE))) {
                $c = self::findClassInNextTokens($this->file->tokens, $i + 2);
                // Check if use statement is after class opening bracket
                if ($c && $this->isAfterOpeningBracket($c)) {
                    yield $c;
                }
            } elseif (TokenUtils::isTokenType($t, array(T_PAAMAYIM_NEKUDOTAYIM, T_DOUBLE_COLON))) {
                $c = self::findClassInPreviousTokens($this->file->tokens, $i - 1);
                if ($c) {
                    yield $c;
                }
            } elseif (TokenUtils::isTokenType($t, array(T_CATCH))) {
                // +3 to consume brace and whitespace
                $c = self::findClassInNextTokens($this->file->tokens, $i + 3);
                if ($c) {
                    yield $c;
                }
            } elseif (TokenUtils::isTokenType($t, array(T_INSTANCEOF))) {
                $c = self::findClassInNextTokens($this->file->tokens, $i + 2);
                if ($c) {
                    yield $c;
                }
            } elseif (TokenUtils::isTokenType($t, array(T_FUNCTION))) {
                $j = $i + 4;
                $j = TokenUtils::isTokenType($this->file->tokens[$j], T_WHITESPACE) ? $j + 1 : $j;

                // +4 to consume whitespace, name, open brace
                $c = self::findClassInNextTokens($this->file->tokens, $j);
                if ($c) {
                    yield $c;
                    $j = $c->to;
                }

                while ($this->file->tokens[$j] != ")") {
                    if ($this->file->tokens[$j] == ',') {
                        // +2 to consume comma and whitespace
                        $c = self::findClassInNextTokens($this->file->tokens, $j + 2);
                        if ($c) {
                            yield $c;
                            $j = $c->to;
                        }
                    }
                    $j++;
                }

            }
        }
    }
767557
def _handle_splits(_str):
    """"""
    _str = _str.replace('/', '-')
    _tmp_dict = {}

    if '-' in _str:
        start, stop = _str.split('-')
        if _check_number(start):
            start = regex.sub(r'[0-9]+\?*', start, stop)
        elif _check_number(stop):
            stop = regex.sub(r'[0-9]+\?*', stop, start)
    else:
        start = _str
        stop = _str
    _tmp_dict['start_raw'] = start
    _tmp_dict['stop_raw'] = stop

    _tmp_dict['start_epoch'] = _get_epoch(start)
    _tmp_dict['stop_epoch'] = _get_epoch(stop)

    return _tmp_dict
411213
def extract_tokens(representation, separators=SEPARATOR_CHARACTERS):
    
    buff = ""
    elements = []
    last_index = 0
    last_token = None

    for index, c in enumerate(representation):
        # if separator character is found, push
        # the content of the buffer in the elements list
        if c in separators:
            if buff:
                # If the last found token is invalid,
                # raise and InvalidTokenError
                if not valid_token(buff):
                    raise InvalidTokenError(
                        "Duration representation {0} contains "
                        "an invalid token: {1}".format(representation, buff)
                    )

                # If buffer content is a separator word, for example
                # "and", just ignore it
                if not buff.strip() in SEPARATOR_TOKENS:
                    elements.append(buff)

            # Anyway, reset buffer and last token marker
            # to their zero value
            buff = ""
            last_token = None
        else:
            token = compute_char_token(c)
            if (token is not None and last_token is not None and token != last_token):
                elements.append(buff)
                buff = c
            else:
                buff += c

            last_token = token

    # push the content left in representation
    # in the elements list
    elements.append(buff)

    return list(zip(elements[::2], elements[1::2]))
296831
def parse_expr(tokens, options):
    """"""
    seq = parse_seq(tokens, options)
    if tokens.current() != '|':
        return seq
    result = [Required(*seq)] if len(seq) > 1 else seq
    while tokens.current() == '|':
        tokens.move()
        seq = parse_seq(tokens, options)
        result += [Required(*seq)] if len(seq) > 1 else seq
    return [Either(*result)] if len(result) > 1 else result
701444
protected static function _splitConditions($token)
    {
        $conditions = false;
        $position = strpos($token, '[');
        if ($position !== false) {
            $conditions = substr($token, $position);
            $token = substr($token, 0, $position);
        }

        return [$token, $conditions];
    }