Returns a {@link Tokens}.
347168
public CompletableFuture<Tokens> getTokens() {
        return tokenRepo.fetch(INTERNAL_PROJ, Project.REPO_DOGMA, TOKEN_JSON)
                        .thenApply(HolderWithRevision::object);
    }
104403
public java.util.List<String> getTagKeys() {
        if (tagKeys == null) {
            tagKeys = new com.amazonaws.internal.SdkInternalList<String>();
        }
        return tagKeys;
    }
445859
public Set tokens() {
        if (tokenSet == null) {
            tokenSet = new AbstractSetView() {
                public long size() {
                    return 0;
                }

                public Iterator iterator() {
                    return new Iterator() {

                        public boolean hasNext()
                        {
                            return false;
                        } // hasNext().

                        public Object next()
                        {
                            throw new java.util.NoSuchElementException();
                        } // next().

                        public boolean hasNext(Transaction transaction)
                                        throws ObjectManagerException
                        {
                            throw new UnsupportedOperationException();
                        } // hasNext().

                        public Object next(Transaction transaction)
                                        throws ObjectManagerException
                        {
                            throw new UnsupportedOperationException();
                        } // next().

                        public Object remove(Transaction transaction)
                                        throws ObjectManagerException
                        {
                            throw new UnsupportedOperationException();
                        } // remove().

                    }; // new Iterator().
                } // iterator().  
            }; // new AbstractSetView().
        } // if (tokenSet == null). 
        return tokenSet;
    }
1090544
@Deprecated
  public List<SToken> getToken()
  {
    if(this.cachedToken == null)
    {
      this.cachedToken = getSResult().getDocumentGraph().getSortedTokenByText();
    }
    return this.cachedToken;
  }
155247
public char getToken() {
    int index = getPosition();
    String document = getDocument();
    return index >= document.length() ? NodewalkerCodec.END_OF_STRING : document.charAt(index);
  }
1299636
public final String[][] getAllPosTags(final String[] tokens) {
    final String[][] allPosTags = this.posTagger.tag(13, tokens);
    return allPosTags;
  }
447590
public List<Charset> getAcceptCharset() {
		List<Charset> result = new ArrayList<Charset>();
		String value = getFirst(ACCEPT_CHARSET);
		if (value != null) {
			String[] tokens = value.split(",\\s*");
			for (String token : tokens) {
				int paramIdx = token.indexOf(';');
				String charsetName;
				if (paramIdx == -1) {
					charsetName = token;
				}
				else {
					charsetName = token.substring(0, paramIdx);
				}
				if (!charsetName.equals("*")) {
					result.add(Charset.forName(charsetName));
				}
			}
		}
		return result;
	}
629177
public static List<TokensLine> convert(List<TokenEntry> tokens) {
    List<TokensLine> result = new ArrayList<>();
    StringBuilder sb = new StringBuilder();
    int startLine = Integer.MIN_VALUE;
    int startIndex = 0;
    int currentIndex = 0;
    for (TokenEntry token : tokens) {
      if (token != TokenEntry.EOF) {
        String value = token.getValue();
        int line = token.getBeginLine();
        if (line != startLine) {
          addNewTokensLine(result, startIndex, currentIndex, startLine, sb);
          startIndex = currentIndex + 1;
          startLine = line;
        }
        currentIndex++;
        sb.append(value);
      }
    }
    addNewTokensLine(result, startIndex, currentIndex, startLine, sb);
    return result;
  }
1072113
public Set<Character> tokens() {
    return root().getChildrenMap().keySet().stream()
      .filter(c -> c != END_OF_STRING && c != FALLBACK && c != ESCAPE)
      .collect(Collectors.toSet());
  }
937905
static Parser<Token[]> tokens(final Parser<? extends Collection<Token>> parser) {
    return parser.map(list -> list.toArray(new Token[list.size()]));
  }
600276
public Set tokens() {
        if (tokenSet == null) {
            tokenSet = new AbstractSetView() {

                public long size() throws ObjectManagerException {
                    return directory.size();
                } // size().

                public Iterator iterator() throws ObjectManagerException {

                    final Iterator storeAreaIterator = directory.entrySet().iterator();
                    return new Iterator() {

                        public boolean hasNext() throws ObjectManagerException
                        {
                            return storeAreaIterator.hasNext();
                        } // hasNext().

                        public Object next() throws ObjectManagerException
                        {
                            Directory.StoreArea storeArea = (Directory.StoreArea) storeAreaIterator.next();
                            Token token = new Token(AbstractSingleFileObjectStore.this, storeArea.identifier);
                            return like(token);
                        } // next().   

                        public boolean hasNext(Transaction transaction)
                                        throws ObjectManagerException
                        {
                            throw new UnsupportedOperationException();
                        } // hasNext().

                        public Object next(Transaction transaction)
                                        throws ObjectManagerException
                        {
                            throw new UnsupportedOperationException();
                        } // next().

                        public Object remove(Transaction transaction)
                                        throws ObjectManagerException
                        {
                            throw new UnsupportedOperationException();
                        } // remove().
                    }; // new Iterator().
                } // iterator().  

            }; // new AbstractSetView().
        } // if (tokenSet == null). 
        return tokenSet;
    }
733090
def get_tokens(self):
        
        tokens_snapshot = self.db.prefixed_db(NotificationPrefix.PREFIX_TOKEN).snapshot()
        results = []
        for val in tokens_snapshot.iterator(include_key=False):
            event = SmartContractEvent.FromByteArray(val)
            results.append(event)
        return results
449405
protected function getTokens() {
        if (!$this->_tokens) {
            $this->_tokens = new TokenList($this->version, $this->solution['sid']);
        }

        return $this->_tokens;
    }
769339
public String getConcatenatedToken() {
        
        StringBuilder sb = new StringBuilder();
        
        for(Token token : tokens) {
            sb.append(token.getValue());
        }
        
        return sb.toString();
        
    }
514366
public synchronized List<Token> tokens(String text) {
        requireValidHandle();
        List<Token> allTokens = new ArrayList<Token>();
        int lastStart = 0;
        for (int i = indexOfSpecialUnknown(text, 0); i != -1; i = indexOfSpecialUnknown(text, i + 1)) {
            allTokens.addAll(tokensNonNull(text.substring(lastStart, i), lastStart));
            allTokens.add(new Token(TokenType.UNKNOWN, Character.toString(text.charAt(i)), i));
            lastStart = i + 1;
        }
        allTokens.addAll(tokensNonNull(text.substring(lastStart), lastStart));
        return allTokens;
    }
498052
public static Token[] makeTokens(String text, boolean tokenize) {
        if (text == null) {
            return null;
        }
        Token[] tokens;
        if (tokenize) {
            tokens = parse(text);
        } else {
            tokens = new Token[1];
            tokens[0] = new Token(text);
        }
        return tokens;
    }
206723
public static String generateToken() {
        String timestampByBase64 = Utils.rightTrim(Base64.encode(String.valueOf(System.currentTimeMillis())), "=");
        return (RandomUtil.simpleUUID() + timestampByBase64).toLowerCase();
    }