<p>
The syntax tokens for the words in the document, one token for each word.
</p>

@param syntaxTokens
The syntax tokens for the words in the document, one token for each word.
359816
public void setSyntaxTokens(java.util.Collection<SyntaxToken> syntaxTokens) {
        if (syntaxTokens == null) {
            this.syntaxTokens = null;
            return;
        }

        this.syntaxTokens = new java.util.ArrayList<SyntaxToken>(syntaxTokens);
    }
1252487
public void setDefaultTokens(final List<HttpSessionToken> tokens) {
		this.defaultTokens = tokens;
		
		saveDefaultTokens();
		this.defaultTokensEnabled = defaultTokens.stream()
				.filter(HttpSessionToken::isEnabled)
				.map(HttpSessionToken::getName)
				.collect(Collectors.toList());
	}
86588
private boolean tokenTextsDoNotMatch(Token token1, Token token2) {
        return !token1.getText().equals(token2.getText());
    }
1281768
@Override
  public List<AnalyzedTokenReadings> tag(List<String> sentenceTokens)
  throws IOException {
    List<AnalyzedToken> taggerTokens;
    List<AnalyzedToken> lowerTaggerTokens;
    List<AnalyzedToken> upperTaggerTokens;
    List<AnalyzedTokenReadings> tokenReadings = new ArrayList<>();
    int pos = 0;

    Matcher matcher;
    for (String word : sentenceTokens) {
      String probeWord = word;

      // This loop happens when we need to retry probing the dictionary
      // which happens rarely when trying to remove suffixes -mañ, -se, etc.
      for (;;) {
        List<AnalyzedToken> l = new ArrayList<>();
        String lowerWord = probeWord.toLowerCase(conversionLocale);
        taggerTokens = asAnalyzedTokenListForTaggedWords(word, getWordTagger().tag(probeWord));
        lowerTaggerTokens = asAnalyzedTokenListForTaggedWords(word, getWordTagger().tag(lowerWord));
        boolean isLowercase = probeWord.equals(lowerWord);

        // Normal case.
        addTokens(taggerTokens, l);

        if (!isLowercase) {
          // Lowercase.
          addTokens(lowerTaggerTokens, l);
        }

        // Uppercase.
        if (lowerTaggerTokens.isEmpty() && taggerTokens.isEmpty()) {
          if (isLowercase) {
            upperTaggerTokens = asAnalyzedTokenListForTaggedWords(word,
                getWordTagger().tag(StringTools.uppercaseFirstChar(probeWord)));
            if (!upperTaggerTokens.isEmpty()) {
              addTokens(upperTaggerTokens, l);
            }
          }
          if (l.isEmpty()) {
            if ((matcher = patternSuffix.matcher(probeWord)).find()) {
              // Remove the suffix and probe dictionary again.
              // So given a word such as "xxx-mañ", we're going to
              // try to probe the dictionary again with "xxx" this time.
              probeWord = matcher.group(1);
              continue;
            }
            l.add(new AnalyzedToken(word, null, null));
          }
        }
        tokenReadings.add(new AnalyzedTokenReadings(l, pos));
        pos += word.length();
        break;
      }
    }

    return tokenReadings;

  }
544778
public void skipUntil(final TokenType... tokenTypes) {
        Set<TokenType> tokenTypeSet = Sets.newHashSet(tokenTypes);
        tokenTypeSet.add(Assist.END);
        while (!tokenTypeSet.contains(lexer.getCurrentToken().getType())) {
            lexer.nextToken();
        }
    }
455113
private void addTokens(final Token curToken, final List<Token> tokens) {
    if (curToken.tokenLength() != 0) {
      if (this.unTokenizable) {
        tokens.add(curToken);
      } else if (!this.unTokenizable) {
        if (!replacement.matcher(curToken.getTokenValue()).matches()) {
          tokens.add(curToken);
        }
      }
    }
  }
930508
public void setValue(Token[] tokens) {
        if (type == null) {
            type = ItemType.SINGLE;
        }
        if (type != ItemType.SINGLE) {
            throw new IllegalArgumentException("The type of this item must be 'single'");
        }
        this.tokens = tokens;
    }
923105
public void addWords(Collection<String> words){
		if(false == (words instanceof Set)){
			words = new HashSet<>(words);
		}
		for (String word : words) {
			addWord(word);
		}
	}
1066666
public void setTagKeys(java.util.Collection<String> tagKeys) {
        if (tagKeys == null) {
            this.tagKeys = null;
            return;
        }

        this.tagKeys = new java.util.ArrayList<String>(tagKeys);
    }
235024
public void setTokenizer(MultiAutoCompleteTextView.Tokenizer t) {
        if(mAutoCompleteMode != AUTOCOMPLETE_MODE_MULTI)
            return;
        ((MultiAutoCompleteTextView)mInputView).setTokenizer(t);
    }
85319
public PatternTokenizer setExtraQuotingCharacters(UnicodeSet syntaxCharacters) {
        this.extraQuotingCharacters = (UnicodeSet) syntaxCharacters.clone();
        needingQuoteCharacters = null;
        return this;
    }
1229171
private static List<Tokenizer.Token> toReversePolishNotation(List<Tokenizer.Token> tokens) {
        LinkedList<Tokenizer.Token> queue = new LinkedList<Tokenizer.Token>();
        Stack<Tokenizer.Token> stack = new Stack<Tokenizer.Token>();

        for (int i = 0; i < tokens.size(); i++) {
            Tokenizer.Token token = tokens.get(i);
            switch (token.type) {
                case VERSION:
                    queue.add(token);
                    break;
                case CLOSING:
                    while (stack.peek().type != Tokenizer.TokenType.OPENING) {
                        queue.add(stack.pop());
                    }
                    stack.pop();
                    if (stack.size() > 0 && stack.peek().type.isUnary()) {
                        queue.add(stack.pop());
                    }
                    break;
                default:
                    if (token.type.isUnary()) {
                        queue.push(token);
                        i++;
                        queue.push(tokens.get(i));
                    } else {
                        stack.push(token);
                    }
                    break;
            }
        }

        while (!stack.isEmpty()) {
            queue.add(stack.pop());
        }

        Collections.reverse(queue);
        return queue;
    }
698697
public void setFeats(GrammaticalFeats v) {
    if (Token_Type.featOkTst && ((Token_Type)jcasType).casFeat_feats == null)
      jcasType.jcas.throwFeatMissing("feats", "de.julielab.jules.types.Token");
    jcasType.ll_cas.ll_setRefValue(addr, ((Token_Type)jcasType).casFeatCode_feats, jcasType.ll_cas.ll_getFSRef(v));}
1224140
public void setTaskKeywords(java.util.Collection<String> taskKeywords) {
        if (taskKeywords == null) {
            this.taskKeywords = null;
            return;
        }

        this.taskKeywords = new java.util.ArrayList<String>(taskKeywords);
    }
1366412
private void constructTokenCollection(final Reader reader)
      throws MtasConfigException, MtasParserException {
    tokenCollection = null;
    try {
      Constructor<?> c = Class.forName(parserName)
          .getDeclaredConstructor(MtasConfiguration.class);      
      Object p = c.newInstance(parserConfiguration);
      if (p instanceof MtasParser) {
        MtasParser parser = (MtasParser) p;
        tokenCollection = parser.createTokenCollection(reader);
        return;
      } else {
        throw new MtasConfigException("no instance of MtasParser");
      }
    } catch (MtasParserException e) {
      log.debug(e);
      tokenCollection = new MtasTokenCollection();
      throw new MtasParserException(e.getMessage());
    } catch (NoSuchMethodException | InvocationTargetException
        | IllegalAccessException | ClassNotFoundException
        | InstantiationException e) {
      log.debug(e);
      throw new MtasConfigException(
          e.getClass().getName() + " : '" + e.getMessage() + "'");
    }

  }
606116
public Document fromTokens(@NonNull String... tokens) {
      return fromTokens(Arrays.asList(tokens), getDefaultLanguage());
   }
197858
@SuppressWarnings("unchecked")
	public void addWords(Object document, Collection<String> collection) {
		if (document instanceof Collection<?>){
			collection.addAll((Collection<String>)document);
		}else if (document instanceof AnalyzedText){
			collection.addAll(((AnalyzedText)document).getWords());
		}else {
			throw new IllegalArgumentException("Only instances of AnalyzedText can be handled.");
		}
	}