Iterate over a list of {@link Token}s and update their counts of how many tokens make up each component.

@param tokens not be updated.
348973
public static void updateComponentTokenCounts(final List<Token> tokens)
    {
        final Map<String, Deque<Integer>> map = new HashMap<>();

        for (int i = 0, size = tokens.size(); i < size; i++)
        {
            final Token token = tokens.get(i);
            final Signal signal = token.signal();

            if (signal.name().startsWith("BEGIN_"))
            {
                final String componentType = signal.name().substring(6);
                map.computeIfAbsent(componentType, (key) -> new LinkedList<>()).push(i);
            }
            else if (signal.name().startsWith("END_"))
            {
                final String componentType = signal.name().substring(4);
                final int beginIndex = map.get(componentType).pop();

                final int componentTokenCount = (i - beginIndex) + 1;
                tokens.get(beginIndex).componentTokenCount(componentTokenCount);
                token.componentTokenCount(componentTokenCount);
            }
        }
    }
17286
public void setTags( Collection<String> tags ) {

		synchronized( this.tags ) {
			this.tags.clear();
			if( tags != null )
				this.tags.addAll( tags );
		}
	}
1010459
public Parser<Token> token(String... tokenNames) {
    if (tokenNames.length == 0) return Parsers.never();
    @SuppressWarnings("unchecked")
    Parser<Token>[] ps = new Parser[tokenNames.length];
    for(int i = 0; i < tokenNames.length; i++) {
      ps[i] = Parsers.token(InternalFunctors.tokenWithSameValue(word(tokenNames[i])));
    }
    return Parsers.or(ps);
  }
384279
public synchronized @Nonnull SingleTokenStats updateUsageForId(@Nonnull String tokenUuid) {
        if(areStatsDisabled()){
            return new SingleTokenStats(tokenUuid);
        }
        
        SingleTokenStats stats = findById(tokenUuid)
                .orElseGet(() -> {
                    SingleTokenStats result = new SingleTokenStats(tokenUuid);
                    tokenStats.add(result);
                    return result;
                });
        
        stats.notifyUse();
        save();
        
        return stats;
    }
1321027
protected boolean ignoreToken(AnalyzedTokenReadings[] tokens, int idx) throws IOException {
    List<String> words = new ArrayList<>();
    for (AnalyzedTokenReadings token : tokens) {
      words.add(token.getToken());
    }
    return ignoreWord(words, idx);
  }
933857
public void setTagKeys(java.util.Collection<String> tagKeys) {
        if (tagKeys == null) {
            this.tagKeys = null;
            return;
        }

        this.tagKeys = new com.amazonaws.internal.SdkInternalList<String>(tagKeys);
    }
1362214
private static boolean areTokensValid(Token... tokens)
    {
        for (Token token : tokens)
        {
            if (!isTokenValid(token))
                return false;
        }
        return true;
    }
882574
private void process(Iterator<String> tokens) {
        // NOTE: this method is intentionally private to ensure that the
        // IteratorFactory.tokenize() tokenization scheme is enforced on the
        // input data
        long numTokens = 0;
        while (tokens.hasNext()) {
            String token = tokens.next();
            if (doLowerCasing)
                token = token.toLowerCase();
            if (token.matches("[0-9]+"))
                token = "<NUM>";
            if (token.matches("[^\\w\\s;:\\(\\)\\[\\]'!/&?\",\\.<>]"))
                continue;

            Integer count = tokenToCount.get(token);
            tokenToCount.put(token, (count == null) ? 1 : 1 + count);
            numTokens++;
            if (numTokens % UPDATE_INTERVAL == 0 
                    && LOGGER.isLoggable(Level.FINE))
                LOGGER.fine("Processed " + numTokens + " tokens.  Currently " 
                            + tokenToCount.size() + " unique tokens");
        }
    }
366331
public Set<AuthorisationData> convertIdentifyingTokenList(Set<IdentifyingToken> tokens) {
        Set<AuthorisationData> resultList = new HashSet<>();

        for (IdentifyingToken token : tokens) {
            resultList.add(convertIdentifyingToken(token));
        }

        return resultList;
    }
972409
@Api
	public void setAuthentications(String token, List<Authentication> authentications) {
		this.token = token;
		this.authentications.clear();
		if (null != authentications) {
			for (Authentication auth : authentications) {
				for (BaseAuthorization ba : auth.getAuthorizations()) {
					if (ba instanceof AuthorizationNeedsWiring) {
						((AuthorizationNeedsWiring) ba).wire(applicationContext);
					}
				}
				this.authentications.add(auth);
			}
		}
		userInfoInit();
	}
952741
public void addValue(Token[] tokens) {
        if (type == null) {
            type = ItemType.LIST;
        }
        if (!isListableType()) {
            throw new IllegalArgumentException("The type of this item must be 'array', 'list' or 'set'");
        }
        if (tokensList == null) {
            tokensList = new ArrayList<>();
        }
        tokensList.add(tokens);
    }
660594
private static List<Token> createTokens(final Map<String, String> properties) {
		List<Token> tokens = new ArrayList<Token>();
		for (Entry<String, String> entry : properties.entrySet()) {
			if (entry.getKey().toLowerCase(Locale.ROOT).startsWith(FIELD_PREFIX)) {
				tokens.add(FormatPatternParser.parse(entry.getValue()));
			}
		}
		return tokens;
	}
930508
public void setValue(Token[] tokens) {
        if (type == null) {
            type = ItemType.SINGLE;
        }
        if (type != ItemType.SINGLE) {
            throw new IllegalArgumentException("The type of this item must be 'single'");
        }
        this.tokens = tokens;
    }
60439
public final Span[] seqToSpans(final String[] tokens) {
    final Span[] annotatedText = this.sequenceLabeler.tag(tokens);
    final List<Span> probSpans = new ArrayList<Span>(
        Arrays.asList(annotatedText));
    return probSpans.toArray(new Span[probSpans.size()]);
  }
763778
public void setInstanceCounts(java.util.Collection<InstanceCount> instanceCounts) {
        if (instanceCounts == null) {
            this.instanceCounts = null;
            return;
        }

        this.instanceCounts = new com.amazonaws.internal.SdkInternalList<InstanceCount>(instanceCounts);
    }
705061
public void push( Token token ) {
        size++;
        if( first == null ) {
            first = token;
            last = token;
            token.previous = null;
            token.next = null;
        } else {
            last.next = token;
            token.previous = last;
            token.next = null;
            last = token;
        }
    }
767200
private boolean advance(int tokens) {
	while (buffer.size() < tokens && tokenizer.hasNext())
	    buffer.add(tokenizer.next());
	return buffer.size() >= tokens;
    }