Add new tokens to list.
775306
def add(self, *tokens: str) -> None:
        """"""
        from wdom.web_node import WdomElement
        _new_tokens = []
        for token in tokens:
            self._validate_token(token)
            if token and token not in self:
                self._list.append(token)
                _new_tokens.append(token)
        if isinstance(self._owner, WdomElement) and _new_tokens:
            self._owner.js_exec('addClass', _new_tokens)
687325
def add_texts(self, reference_id, texts):
        
        self.add_words(reference_id, chain(*(self._tokenize(t) for t in texts)))
320556
def __add_text(self, text):
        """"""
        if text is not None and not isinstance(text, six.text_type):
            raise TypeError('Local symbol definition must be a Unicode sequence or None: %r' % text)
        sid = self.__new_sid()
        location = None
        if self.table_type.is_shared:
            location = self.__import_location(sid)
        token = SymbolToken(text, sid, location)
        self.__add(token)
        return token
1184397
public ArgumentListBuilder addTokenized(String s) {
        if(s==null) return this;
        add(Util.tokenize(s));
        return this;
    }
1185828
def join(tokens, start, result):
    """"""
    texts = []
    if len(result) > 0:
        for e in result:
            for child in e.iter():
                if child.text is not None:
                    texts.append(child.text)
        return [E(result[0].tag, ' '.join(texts))]
287085
def add_tokens_for_single(self, ignore=False):
        """"""
        args = self.single.args
        name = self.single.python_name

        # Reset indentation to proper amount and add signature
        self.reset_indentation(self.indent_type * self.single.indent)
        self.result.extend(self.tokens.make_single(name, args))

        # Add skip if necessary
        if ignore:
            self.single.skipped = True
            self.result.extend(self.tokens.test_skip)

        self.groups.finish_signature()
455113
private void addTokens(final Token curToken, final List<Token> tokens) {
    if (curToken.tokenLength() != 0) {
      if (this.unTokenizable) {
        tokens.add(curToken);
      } else if (!this.unTokenizable) {
        if (!replacement.matcher(curToken.getTokenValue()).matches()) {
          tokens.add(curToken);
        }
      }
    }
  }
21798
def add_token_to_namespace(self, token: str, namespace: str = 'tokens') -> int:
        
        if not isinstance(token, str):
            raise ValueError("Vocabulary tokens must be strings, or saving and loading will break."
                             "  Got %s (with type %s)" % (repr(token), type(token)))
        if token not in self._token_to_index[namespace]:
            index = len(self._token_to_index[namespace])
            self._token_to_index[namespace][token] = index
            self._index_to_token[namespace][index] = token
            return index
        else:
            return self._token_to_index[namespace][token]
878415
public void insertAfter(Token before, TokenList list ) {
        Token after = before.next;

        before.next = list.first;
        list.first.previous = before;
        if( after == null ) {
            last = list.last;
        } else {
            after.previous = list.last;
            list.last.next = after;
        }
        size += list.size;
    }
40545
def _init_from_list(self, subwords):
    """"""
    subwords = [tf.compat.as_text(s) for s in subwords if s]
    self._subwords = subwords
    # Note that internally everything is 0-indexed. Padding is dealt with at the
    # end of encode and the beginning of decode.
    self._subword_to_id = {s: i for i, s in enumerate(subwords)}

    # We remember the maximum length of any subword to avoid having to
    # check arbitrarily long strings.
    self._max_subword_len = max(
        len(_UNDERSCORE_REPLACEMENT), max([len(s) for s in subwords] or [1]))

    # Initialize the cache
    self._cache_size = 2**20
    self._token_to_ids_cache = [(None, None)] * self._cache_size

    # Setup tokenizer
    # Reserved tokens are all tokens that are mixed alphanum and non-alphanum.
    reserved_tokens = set([_UNDERSCORE_REPLACEMENT])
    for t in self._subwords:
      if text_encoder.is_mixed_alphanum(t):
        reserved_tokens.add(t)
    self._tokenizer = text_encoder.Tokenizer(
        alphanum_only=False, reserved_tokens=reserved_tokens)
29517
public static synchronized void updateTokens(Collection<Token> tokens)
    {
        assert !tokens.isEmpty() : "removeEndpoint should be used instead";
        String req = "INSERT INTO system.%s (key, tokens) VALUES ('%s', ?)";
        executeInternal(String.format(req, LOCAL_CF, LOCAL_KEY), tokensAsSet(tokens));
        forceBlockingFlush(LOCAL_CF);
    }
88598
def new_from
      source_tokens = [ :STRING ]
      from_tok = Tree::From.new(basic_options)

      # read in all the derived columns - search sources:
      while peek_token and source_tokens.include? peek_token.token
        src_token = read_token
        # check if the provider is registered in the given list:
        if provider_exists? src_token.lexeme.downcase
          from_tok.sources[src_token.lexeme] = @providers[src_token.lexeme.downcase.to_sym]
        else raise UnregisteredProviderError, "#{src_token.lexeme} is not a registered provider"
        end
      end

      if from_tok.sources.count == 0
        raise SyntaxError, "No search sources given (#{error_location})"
      end

      from_tok
    end
1049181
def extend_node_list(acc, new):
        """"""
        if new is None:
            new = []
        elif not isinstance(new, list):
            new = [new]
        return acc + new
625338
def append(self, tag):
        '''
            
        '''
        list.append(self, tag)
        self.uids.add(tag.uid)
738008
def _add_from_list(self, l):
        

        for item in l:
            if isinstance(item, (str, ustr)):
                self._add_from_str(item)
            elif isinstance(item, (list, tuple)):
                location = Location(item)
                self._add_location(location)
627979
def _listify(collection):
        """"""
        new_list = []
        for index in range(len(collection)):
            new_list.append(collection[index])
        return new_list
374492
def add_new(self, command):
        """"""
        self.queue[self.next_key] = command
        self.queue[self.next_key]['status'] = 'queued'
        self.queue[self.next_key]['returncode'] = ''
        self.queue[self.next_key]['stdout'] = ''
        self.queue[self.next_key]['stderr'] = ''
        self.queue[self.next_key]['start'] = ''
        self.queue[self.next_key]['end'] = ''

        self.next_key += 1
        self.write()