Return the set of tokens that are referenced but not defined.
177243
def undefined_entries(self) -> Set[str]:
        
        return as_set([[d for d in self.dependencies(k) if d not in self.grammarelts]
                       for k in self.grammarelts.keys()])
80209
def unpack(tokens):
    """"""
    logger.log_tag("unpack", tokens)
    if use_computation_graph:
        tokens = evaluate_tokens(tokens)
    if isinstance(tokens, ParseResults) and len(tokens) == 1:
        tokens = tokens[0]
    return tokens
1329160
def tokenlist(item, sep, suppress=True):
    """"""
    if suppress:
        sep = sep.suppress()
    return item + ZeroOrMore(sep + item) + Optional(sep)
626611
def token_name(tokens, expected):
    """"""
    try:
        token = next(iter(tokens))
    except StopIteration:
        return

    if token and token.name == expected:
        return TokenMatch(None, token.value, (token,))
436984
def get_tokens_unprocessed(self, text):
        
        tokens = self._block_re.split(text)
        tokens.reverse()
        state = idx = 0
        try:
            while True:
                # text
                if state == 0:
                    val = tokens.pop()
                    yield idx, Other, val
                    idx += len(val)
                    state = 1
                # block starts
                elif state == 1:
                    tag = tokens.pop()
                    # literals
                    if tag in ('<%%', '%%>'):
                        yield idx, Other, tag
                        idx += 3
                        state = 0
                    # comment
                    elif tag == '<%#':
                        yield idx, Comment.Preproc, tag
                        val = tokens.pop()
                        yield idx + 3, Comment, val
                        idx += 3 + len(val)
                        state = 2
                    # blocks or output
                    elif tag in ('<%', '<%=', '<%-'):
                        yield idx, Comment.Preproc, tag
                        idx += len(tag)
                        data = tokens.pop()
                        r_idx = 0
                        for r_idx, r_token, r_value in \
                                self.ruby_lexer.get_tokens_unprocessed(data):
                            yield r_idx + idx, r_token, r_value
                        idx += len(data)
                        state = 2
                    elif tag in ('%>', '-%>'):
                        yield idx, Error, tag
                        idx += len(tag)
                        state = 0
                    # % raw ruby statements
                    else:
                        yield idx, Comment.Preproc, tag[0]
                        r_idx = 0
                        for r_idx, r_token, r_value in \
                                self.ruby_lexer.get_tokens_unprocessed(tag[1:]):
                            yield idx + 1 + r_idx, r_token, r_value
                        idx += len(tag)
                        state = 0
                # block ends
                elif state == 2:
                    tag = tokens.pop()
                    if tag not in ('%>', '-%>'):
                        yield idx, Other, tag
                    else:
                        yield idx, Comment.Preproc, tag
                    idx += len(tag)
                    state = 0
        except IndexError:
            return
186672
def missingkeys_nonstandard(block, commdct, dtls, objectlist, afield='afiled %s'):
    """"""
    afield = 'afield %s'
    for key_txt in objectlist:
        key_i = dtls.index(key_txt.upper())
        comm = commdct[key_i]
        if block:
            blk = block[key_i]
        for i, cmt in enumerate(comm):
            if cmt == {}:
                first_i = i
                break
        for i, cmt in enumerate(comm):
            if i >= first_i:
                if block:
                    comm[i]['field'] = ['%s' % (blk[i])]
                else:
                    comm[i]['field'] = [afield % (i - first_i + 1,),]
366837
def detect_spans(self, tokens):
        
        candidates = self._get_candidates(tokens)
        results = self._filter_candidates(tokens, candidates)
        return results
1175960
def _subtoken_to_tokens(self, subtokens):
        
        concatenated = "".join(subtokens)
        split = concatenated.split("_")
        return [_unescape_token(t + "_") for t in split if t]
1227458
def find_unresolved_and_unreferenced_symbols(self):
        
        unresolved = set()
        unreferenced = self._definitions.copy()
        self._collect_unresolved_and_unreferenced(set(), set(), unresolved, unreferenced,
                                                  frozenset(self._definitions), start=True)
        return unresolved, unreferenced - Scope.ALL_BUILTINS
1006301
def _clean_inheritance_tokens(self):
        """"""

        c = self._copy()
        x = c._data
        x.pop('self', None)
        x.pop('parent', None)
        x.pop('next', None)
        return c
593603
def _subtoken_ids_to_tokens(self, subtokens):
    """"""
    escaped_tokens = "".join([
        self.subtoken_list[s] for s in subtokens
        if s < len(self.subtoken_list)])
    escaped_tokens = escaped_tokens.split("_")

    # All tokens in the vocabulary list have been escaped (see _escape_token())
    # so each token must be unescaped when decoding.
    ret = []
    for token in escaped_tokens:
      if token:
        ret.append(_unescape_token(token))
    return ret
779674
def get_unlabeled_entries(self):
        
        return [
            (idx, entry[0]) for idx, entry in enumerate(self.data)
            if entry[1] is None
        ]
311039
def get_unaligned_lines(self):
    """"""
    sys.stderr.write("error unimplemented get_unaligned_lines\n")
    sys.exit()
    return [self._lines[x-1] for x in self._unaligned]
1072113
public Set<Character> tokens() {
    return root().getChildrenMap().keySet().stream()
      .filter(c -> c != END_OF_STRING && c != FALLBACK && c != ESCAPE)
      .collect(Collectors.toSet());
  }
157699
def _segment_with_tokens(text, tokens):
        """"""
        list_form = []
        text_ptr = 0
        for token in tokens:
            inter_token_string = []
            while not text[text_ptr:].startswith(token):
                inter_token_string.append(text[text_ptr])
                text_ptr += 1
                if text_ptr >= len(text):
                    raise ValueError("Tokenization produced tokens that do not belong in string!")
            text_ptr += len(token)
            if inter_token_string:
                list_form.append(''.join(inter_token_string))
            list_form.append(token)
        if text_ptr < len(text):
            list_form.append(text[text_ptr:])
        return list_form
707082
def unusedAssignments(self):
        """"""
        for name, binding in self.items():
            if (not binding.used and name not in self.globals
                    and not self.uses_locals
                    and isinstance(binding, Assignment)):
                yield name, binding
568636
def list_tokens(opts):
    '''
    
    '''
    ret = []
    for (dirpath, dirnames, filenames) in salt.utils.path.os_walk(opts['token_dir']):
        for token in filenames:
            ret.append(token)
    return ret