Parameters
        ----------
        category_doc_iter : iterator of (string category name, spacy.tokens.doc.Doc) pairs

        Returns
        ----------
        t : TermDocMatrix
351242
def _build_from_category_spacy_doc_iter(self, category_doc_iter):
        '''
        
        '''
        term_idx_store = IndexStore()
        category_idx_store = IndexStore()
        metadata_idx_store = IndexStore()
        X, mX, y = self._get_features_and_labels_from_documents_and_indexes \
            (category_doc_iter,
             category_idx_store,
             term_idx_store,
             metadata_idx_store)
        return TermDocMatrix(X,
                             mX,
                             y,
                             term_idx_store=term_idx_store,
                             category_idx_store=category_idx_store,
                             metadata_idx_store=metadata_idx_store)
745844
def category(soup):
    
    category = []

    tags = raw_parser.category(soup)
    for tag in tags:
        category.append(node_text(tag))

    return category
265582
def apply(self, doc, clear, **kwargs):
        

        # Reattach doc with the current session or DetachedInstanceError happens
        doc = self.session.merge(doc)
        # Iterate over each mention class
        for i, mention_class in enumerate(self.mention_classes):
            tc_to_insert = defaultdict(list)
            # Generate TemporaryContexts that are children of the context using
            # the mention_space and filtered by the Matcher
            self.child_context_set.clear()
            for tc in self.matchers[i].apply(self.mention_spaces[i].apply(doc)):
                rec = tc._load_id_or_insert(self.session)
                if rec:
                    tc_to_insert[tc._get_table()].append(rec)
                self.child_context_set.add(tc)

            # Bulk insert temporary contexts
            for table, records in tc_to_insert.items():
                stmt = insert(table.__table__).values(records)
                self.session.execute(stmt)

            # Generates and persists mentions
            mention_args = {"document_id": doc.id}
            for child_context in self.child_context_set:
                # Assemble mention arguments
                for arg_name in mention_class.__argnames__:
                    mention_args[arg_name + "_id"] = child_context.id

                # Checking for existence
                if not clear:
                    q = select([mention_class.id])
                    for key, value in list(mention_args.items()):
                        q = q.where(getattr(mention_class, key) == value)
                    mention_id = self.session.execute(q).first()
                    if mention_id is not None:
                        continue

                # Add Mention to session
                yield mention_class(**mention_args)
1288561
private static List<List<String>> convertSentenceListToDocument(List<String> sentenceList)
    {
        List<List<String>> docs = new ArrayList<List<String>>(sentenceList.size());
        for (String sentence : sentenceList)
        {
            List<Term> termList = StandardTokenizer.segment(sentence.toCharArray());
            List<String> wordList = new LinkedList<String>();
            for (Term term : termList)
            {
                if (CoreStopWordDictionary.shouldInclude(term))
                {
                    wordList.add(term.word);
                }
            }
            docs.add(wordList);
        }
        return docs;
    }
1361320
def get_texts(self):
		'''
		
		'''
		if self._document_category_df is None:
			return pd.np.array([])
		return self._document_category_df.text.values
1246019
def tokenize_to_spacy_doc(self, text: str) -> Doc:
        
        if not self.keep_multi_space:
            text = re.sub(' +', ' ', text)
        doc = self.nlp(text, disable=['parser'])
        for a_token in doc:
            self.custom_token(a_token)

        return doc
1061061
function oncharincategory(lexer, lookahead, categoryREs) {
  var c = lexer.peek_char(lookahead);
  return categoryREs.find(function(re) {
    // because we're matching a single char...
    re.lastIndex = 0; // make sure we start from the start
    return re.test(c);
  });
}
1318483
def categories(self):
        
        r = [ch for ch in self._channels.values() if isinstance(ch, CategoryChannel)]
        r.sort(key=lambda c: (c.position, c.id))
        return r
1151684
def iter_sparts(self):
        
        for s in self._sparts:
            yield (s.partname, s.content_type, s.reltype, s.blob)
961167
def iter_documents(self, fileids=None, categories=None, _destroy=False):
        
        doc_ids = self._filter_ids(fileids, categories)
        for doc in imap(self.get_document, doc_ids):
            yield doc
            if _destroy:
                doc.destroy()
1325882
def iter_block_items(self):
        
        block_item_tags = (qn('w:p'), qn('w:tbl'), qn('w:sdt'))
        for child in self:
            if child.tag in block_item_tags:
                yield child
1332658
def iter(self):
        
        count = 0
        for _file in self._files:
            with asHandle(_file) as fp:
                # Duplicate some code here so as not to test
                # self._upperCase in the loop.
                if self._upperCase:
                    for seq in SeqIO.parse(fp, 'fasta'):
                        read = self._readClass(seq.description,
                                               str(seq.seq.upper()))
                        yield read
                        count += 1
                else:
                    for seq in SeqIO.parse(fp, 'fasta'):
                        read = self._readClass(seq.description, str(seq.seq))
                        yield read
                        count += 1
1147363
def get_new_term_doc_mat(self, doc_domains):
		'''
		

		'''
		assert len(doc_domains) == self.term_doc_matrix.get_num_docs()
		doc_domain_set = set(doc_domains)
		num_terms = self.term_doc_matrix.get_num_terms()
		num_domains = len(doc_domain_set)
		domain_mat = lil_matrix((num_domains, num_terms), dtype=int)
		X = self.term_doc_matrix.get_term_doc_mat()
		for i, domain in enumerate(doc_domain_set):
			domain_mat[i, :] = X[np.array(doc_domains == domain)].sum(axis=0)
		return domain_mat.tocsr()
199106
def category(self) -> str:
        '''

        '''

        logger.debug('dir(self.__module__): %s', dir(self.__module__))

        return self.__module__.__name__.rsplit('.', 2)[-2].replace('test_', '')
325762
public static <TERM> Map<TERM, Double> idf(Iterable<Iterable<TERM>> documentVocabularies,
                                               boolean smooth, boolean addOne)
    {
        Map<TERM, Integer> df = new HashMap<TERM, Integer>();
        int d = smooth ? 1 : 0;
        int a = addOne ? 1 : 0;
        int n = d;
        for (Iterable<TERM> documentVocabulary : documentVocabularies)
        {
            n += 1;
            for (TERM term : documentVocabulary)
            {
                Integer t = df.get(term);
                if (t == null) t = d;
                df.put(term, t + 1);
            }
        }
        Map<TERM, Double> idf = new HashMap<TERM, Double>();
        for (Map.Entry<TERM, Integer> e : df.entrySet())
        {
            TERM term = e.getKey();
            double f = e.getValue();
            idf.put(term, Math.log(n / f) + a);
        }
        return idf;
    }
660328
def load_wordlist(name: str) -> Iterator[str]:
    
    data = pkgutil.get_data('pydocstyle', 'data/' + name)
    if data is not None:
        text = data.decode('utf8')
        for line in text.splitlines():
            line = COMMENT_RE.sub('', line).strip()
            if line:
                yield line
567192
def _parse_section(token_iter):
    '''
    '''
    sexp = []
    for token in token_iter:
        if token == '(':
            new_sexp = _parse_section(token_iter)
            if not _match_section(new_sexp, UNWANTED_SECTIONS):
                sexp.append(new_sexp)
        elif token == ')':
            return sexp
        else:
            sexp.append(token)
    return sexp