获取语料库

        Return:
        corpus               -- 语料库，str类型
173426
def get_corpus(self):
        

        # 正向判定
        corpus = []
        cd = 0
        tag = None
        for i in range(0, self.init_corpus[0][0]):
            init_unit = self.unit_raw[self.init_corpus[0][0] - i]
            cdm = CDM(init_unit)
            alpha = cdm.get_alpha()

            if cd <= self.cd_min and cdm.NC is not 0:
                tag = True
            if cd > self.cd_max or cdm.NC == 0:
                tag = False
            if cd in range(self.cd_min + 1, self.cd_max) and cdm.NC is not 0:
                if alpha > 0:
                    tag = True
                else:
                    tag = False

            if cdm.NC == 0:
                cd += 1
            else:
                cd = 0

            if tag == True:
                corpus.append(init_unit)
            elif tag == False:
                if alpha < 0 or cd > self.cd_max:
                    break
                else:
                    continue
        corpus = list(reversed(corpus))
        try:
            self.index = self.init_corpus[0][0] - i + 1
        except UnboundLocalError:
            log('err', '正向判定完成，索引定位出错')
            self.index = self.init_corpus[0][0]

        # 反向判定
        cd = 0
        tag = None
        for i in range(1, len(self.unit_raw) - self.init_corpus[0][0]):
            init_unit = self.unit_raw[self.init_corpus[0][0] + i]
            cdm = CDM(init_unit)
            alpha = cdm.get_alpha()

            if cd <= self.cd_min and cdm.NC is not 0:
                tag = True
            if cd > self.cd_max or cdm.NC == 0:
                tag = False
            if cd in range(self.cd_min + 1, self.cd_max) and cdm.NC is not 0:
                if alpha > 0:
                    tag = True
                else:
                    tag = False

            if cdm.NC == 0:
                cd += 1
            else:
                cd = 0

            if tag == True:
                corpus.append(init_unit)
            elif tag == False:
                if alpha < 0 or cd > self.cd_max:
                    break
                else:
                    continue
        log('debug', '\n获取语料库成功:【{}】\n'.format(corpus))
        return ''.join(corpus)
73527
def corpus2clusters(corpus, index,params_clustermap={'vmin':0,'vmax':1,'figsize':[6,6]}):
    
    from sklearn.feature_extraction.text import TfidfVectorizer
    vect = TfidfVectorizer()
    tfidf = vect.fit_transform(corpus)

    df=pd.DataFrame((tfidf * tfidf.T).A,
                index=index,
                columns=index,
                )
    clustergrid=sns.clustermap(df,**params_clustermap
    #                                method='complete', metric='canberra',
                                  )
    dclusters=get_clusters(clustergrid,axis=0,criterion='maxclust',clusters_fraction=0.25)
    return dclusters
865017
def text(self):
        
        data = self.content
        return data.decode(self.encoding or 'utf-8') if data else ''
1254049
def corpus(project_id, name, opts = {})
      data, _status_code, _headers = corpus_with_http_info(project_id, name, opts)
      return data
    end
801516
def string(self):
        
        for term in self._terms:
            if isinstance(term, String):
                return str(term)
        return None
419843
def to_dict(self):
        
        return {
            "title": self.title,
            "language" : self.language,
            "description": self.description,
            "node_id": self.get_node_id().hex,
            "content_id": self.get_content_id().hex,
            "source_domain": self.domain_ns.hex,
            "source_id": self.source_id,
            "author": self.author,
            "aggregator": self.aggregator,
            "provider": self.provider,
            "files" : [f.to_dict() for f in self.files if f and f.filename], # Filter out failed downloads
            "tags": self.tags,
            "kind": self.kind,
            "license": None,
            "license_description": None,
            "copyright_holder": "",
            "questions": [],
            "extra_fields": {},
        }
1296111
def _get_path(fname: str) -> str:
    
    path = get_corpus_path(fname)
    if not path:
        download(fname)
        path = get_corpus_path(fname)
    return path
811278
def tag(words: List[str], corpus: str = "pud") -> List[Tuple[str, str]]:
    
    if not words:
        return []

    if corpus == "orchid":
        tagger = _ORCHID_TAGGER
        i = 0
        while i < len(words):
            if words[i] == " ":
                words[i] = "<space>"
            elif words[i] == "+":
                words[i] = "<plus>"
            elif words[i] == "-":
                words[i] = "<minus>"
            elif words[i] == "=":
                words[i] = "<equal>"
            elif words[i] == ",":
                words[i] = "<comma>"
            elif words[i] == "$":
                words[i] = "<dollar>"
            elif words[i] == ".":
                words[i] = "<full_stop>"
            elif words[i] == "(":
                words[i] = "<left_parenthesis>"
            elif words[i] == ")":
                words[i] = "<right_parenthesis>"
            elif words[i] == '"':
                words[i] = "<quotation>"
            elif words[i] == "@":
                words[i] = "<at_mark>"
            elif words[i] == "&":
                words[i] = "<ampersand>"
            elif words[i] == "{":
                words[i] = "<left_curly_bracket>"
            elif words[i] == "^":
                words[i] = "<circumflex_accent>"
            elif words[i] == "?":
                words[i] = "<question_mark>"
            elif words[i] == "<":
                words[i] = "<less_than>"
            elif words[i] == ">":
                words[i] = "<greater_than>"
            elif words[i] == "=":
                words[i] = "<equal>"
            elif words[i] == "!":
                words[i] = "<exclamation>"
            elif words[i] == "’":
                words[i] = "<apostrophe>"
            elif words[i] == ":":
                words[i] = "<colon>"
            elif words[i] == "*":
                words[i] = "<asterisk>"
            elif words[i] == ";":
                words[i] = "<semi_colon>"
            elif words[i] == "/":
                words[i] = "<slash>"
            i += 1
        t2 = tagger.tag(words)
        t = []
        i = 0
        while i < len(t2):
            word = t2[i][0]
            tag = t2[i][1]
            if word == "<space>":
                word = " "
            elif word == "<plus>":
                word = "+"
            elif word == "<minus>":
                word = "-"
            elif word == "<equal>":
                word = "="
            elif word == "<comma>":
                word = ","
            elif word == "<dollar>":
                word = "$"
            elif word == "<full_stop>":
                word = "."
            elif word == "<left_parenthesis>":
                word = "("
            elif word == "<right_parenthesis>":
                word = ")"
            elif word == "<quotation>":
                word = '"'
            elif word == "<at_mark>":
                word = "@"
            elif word == "<ampersand>":
                word = "&"
            elif word == "<left_curly_bracket>":
                word = "{"
            elif word == "<circumflex_accent>":
                word = "^"
            elif word == "<question_mark>":
                word = "?"
            elif word == "<less_than>":
                word = "<"
            elif word == "<greater_than>":
                word = ">"
            elif word == "<equal>":
                word = "="
            elif word == "<exclamation>":
                word = "!"
            elif word == "<apostrophe>":
                word = "’"
            elif word == "<colon>":
                word = ":"
            elif word == "<asterisk>":
                word = "*"
            elif word == "<semi_colon>":
                word = ";"
            elif word == "<slash>":
                word = "/"
            t.append((word, tag))
            i += 1
    else:  # default, use "pud" as a corpus
        tagger = _PUD_TAGGER
        t = tagger.tag(words)

    return t
57874
def text(self, encoding = 'UTF-8', language = 'en'):
        
        self.ndef_type = _NDEF_TEXT_TYPE
        self.ndef_text_lang = language
        self.ndef_text_enc = encoding
        return self
634882
def text(self):
        
        t = self.t
        if t is None:
            return u''
        text = t.text
        return to_unicode(text) if text is not None else u''
607539
def corpusindex():
        """"""
        corpora = []
        for f in glob.glob(settings.ROOT + "corpora/*"):
            if os.path.isdir(f):
                corpora.append(os.path.basename(f))
        return corpora
1252936
def list_corpora(self):
        """"""
        try:
            # corpora = LANGUAGE_CORPORA[self.language]
            corpora = self.all_corpora
            corpus_names = [corpus['name'] for corpus in corpora]
            return corpus_names
        except (NameError, KeyError) as error:
            msg = 'Corpus not available for language "{}": {}'.format(self.language, error)
            logger.error(msg)
            raise CorpusImportError(msg)
137796
def _render_corpus_row(self, label, ngrams):
        """"""
        row = ('<tr>\n<td>{label}</td>\n<td>{first}</td>\n<td>{only}</td>\n'
               '<td>{last}</td>\n</tr>')
        cell_data = {'label': label}
        for period in ('first', 'only', 'last'):
            cell_data[period] = ', '.join(ngrams[label][period])
        return row.format(**cell_data)
21610
def tostring(self, *args, **kwargs):
        
        return etree.tostring(self.resource, *args, **kwargs)
740381
def get_builder(self, corpus):
        '''  '''
        builder = WordBuilder(chunk_size=self.chunk_size)
        builder.ingest(corpus)
        return builder
485593
def raw_corpus_rougel(hypotheses: Iterable[str], references: Iterable[str]) -> float:
    
    return rouge.rouge_l(hypotheses, references)
260512
def tfidf_search(text, corpus=tfidf_dense, corpus_text=corpus):
    
    tokens = tokenize(text, vocabulary=corpus.columns)
    tfidf_vector_query = np.array(tfidfer.transform([' '.join(tokens)]).todense())[0]
    query_series = pd.Series(tfidf_vector_query, index=corpus.columns)

    return corpus_text[query_series.dot(corpus.T).values.argmax()]