Get entities and sentiment analysis of text
774091
public function polyglot_entities( $text, $language = null )
	{
		$data = $this->post_call('/polyglot/entities', ['text' => $text, 'lang' => $language] );
		$this->msg( $data );
		return new \Web64\Nlp\Classes\PolyglotResponse( $data['polyglot'] );
	}
1144798
def analyze(text, features = default_features)
      if text.nil? || features.nil?
        raise ArgumentError.new(NIL_ARGUMENT_ERROR)
      end

      response = self.class.post(
        "#{@url}/analyze?version=#{@version}",
        body: {
          text: text.to_s,
          features: features
        }.to_json,
        basic_auth: auth,
        headers: {
          "Content-Type" => CONTENT_TYPE
        }
      )

      response.parsed_response
    end
912226
def extract(self, text: str, get_attr=['PERSON', 'ORG', 'GPE']) -> List[Extraction]:
        
        doc = self.__nlp(text)
        attr_list = list()
        for ent in doc.ents:
            if ent.label_ in get_attr:
                attr_list.append(Extraction(extractor_name=self.name,
                                            start_char=int(ent.start_char),
                                            end_char=int(ent.end_char),
                                            value=ent.text,
                                            tag=ent.label_,
                                            start_token=ent.start,
                                            end_token=ent.end))
        return attr_list
1300435
def _get_features(self, i, context, prev, prev2):
        """"""
        w = self.lexicon[context[i]]
        features = [
            'bias',
            'w:shape=%s' % w.shape,
            'w:lower=%s' % w.lower,
            'p1:tag=%s' % prev,
            'p2:tag=%s' % prev2,
            'p1:tag+w:lower=%s+%s' % (prev, w.lower),
            'p1:tag+p2:tag=%s+%s' % (prev, prev2),
        ]
        if w.like_number:
            features.append('w:like_number')
        elif w.is_punct:
            features.append('w:is_punct')
        elif w.like_url:
            features.append('w:like_url')
        else:
            features.extend([
                'w:suffix2=%s' % w.lower[-2:],
                'w:suffix3=%s' % w.lower[-3:],
                'w:suffix4=%s' % w.lower[-4:],
                'w:suffix5=%s' % w.lower[-5:],
                'w:prefix1=%s' % w.lower[:1],
                'w:prefix2=%s' % w.lower[:2],
                'w:prefix3=%s' % w.lower[:3],
            ])
            if w.is_alpha:
                features.append('w:is_alpha')
            elif w.is_hyphenated:
                features.append('w:is_hyphenated')
            if w.is_upper:
                features.append('w:is_upper')
            elif w.is_lower:
                features.append('w:is_lower')
            elif w.is_title:
                features.append('w:is_title')
        if self.clusters and w.cluster:
            features.extend([
                'w:cluster4=%s' % w.cluster[:4],
                'w:cluster6=%s' % w.cluster[:6],
                'w:cluster10=%s' % w.cluster[:10],
                'w:cluster20=%s' % w.cluster[:20],
            ])
        # Add features for previous tokens if present
        if i > 0:
            p1 = self.lexicon[context[i-1]]
            features.extend([
                'p1:lower=%s' % p1.lower,
                'p1:shape=%s' % p1.shape,
            ])
            if not (p1.like_number or p1.is_punct or p1.like_url):
                features.append('p1:suffix3=%s' % p1.lower[-3:])
            if self.clusters and p1.cluster:
                features.extend([
                    'p1:cluster4=%s' % p1.cluster[:4],
                    'p1:cluster6=%s' % p1.cluster[:6],
                    'p1:cluster10=%s' % p1.cluster[:10],
                    'p1:cluster20=%s' % p1.cluster[:20],
                ])
            if i > 1:
                p2 = self.lexicon[context[i-2]]
                features.extend([
                    'p2:lower=%s' % p2.lower,
                    'p2:shape=%s' % p2.shape,
                ])
                if self.clusters and p2.cluster:
                    features.extend([
                        'p2:cluster4=%s' % p2.cluster[:4],
                        'p2:cluster6=%s' % p2.cluster[:6],
                        'p2:cluster10=%s' % p2.cluster[:10],
                        'p2:cluster20=%s' % p2.cluster[:20],
                    ])
        # Add features for next tokens if present
        end = len(context) - 1
        if i < end:
            n1 = self.lexicon[context[i+1]]
            features.extend([
                'n1:lower=%s' % n1.lower,
                'n1:shape=%s' % n1.shape
            ])
            if not (n1.like_number or n1.is_punct or n1.like_url):
                features.append('n1:suffix3=%s' % n1.lower[-3:])
            if self.clusters and n1.cluster:
                features.extend([
                    'n1:cluster4=%s' % n1.cluster[:4],
                    'n1:cluster6=%s' % n1.cluster[:6],
                    'n1:cluster10=%s' % n1.cluster[:10],
                    'n1:cluster20=%s' % n1.cluster[:20],
                ])
            if i < end - 1:
                n2 = self.lexicon[context[i+2]]
                features.extend([
                    'n2:lower=%s' % n2.lower,
                    'n2:shape=%s' % n2.shape
                ])
                if self.clusters and n2.cluster:
                    features.extend([
                        'n2:cluster4=%s' % n2.cluster[:4],
                        'n2:cluster6=%s' % n2.cluster[:6],
                        'n2:cluster10=%s' % n2.cluster[:10],
                        'n2:cluster20=%s' % n2.cluster[:20],
                    ])
        # Add position features
        if i == 0:
            features.append('-firsttoken-')
        elif i == 1:
            features.append('-secondtoken-')
        elif i == end - 1:
            features.append('-secondlasttoken-')
        elif i == end:
            features.append('-lasttoken-')
        return features
1062129
def _from_dict(cls, _dict):
        """"""
        args = {}
        if 'text' in _dict:
            args['text'] = _dict.get('text')
        if 'entities' in _dict:
            args['entities'] = [
                SemanticRolesEntity._from_dict(x)
                for x in (_dict.get('entities'))
            ]
        if 'keywords' in _dict:
            args['keywords'] = [
                SemanticRolesKeyword._from_dict(x)
                for x in (_dict.get('keywords'))
            ]
        return cls(**args)
1080209
def load_textcat_data(limit=0):
    """"""
    # Partition off part of the train data for evaluation
    train_data, eval_data = thinc.extra.datasets.imdb()
    random.shuffle(train_data)
    train_data = train_data[-limit:]
    texts, labels = zip(*train_data)
    eval_texts, eval_labels = zip(*eval_data)
    cats = [{"POSITIVE": bool(y), "NEGATIVE": not bool(y)} for y in labels]
    eval_cats = [{"POSITIVE": bool(y), "NEGATIVE": not bool(y)} for y in eval_labels]
    return (texts, cats), (eval_texts, eval_cats)
1316616
def babelfy(self, text, params=None):
        
        self._entities = list()
        self._all_entities = list()
        self._merged_entities = list()
        self._all_merged_entities = list()
        self._text = " ".join(word.strip() for word in text.split())

        params = params or self._params
        params['key'] = self._api_key
        params['text'] = self._text
        if (sys.version < '3' and isinstance(params['text'], unicode)) or (sys.version >= '3' and isinstance(params['text'], bytes)): #pylint: disable=undefined-variable
            params['text'] = params['text'].encode('utf-8')
        url = BABELFY_API_URL + '?' + urlencode(params)

        request = Request(url)
        request.add_header('Accept-encoding', 'gzip')
        response = urlopen(request)
        if sys.version < '3':
            buf = StringIO(response.read())
        else:
            buf = BytesIO(response.read())
        f = gzip.GzipFile(fileobj=buf)
        reader = codecs.getreader('utf-8')
        self._data = json.load(reader(f))
554774
function parseEntityBotCommand(text, entity) {
    let result = RE_BOT_COMMAND.exec(text.substr(
        entity.offset,
        entity.length
    ));

    return {
        type: entity.type,
        name: result[1],
        mention: result[2],
    };
}
407288
def get_statements(self):
        """"""
        for k, v in self.reader_output.items():
            for interaction in v['interactions']:
                self._process_interaction(k, interaction, v['text'], self.pmid,
                                          self.extra_annotations)
43288
def extract_named_entities(text_blocks):
    
    sentences = []
    for text in text_blocks:
        sentences.extend(nltk.sent_tokenize(text))

    tokenized_sentences = [nltk.word_tokenize(sentence) for sentence in sentences]
    tagged_sentences = [nltk.pos_tag(sentence) for sentence in tokenized_sentences]
    chunked_sentences = nltk.ne_chunk_sents(tagged_sentences, binary=True)

    def extract_entity_names(t):
        entity_names = []

        if hasattr(t, 'label'):
            if t.label() == 'NE':
                entity_names.append(' '.join([child[0] for child in t]))
            else:
                for child in t:
                    entity_names.extend(extract_entity_names(child))

        return entity_names

    entity_names = []
    for tree in chunked_sentences:
        entity_names.extend(extract_entity_names(tree))

    return set(entity_names)
830147
def extract_objects_from_source(self, text, type_filter=None):
    '''
    '''
    objects = parse_verilog(text)

    if type_filter:
      objects = [o for o in objects if isinstance(o, type_filter)]

    return objects
1243795
def extract_objects_from_source(self, text, type_filter=None):
    '''
    '''
    objects = parse_vhdl(text)
    self._register_array_types(objects)

    if type_filter:
      objects = [o for o in objects if isinstance(o, type_filter)]

    return objects
533689
def _filter(self, text, context, encoding):
        """"""

        docstrings = []
        strings = []
        comments = []
        prev_token_type = tokenize.NEWLINE
        indent = ''
        name = None
        stack = [(context, 0, self.MODULE)]
        last_comment = False
        possible_fmt_str = None

        src = io.StringIO(text)

        for token in tokenizer(src.readline):
            token_type = token[0]
            value = token[1]
            line = str(token[2][0])
            line_num = token[2][0]

            value = token[1]
            line = str(token[2][0])

            # Track function and class ancestry
            if token_type == tokenize.NAME:
                possible_fmt_str = None
                if value in ('def', 'class'):
                    name = value
                elif name:
                    parent = stack[-1][2]
                    prefix = ''
                    if parent != self.MODULE:
                        prefix = '.' if parent == self.CLASS else ', '
                    if name == 'class':
                        stack.append(('%s%s' % (prefix, value), len(indent), self.CLASS))
                    elif name == 'def':
                        stack.append(('%s%s()' % (prefix, value), len(indent), self.FUNCTION))
                    name = None
                elif not F_SUPPORT and value in FMT_STR:
                    possible_fmt_str = (prev_token_type, value)
            elif token_type != tokenize.STRING:
                possible_fmt_str = None

            if token_type == tokenize.COMMENT and self.comments:
                # Capture comments
                if (
                    self.group_comments and
                    last_comment and
                    comments and (comments[-1][2] + 1) == line_num
                ):
                    # Group multiple consecutive comments
                    comments[-1][0] += '\n' + value[1:]
                    comments[-1][2] = line_num
                else:
                    if len(stack) > 1:
                        loc = "%s(%s): %s" % (stack[0][0], line, ''.join([crumb[0] for crumb in stack[1:]]))
                    else:
                        loc = "%s(%s)" % (stack[0][0], line)
                    comments.append([value[1:], loc, line_num])
            if token_type == tokenize.STRING:
                # Capture docstrings.
                # If we captured an `INDENT` or `NEWLINE` previously we probably have a docstring.
                # `NL` means end of line, but not the end of the Python code line (line continuation).
                if (
                    (prev_token_type in PREV_DOC_TOKENS) or
                    (possible_fmt_str and possible_fmt_str[0] in PREV_DOC_TOKENS)
                ):
                    if self.docstrings:
                        value = value.strip()
                        if possible_fmt_str and value.startswith(("'", "\"")):
                            value = possible_fmt_str[1] + value
                        string, is_bytes = self.process_strings(value, docstrings=True)
                        if string:
                            loc = "%s(%s): %s" % (stack[0][0], line, ''.join([crumb[0] for crumb in stack[1:]]))
                            docstrings.append(
                                filters.SourceText(string, loc, 'utf-8', 'py-docstring')
                            )
                elif self.strings:
                    value = value.strip()
                    if possible_fmt_str and value.startswith(("'", "\"")):
                        value = possible_fmt_str[1] + value
                    string, is_bytes = self.process_strings(value)
                    if string:
                        loc = "%s(%s): %s" % (stack[0][0], line, ''.join([crumb[0] for crumb in stack[1:]]))
                        strings.append(filters.SourceText(string, loc, 'utf-8', 'py-string'))

            if token_type == tokenize.INDENT:
                indent = value
            elif token_type == tokenize.DEDENT:
                indent = indent[:-4]
                if len(stack) > 1 and len(indent) <= stack[-1][1]:
                    stack.pop()

            # We purposefully avoid storing comments as comments can come before docstrings,
            # and that can mess up our logic. So if the token is a comment we won't track it,
            # and comments are always followed with `NL` so we ignore that as well.
            # We only care that docstrings are preceded by `NEWLINE`.
            if token_type != tokenize.COMMENT and not (last_comment and token_type == tokenize.NL):
                prev_token_type = token_type
                last_comment = False
            else:
                last_comment = True

        final_comments = []
        for comment in comments:
            final_comments.append(filters.SourceText(textwrap.dedent(comment[0]), comment[1], encoding, 'py-comment'))

        return docstrings + final_comments + strings
1182402
def contains_entity(entity, text):
        

        try:
            entity = re.escape(entity)
            entity = entity.replace("\ ", "([^\w])?")
            pattern = "(\ |-|\\\|/|\.|,|^)%s(\ |\-|\\\|/|\.|,|$)" % entity
            found = len(re.findall(pattern, text, re.I | re.M))
        except Exception as e:
            found = False

        return found
929374
def get_sentences(corpus):
		'''
		
		'''
		assert isinstance(corpus, ParsedCorpus)
		return itertools.chain(*[[[corpus._term_idx_store.getidxstrict(t.lower_) for t in sent
		                           if not t.is_punct]
		                          for sent in doc.sents]
		                         for doc in corpus.get_parsed_docs()])
774248
def _getPhrase( self, i, sentence, NPlabels ):
        ''' 
        ''' 
        phrase  = []
        indices = []
        if 0 <= i and i < len(sentence) and NPlabels[i] == 'B':
            phrase  = [ sentence[i] ]
            indices = [ i ]
            j = i + 1
            while ( j < len(sentence) ):
                if NPlabels[j] in ['B', '']:
                    break
                else:
                    phrase.append( sentence[j] )
                    indices.append( j )
                j += 1
        return phrase, indices
555992
def predict(self, text):
        
        pred = self.predict_proba(text)
        tags = self._get_tags(pred)

        return tags