Fetches the full length phrase from the position i
            based on the existing NP phrase annotations (from 
            NPlabels);
            Returns list of sentence tokens in the phrase, and 
            indices of the phrase;
774248
def _getPhrase( self, i, sentence, NPlabels ):
        ''' 
        ''' 
        phrase  = []
        indices = []
        if 0 <= i and i < len(sentence) and NPlabels[i] == 'B':
            phrase  = [ sentence[i] ]
            indices = [ i ]
            j = i + 1
            while ( j < len(sentence) ):
                if NPlabels[j] in ['B', '']:
                    break
                else:
                    phrase.append( sentence[j] )
                    indices.append( j )
                j += 1
        return phrase, indices
420304
def patterns(self):
        
        all_patterns = []
        for label, patterns in self.token_patterns.items():
            for pattern in patterns:
                all_patterns.append({"label": label, "pattern": pattern})
        for label, patterns in self.phrase_patterns.items():
            for pattern in patterns:
                all_patterns.append({"label": label, "pattern": pattern.text})
        return all_patterns
262274
public static HMMPOSTagger learn(String[][] sentences, PennTreebankPOS[][] labels) {
        int index = 1;
        int suffixIndex = 0;
        Map<String, Integer> symbol = new HashMap<>();
        Map<String, Integer> suffix = new HashMap<>();
        for (int i = 0; i < sentences.length; i++) {
            for (int j = 0; j < sentences[i].length; j++) {
                Integer sym = symbol.get(sentences[i][j]);
                if (sym == null) {
                    symbol.put(sentences[i][j], index++);
                }
                
                if (sentences[i][j].length() > 2) {
                    String s = sentences[i][j].substring(sentences[i][j].length() - 2);
                    sym = suffix.get(s);
                    if (sym == null) {
                        suffix.put(s, suffixIndex++);
                    }
                }
            }
        }

        int numStates = PennTreebankPOS.values().length;

        double[] pi = new double[numStates];
        double[][] a = new double[numStates][numStates];
        double[][] b = new double[numStates][symbol.size() + 1];
        double[][] c = new double[numStates][suffix.size()];
        
        PennTreebankPOS[] tags = PennTreebankPOS.values();
        for (int i = 0; i < numStates; i++) {
            if (tags[i].open) {
                b[i][0] = 1.0;
            }
        }

        for (int i = 0; i < sentences.length; i++) {
            int[] tag = translate(labels[i]);
            int[][] obs = translate(symbol, suffix, sentences[i]);

            pi[tag[0]]++;
            b[tag[0]][obs[0][0]]++;            
            if (obs[0][1] >= 0) {
                c[tag[0]][obs[0][1]]++;
            }
            
            for (int j = 1; j < obs.length; j++) {
                a[tag[j-1]][tag[j]]++;
                b[tag[j]][obs[j][0]]++;
                if (obs[j][1] >= 0) {
                    c[tag[j]][obs[j][1]]++;
                }
            }                
        }

        Math.unitize1(pi);
        for (int i = 0; i < numStates; i++) {
            Math.unitize1(a[i]);
            Math.unitize1(b[i]);
            Math.unitize1(c[i]);
        }
        
        return new HMMPOSTagger(symbol, suffix, pi, a, b, c);
    }
1340724
def tokens(self):
        
        for content in self.contents:
            if isinstance(content, TokenWithPosition):
                for word in content.split():
                    yield word
            else:
                yield content
1023011
def get_deepest_phrase_for_termid(self,termid):
        
        terminal_id = self.terminal_for_term.get(termid)
        label = None
        subsumed = []
        if terminal_id is not None:
            first_path = self.paths_for_terminal[terminal_id][0]
            first_phrase_id = first_path[1]
            label = self.label_for_nonter.get(first_phrase_id)
            subsumed = self.terms_subsumed_by_nonter.get(first_phrase_id,[])
        return label,sorted(list(subsumed))
488430
def __get_sentence(counts, sentence_id=None):
    

    # First of all we need a cursor and a query to retrieve our ID's
    cursor = CONN.cursor()
    check_query = "select sen_id from sursentences"

    # Now we fetch the result of the query and save it into check_result
    cursor.execute(check_query)
    check_result = cursor.fetchall()

    # declare an empty list to be populated below
    id_list = []
    id_to_fetch = None

    # Populate the id_list variable with all of the ID's we retrieved from the database query.
    for row in check_result:
        id_list.append(row[0])

    if sentence_id is not None:
        if type(sentence_id) is int:
            id_to_fetch = sentence_id
    else:
        id_to_fetch = random.randint(1, counts['max_sen'])

        while id_to_fetch not in id_list:
            id_to_fetch = random.randint(1, counts['max_sen'])

    query = ("select * from sursentences where sen_id = {0}".format(id_to_fetch))
    cursor.execute(query)
    result = cursor.fetchone()
    # cursor.close()

    return result
52858
private List<Map.Entry<Sentence, Map.Entry<Integer, Integer>>> getRelevantSentences(List<String> inputs, String token, int maxSentences) throws IOException {
    List<Map.Entry<Sentence, Map.Entry<Integer, Integer>>> sentences = new ArrayList<>();
    for (String input : inputs) {
      if (new File(input).isDirectory()) {
        File file = new File(input, token + ".txt");
        if (!file.exists()) {
          throw new RuntimeException("File with example sentences not found: " + file);
        }
        try (FileInputStream fis = new FileInputStream(file)) {
          SentenceSource sentenceSource = new PlainTextSentenceSource(fis, language);
          sentences = getSentencesFromSource(inputs, token, maxSentences, sentenceSource);
        }
      } else {
        SentenceSource sentenceSource = MixingSentenceSource.create(inputs, language);
        sentences = getSentencesFromSource(inputs, token, maxSentences, sentenceSource);
      }
    }
    return sentences;
  }
18236
def from_tokens(cls, text, toks):
        
        sentence_pb = CoreNLP_pb2.Sentence()
        sentence_pb.characterOffsetBegin = 0
        sentence_pb.characterOffsetEnd = len(text)
        sentence_pb.sentenceIndex = 0
        sentence_pb.tokenOffsetBegin = 0
        sentence_pb.tokenOffsetEnd = len(toks)

        # Track progress in sentence and tokens.
        char_idx = 0
        tok_idx = 0

        buf = ""
        token_pb = None
        while char_idx < len(text):
            tok = toks[tok_idx]
            # Scan to the beginning of the token.
            if text[char_idx] != tok[0]:
                buf += text[char_idx]
                char_idx += 1
            # Aha! we have found the token. Assert that they match.
            else:
                assert text[char_idx:char_idx+len(tok)] == tok, "text did not match a token"
                # Create a new Token from this text.
                if token_pb: token_pb.after = buf
                token_pb = sentence_pb.token.add()
                token_pb.before = buf

                token_pb.beginChar = char_idx
                token_pb.endChar = char_idx + len(tok)
                # TODO(chaganty): potentially handle -LRB-?
                token_pb.value = tok
                token_pb.word = tok
                token_pb.originalText = text[char_idx:char_idx+len(tok)]

                buf = ""
                char_idx += len(tok)
                tok_idx += 1
        if token_pb: token_pb.after = buf
        assert tok_idx == len(toks), "text does not match all tokens"
        return AnnotatedSentence.from_pb(sentence_pb)
983883
def readAnnotations(self):
        
        annot = self.read_annotation()
        annot = np.array(annot)
        if (annot.shape[0] == 0):
            return np.array([]), np.array([]), np.array([])
        ann_time = self._get_float(annot[:, 0])
        ann_text = annot[:, 2]
        ann_text_out = ["" for x in range(len(annot[:, 1]))]
        for i in np.arange(len(annot[:, 1])):
            ann_text_out[i] = self._convert_string(ann_text[i])
            if annot[i, 1] == '':
                annot[i, 1] = '-1'
        ann_duration = self._get_float(annot[:, 1])
        return ann_time/10000000, ann_duration, np.array(ann_text_out)
707322
def find_tracked_words(self, tokens):
        
        tracked_words = []

        for i in range(len(tokens)):
            for j in range(i + 1, len(tokens) + 1):
                phrase = tokens[i:j]
                status = self.has_tokens(phrase)

                if status is not None:
                    if status is True:
                        tracked_words.append(TokenTrie.Token(phrase, i, j - 1))
                    elif status is False:
                        break

        return tracked_words
890968
def extract(self, text):
        
        _extracted = []
        if text.strip() == "":
            return _extracted
        parsed_sentences = self._parse_text(text)
        for s in parsed_sentences:
            tokens = s.split()
            new_np = []
            for t in tokens:
                w, tag, phrase, role = t.split('/')
                # exclude some parser errors (e.g. VB within NP),
                # extend startswith tuple if necessary
                if 'NP' in phrase and not self._is_verb(w, tag):
                    if len(new_np) > 0 and w.lower() in START_NEW_NP:
                        _extracted.append(" ".join(new_np))
                        new_np = [w]
                    else:
                        # normalize capitalisation of sentence starters, except
                        # for nouns
                        new_np.append(w.lower() if tokens[0].startswith(w) and
                                      not tag.startswith('N') else w)
                else:
                    if len(new_np) > 0:
                        _extracted.append(" ".join(new_np))
                    new_np = []
        return self._filter_extracted(_extracted)
659075
def tag_clauses(self):
        """"""
        if not self.is_tagged(CLAUSE_ANNOTATION):
            self.tag_clause_annotations()

        def from_sentence(words):
            """Function that extracts clauses from a signle sentence."""
            clauses = defaultdict(list)
            start = words[0][START]
            end = words[0][END]
            clause = words[0][CLAUSE_IDX]
            for word in words:
                if word[CLAUSE_IDX] != clause:
                    clauses[clause].append((start, end))
                    start, clause = word[START], word[CLAUSE_IDX]
                end = word[END]
            clauses[clause].append((start, words[-1][END]))
            clauses = [(key, {START: [s for s, e in clause], END: [e for s, e in clause]}) for key, clause in clauses.items()]
            return [v for k, v in sorted(clauses)]

        clauses = []
        sentences = self.divide()
        for sentence in sentences:
            clauses.extend(from_sentence(sentence))

        self[CLAUSES] = clauses
        return self
322330
def tags(self):
        
        # See also. Sentence.__repr__().
        ch, I,O,B = self.chunk, INSIDE+"-", OUTSIDE, BEGIN+"-"
        tags = [OUTSIDE for i in range(len(self.sentence.token))]
        for i, tag in enumerate(self.sentence.token): # Default: [WORD, POS, CHUNK, PNP, RELATION, ANCHOR, LEMMA]
            if tag == WORD:
                tags[i] = encode_entities(self.string)
            elif tag == POS and self.type:
                tags[i] = self.type
            elif tag == CHUNK and ch and ch.type:
                tags[i] = (self == ch[0] and B or I) + ch.type
            elif tag == PNP and self.pnp:
                tags[i] = (self == self.pnp[0] and B or I) + "PNP"
            elif tag == REL and ch and len(ch.relations) > 0:
                tags[i] = ["-".join([str(x) for x in [ch.type]+list(reversed(r)) if x]) for r in ch.relations]
                tags[i] = "*".join(tags[i])
            elif tag == ANCHOR and ch:
                tags[i] = ch.anchor_id or OUTSIDE
            elif tag == LEMMA:
                tags[i] = encode_entities(self.lemma or "")
            elif tag in self.custom_tags:
                tags[i] = self.custom_tags.get(tag) or OUTSIDE
        return tags
625682
def get_strings_from_utterance(tokenized_utterance: List[Token]) -> Dict[str, List[int]]:
    
    string_linking_scores: Dict[str, List[int]] = defaultdict(list)

    for index, token in enumerate(tokenized_utterance):
        for string in ATIS_TRIGGER_DICT.get(token.text.lower(), []):
            string_linking_scores[string].append(index)

    token_bigrams = bigrams([token.text for token in tokenized_utterance])
    for index, token_bigram in enumerate(token_bigrams):
        for string in ATIS_TRIGGER_DICT.get(' '.join(token_bigram).lower(), []):
            string_linking_scores[string].extend([index,
                                                  index + 1])

    trigrams = ngrams([token.text for token in tokenized_utterance], 3)
    for index, trigram in enumerate(trigrams):
        if trigram[0] == 'st':
            natural_language_key = f'st. {trigram[2]}'.lower()
        else:
            natural_language_key = ' '.join(trigram).lower()
        for string in ATIS_TRIGGER_DICT.get(natural_language_key, []):
            string_linking_scores[string].extend([index,
                                                  index + 1,
                                                  index + 2])
    return string_linking_scores
221198
def get_anki_phrases_english(limit=None):
    
    texts = set()
    for lang in ANKI_LANGUAGES:
        df = get_data(lang)
        phrases = df.eng.str.strip().values
        texts = texts.union(set(phrases))
        if limit and len(texts) >= limit:
            break
    return sorted(texts)
615676
public static Map<Integer, NerClassification> positionClassificationMap(final NerResultSet nerResultSet) {
        final Map<Integer, NerClassification> m = new HashMap<>();

        for (final List<Phrase> sentence : nerResultSet.phrases) {
            for (final Phrase p : sentence) {
                final int phraseStartIndex = p.phrase.get(0).startIndex;
                for (final Token t : p.phrase) {
                    final NerClassification clas =
                            new NerClassification(t.text, p.phraseType, phraseStartIndex, p.score);
                    final NerClassification replaced = m.put(t.startIndex, clas);
                    if (replaced != null) {
                        // since modifying the contents of the map this error should now never happen
                        System.err.println("########### Error start");
                        System.err.print(nerResultSet);
                        throw new IllegalStateException("Tried to add a Token to the position classification map " +
                                                        "with startIndex " + t.startIndex + " that is already there!");
                    }
                }
            }
        }

        return m;
    }
191546
def _intertext_score(full_text):
    ''''''

    sentences = sentence_tokenizer(full_text)
    norm = _normalize(sentences)
    similarity_matrix = pairwise_kernels(norm, metric='cosine')
    scores = _textrank(similarity_matrix)
    scored_sentences = []
    for i, s in enumerate(sentences):
        scored_sentences.append((scores[i],i,s))
    top_scorers = sorted(scored_sentences,
                         key=lambda tup: tup[0], 
                         reverse=True)
    return top_scorers