Parameters
        ----------
        sentence : etree.Element
            etree representation of a sentence
            (syntax tree with coreference annotation)
773625
def add_sentence(self, sentence):
        
        sent_root_id = sentence.attrib[add_ns('id')]
        # add edge from document root to sentence root
        self.add_edge(self.root, sent_root_id, edge_type=dg.EdgeTypes.dominance_relation)
        self.sentences.append(sent_root_id)

        sentence_token_ids = []

        if 'span' in sentence.attrib:
            # the sentence element looks like this:
            # <sentence xml:id="s144" span="s144_1..s144_23">, which means that
            # there might be <word> elements which belong to this sentence but
            # occur after the closing </sentence> element
            span_str = sentence.attrib['span']
            sentence_token_ids.extend(convert_spanstring(span_str))
        else:  # a normal sentence element, i.e. <sentence xml:id="s143">
            for descendant in sentence.iterdescendants('word'):
                sentence_token_ids.append(self.get_element_id(descendant))

        self.node[sent_root_id]['tokens'] = sentence_token_ids
323498
def sentences(obj: BioCCollection or BioCDocument or BioCPassage) \
        -> Generator[BioCSentence, None, None]:
    
    if isinstance(obj, BioCCollection):
        for document in obj.documents:
            yield from sentences(document)
    elif isinstance(obj, BioCDocument):
        for passage in obj.passages:
            yield from sentences(passage)
    elif isinstance(obj, BioCPassage):
        yield from obj.sentences
    else:
        raise TypeError(f'Object of type {obj.__class__.__name__} must be BioCCollection, '
                        f'BioCDocument, BioCPassage, or BioCSentence')
96631
def p_contextualize_sentence(self, t):
        """"""

        if len(t) == 5:
            t[0] = contextualize(t[3], line=t.lineno(1))
        else:
            t[0] = contextualize(t[4], t[2], line=t.lineno(1))
453727
def sentence_similarity(sentence1, sentence2):
    
    # Tokenize and tag
    sentence1 = pos_tag(word_tokenize(sentence1))
    sentence2 = pos_tag(word_tokenize(sentence2))

    # Get the synsets for the tagged words
    synsets1 = [tagged_to_synset(*tagged_word) for tagged_word in sentence1]
    synsets2 = [tagged_to_synset(*tagged_word) for tagged_word in sentence2]

    # Filter out the Nones
    synsets1 = [ss for ss in synsets1 if ss]
    synsets2 = [ss for ss in synsets2 if ss]
    #print(synsets1)
    #print(synsets2)
    score, count = 0.0, 0.0

    # For each word in the first sentence
    for synset in synsets1:
        # Get the similarity value of the most similar word in the other sentence

        best_score=[synset.path_similarity(ss) for ss in synsets2 if synset.path_similarity(ss)]

        # Check that the similarity could have been computed
        if best_score:
            score += max(best_score)
            count += 1

    # Average the values
    if count > 0:
        score /= count
    else:
        score = 0

    return score
31420
def etree_to_dict(tree):
    
    d = {tree.tag.split('}')[1]: map(
        etree_to_dict, tree.iterchildren()
    ) or tree.text}

    return d
1077156
public Pair<CoreMap, GrammaticalStructure> parseForCoref(String sentence) {
        List<CoreLabel> tokens = stanfordTokenize(sentence);
        Tree tree = parser.parse(tokens);
        GrammaticalStructure gs = tagDependencies(tree, true);
        tagPOS(tokens);
        tagLemma(tokens);
        tagNamedEntity(tokens);

        CoreMap result = new ArrayCoreMap();
        result.set(CoreAnnotations.TokensAnnotation.class, tokens);
        result.set(TreeCoreAnnotations.TreeAnnotation.class, tree);

        GrammaticalStructure.Extras extras = GrammaticalStructure.Extras.NONE;
        SemanticGraph deps = SemanticGraphFactory.generateCollapsedDependencies(gs, extras);
        SemanticGraph uncollapsedDeps = SemanticGraphFactory.generateUncollapsedDependencies(gs, extras);
        SemanticGraph ccDeps = SemanticGraphFactory.generateCCProcessedDependencies(gs, extras);

        result.set(SemanticGraphCoreAnnotations.CollapsedDependenciesAnnotation.class, deps);
        result.set(SemanticGraphCoreAnnotations.BasicDependenciesAnnotation.class, uncollapsedDeps);
        result.set(SemanticGraphCoreAnnotations.CollapsedCCProcessedDependenciesAnnotation.class, ccDeps);
        return new ImmutablePair<>(result, gs);
    }
951157
private void processLeaf(Leaf leaf, List<String> sentence, List<Span> names) {

    if (leaf != null && leftContractionPart == null) {

      String namedEntityTag = null;
      int startOfNamedEntity = -1;

      String leafTag = leaf.getSecondaryTag();

      if (leafTag != null) {
        if (leafTag.contains("<sam->")) {
          String[] lexemes = leaf.getLexeme().split("_");
          if (lexemes.length > 1) {
            for (int i = 0; i < lexemes.length - 1; i++) {
              sentence.add(lexemes[i]);
            }
          }
          leftContractionPart = lexemes[lexemes.length - 1];
          return;
        }
        if (leaf.getLexeme().contains("_") && leaf.getLexeme().length() > 3) {
          String tag = leaf.getFunctionalTag();
          if (tags != null) {
            if (tags.contains(tag)) {
              namedEntityTag = leaf.getFunctionalTag();
            }
          } else {
            namedEntityTag = leaf.getFunctionalTag();
          }
        }
      }

      if (namedEntityTag != null) {
        startOfNamedEntity = sentence.size();
      }

      sentence.addAll(Arrays.asList(leaf.getLexeme().split("_")));

      if (namedEntityTag != null) {
        names
            .add(new Span(startOfNamedEntity, sentence.size(), namedEntityTag));
      }

    } else {
      // will handle the contraction
      String tag = leaf.getSecondaryTag();
      String right = leaf.getLexeme();
      if (tag != null && tag.contains("<-sam>")) {
        right = leaf.getLexeme();
        String c = PortugueseContractionUtility.toContraction(
            leftContractionPart, right);

        if (c != null) {
          sentence.add(c);
        } else {
          System.err.println("missing " + leftContractionPart + " + " + right);
          sentence.add(leftContractionPart);
          sentence.add(right);
        }

      } else {
        System.err.println("unmatch" + leftContractionPart + " + " + right);
      }
      leftContractionPart = null;
    }

  }
284451
def indexableText(self, tree):
        
        rval = set()
        root = tree.getroot()
        for txp in self.text_elts_xpaths:
            elts = txp(root)
            texts = []
            # Texts in element may be empty
            for elt in elts:
                text = self.text_extract_xpath(elt)
                if len(text) > 0:
                    texts.append(text[0])
            texts = self.separator.join(texts)
            texts = [toUnicode(x) for x in self.wordssearch_rx.findall(texts)
                     if len(x) > 0]
            rval |= set(texts)
        return rval
727780
def from_etree(
    el, node=None, node_cls=None,
    tagsub=functools.partial(re.sub, r'\{.+?\}', ''),
    Node=Node):
    '''
    '''
    node_cls = node_cls or Node
    if node is None:
        node = node_cls()
    tag = tagsub(el.tag)
    attrib = dict((tagsub(k), v) for (k, v) in el.attrib.items())
    node.update(attrib, tag=tag)

    if el.text:
        node['text'] = el.text
    for child in el:
        child = from_etree(child, node_cls=node_cls)
        node.append(child)
    if el.tail:
        node['tail'] = el.tail
    return node
1174106
def coreferences(self):
        
        if self._coreferences is None:
            coreferences = self._xml.xpath('/root/document/coreference/coreference')
            if len(coreferences) > 0:
                self._coreferences = [Coreference(self, element) for element in coreferences]
        return self._coreferences
769757
@Override
    protected List<Term> segSentence(char[] sentence)
    {
        if (sentence.length == 0) return Collections.emptyList();
        List<Term> termList = roughSegSentence(sentence);
        if (!(config.ner || config.useCustomDictionary || config.speechTagging))
            return termList;
        List<Vertex> vertexList = toVertexList(termList, true);
        if (config.speechTagging)
        {
            Viterbi.compute(vertexList, CoreDictionaryTransformMatrixDictionary.transformMatrixDictionary);
            int i = 0;
            for (Term term : termList)
            {
                if (term.nature != null) term.nature = vertexList.get(i + 1).guessNature();
                ++i;
            }
        }
        if (config.useCustomDictionary)
        {
            combineByCustomDictionary(vertexList);
            termList = convert(vertexList, config.offset);
        }
        return termList;
    }
959811
def _create_sentence_objects(self):
    '''
    '''
    sentence_objects = []
    sent_tokenizer = SentenceTokenizer(locale=self.language.code)
    seq = Sequence(self.raw)
    seq = sent_tokenizer.transform(seq)
    for start_index, end_index in zip(seq.idx[:-1], seq.idx[1:]):
      # Sentences share the same models as their parent blob
      sent = seq.text[start_index: end_index].strip()
      if not sent: continue
      s = Sentence(sent, start_index=start_index, end_index=end_index)
      s.detected_languages = self.detected_languages
      sentence_objects.append(s)
    return sentence_objects
257350
def etree_to_dict(element):
    
    i = dict(element.items())
    content = element.text.strip() if element.text else None
    i.update({'content': content}) if content else None

    if len(element.getchildren()):
        for child in element.iterchildren():
            tag = child.tag.split('}', 1)[-1]
            new = etree_to_dict(child)
            content = _make_content(i, tag, new)
            i.update({tag: content}) if content else None

            tag = 'content'
            new = child.tail.strip() if child.tail else None
            content = _make_content(i, tag, new)
            i.update({tag: content}) if content else None
    elif content and not set(i).difference(['content']):
        # element is leaf node and doesn't have attributes
        i = content

    return i
1333409
def ln_from_etree(self, ln_element, context=''):
        
        ln = {}
        # grab all understood attributes into ln dict
        for att in ('hash', 'href', 'length', 'modified',
                    'path', 'rel', 'pri', 'mime_type'):
            xml_att = self._xml_att_name(att)
            val = ln_element.attrib.get(xml_att, None)
            if (val is not None):
                ln[att] = val
        # now do some checks and conversions...
        # href (MANDATORY)
        if ('href' not in ln):
            raise SitemapParseError(
                "Missing href in <rs:ln> in %s" %
                (context))
        # rel (MANDATORY)
        if ('rel' not in ln):
            raise SitemapParseError("Missing rel in <rs:ln> in %s" % (context))
        # length in bytes
        if ('length' in ln):
            try:
                ln['length'] = int(ln['length'])
            except ValueError as e:
                raise SitemapParseError(
                    "Invalid length attribute value in <rs:ln> for %s" %
                    (context))
        # pri - priority, must be a number between 1 and 999999
        if ('pri' in ln):
            try:
                ln['pri'] = int(ln['pri'])
            except ValueError as e:
                raise SitemapParseError(
                    "Invalid pri attribute in <rs:ln> for %s" %
                    (context))
            if (ln['pri'] < 1 or ln['pri'] > 999999):
                raise SitemapParseError(
                    "Bad pri attribute value in <rs:ln> for %s" %
                    (context))
        return(ln)
207298
function tokenize_sentence(sentence) {
  var tokenized = tokenizer.tokenize(sentence);
  logger.info("tokenize_sentence: " + tokenized);
  return(tokenized);
}
200092
private void processLeaf(Leaf leaf, List<String> sentence, List<Span> names) {

    if (leaf != null && leftContractionPart == null) {

      int startOfNamedEntity = -1;
      String leafTag = leaf.getSecondaryTag();

      if (leafTag != null) {
        if (leafTag.contains("<sam->")) {
          String[] lexemes = underlinePattern.split(leaf.getLexeme());
          if (lexemes.length > 1) {
            for (int i = 0; i < lexemes.length - 1; i++) {
              sentence.add(lexemes[i]);
              
              String[] conts = ContractionUtility.expand(lexemes[i]);
              if(conts != null) {
                int end = sentence.size();
                int start = end - 1;
                Span s = new Span(start, end, "default");
                names.add(s);
                {
                  Span[] ss = {s};
                  System.out.println(Arrays.toString(Span.spansToStrings(ss, sentence.toArray(new String[sentence.size()]))));
                }
              }
            }
          }
          leftContractionPart = lexemes[lexemes.length - 1];
          return;
        }
        // if (leaf.getLexeme().contains("_") && leaf.getLexeme().length() > 3)
        // {
        // String tag = leaf.getFunctionalTag();
        // if (tags != null) {
        // if (tags.contains(tag)) {
        // namedEntityTag = leaf.getFunctionalTag();
        // }
        // } else {
        // namedEntityTag = leaf.getFunctionalTag();
        // }
        // }
      }

      // if (contraction) {
      // startOfNamedEntity = sentence.size();
      // }
      //
      sentence.addAll(Arrays.asList(leaf.getLexeme()));// .split("_")
      //
      // if (contraction) {
      // names
      // .add(new Span(startOfNamedEntity, sentence.size()));
      // }

    } else {
      // will handle the contraction
      String tag = leaf.getSecondaryTag();
      String right = leaf.getLexeme();
      if (tag != null && tag.contains("<-sam>")) {
        String[] parts = underlinePattern.split(leaf.getLexeme());
        if(parts != null) {
          // try to join only the first
          String c = PortugueseContractionUtility.toContraction(
              leftContractionPart, parts[0]);

          if (c != null) {
            sentence.add(c);
            names.add(new Span(sentence.size() - 1, sentence.size(), "default"));
          }
          
          for (int i = 1; i < parts.length; i++) {
            sentence.add(parts[i]);
          }
        } else {
          right = leaf.getLexeme();
          String c = PortugueseContractionUtility.toContraction(
              leftContractionPart, right);

          if (c != null) {
            sentence.add(c);
            names.add(new Span(sentence.size() - 1, sentence.size(), "default"));
          } else {
            System.err.println("missing " + leftContractionPart + " + " + right);
            sentence.add(leftContractionPart);
            sentence.add(right);
          }
        }
      } else {
        System.err.println("unmatch" + leftContractionPart + " + " + right);
      }
      leftContractionPart = null;
    }

  }
762574
def load_grammar(self, grammar_text, grammar_name='<?>'):
        ""

        try:
            tree = self.canonize_tree.transform( self.parser.parse(grammar_text+'\n') )
        except UnexpectedCharacters as e:
            context = e.get_context(grammar_text)
            raise GrammarError("Unexpected input at line %d column %d in %s: \n\n%s" %
                               (e.line, e.column, grammar_name, context))
        except UnexpectedToken as e:
            context = e.get_context(grammar_text)
            error = e.match_examples(self.parser.parse, {
                'Unclosed parenthesis': ['a: (\n'],
                'Umatched closing parenthesis': ['a: )\n', 'a: [)\n', 'a: (]\n'],
                'Expecting rule or terminal definition (missing colon)': ['a\n', 'a->\n', 'A->\n', 'a A\n'],
                'Alias expects lowercase name': ['a: -> "a"\n'],
                'Unexpected colon': ['a::\n', 'a: b:\n', 'a: B:\n', 'a: "a":\n'],
                'Misplaced operator': ['a: b??', 'a: b(?)', 'a:+\n', 'a:?\n', 'a:*\n', 'a:|*\n'],
                'Expecting option ("|") or a new rule or terminal definition': ['a:a\n()\n'],
                '%import expects a name': ['%import "a"\n'],
                '%ignore expects a value': ['%ignore %import\n'],
            })
            if error:
                raise GrammarError("%s at line %s column %s\n\n%s" % (error, e.line, e.column, context))
            elif 'STRING' in e.expected:
                raise GrammarError("Expecting a value at line %s column %s\n\n%s" % (e.line, e.column, context))
            raise

        tree = PrepareGrammar().transform(tree)

        # Extract grammar items
        defs = classify(tree.children, lambda c: c.data, lambda c: c.children)
        term_defs = defs.pop('term', [])
        rule_defs = defs.pop('rule', [])
        statements = defs.pop('statement', [])
        assert not defs

        term_defs = [td if len(td)==3 else (td[0], 1, td[1]) for td in term_defs]
        term_defs = [(name.value, (t, int(p))) for name, p, t in term_defs]
        rule_defs = [options_from_rule(*x) for x in rule_defs]

        # Execute statements
        ignore, imports = [], {}
        for (stmt,) in statements:
            if stmt.data == 'ignore':
                t ,= stmt.children
                ignore.append(t)
            elif stmt.data == 'import':
                if len(stmt.children) > 1:
                    path_node, arg1 = stmt.children
                else:
                    path_node, = stmt.children
                    arg1 = None

                if isinstance(arg1, Tree):  # Multi import
                    dotted_path = tuple(path_node.children)
                    names = arg1.children
                    aliases = dict(zip(names, names))  # Can't have aliased multi import, so all aliases will be the same as names
                else:  # Single import
                    dotted_path = tuple(path_node.children[:-1])
                    name = path_node.children[-1]  # Get name from dotted path
                    aliases = {name: arg1 or name}  # Aliases if exist

                if path_node.data == 'import_lib':  # Import from library
                    base_paths = []
                else:  # Relative import
                    if grammar_name == '<string>':  # Import relative to script file path if grammar is coded in script
                        try:
                            base_file = os.path.abspath(sys.modules['__main__'].__file__)
                        except AttributeError:
                            base_file = None
                    else:
                        base_file = grammar_name  # Import relative to grammar file path if external grammar file
                    if base_file:
                        base_paths = [os.path.split(base_file)[0]]
                    else:
                        base_paths = [os.path.abspath(os.path.curdir)]

                try:
                    import_base_paths, import_aliases = imports[dotted_path]
                    assert base_paths == import_base_paths, 'Inconsistent base_paths for %s.' % '.'.join(dotted_path)
                    import_aliases.update(aliases)
                except KeyError:
                    imports[dotted_path] = base_paths, aliases

            elif stmt.data == 'declare':
                for t in stmt.children:
                    term_defs.append([t.value, (None, None)])
            else:
                assert False, stmt

        # import grammars
        for dotted_path, (base_paths, aliases) in imports.items():
            grammar_path = os.path.join(*dotted_path) + EXT
            g = import_grammar(grammar_path, base_paths=base_paths)
            new_td, new_rd = import_from_grammar_into_namespace(g, '__'.join(dotted_path), aliases)

            term_defs += new_td
            rule_defs += new_rd

        # Verify correctness 1
        for name, _ in term_defs:
            if name.startswith('__'):
                raise GrammarError('Names starting with double-underscore are reserved (Error at %s)' % name)

        # Handle ignore tokens
        # XXX A slightly hacky solution. Recognition of %ignore TERMINAL as separate comes from the lexer's
        #     inability to handle duplicate terminals (two names, one value)
        ignore_names = []
        for t in ignore:
            if t.data=='expansions' and len(t.children) == 1:
                t2 ,= t.children
                if t2.data=='expansion' and len(t2.children) == 1:
                    item ,= t2.children
                    if item.data == 'value':
                        item ,= item.children
                        if isinstance(item, Token) and item.type == 'TERMINAL':
                            ignore_names.append(item.value)
                            continue

            name = '__IGNORE_%d'% len(ignore_names)
            ignore_names.append(name)
            term_defs.append((name, (t, 1)))

        # Verify correctness 2
        terminal_names = set()
        for name, _ in term_defs:
            if name in terminal_names:
                raise GrammarError("Terminal '%s' defined more than once" % name)
            terminal_names.add(name)

        if set(ignore_names) > terminal_names:
            raise GrammarError("Terminals %s were marked to ignore but were not defined!" % (set(ignore_names) - terminal_names))

        resolve_term_references(term_defs)

        rules = rule_defs

        rule_names = set()
        for name, _x, _o in rules:
            if name.startswith('__'):
                raise GrammarError('Names starting with double-underscore are reserved (Error at %s)' % name)
            if name in rule_names:
                raise GrammarError("Rule '%s' defined more than once" % name)
            rule_names.add(name)

        for name, expansions, _o in rules:
            used_symbols = {t for x in expansions.find_data('expansion')
                              for t in x.scan_values(lambda t: t.type in ('RULE', 'TERMINAL'))}
            for sym in used_symbols:
                if sym.type == 'TERMINAL':
                    if sym not in terminal_names:
                        raise GrammarError("Token '%s' used but not defined (in rule %s)" % (sym, name))
                else:
                    if sym not in rule_names:
                        raise GrammarError("Rule '%s' used but not defined (in rule %s)" % (sym, name))

        return Grammar(rules, term_defs, ignore_names)