Generate dependency parse in {'words': [], 'arcs': []} format.

    doc (Doc): Document do parse.
    RETURNS (dict): Generated dependency parse keyed by words and arcs.
511298
def parse_deps(orig_doc, options={}):
    
    doc = Doc(orig_doc.vocab).from_bytes(orig_doc.to_bytes())
    if not doc.is_parsed:
        user_warning(Warnings.W005)
    if options.get("collapse_phrases", False):
        with doc.retokenize() as retokenizer:
            for np in list(doc.noun_chunks):
                attrs = {
                    "tag": np.root.tag_,
                    "lemma": np.root.lemma_,
                    "ent_type": np.root.ent_type_,
                }
                retokenizer.merge(np, attrs=attrs)
    if options.get("collapse_punct", True):
        spans = []
        for word in doc[:-1]:
            if word.is_punct or not word.nbor(1).is_punct:
                continue
            start = word.i
            end = word.i + 1
            while end < len(doc) and doc[end].is_punct:
                end += 1
            span = doc[start:end]
            spans.append((span, word.tag_, word.lemma_, word.ent_type_))
        with doc.retokenize() as retokenizer:
            for span, tag, lemma, ent_type in spans:
                attrs = {"tag": tag, "lemma": lemma, "ent_type": ent_type}
                retokenizer.merge(span, attrs=attrs)
    if options.get("fine_grained"):
        words = [{"text": w.text, "tag": w.tag_} for w in doc]
    else:
        words = [{"text": w.text, "tag": w.pos_} for w in doc]
    arcs = []
    for word in doc:
        if word.i < word.head.i:
            arcs.append(
                {"start": word.i, "end": word.head.i, "label": word.dep_, "dir": "left"}
            )
        elif word.i > word.head.i:
            arcs.append(
                {
                    "start": word.head.i,
                    "end": word.i,
                    "label": word.dep_,
                    "dir": "right",
                }
            )
    return {"words": words, "arcs": arcs, "settings": get_doc_settings(orig_doc)}
550837
def iter_doc_objs(self, **kwargs):
        
        _LOG = get_logger('TypeAwareGitShard')
        try:
            for doc_id, fp in self.iter_doc_filepaths(**kwargs):
                if not self._is_alias(doc_id):
                    # TODO:hook for type-specific parser?
                    with codecs.open(fp, 'r', 'utf-8') as fo:
                        try:
                            nex_obj = anyjson.loads(fo.read())
                            yield (doc_id, nex_obj)
                        except Exception:
                            pass
        except Exception as x:
            f = 'iter_doc_filepaths FAILED with this error:\n{}'
            f = f.format(str(x))
            _LOG.warn(f)
986726
def noun_chunks(obj):
    
    # It follows the logic of the noun chunks finder of English language,
    # adjusted to some Greek language special characteristics.
    # obj tag corrects some DEP tagger mistakes.
    # Further improvement of the models will eliminate the need for this tag.
    labels = ["nsubj", "obj", "iobj", "appos", "ROOT", "obl"]
    doc = obj.doc  # Ensure works on both Doc and Span.
    np_deps = [doc.vocab.strings.add(label) for label in labels]
    conj = doc.vocab.strings.add("conj")
    nmod = doc.vocab.strings.add("nmod")
    np_label = doc.vocab.strings.add("NP")
    seen = set()
    for i, word in enumerate(obj):
        if word.pos not in (NOUN, PROPN, PRON):
            continue
        # Prevent nested chunks from being produced
        if word.i in seen:
            continue
        if word.dep in np_deps:
            if any(w.i in seen for w in word.subtree):
                continue
            flag = False
            if word.pos == NOUN:
                #  check for patterns such as γραμμή παραγωγής
                for potential_nmod in word.rights:
                    if potential_nmod.dep == nmod:
                        seen.update(
                            j for j in range(word.left_edge.i, potential_nmod.i + 1)
                        )
                        yield word.left_edge.i, potential_nmod.i + 1, np_label
                        flag = True
                        break
            if flag is False:
                seen.update(j for j in range(word.left_edge.i, word.i + 1))
                yield word.left_edge.i, word.i + 1, np_label
        elif word.dep == conj:
            # covers the case: έχει όμορφα και έξυπνα παιδιά
            head = word.head
            while head.dep == conj and head.head.i < head.i:
                head = head.head
            # If the head is an NP, and we're coordinated to it, we're an NP
            if head.dep in np_deps:
                if any(w.i in seen for w in word.subtree):
                    continue
                seen.update(j for j in range(word.left_edge.i, word.i + 1))
                yield word.left_edge.i, word.i + 1, np_label
417664
def yield_deps(doc):
    

    this_node = DependencyNode(doc, None)

    for r in doc.references():

        other_node = DependencyNode(r, this_node)

        yield (this_node, other_node)

        if isinstance(other_node, PackageDependencyNode):
            yield from yield_deps(other_node.doc)
893885
def get_feats(self, doc):
		'''
		
		'''
		# ngram_counter = phrasemachine.get_phrases(str(doc), tagger='spacy')['counts']
		ngram_counter = Counter()
		for sent in doc.sents:
			unigrams = self._get_unigram_feats(sent)
			ngram_counter += Counter(unigrams) + _phrase_counts(sent)
		return ngram_counter
635380
def parse_document(lexer: Lexer) -> DocumentNode:
    """"""
    start = lexer.token
    return DocumentNode(
        definitions=many_nodes(lexer, TokenKind.SOF, parse_definition, TokenKind.EOF),
        loc=loc(lexer, start),
    )
311722
def _docs(self):
        
        iterdocs = iter(self.docs())
        first = next(iterdocs)
        needs_parsing = False
        if isinstance(first, six.string_types):
            needs_parsing = True
        elif isinstance(first, dict):
            pass
        else:
            raise RuntimeError('Document must be either JSON strings or dict.')
        for doc in itertools.chain([first], iterdocs):
            if needs_parsing:
                doc = json.loads(doc)
            if '_index' not in doc:
                doc['_index'] = self.index
            if '_type' not in doc:
                doc['_type'] = self.doc_type
            yield doc
514818
def parsecommonarguments(object, doc, annotationtype, required, allowed, **kwargs):
    """"""

    object.doc = doc #The FoLiA root document

    if required is None:
        required = tuple()
    if allowed is None:
        allowed = tuple()

    supported = required + allowed


    if 'generate_id_in' in kwargs:
        try:
            kwargs['id'] = kwargs['generate_id_in'].generate_id(object.__class__)
        except GenerateIDException:
            pass #ID could not be generated, just skip
        del kwargs['generate_id_in']



    if 'id' in kwargs:
        if Attrib.ID not in supported:
            raise ValueError("ID is not supported on " + object.__class__.__name__)
        isncname(kwargs['id'])
        object.id = kwargs['id']
        del kwargs['id']
    elif Attrib.ID in required:
        raise ValueError("ID is required for " + object.__class__.__name__)
    else:
        object.id = None

    if 'set' in kwargs:
        if Attrib.CLASS not in supported and not object.SETONLY:
            raise ValueError("Set is not supported on " + object.__class__.__name__)
        if not kwargs['set']:
            object.set ="undefined"
        else:
            object.set = kwargs['set']
        del kwargs['set']

        if object.set:
            if doc and (not (annotationtype in doc.annotationdefaults) or not (object.set in doc.annotationdefaults[annotationtype])):
                if object.set in doc.alias_set:
                    object.set = doc.alias_set[object.set]
                elif doc.autodeclare:
                    doc.annotations.append( (annotationtype, object.set ) )
                    doc.annotationdefaults[annotationtype] = {object.set: {} }
                else:
                    raise ValueError("Set '" + object.set + "' is used for " + object.__class__.__name__ + ", but has no declaration!")
    elif annotationtype in doc.annotationdefaults and len(doc.annotationdefaults[annotationtype]) == 1:
        object.set = list(doc.annotationdefaults[annotationtype].keys())[0]
    elif object.ANNOTATIONTYPE == AnnotationType.TEXT:
        object.set = "undefined" #text content needs never be declared (for backward compatibility) and is in set 'undefined'
    elif Attrib.CLASS in required: #or (hasattr(object,'SETONLY') and object.SETONLY):
        raise ValueError("Set is required for " + object.__class__.__name__)


    if 'class' in kwargs:
        if not Attrib.CLASS in supported:
            raise ValueError("Class is not supported for " + object.__class__.__name__)
        object.cls = kwargs['class']
        del kwargs['class']
    elif 'cls' in kwargs:
        if not Attrib.CLASS in supported:
            raise ValueError("Class is not supported on " + object.__class__.__name__)
        object.cls = kwargs['cls']
        del kwargs['cls']
    elif Attrib.CLASS in required:
        raise ValueError("Class is required for " + object.__class__.__name__)

    if object.cls and not object.set:
        if doc and doc.autodeclare:
            if not (annotationtype, 'undefined') in doc.annotations:
                doc.annotations.append( (annotationtype, 'undefined') )
                doc.annotationdefaults[annotationtype] = {'undefined': {} }
            object.set = 'undefined'
        else:
            raise ValueError("Set is required for " + object.__class__.__name__ +  ". Class '" + object.cls + "' assigned without set.")





    if 'annotator' in kwargs:
        if not Attrib.ANNOTATOR in supported:
            raise ValueError("Annotator is not supported for " + object.__class__.__name__)
        object.annotator = kwargs['annotator']
        del kwargs['annotator']
    elif doc and annotationtype in doc.annotationdefaults and object.set in doc.annotationdefaults[annotationtype] and 'annotator' in doc.annotationdefaults[annotationtype][object.set]:
        object.annotator = doc.annotationdefaults[annotationtype][object.set]['annotator']
    elif Attrib.ANNOTATOR in required:
        raise ValueError("Annotator is required for " + object.__class__.__name__)


    if 'annotatortype' in kwargs:
        if not Attrib.ANNOTATOR in supported:
            raise ValueError("Annotatortype is not supported for " + object.__class__.__name__)
        if kwargs['annotatortype'] == 'auto' or kwargs['annotatortype'] == AnnotatorType.AUTO:
            object.annotatortype = AnnotatorType.AUTO
        elif kwargs['annotatortype'] == 'manual' or kwargs['annotatortype']  == AnnotatorType.MANUAL:
            object.annotatortype = AnnotatorType.MANUAL
        else:
            raise ValueError("annotatortype must be 'auto' or 'manual', got "  + repr(kwargs['annotatortype']))
        del kwargs['annotatortype']
    elif doc and annotationtype in doc.annotationdefaults and object.set in doc.annotationdefaults[annotationtype] and 'annotatortype' in doc.annotationdefaults[annotationtype][object.set]:
        object.annotatortype = doc.annotationdefaults[annotationtype][object.set]['annotatortype']
    elif Attrib.ANNOTATOR in required:
        raise ValueError("Annotatortype is required for " + object.__class__.__name__)

    if 'confidence' in kwargs:
        if not Attrib.CONFIDENCE in supported:
            raise ValueError("Confidence is not supported")
        if kwargs['confidence'] is not None:
            try:
                object.confidence = float(kwargs['confidence'])
                assert object.confidence >= 0.0 and object.confidence <= 1.0
            except:
                raise ValueError("Confidence must be a floating point number between 0 and 1, got " + repr(kwargs['confidence']) )
        del kwargs['confidence']
    elif Attrib.CONFIDENCE in required:
        raise ValueError("Confidence is required for " + object.__class__.__name__)



    if 'n' in kwargs:
        if not Attrib.N in supported:
            raise ValueError("N is not supported for " + object.__class__.__name__)
        object.n = kwargs['n']
        del kwargs['n']
    elif Attrib.N in required:
        raise ValueError("N is required for " + object.__class__.__name__)

    if 'datetime' in kwargs:
        if not Attrib.DATETIME in supported:
            raise ValueError("Datetime is not supported")
        if isinstance(kwargs['datetime'], datetime):
            object.datetime = kwargs['datetime']
        else:

            #try:
            object.datetime = parse_datetime(kwargs['datetime'])
            #except:
            #    raise ValueError("Unable to parse datetime: " + str(repr(kwargs['datetime'])))
        del kwargs['datetime']
    elif doc and annotationtype in doc.annotationdefaults and object.set in doc.annotationdefaults[annotationtype] and 'datetime' in doc.annotationdefaults[annotationtype][object.set]:
        object.datetime = doc.annotationdefaults[annotationtype][object.set]['datetime']
    elif Attrib.DATETIME in required:
        raise ValueError("Datetime is required for " + object.__class__.__name__)

    if 'src' in kwargs:
        if not Attrib.SRC in supported:
            raise ValueError("Source is not supported for " + object.__class__.__name__)
        object.src = kwargs['src']
        del kwargs['src']
    elif Attrib.SRC in required:
        raise ValueError("Source is required for " + object.__class__.__name__)

    if 'begintime' in kwargs:
        if not Attrib.BEGINTIME in supported:
            raise ValueError("Begintime is not supported for " + object.__class__.__name__)
        object.begintime = parsetime(kwargs['begintime'])
        del kwargs['begintime']
    elif Attrib.BEGINTIME in required:
        raise ValueError("Begintime is required for " + object.__class__.__name__)

    if 'endtime' in kwargs:
        if not Attrib.ENDTIME in supported:
            raise ValueError("Endtime is not supported for " + object.__class__.__name__)
        object.endtime = parsetime(kwargs['endtime'])
        del kwargs['endtime']
    elif Attrib.ENDTIME in required:
        raise ValueError("Endtime is required for " + object.__class__.__name__)


    if 'speaker' in kwargs:
        if not Attrib.SPEAKER in supported:
            raise ValueError("Speaker is not supported for " + object.__class__.__name__)
        object.speaker = kwargs['speaker']
        del kwargs['speaker']
    elif Attrib.SPEAKER in required:
        raise ValueError("Speaker is required for " + object.__class__.__name__)

    if 'auth' in kwargs:
        if kwargs['auth'] in ('no','false'):
            object.auth = False
        else:
            object.auth = bool(kwargs['auth'])
        del kwargs['auth']
    else:
        object.auth = object.__class__.AUTH



    if 'text' in kwargs:
        if kwargs['text']:
            object.settext(kwargs['text'])
        del kwargs['text']

    if 'phon' in kwargs:
        if kwargs['phon']:
            object.setphon(kwargs['phon'])
        del kwargs['phon']

    if 'textclass' in kwargs:
        if not Attrib.TEXTCLASS in supported:
            raise ValueError("Textclass is not supported for " + object.__class__.__name__)
        object.textclass = kwargs['textclass']
        del kwargs['textclass']
    else:
        if Attrib.TEXTCLASS in supported:
            object.textclass = "current"

    if 'metadata' in kwargs:
        if not Attrib.METADATA in supported:
            raise ValueError("Metadata is not supported for " + object.__class__.__name__)
        object.metadata = kwargs['metadata']
        if doc:
            try:
                doc.submetadata[kwargs['metadata']]
            except KeyError:
                raise KeyError("No such metadata defined: " + kwargs['metadata'])
        del kwargs['metadata']

    if object.XLINK:
        if 'href' in kwargs:
            object.href =kwargs['href']
            del kwargs['href']
        if 'xlinktype' in kwargs:
            object.xlinktype = kwargs['xlinktype']
            del kwargs['xlinktype']
        if 'xlinkrole' in kwargs:
            object.xlinkrole = kwargs['xlinkrole']
            del kwargs['xlinkrole']
        if 'xlinklabel' in kwargs:
            object.xlinklabel = kwargs['xlinklabel']
            del kwargs['xlinklabel']
        if 'xlinkshow' in kwargs:
            object.xlinkshow = kwargs['xlinkshow']
            del kwargs['xlinklabel']
        if 'xlinktitle' in kwargs:
            object.xlinktitle = kwargs['xlinktitle']
            del kwargs['xlinktitle']

    if doc and doc.debug >= 2:
        print("   @id           = ", repr(object.id),file=stderr)
        print("   @set          = ", repr(object.set),file=stderr)
        print("   @class        = ", repr(object.cls),file=stderr)
        print("   @annotator    = ", repr(object.annotator),file=stderr)
        print("   @annotatortype= ", repr(object.annotatortype),file=stderr)
        print("   @confidence   = ", repr(object.confidence),file=stderr)
        print("   @n            = ", repr(object.n),file=stderr)
        print("   @datetime     = ", repr(object.datetime),file=stderr)



    #set index
    if object.id and doc:
        if object.id in doc.index:
            if doc.debug >= 1: print("[PyNLPl FoLiA DEBUG] Duplicate ID not permitted:" + object.id,file=stderr)
            raise DuplicateIDError("Duplicate ID not permitted: " + object.id)
        else:
            if doc.debug >= 1: print("[PyNLPl FoLiA DEBUG] Adding to index: " + object.id,file=stderr)
            doc.index[object.id] = object

    #Parse feature attributes (shortcut for feature specification for some elements)
    for c in object.ACCEPTED_DATA:
        if issubclass(c, Feature):
            if c.SUBSET in kwargs:
                if kwargs[c.SUBSET]:
                    object.append(c,cls=kwargs[c.SUBSET])
                del kwargs[c.SUBSET]

    return kwargs
195117
def _doc2vec_doc_stream(paths, n, tokenizer=word_tokenize, sentences=True):
    
    i = 0
    p = Progress()
    for path in paths:
        with open(path, 'r') as f:
            for line in f:
                i += 1
                p.print_progress(i/n)

                # We do minimal pre-processing here so the model can learn
                # punctuation
                line = line.lower()

                if sentences:
                    for sent in sent_tokenize(line):
                        tokens = tokenizer(sent)
                        yield LabeledSentence(tokens, ['SENT_{}'.format(i)])
                else:
                    tokens = tokenizer(line)
                    yield LabeledSentence(tokens, ['SENT_{}'.format(i)])
1280257
def lex_document(self, cli, document):
        
        # Cache of already lexed lines.
        cache = {}

        # Pygments generators that are currently lexing.
        line_generators = {}  # Map lexer generator to the line number.

        def get_syntax_sync():
            " The Syntax synchronisation objcet that we currently use. "
            if self.sync_from_start(cli):
                return SyncFromStart()
            else:
                return self.syntax_sync

        def find_closest_generator(i):
            " Return a generator close to line 'i', or None if none was fonud. "
            for generator, lineno in line_generators.items():
                if lineno < i and i - lineno < self.REUSE_GENERATOR_MAX_DISTANCE:
                    return generator

        def create_line_generator(start_lineno, column=0):
            """
            Create a generator that yields the lexed lines.
            Each iteration it yields a (line_number, [(token, text), ...]) tuple.
            """
            def get_tokens():
                text = '\n'.join(document.lines[start_lineno:])[column:]

                # We call `get_tokens_unprocessed`, because `get_tokens` will
                # still replace \r\n and \r by \n.  (We don't want that,
                # Pygments should return exactly the same amount of text, as we
                # have given as input.)
                for _, t, v in self.pygments_lexer.get_tokens_unprocessed(text):
                    yield t, v

            return enumerate(split_lines(get_tokens()), start_lineno)

        def get_generator(i):
            """
            Find an already started generator that is close, or create a new one.
            """
            # Find closest line generator.
            generator = find_closest_generator(i)
            if generator:
                return generator

            # No generator found. Determine starting point for the syntax
            # synchronisation first.

            # Go at least x lines back. (Make scrolling upwards more
            # efficient.)
            i = max(0, i - self.MIN_LINES_BACKWARDS)

            if i == 0:
                row = 0
                column = 0
            else:
                row, column = get_syntax_sync().get_sync_start_position(document, i)

            # Find generator close to this point, or otherwise create a new one.
            generator = find_closest_generator(i)
            if generator:
                return generator
            else:
                generator = create_line_generator(row, column)

            # If the column is not 0, ignore the first line. (Which is
            # incomplete. This happens when the synchronisation algorithm tells
            # us to start parsing in the middle of a line.)
            if column:
                next(generator)
                row += 1

            line_generators[generator] = row
            return generator

        def get_line(i):
            " Return the tokens for a given line number. "
            try:
                return cache[i]
            except KeyError:
                generator = get_generator(i)

                # Exhaust the generator, until we find the requested line.
                for num, line in generator:
                    cache[num] = line
                    if num == i:
                        line_generators[generator] = i

                        # Remove the next item from the cache.
                        # (It could happen that it's already there, because of
                        # another generator that started filling these lines,
                        # but we want to synchronise these lines with the
                        # current lexer's state.)
                        if num + 1 in cache:
                            del cache[num + 1]

                        return cache[num]
            return []

        return get_line
1086773
def apply(self, doc, split, train, **kwargs):
        
        logger.debug(f"Document: {doc}")

        # Get all the candidates in this doc that will be featurized
        cands_list = get_cands_list_from_split(
            self.session, self.candidate_classes, doc, split
        )

        feature_map = dict()

        # Make a flat list of all candidates from the list of list of
        # candidates. This helps reduce the number of queries needed to update.
        all_cands = itertools.chain.from_iterable(cands_list)
        records = list(
            get_mapping(self.session, Feature, all_cands, get_all_feats, feature_map)
        )
        batch_upsert_records(self.session, Feature, records)

        # Insert all Feature Keys
        if train:
            upsert_keys(self.session, FeatureKey, feature_map)

        # This return + yield makes a completely empty generator
        return
        yield
22788
def parse_doc(doc):
    
    if not doc:
        return {}
    out = {}
    for s in doc.split('\n'):
        s = s.strip().split(':', maxsplit=1)
        if len(s) == 2:
            out[s[0]] = s[1]
    return out
834699
def process_document(self, doc: Document):
        

        text_segments = doc.select_segments("lexisnexis.doc_description")
        for text_segment in text_segments:
            split_sentences = doc.extract(self.sentence_extractor, text_segment)
            doc.store(split_sentences, 'split_sentences')
        # for t, u in zip(text_to_be_split, units_of_text):
        #     split_sentences = doc.extract(self.sentence_extractor, t)
        #     u.store(split_sentences, "split_sentences")
        return list()
680591
function parseAsObject(docblock) {
  var pairs = parse(docblock);
  var result = {};
  for (var i = 0; i < pairs.length; i++) {
    result[pairs[i][0]] = pairs[i][1];
  }
  return result;
}
622860
def proceed(self):
        
        self._parse()
        for e in self.docs_list:
            e['docs'].generate_docs()
        return self.docs_list
252252
def parse(str)
      @document = AST::Document.new(file_path: file_path)
      @root = @document.root

      @dependency_paths = []

      if str.is_a?(Array)
        reset(str, [[@root]])
      else
        reset(str.split(/\r?\n/, -1), [[@root]]) # -1 is for not suppressing empty lines
      end

      parse_line while next_line

      reset

      @document
    end
1359159
def parse(self, sentence):
        
        words = np.zeros((len(sentence) + 1, 1), np.int32)
        tags = np.zeros((len(sentence) + 1, 1), np.int32)
        words[0, 0] = ParserVocabulary.ROOT
        tags[0, 0] = ParserVocabulary.ROOT
        vocab = self._vocab

        for i, (word, tag) in enumerate(sentence):
            words[i + 1, 0], tags[i + 1, 0] = vocab.word2id(word.lower()), vocab.tag2id(tag)

        with mx.Context(mxnet_prefer_gpu()):
            outputs = self._parser.forward(words, tags)
        words = []
        for arc, rel, (word, tag) in zip(outputs[0][0], outputs[0][1], sentence):
            words.append(ConllWord(id=len(words) + 1, form=word, pos=tag, head=arc, relation=vocab.id2rel(rel)))
        return ConllSentence(words)