If possible, return a dict with preprocessed document and a list
    of spans with weights, corresponding to features in the document.
355836
def get_weighted_spans(doc, vec, feature_weights):
    # type: (Any, Any, FeatureWeights) -> Optional[WeightedSpans]
    
    if isinstance(vec, FeatureUnion):
        return _get_weighted_spans_from_union(doc, vec, feature_weights)
    else:
        result = _get_doc_weighted_spans(doc, vec, feature_weights)
        if result is not None:
            found_features, doc_weighted_spans = result
            return WeightedSpans(
                [doc_weighted_spans],
                other=_get_other(feature_weights, [('', found_features)]),
            )
    return None
1102044
def embedding(indices, vocab_dim, output_dim, variable_dtype, name="embedding"):
  """"""
  weights = embedding_weights(
      indices.mesh, vocab_dim, output_dim, variable_dtype, name)
  return mtf.gather(weights, indices, vocab_dim)
366837
def detect_spans(self, tokens):
        
        candidates = self._get_candidates(tokens)
        results = self._filter_candidates(tokens, candidates)
        return results
331563
def simple_preprocess(doc, deacc=False, min_len=2, max_len=15):
    
    tokens = [
        token for token in tokenize(doc, lower=True, deacc=deacc, errors='ignore')
        if min_len <= len(token) <= max_len and not token.startswith('_')
    ]
    return tokens
786880
def multipartite(corpus, featureset_names, min_weight=1, filters={}):
    

    pairs = Counter()
    node_type = {corpus._generate_index(p): {'type': 'paper'}
                 for p in corpus.papers}
    for featureset_name in featureset_names:
        ftypes = {}

        featureset = _get_featureset(corpus, featureset_name)
        for paper, feature in featureset.iteritems():
            if featureset_name in filters:
                if not filters[featureset_name](featureset, feature):
                    continue
            if len(feature) < 1:
                continue
            for f in list(zip(*feature))[0]:
                ftypes[f] = {'type': featureset_name}
                pairs[(paper, f)] += 1
        node_type.update(ftypes)

    return _generate_graph(nx.DiGraph, pairs, node_attrs=node_type,
                           min_weight=min_weight)
1366333
def vectorize( self, docs ):
        '''
        
        '''

        if type(docs) == dict:
            docs = docs.items()

        if self.model == None:
            self.train(docs)

        asset_id2vector = {}

        unfound = []
        for item in docs:
            ## iterate through the items in docs and check if any are already in the model.
            asset_id, _ = item
            label = 'DOC_' + str(asset_id)
            if label in self.model:
                asset_id2vector.update({asset_id: self.model['DOC_' + str(asset_id)]})
            else:
                unfound.append(item)

        if len(unfound) > 0:
            ## for all assets not in the model, update the model and then get their sentence vectors.
            sentences = [self._gen_sentence(item) for item in unfound]
            self.update_model(sentences, train=self.stream_train)
            asset_id2vector.update({item[0]: self.model['DOC_' + str(item[0])] for item in unfound})

        return asset_id2vector
538327
def tokenize_documents(docs):
    """"""
    sep = '\n\n'
    texts = []
    for doc in docs:
        text = '\n\n'.join(['\n'.join(para[SENTENCES]) for para in doc[PARAGRAPHS]])
        doc[TEXT] = text
        del doc[PARAGRAPHS]
        texts.append(Text(doc))
    return texts
467275
def _to_dict(self):
        """"""
        _dict = {}
        if hasattr(self, 'features') and self.features is not None:
            _dict['features'] = self.features
        if hasattr(self,
                   'text_characters') and self.text_characters is not None:
            _dict['text_characters'] = self.text_characters
        if hasattr(self, 'text_units') and self.text_units is not None:
            _dict['text_units'] = self.text_units
        return _dict
145248
def _to_dict(self):
        """"""
        _dict = {}
        if hasattr(self, 'model_id') and self.model_id is not None:
            _dict['model_id'] = self.model_id
        if hasattr(self, 'model_version') and self.model_version is not None:
            _dict['model_version'] = self.model_version
        if hasattr(self, 'documents') and self.documents is not None:
            _dict['documents'] = [x._to_dict() for x in self.documents]
        if hasattr(self,
                   'aligned_elements') and self.aligned_elements is not None:
            _dict['aligned_elements'] = [
                x._to_dict() for x in self.aligned_elements
            ]
        if hasattr(
                self,
                'unaligned_elements') and self.unaligned_elements is not None:
            _dict['unaligned_elements'] = [
                x._to_dict() for x in self.unaligned_elements
            ]
        return _dict
901742
def rintersects(self, span):
        
        if isinstance(span, list):
            return [sp for sp in span if self._rintersects(sp)]

        return self._rintersects(span)
486391
def to_document(self):
        
        return dict(
            info=thaw(self.info),
            paths=thaw(self.paths),
            definitions=thaw(self.definitions),
            securityDefinitions=thaw(self.securityDefinitions),
            security=thaw(self.security),
            swagger=thaw(self.swagger),
        )
305495
def tablelib_unary_features(span):
    
    if not span.sentence.is_tabular():
        return
    sentence = span.sentence
    for attrib in settings["featurization"]["table"]["unary_features"]["attrib"]:
        for ngram in get_cell_ngrams(
            span,
            n_max=settings["featurization"]["table"]["unary_features"][
                "get_cell_ngrams"
            ]["max"],
            attrib=attrib,
        ):
            yield f"CELL_{attrib.upper()}_[{ngram}]", DEF_VALUE
        for row_num in range(sentence.row_start, sentence.row_end + 1):
            yield f"ROW_NUM_[{row_num}]", DEF_VALUE
        for col_num in range(sentence.col_start, sentence.col_end + 1):
            yield f"COL_NUM_[{col_num}]", DEF_VALUE
        # NOTE: These two features could be accounted for by HTML_ATTR in
        # structural features
        yield f"ROW_SPAN_[{num_rows(sentence)}]", DEF_VALUE
        yield f"COL_SPAN_[{num_cols(sentence)}]", DEF_VALUE
        for axis in ["row", "col"]:
            for ngram in get_head_ngrams(
                span,
                axis,
                n_max=settings["featurization"]["table"]["unary_features"][
                    "get_head_ngrams"
                ]["max"],
                attrib=attrib,
            ):
                yield f"{axis.upper()}_HEAD_{attrib.upper()}_[{ngram}]", DEF_VALUE
        for ngram in get_row_ngrams(
            span,
            n_max=settings["featurization"]["table"]["unary_features"][
                "get_row_ngrams"
            ]["max"],
            attrib=attrib,
        ):
            yield f"ROW_{attrib.upper()}_[{ngram}]", DEF_VALUE
        for ngram in get_col_ngrams(
            span,
            n_max=settings["featurization"]["table"]["unary_features"][
                "get_col_ngrams"
            ]["max"],
            attrib=attrib,
        ):
            yield f"COL_{attrib.upper()}_[{ngram}]", DEF_VALUE
1173166
def _from_dict(cls, _dict):
        """"""
        args = {}
        if 'count' in _dict:
            args['count'] = _dict.get('count')
        if 'relevance' in _dict:
            args['relevance'] = _dict.get('relevance')
        if 'text' in _dict:
            args['text'] = _dict.get('text')
        if 'emotion' in _dict:
            args['emotion'] = EmotionScores._from_dict(_dict.get('emotion'))
        if 'sentiment' in _dict:
            args['sentiment'] = FeatureSentimentResults._from_dict(
                _dict.get('sentiment'))
        return cls(**args)
750720
def _compute_weights(self, X, rs):
        """"""

        # use supplied weights if present
        weights = self._get_user_components('weights')
        if (weights is None):
            n_features = X.shape[1]
            hw_size = (n_features, self.n_hidden)
            weights = rs.normal(size=hw_size)

        self.components_['weights'] = weights
437384
def to_dict(self, remove_nones=False):
        

        items = []

        # attempt to replace each item with its dictionary representation if possible
        for item in self.items:
            if hasattr(item, 'to_dict'):
                items.append(item.to_dict(remove_nones=remove_nones))
            else:
                items.append(item)

        return {
            'items': items,
            'pageNumber': self.page_number,
            'pageSize': self.page_size,
            'totalElements': self.total_elements,
            'hasNext': self.has_next
        }
1342852
def parse_doc(doc):
    """"""
    word_spans = []
    sentence_spans = []
    sentence_chunks = doc.split('\n\n')
    sentences = []
    for chunk in sentence_chunks:
        sent_texts, sent_labels = get_texts_and_labels(chunk.strip())
        sentences.append(list(zip(sent_texts, sent_labels)))
    return sentences
311722
def _docs(self):
        
        iterdocs = iter(self.docs())
        first = next(iterdocs)
        needs_parsing = False
        if isinstance(first, six.string_types):
            needs_parsing = True
        elif isinstance(first, dict):
            pass
        else:
            raise RuntimeError('Document must be either JSON strings or dict.')
        for doc in itertools.chain([first], iterdocs):
            if needs_parsing:
                doc = json.loads(doc)
            if '_index' not in doc:
                doc['_index'] = self.index
            if '_type' not in doc:
                doc['_type'] = self.doc_type
            yield doc