Translate golr association documents to a compact representation
170434
def translate_docs_compact(self, ds, field_mapping=None, slim=None, map_identifiers=None, invert_subject_object=False, **kwargs):
        
        amap = {}
        logging.info("Translating docs to compact form. Slim={}".format(slim))
        for d in ds:
            self.map_doc(d, field_mapping, invert_subject_object=invert_subject_object)

            subject = d[M.SUBJECT]
            subject_label = d[M.SUBJECT_LABEL]

            # TODO: use a more robust method; we need equivalence as separate field in solr
            if map_identifiers is not None:
                if M.SUBJECT_CLOSURE in d:
                    subject = self.map_id(subject, map_identifiers, d[M.SUBJECT_CLOSURE])
                else:
                    logging.debug("NO SUBJECT CLOSURE IN: "+str(d))

            rel = d.get(M.RELATION)
            skip = False

            # TODO
            if rel == 'not' or rel == 'NOT':
                skip = True

            # this is a list in GO
            if isinstance(rel,list):
                if 'not' in rel or 'NOT' in rel:
                    skip = True
                if len(rel) > 1:
                    logging.warn(">1 relation: {}".format(rel))
                rel = ";".join(rel)

            if skip:
                logging.debug("Skipping: {}".format(d))
                continue

            subject = self.make_canonical_identifier(subject)
            #if subject.startswith('MGI:MGI:'):
            #    subject = subject.replace('MGI:MGI:','MGI:')

            k = (subject,rel)
            if k not in amap:
                amap[k] = {'subject':subject,
                           'subject_label':subject_label,
                           'relation':rel,
                           'objects': []}
            if slim is not None and len(slim)>0:
                mapped_objects = [x for x in d[M.OBJECT_CLOSURE] if x in slim]
                logging.debug("Mapped objects: {}".format(mapped_objects))
                amap[k]['objects'] += mapped_objects
            else:
                amap[k]['objects'].append(d[M.OBJECT])
        for k in amap.keys():
            amap[k]['objects'] = list(set(amap[k]['objects']))

        return list(amap.values())
812974
private static Collection<?> getRowsFromMapAssociation(AssociationKey associationKey, Document value) {
		String rowKeyIndexColumn = associationKey.getMetadata().getRowKeyIndexColumnNames()[0];
		List<Document> rows = new ArrayList<Document>();

		String[] associationKeyColumns = associationKey.getMetadata()
				.getAssociatedEntityKeyMetadata()
				.getAssociationKeyColumns();

		// Omit shared prefix of compound ids, will be handled in the row accessor
		String prefix = DocumentHelpers.getColumnSharedPrefix( associationKeyColumns );
		prefix = prefix == null ? "" : prefix + ".";

		String embeddedValueColumnPrefix = associationKey.getMetadata().getCollectionRole() + EMBEDDABLE_COLUMN_PREFIX;

		// restore the list representation
		for ( String rowKey : value.keySet() ) {
			Object mapRow = value.get( rowKey );

			// include the row key index column
			Document row = new Document();
			row.put( rowKeyIndexColumn, rowKey );

			// several value columns, copy them all
			if ( mapRow instanceof Document ) {
				for ( String column : associationKey.getMetadata().getAssociatedEntityKeyMetadata().getAssociationKeyColumns() ) {
					// The column is part of an element collection; Restore the "value" node in the hierarchy
					if ( column.startsWith( embeddedValueColumnPrefix ) ) {
						MongoHelpers.setValue(
								row,
								column.substring( associationKey.getMetadata().getCollectionRole().length() + 1 ),
								( (Document) mapRow ).get( column.substring( embeddedValueColumnPrefix.length() ) )
						);
					}
					else {
						row.put(
								column.substring( prefix.length() ),
								( (Document) mapRow ).get( column.substring( prefix.length() ) )
						);
					}
				}
			}
			// single value column
			else {
				row.put( associationKey.getMetadata().getAssociatedEntityKeyMetadata().getAssociationKeyColumns()[0], mapRow );
			}

			rows.add( row );
		}

		return rows;
	}
1310003
def fmt_ria(ria, verbose=True, mip=False):
    """"""
    if verbose:
        mechanism = 'Mechanism: {}\n'.format(
            fmt_mechanism(ria.mechanism, ria.node_labels))
        direction = '\nDirection: {}'.format(ria.direction)
    else:
        mechanism = ''
        direction = ''

    if config.REPR_VERBOSITY is HIGH:
        partition = '\n{}:\n{}'.format(
            ('MIP' if mip else 'Partition'),
            indent(fmt_partition(ria.partition)))
        repertoire = '\nRepertoire:\n{}'.format(
            indent(fmt_repertoire(ria.repertoire)))
        partitioned_repertoire = '\nPartitioned repertoire:\n{}'.format(
            indent(fmt_repertoire(ria.partitioned_repertoire)))
    else:
        partition = ''
        repertoire = ''
        partitioned_repertoire = ''

    # TODO? print the two repertoires side-by-side
    return (
        '{SMALL_PHI} = {phi}\n'
        '{mechanism}'
        'Purview = {purview}'
        '{direction}'
        '{partition}'
        '{repertoire}'
        '{partitioned_repertoire}').format(
            SMALL_PHI=SMALL_PHI,
            mechanism=mechanism,
            purview=fmt_mechanism(ria.purview, ria.node_labels),
            direction=direction,
            phi=fmt_number(ria.phi),
            partition=partition,
            repertoire=repertoire,
            partitioned_repertoire=partitioned_repertoire)
1053115
def _translate(self, embedding):
        ""
        if embedding is None:
            return None
        if not self._linear:
            return embedding
        return [_bulk_to_linear(self.M, self.N, self.L, chain) for chain in embedding]
571120
def rdtxt_gos(go_file, prt):
        """"""
        goids_all = set()
        if not os.path.exists(go_file):
            raise RuntimeError("CAN NOT READ GO FILE: {FILE}\n".format(FILE=go_file))
        re_go = re.compile(r'(GO:\d{7})+?')
        re_com = re.compile(r'^\s*#')  # Lines starting with a '#' are comment lines and ignored
        with open(go_file) as ifstrm:
            for line in ifstrm:
                # Skip lines that are comments
                if re_com.search(line):
                    continue
                # Search for GO IDs on the line
                goids_found = re_go.findall(line)
                if goids_found:
                    goids_all.update(goids_found)
            if prt:
                prt.write("  {N} GO IDs READ: {TXT}\n".format(N=len(goids_all), TXT=go_file))
        return goids_all
453102
def get_go2txt(grprobj_cur, grp_go2color, grp_go2bordercolor):
        """"""
        goids_main = set(o.id for o in grprobj_cur.gosubdag.go2obj.values())
        hdrobj = grprobj_cur.hdrobj
        grprobj_all = Grouper("all",
                              grprobj_cur.usrgos.union(goids_main), hdrobj, grprobj_cur.gosubdag)
        # Adds section text to all GO terms in plot (misses middle GO terms)
        _secdflt = hdrobj.secdflt
        _hilight = set(grp_go2color.keys()).union(grp_go2bordercolor)
        ret_go2txt = {}
        # Keep sections text only if GO header, GO user, or not Misc.
        if hdrobj.sections:
            for goid, txt in grprobj_all.get_go2sectiontxt().items():
                if txt == 'broad':
                    continue
                if txt != _secdflt or goid in _hilight:
                    ret_go2txt[goid] = txt
        return ret_go2txt
441911
def align_entities(extracted: List[str],
                   literals: JsonDict,
                   stemmer: NltkPorterStemmer) -> List[str]:
    
    literal_keys = list(literals.keys())
    literal_values = list(literals.values())
    overlaps = [get_stem_overlaps(extract, literal_values, stemmer) for extract in extracted]
    worlds = []
    for overlap in overlaps:
        if overlap[0] > overlap[1]:
            worlds.append(literal_keys[0])
        elif overlap[0] < overlap[1]:
            worlds.append(literal_keys[1])
        else:
            worlds.append(None)
    return worlds
1319583
def _pprint_dict(seq, _nest_lvl=0, max_seq_items=None, **kwds):
    
    fmt = u("{%s}")
    pairs = []

    pfmt = u("%s: %s")

    if max_seq_items is False:
        nitems = len(seq)
    else:
        nitems = max_seq_items or options.display.max_seq_items or len(seq)

    for k, v in list(seq.items())[:nitems]:
        pairs.append(pfmt % (pprint_thing(k, _nest_lvl + 1, max_seq_items=max_seq_items, **kwds),
                             pprint_thing(v, _nest_lvl + 1, max_seq_items=max_seq_items, **kwds)))

    if nitems < len(seq):
        return fmt % (", ".join(pairs) + ", ...")
    else:
        return fmt % ", ".join(pairs)
907451
def awesome_mongo_mapper_association(object)
      return object.inspect if !defined?(::ActiveSupport::OrderedHash)
      return awesome_object(object) if @options[:raw]

      association = object.class.name.split('::').last.titleize.downcase.sub(/ association$/, '')
      association = "embeds #{association}" if object.embeddable?
      class_name = object.class_name

      "#{colorize(association, :assoc)} #{colorize(class_name, :class)}"
    end
528413
def _dereference(cls, documents, references):
        """"""

        # Dereference each reference
        for path, projection in references.items():

            # Check there is a $ref in the projection, else skip it
            if '$ref' not in projection:
                continue

            # Collect Ids of documents to dereference
            ids = set()
            for document in documents:
                value = cls._path_to_value(path, document)
                if not value:
                    continue

                if isinstance(value, list):
                    ids.update(value)

                elif isinstance(value, dict):
                    ids.update(value.values())

                else:
                    ids.add(value)

            # Find the referenced documents
            ref = projection.pop('$ref')
            frames = ref.many(
                {'_id': {'$in': list(ids)}},
                projection=projection
                )
            frames = {f._id: f for f in frames}

            # Add dereferenced frames to the document
            for document in documents:
                value = cls._path_to_value(path, document)
                if not value:
                    continue

                if isinstance(value, list):
                    # List of references
                    value = [frames[id] for id in value if id in frames]

                elif isinstance(value, dict):
                    # Dictionary of references
                    value = {key: frames.get(id) for key, id in value.items()}

                else:
                    value = frames.get(value, None)

                child_document = document
                keys = cls._path_to_keys(path)
                for key in keys[:-1]:
                    child_document = child_document[key]
                child_document[keys[-1]] = value
786880
def multipartite(corpus, featureset_names, min_weight=1, filters={}):
    

    pairs = Counter()
    node_type = {corpus._generate_index(p): {'type': 'paper'}
                 for p in corpus.papers}
    for featureset_name in featureset_names:
        ftypes = {}

        featureset = _get_featureset(corpus, featureset_name)
        for paper, feature in featureset.iteritems():
            if featureset_name in filters:
                if not filters[featureset_name](featureset, feature):
                    continue
            if len(feature) < 1:
                continue
            for f in list(zip(*feature))[0]:
                ftypes[f] = {'type': featureset_name}
                pairs[(paper, f)] += 1
        node_type.update(ftypes)

    return _generate_graph(nx.DiGraph, pairs, node_attrs=node_type,
                           min_weight=min_weight)
929131
def _items_to_rela_paths(self, items):
        """"""
        paths = []
        for item in items:
            if isinstance(item, (BaseIndexEntry, (Blob, Submodule))):
                paths.append(self._to_relative_path(item.path))
            elif isinstance(item, string_types):
                paths.append(self._to_relative_path(item))
            else:
                raise TypeError("Invalid item type: %r" % item)
        # END for each item
        return paths
1051311
def formatLij(Lij0,Ne):
	""""""
	#We create Lij as a matrix of lists of laser indices
	global Lij
	Lij=[]
	for i in range(Ne):
		fila=[]
		for j in range(Ne):
			band=False
			for tri in Lij0:
				if [i+1,j+1] == tri[:2]:
					band=True
					break
				elif [j+1,i+1] == tri[:2]:
					band=True
					break
			if band:
				fila+=[tri[2]]
			else:
				fila+=[[]]
		Lij+=[fila]
	return Lij
1009163
def _spec_to_globs(address_mapper, specs):
  """"""
  patterns = set()
  for spec in specs:
    patterns.update(spec.make_glob_patterns(address_mapper))
  return PathGlobs(include=patterns, exclude=address_mapper.build_ignore_patterns)
727212
def _get_grprobj(self, goids, sections):
        """"""
        grprdflt = GrouperDflts(self.gosubdag, "goslim_generic.obo")
        hdrobj = HdrgosSections(self.gosubdag, grprdflt.hdrgos_dflt, sections)
        return Grouper("sections", goids, hdrobj, self.gosubdag)
1330658
def _from_dict(cls, _dict):
        """"""
        args = {}
        if 'model_id' in _dict:
            args['model_id'] = _dict.get('model_id')
        if 'model_version' in _dict:
            args['model_version'] = _dict.get('model_version')
        if 'documents' in _dict:
            args['documents'] = [
                Document._from_dict(x) for x in (_dict.get('documents'))
            ]
        if 'aligned_elements' in _dict:
            args['aligned_elements'] = [
                AlignedElement._from_dict(x)
                for x in (_dict.get('aligned_elements'))
            ]
        if 'unaligned_elements' in _dict:
            args['unaligned_elements'] = [
                UnalignedElement._from_dict(x)
                for x in (_dict.get('unaligned_elements'))
            ]
        return cls(**args)
428958
def convert_examples_to_features(examples, tokenizer, max_seq_length,
                                 doc_stride, max_query_length, is_training):
    """"""

    unique_id = 1000000000

    features = []
    for (example_index, example) in enumerate(examples):
        query_tokens = tokenizer.tokenize(example.question_text)

        if len(query_tokens) > max_query_length:
            query_tokens = query_tokens[0:max_query_length]

        tok_to_orig_index = []
        orig_to_tok_index = []
        all_doc_tokens = []
        for (i, token) in enumerate(example.doc_tokens):
            orig_to_tok_index.append(len(all_doc_tokens))
            sub_tokens = tokenizer.tokenize(token)
            for sub_token in sub_tokens:
                tok_to_orig_index.append(i)
                all_doc_tokens.append(sub_token)

        tok_start_position = None
        tok_end_position = None
        if is_training and example.is_impossible:
            tok_start_position = -1
            tok_end_position = -1
        if is_training and not example.is_impossible:
            tok_start_position = orig_to_tok_index[example.start_position]
            if example.end_position < len(example.doc_tokens) - 1:
                tok_end_position = orig_to_tok_index[example.end_position + 1] - 1
            else:
                tok_end_position = len(all_doc_tokens) - 1
            (tok_start_position, tok_end_position) = _improve_answer_span(
                all_doc_tokens, tok_start_position, tok_end_position, tokenizer,
                example.orig_answer_text)

        # The -3 accounts for [CLS], [SEP] and [SEP]
        max_tokens_for_doc = max_seq_length - len(query_tokens) - 3

        # We can have documents that are longer than the maximum sequence length.
        # To deal with this we do a sliding window approach, where we take chunks
        # of the up to our max length with a stride of `doc_stride`.
        _DocSpan = collections.namedtuple(  # pylint: disable=invalid-name
            "DocSpan", ["start", "length"])
        doc_spans = []
        start_offset = 0
        while start_offset < len(all_doc_tokens):
            length = len(all_doc_tokens) - start_offset
            if length > max_tokens_for_doc:
                length = max_tokens_for_doc
            doc_spans.append(_DocSpan(start=start_offset, length=length))
            if start_offset + length == len(all_doc_tokens):
                break
            start_offset += min(length, doc_stride)

        for (doc_span_index, doc_span) in enumerate(doc_spans):
            tokens = []
            token_to_orig_map = {}
            token_is_max_context = {}
            segment_ids = []
            tokens.append("[CLS]")
            segment_ids.append(0)
            for token in query_tokens:
                tokens.append(token)
                segment_ids.append(0)
            tokens.append("[SEP]")
            segment_ids.append(0)

            for i in range(doc_span.length):
                split_token_index = doc_span.start + i
                token_to_orig_map[len(tokens)] = tok_to_orig_index[split_token_index]

                is_max_context = _check_is_max_context(doc_spans, doc_span_index,
                                                       split_token_index)
                token_is_max_context[len(tokens)] = is_max_context
                tokens.append(all_doc_tokens[split_token_index])
                segment_ids.append(1)
            tokens.append("[SEP]")
            segment_ids.append(1)

            input_ids = tokenizer.convert_tokens_to_ids(tokens)

            # The mask has 1 for real tokens and 0 for padding tokens. Only real
            # tokens are attended to.
            input_mask = [1] * len(input_ids)

            # Zero-pad up to the sequence length.
            while len(input_ids) < max_seq_length:
                input_ids.append(0)
                input_mask.append(0)
                segment_ids.append(0)

            assert len(input_ids) == max_seq_length
            assert len(input_mask) == max_seq_length
            assert len(segment_ids) == max_seq_length

            start_position = None
            end_position = None
            if is_training and not example.is_impossible:
                # For training, if our document chunk does not contain an annotation
                # we throw it out, since there is nothing to predict.
                doc_start = doc_span.start
                doc_end = doc_span.start + doc_span.length - 1
                out_of_span = False
                if not (tok_start_position >= doc_start and
                        tok_end_position <= doc_end):
                    out_of_span = True
                if out_of_span:
                    start_position = 0
                    end_position = 0
                else:
                    doc_offset = len(query_tokens) + 2
                    start_position = tok_start_position - doc_start + doc_offset
                    end_position = tok_end_position - doc_start + doc_offset
            if is_training and example.is_impossible:
                start_position = 0
                end_position = 0
            if example_index < 20:
                logger.info("*** Example ***")
                logger.info("unique_id: %s" % (unique_id))
                logger.info("example_index: %s" % (example_index))
                logger.info("doc_span_index: %s" % (doc_span_index))
                logger.info("tokens: %s" % " ".join(tokens))
                logger.info("token_to_orig_map: %s" % " ".join([
                    "%d:%d" % (x, y) for (x, y) in token_to_orig_map.items()]))
                logger.info("token_is_max_context: %s" % " ".join([
                    "%d:%s" % (x, y) for (x, y) in token_is_max_context.items()
                ]))
                logger.info("input_ids: %s" % " ".join([str(x) for x in input_ids]))
                logger.info(
                    "input_mask: %s" % " ".join([str(x) for x in input_mask]))
                logger.info(
                    "segment_ids: %s" % " ".join([str(x) for x in segment_ids]))
                if is_training and example.is_impossible:
                    logger.info("impossible example")
                if is_training and not example.is_impossible:
                    answer_text = " ".join(tokens[start_position:(end_position + 1)])
                    logger.info("start_position: %d" % (start_position))
                    logger.info("end_position: %d" % (end_position))
                    logger.info(
                        "answer: %s" % (answer_text))

            features.append(
                InputFeatures(
                    unique_id=unique_id,
                    example_index=example_index,
                    doc_span_index=doc_span_index,
                    tokens=tokens,
                    token_to_orig_map=token_to_orig_map,
                    token_is_max_context=token_is_max_context,
                    input_ids=input_ids,
                    input_mask=input_mask,
                    segment_ids=segment_ids,
                    start_position=start_position,
                    end_position=end_position,
                    is_impossible=example.is_impossible))
            unique_id += 1

    return features