Condense the data set to the distinct intervals based on the pseudo key.
770077
def condense(self):
        
        for pseudo_key, rows in self._rows.items():
            tmp1 = []
            intervals = sorted(self._derive_distinct_intervals(rows))
            for interval in intervals:
                tmp2 = dict(zip(self._pseudo_key, pseudo_key))
                tmp2[self._key_start_date] = interval[0]
                tmp2[self._key_end_date] = interval[1]
                tmp1.append(tmp2)

            self._rows[pseudo_key] = tmp1
253461
def _join_results(self, results, coarse, join):
    '''
    
    '''
    rval = OrderedDict()
    i_keys = set()
    for res in results:
      i_keys.update( res.keys() )
    for i_key in sorted(i_keys):
      if coarse:
        rval[i_key] = join( [res.get(i_key) for res in results] )
      else:
        rval[i_key] = OrderedDict()
        r_keys = set()
        for res in results:
          r_keys.update( res.get(i_key,{}).keys() )
        for r_key in sorted(r_keys):
          rval[i_key][r_key] = join( [res.get(i_key,{}).get(r_key) for res in results] )
    return rval
457659
def keys(self):
    """"""
    columns = set()
    for name in vars(self):
      if (not name) or name[0] == "_":
        continue
      columns.add(name)
    return columns
463639
def massage_keys(keys):
    """"""
    m_keys = []
    for key in keys:
        if key.startswith('keybase:'):
            m_keys.append(cryptorito.key_from_keybase(key[8:])['fingerprint'])
        else:
            m_keys.append(key)

    return m_keys
639013
def _find_key_cols(df):
    """"""
    
    keys = []
    for col in df:
        if len(df[col].unique()) == len(df[col]):
            keys.append(col)
    return keys
75828
def split_key(key):
    """"""
    if key == KEY_SEP:
        return ()
    key_chunks = tuple(key.strip(KEY_SEP).split(KEY_SEP))
    if key_chunks[0].startswith(KEY_SEP):
        return (key_chunks[0][len(KEY_SEP):],) + key_chunks[1:]
    else:
        return key_chunks
953007
def _merge_cwlinputs(items_by_key, input_order, parallel):
    
    items_by_key = _maybe_nest_bare_single(items_by_key, parallel)
    if parallel == "multi-combined":
        items_by_key, input_order = _concat_records(items_by_key, input_order)
    var_items = set([_item_count(items_by_key[tuple(k.split("__"))])
                     for (k, t) in input_order.items() if t == "var"])
    rec_items = set([_item_count(items_by_key[k]) for (k, t) in input_order.items() if t == "record"])
    if var_items:
        num_items = var_items
        if len(num_items) == 2 and 1 in num_items:
            num_items.remove(1)
            items_by_key_test = _check_for_single_nested(num_items.pop(), items_by_key, input_order)
            var_items = set([_item_count(items_by_key_test[tuple(k.split("__"))])
                             for (k, t) in input_order.items() if t == "var"])
            num_items = var_items
        assert len(num_items) == 1, "Non-consistent variable data counts in CWL input:\n%s" % \
            (pprint.pformat(items_by_key))
        items_by_key, num_items = _nest_vars_in_rec(var_items, rec_items, input_order, items_by_key, parallel)
    else:
        num_items = rec_items
        assert len(num_items) == 1, "Non-consistent record data counts in CWL input:\n%s" % \
            (pprint.pformat(items_by_key))
    target_items = num_items.pop()
    out = [{} for _ in range(target_items)]
    for (cwl_key, cwl_type) in input_order.items():
        if cwl_type == "var":
            cwl_key = tuple(cwl_key.split("__"))
        cur_vals = items_by_key[cwl_key]
        if _is_nested_single(cur_vals, target_items):
            cur_vals = [[x] for x in cur_vals[0]]
        for i, cur_val in enumerate(cur_vals):
            if isinstance(cwl_key, (list, tuple)):
                # nested batches with records
                if (parallel.startswith(("batch", "multi-parallel")) and
                      isinstance(out[i], (list, tuple))):
                    for j in range(len(out[i])):
                        out[i][j] = _update_nested(list(cwl_key), cur_val, out[i][j], allow_overwriting=True)
                else:
                    out[i] = _update_nested(list(cwl_key), cur_val, out[i], allow_overwriting=True)
            elif out[i] == {}:
                out[i] = cur_val
            else:
                # Handle single non-batched records
                if isinstance(cur_val, (list, tuple)) and len(cur_val) == 1:
                    cur_val = cur_val[0]
                assert isinstance(cur_val, dict), (cwl_key, cur_val)
                for k, v in cur_val.items():
                    out[i] = _update_nested([k], v, out[i], allow_overwriting=True)
    return out
1232636
def gen_multi_key(self, values, db='default'):
        """"""
        db = settings.DB_CACHE_KEYS[db]
        if db and len(db) > 100:
            db = db[0:68] + self.gen_key(db[68:])
        return '%s_%s_multi_%s' % (self.prefix, db, self.gen_key(*values))
432718
def __parseKeyValueStore(self, data):
        """"""
        offset = 0
        key_value_store = {}
        while offset != len(data):
            key = get_str(data, offset)
            offset += len(key)+1

            value = get_str(data, offset)
            offset += len(value)+1

            key_value_store[key] = value

        return key_value_store
1097091
def create_record_sets(self, record_set_dicts):
        """"""
        record_set_objects = []
        for record_set_dict in record_set_dicts:
            # pop removes the 'Enabled' key and tests if True.
            if record_set_dict.pop('Enabled', True):
                record_set_objects.append(
                    self.create_record_set(record_set_dict)
                )
        return record_set_objects
246181
def get_interval(note, interval, key='C'):
    
    intervals = map(lambda x: (notes.note_to_int(key) + x) % 12, [
        0,
        2,
        4,
        5,
        7,
        9,
        11,
        ])
    key_notes = keys.get_notes(key)
    for x in key_notes:
        if x[0] == note[0]:
            result = (intervals[key_notes.index(x)] + interval) % 12
    if result in intervals:
        return key_notes[intervals.index(result)] + note[1:]
    else:
        return notes.diminish(key_notes[intervals.index((result + 1) % 12)]
                               + note[1:])
558096
def _compare_rows(from_recs, to_recs, keys):
    ""
    return set(
        k for k in keys
        if sorted(from_recs[k].items()) != sorted(to_recs[k].items())
    )
299188
def distinct_values(t_old, t_new):   
    
    res = []
    res.append([' -- NOT IN check -- '])
    for new_col in t_new.header:
        dist_new = t_new.get_distinct_values_from_cols([new_col])
        #print('NEW Distinct values for ' + new_col + ' = ' + str(dist_new))
        for old_col in t_old.header:
            if old_col == new_col:
                dist_old = t_old.get_distinct_values_from_cols([old_col])
                #print('OLD Distinct values for ' + old_col + ' = ' + str(dist_old))
                
                # Now compare the old and new values to see what is different
                not_in_new = [x for x in dist_old[0] if x not in dist_new[0]]
                if not_in_new != []:
                    #print(old_col + ' not_in_new = ' , not_in_new)
                    res.append(['Not in New', old_col, not_in_new])
                    
                not_in_old = [x for x in dist_new[0] if x not in dist_old[0]]
                if not_in_old != []:
                    #print(new_col + ' not_in_old = ' , not_in_old)
                    res.append(['Not in Old', new_col, not_in_old])
                    
                    
    
    return sorted(res)
1031455
def cache_keys(keys):
    """"""
    d = known_keys()
    known_names = dict(zip(d.values(), d.keys()))
    for k in keys:
        i = (ord(k),) if len(k) == 1 else known_names[k]
        _key_cache.insert(0, i)
917321
def fast_distinct(self):
        
        return self.model.objects.filter(pk__in=self.values_list('pk', flat=True))
1248813
def flatten(d, sep='.'):
    """"""

    def _flatten(e, parent_key='', sep='.'):
        import collections

        prefix = parent_key + sep if parent_key else ''

        if isinstance(e, collections.MutableMapping):
            return tuple((prefix + k2, v2) for k, v in e.items() for k2, v2 in _flatten(v, k, sep))
        elif isinstance(e, collections.MutableSequence):
            return tuple((prefix + k2, v2) for i, v in enumerate(e) for k2, v2 in _flatten(v, str(i), sep))
        else:
            return (parent_key, (e,)),

    return tuple((k, v[0]) for k, v in _flatten(d, '', sep))
404270
def cast(keys, data):
        """"""
        matrix = Matrix()
        matrix.keys = keys
        matrix.data = data
        return matrix