Generate ColumnDataSource dictionary for segment separation lines.
344228
def _get_xmarks_data(self, order_seg, bins_seg):
        

        if not self.xmarks:
            return dict(xs=[], ys=[])

        angles = self._get_markers(self.xmarks, order_seg, bins_seg)

        inner = self.max_radius * self.radius_inner
        outer = self.max_radius

        y_start = np.sin(angles) * inner + self.max_radius
        y_end = np.sin(angles) * outer + self.max_radius

        x_start = np.cos(angles) * inner + self.max_radius
        x_end = np.cos(angles) * outer + self.max_radius

        xs = zip(x_start, x_end)
        ys = zip(y_start, y_end)

        return dict(xs=list(xs), ys=list(ys))
75691
def process_row(cls, data, column_map):
        """"""
        row = {}
        for key,value in data.iteritems():
            if not value:
                value = '-'                
            elif isinstance(value, list):
                value = value[1]
            elif isinstance(value, dict):
                if 'type_name' in value:
                    if 'UnixTimeStamp' in value['type_name']:
                        value = datetime.datetime.utcfromtimestamp(value['epoch'])
                        if value == datetime.datetime(1970, 1, 1, 0, 0):
                            value = '-'

            # Assume the value is somehow well formed when we get here
            row[column_map[key]] = value
        return row
1078323
def datasetHeaderChunk(key, lines):
    
    KEYWORDS = ('DATASET',
                'OBJTYPE',
                'VECTYPE',
                'BEGSCL',
                'BEGVEC',
                'OBJID',
                'ND',
                'NC',
                'NAME')

    TYPE_KEYS = ('BEGSCL', 'BEGVEC')

    result = {'type': None,
              'numberData': None,
              'numberCells': None,
              'name': None,
              'objectID': None,
              'objectType': None,
              'vectorType': None}

    chunks = pt.chunk(KEYWORDS, lines)

    for key, chunkList in iteritems(chunks):

        for chunk in chunkList:
            schunk = pt.splitLine(chunk[0])

            if key == 'ND':
                result['numberData'] = int(schunk[1])

            elif key == 'NC':
                result['numberCells'] = int(schunk[1])

            elif key == 'NAME':
                result['name'] = schunk[1]

            elif key == 'OBJID':
                result['objectID'] = int(schunk[1])

            elif key == 'OBJTYPE':
                result['objectType'] = schunk[1]

            elif key == 'VECTYPE':
                result['vectorType'] = schunk[1]

            elif key in TYPE_KEYS:
                result['type'] = schunk[0]

    return result
919838
def generate_sections(self):
        """"""
        sources = Dataset.objects.values(
            'source', 'source_slug'
        ).annotate(source_count=Count('source_slug'))

        return sorted([
            {
                'slug': source['source_slug'],
                'name': source['source'],
                'count': source['source_count']
            }
            for source in sources
        ], key=lambda k: k['count'], reverse=True)
307834
def mk_dict(results,description):
    """"""
    
    rows=[]
    for row in results:
        row_dict={}
        for idx in range(len(row)):
            col=description[idx][0]
            row_dict[col]=row[idx]
        rows.append(row_dict)
    return rows
422499
def DictKeyColumns(d):
    ''
    return [ColumnItem(k, k, type=deduceType(d[k])) for k in d.keys()]
1289677
def from_columns_dict(cls, columns):
        """"""
        warnings.warn("Table.from_columns_dict is deprecated. Use Table().with_columns(...)", FutureWarning)
        return cls().with_columns(columns.items())
435581
def dataset_splits(self):
    """"""
    return [{
        "split": problem.DatasetSplit.TRAIN,
        "shards": _TRAIN_SHARDS,
    }, {
        "split": problem.DatasetSplit.EVAL,
        "shards": _DEV_SHARDS,
    }]
1046511
def _read_segment_lines(segment_lines):
    
    # Dictionary for segment fields
    segment_fields = {}

    # Each dictionary field is a list
    for field in SEGMENT_SPECS.index:
        segment_fields[field] = [None] * len(segment_lines)

    # Read string fields from signal line
    for i in range(len(segment_lines)):
        (segment_fields['seg_name'][i], segment_fields['seg_len'][i]) = _rx_segment.findall(segment_lines[i])[0]

        # Typecast strings for numerical field
        if field == 'seg_len':
            segment_fields['seg_len'][i] = int(segment_fields['seg_len'][i])

    return segment_fields
903845
def extract_columns(data, *cols):
    
    out = []
    try:
        # for python 2.2:
        # return [ [r[c] for c in cols] for r in data]
        for r in data:
            col = []
            for c in cols:
                col.append(r[c])
            out.append(col)
    except IndexError:
        raise IndexError("data=%s col=%s" % (data, col))
    return out
977653
def columns_dataset(self):
        
        data = {}
        words_total = {}

        for instance, words in self.raw_dataset.items():
            words_item_total = {}
            for word in words:
                words_total.setdefault(word, 0)
                words_item_total.setdefault(word, 0)
                words_total[word] += 1
                words_item_total[word] += 1
            data[instance] = words_item_total

        columns = sorted(words_total.keys(),
                         key=lambda w: words_total[w],
                         reverse=True)[:250]
        columns = sorted(columns)
        dataset = {}
        for instance in data.keys():
            dataset[instance] = [data[instance].get(word, 0)
                                 for word in columns]
        return columns, dataset
1114763
private void splitOn(String... columnNames) {

        List<Column<?>> columns = getSourceTable().columns(columnNames);
        int byteSize = getByteSize(columns);

        byte[] currentKey = null;
        String currentStringKey = null;
        TableSlice view;

        Selection selection = new BitmapBackedSelection();

        for (int row = 0; row < getSourceTable().rowCount(); row++) {

            ByteBuffer byteBuffer = ByteBuffer.allocate(byteSize);
            String newStringKey = "";

            for (int col = 0; col < columnNames.length; col++) {
                if (col > 0) {
                    newStringKey = newStringKey + SPLIT_STRING;
                }

                Column<?> c = getSourceTable().column(columnNames[col]);
                String groupKey = getSourceTable().getUnformatted(row, getSourceTable().columnIndex(c));
                newStringKey = newStringKey + groupKey;
                byteBuffer.put(c.asBytes(row));
            }
            byte[] newKey = byteBuffer.array();
            if (row == 0) {
                currentKey = newKey;
                currentStringKey = newStringKey;
            }
            if (!Arrays.equals(newKey, currentKey)) {
                currentKey = newKey;
                view = new TableSlice(getSourceTable(), selection);
                view.setName(currentStringKey);
                currentStringKey = newStringKey;
                addSlice(view);
                selection = new BitmapBackedSelection();
                selection.add(row);
            } else {
                selection.add(row);
            }
        }
        if (!selection.isEmpty()) {
            view = new TableSlice(getSourceTable(), selection);
            view.setName(currentStringKey);
            addSlice(view);
        }
    }
88586
def normalize_mapping_line(mapping_line, previous_source_column=0):
    

    if not mapping_line:
        return [], previous_source_column

    # Note that while the local record here is also done as a 4-tuple,
    # element 1 and 2 are never used since they are always provided by
    # the segments in the mapping line; they are defined for consistency
    # reasons.

    def regenerate(segment):
        if len(segment) == 5:
            result = (record[0], segment[1], segment[2], record[3], segment[4])
        else:
            result = (record[0], segment[1], segment[2], record[3])
        # Ideally the exact location should still be kept, but given
        # that the sourcemap format is accumulative and permits a lot
        # of inferred positions, resetting all values to 0 is intended.
        record[:] = [0, 0, 0, 0]
        return result

    # first element of the line; sink column (0th element) is always
    # the absolute value, so always use the provided value sourced from
    # the original mapping_line; the source column (3rd element) is
    # never reset, so if a previous counter exists (which is specified
    # by the optional argument), make use of it to generate the initial
    # normalized segment.
    record = [0, 0, 0, previous_source_column]
    result = []
    regen_next = True

    for segment in mapping_line:
        if not segment:
            # ignore empty records
            continue
        # if the line has not changed, and that the increases of both
        # columns are the same, accumulate the column counter and drop
        # the segment.

        # accumulate the current record first
        record[0] += segment[0]
        if len(segment) == 1:
            # Mark the termination, as 1-tuple determines the end of the
            # previous symbol and denote that whatever follows are not
            # in any previous source files.  So if it isn't recorded,
            # make note of this if it wasn't done already.
            if result and len(result[-1]) != 1:
                result.append((record[0],))
                record[0] = 0
                # the next complete segment will require regeneration
                regen_next = True
            # skip the remaining processing.
            continue

        record[3] += segment[3]

        # 5-tuples are always special case with the remapped identifier
        # name element, and to mark the termination the next token must
        # also be explicitly written (in our case, regenerated).  If the
        # filename or source line relative position changed (idx 1 and
        # 2), regenerate it too.  Finally, if the column offsets differ
        # between source and sink, regenerate.
        if len(segment) == 5 or regen_next or segment[1] or segment[2] or (
                record[0] != record[3]):
            result.append(regenerate(segment))
            regen_next = len(segment) == 5

    # must return the consumed/omitted values.
    return result, record[3]
1131047
def _parse_columns(self):
    
        '''  '''
        
    # construct column list
        column_map = {}
        for key, value in self.model.keyMap.items():
            record_key = key[1:]
            if record_key:
                if self.item_key.findall(record_key):
                    pass
                else:
                    if value['value_datatype'] == 'map':
                        continue
                    datatype = value['value_datatype']
                    if value['value_datatype'] == 'number':
                        datatype = 'float'
                        if 'integer_data' in value.keys():
                            if value['integer_data']:
                                datatype = 'integer'
                    replace_key = ''
                    if 'field_metadata' in value.keys():
                        if 'replace_key' in value['field_metadata'].keys():
                            if isinstance(value['field_metadata']['replace_key'], str):
                                replace_key = value['field_metadata']['replace_key']
                    max_length = None
                    if 'max_length' in value.keys():
                        max_length = value['max_length']
                    column_map[record_key] = (record_key, datatype, replace_key, max_length)
        
        return column_map
892778
def extract_col(self, col):
        
        new_col = [row[col] for row in self.grid]
        return new_col
936264
def _construct_auto_distance(feature_names, column_names, column_types, sample):
    

    ## Make a dictionary from the column_names and column_types
    col_type_dict = {k: v for k, v in zip(column_names, column_types)}

    ## Loop through feature names, appending a distance component if the
    #  feature's type is *not* numeric. If the type *is* numeric, append it to
    #  the numeric_cols list, then at the end make a numeric columns distance
    #  component.
    composite_distance_params = []
    numeric_cols = []

    for c in feature_names:
        if col_type_dict[c] == str:
            composite_distance_params.append([[c], _turicreate.distances.levenshtein, 1])
        elif col_type_dict[c] == dict:
            composite_distance_params.append([[c], _turicreate.distances.jaccard, 1])
        elif col_type_dict[c] == array.array:
            composite_distance_params.append([[c], _turicreate.distances.euclidean, 1])
        elif col_type_dict[c] == list:
            only_str_lists = _validate_lists(sample[c], allowed_types=[str])
            if not only_str_lists:
                raise TypeError("Only lists of all str objects are currently supported")
            composite_distance_params.append([[c], _turicreate.distances.jaccard, 1])
        elif col_type_dict[c] in [int, float, array.array, list]:
            numeric_cols.append(c)
        else:
            raise TypeError("Unable to automatically determine a distance "+\
                "for column {}".format(c))

    # Make the standalone numeric column distance component
    if len(numeric_cols) > 0:
        composite_distance_params.append([numeric_cols, _turicreate.distances.euclidean, 1])

    return composite_distance_params
13861
def _split_lines(self):
        ''''''
        parsed_lines = {}
        for rt in all_record_types:
            parsed_lines[rt] = []
        parsed_lines[0] = []

        for line in self.lines:
            linetype = line[0:6]
            if linetype in all_record_types:
                parsed_lines[linetype].append(line)
            else:
                parsed_lines[0].append(line)

        self.parsed_lines = parsed_lines
        self._update_structure_lines()