Shuffle X and Y into n / len(paths) datasets, and save them
    to disk at the locations provided in paths.
512534
def split_and_save_datasets(X, Y, paths):
    
    shuffled_idxs = np.random.permutation(np.arange(len(X)))

    for i in range(len(paths)):
        # Take every len(paths) item, starting at i.
        # len(paths) is 3, so this would be [0::3], [1::3], [2::3]
        X_i = X[shuffled_idxs[i::len(paths)]]
        Y_i = Y[shuffled_idxs[i::len(paths)]]
        np.savez(paths[i], X=X_i, Y=Y_i)
504382
def _shuffle_data(self):
        """"""
        # shuffle index
        np.random.shuffle(self.idx)
        # get the data by corresponding index
        self.data = _getdata_by_idx(self.data, self.idx)
        self.label = _getdata_by_idx(self.label, self.idx)
1342337
def _zip_flatten(x, ys):
    '''
    
    '''
    return itertools.chain.from_iterable(
            zip(itertools.repeat(x), ys))
648249
def build_dataset(filename, max_lines=-1):
    """"""
    encodings = dict(list(map(reversed, enumerate(string.printable))))
    digitize = lambda char: encodings[char] if char in encodings else len(encodings)
    encode_line = lambda line: np.array(list(map(digitize, line)))
    nonblank_line = lambda line: len(line) > 2

    with open(filename) as f:
        lines = f.readlines()

    encoded_lines = list(map(encode_line, list(filter(nonblank_line, lines))[:max_lines]))
    num_outputs = len(encodings) + 1

    return encoded_lines, num_outputs
578253
def _generate_examples(self, filepath):
    
    # Simultaneously iterating through the different data sets in the hdf5
    # file is >100x slower and the data set is small (26.7MB). Hence, we first
    # load everything into memory before yielding the samples.
    image_array, class_array, values_array = _load_data(filepath)
    for image, classes, values in moves.zip(image_array, class_array,
                                            values_array):
      yield dict(
          image=np.expand_dims(image, -1),
          label_shape=classes[1],
          label_scale=classes[2],
          label_orientation=classes[3],
          label_x_position=classes[4],
          label_y_position=classes[5],
          value_shape=values[1],
          value_scale=values[2],
          value_orientation=values[3],
          value_x_position=values[4],
          value_y_position=values[5])
1286730
def downsample(f1, f2, data, N, quick=False):
    
    if quick:
        rand_records = range(N)
    else:
        records = sum(1 for _ in open(f1)) / 4
        N = records if N > records else N
        rand_records = sorted(random.sample(xrange(records), N))

    fh1 = open_possible_gzip(f1)
    fh2 = open_possible_gzip(f2) if f2 else None
    outf1 = os.path.splitext(f1)[0] + ".subset" + os.path.splitext(f1)[1]
    outf2 = os.path.splitext(f2)[0] + ".subset" + os.path.splitext(f2)[1] if f2 else None

    if utils.file_exists(outf1):
        if not outf2:
            return outf1, outf2
        elif utils.file_exists(outf2):
            return outf1, outf2

    out_files = (outf1, outf2) if outf2 else (outf1)

    with file_transaction(out_files) as tx_out_files:
        if isinstance(tx_out_files, six.string_types):
            tx_out_f1 = tx_out_files
        else:
            tx_out_f1, tx_out_f2 = tx_out_files
        sub1 = open_possible_gzip(tx_out_f1, "w")
        sub2 = open_possible_gzip(tx_out_f2, "w") if outf2 else None
        rec_no = - 1
        for rr in rand_records:
            while rec_no < rr:
                rec_no += 1
                for i in range(4): fh1.readline()
                if fh2:
                    for i in range(4): fh2.readline()
            for i in range(4):
                sub1.write(fh1.readline())
                if sub2:
                    sub2.write(fh2.readline())
            rec_no += 1
        fh1.close()
        sub1.close()
        if f2:
            fh2.close()
            sub2.close()

    return outf1, outf2
820177
def to_netcdf(dataset, path_or_file=None, mode='w', format=None, group=None,
              engine=None, encoding=None, unlimited_dims=None, compute=True,
              multifile=False):
    
    if isinstance(path_or_file, Path):
        path_or_file = str(path_or_file)

    if encoding is None:
        encoding = {}

    if path_or_file is None:
        if engine is None:
            engine = 'scipy'
        elif engine != 'scipy':
            raise ValueError('invalid engine for creating bytes with '
                             'to_netcdf: %r. Only the default engine '
                             "or engine='scipy' is supported" % engine)
        if not compute:
            raise NotImplementedError(
                'to_netcdf() with compute=False is not yet implemented when '
                'returning bytes')
    elif isinstance(path_or_file, str):
        if engine is None:
            engine = _get_default_engine(path_or_file)
        path_or_file = _normalize_path(path_or_file)
    else:  # file-like object
        engine = 'scipy'

    # validate Dataset keys, DataArray names, and attr keys/values
    _validate_dataset_names(dataset)
    _validate_attrs(dataset)

    try:
        store_open = WRITEABLE_STORES[engine]
    except KeyError:
        raise ValueError('unrecognized engine for to_netcdf: %r' % engine)

    if format is not None:
        format = format.upper()

    # handle scheduler specific logic
    scheduler = _get_scheduler()
    have_chunks = any(v.chunks for v in dataset.variables.values())

    autoclose = have_chunks and scheduler in ['distributed', 'multiprocessing']
    if autoclose and engine == 'scipy':
        raise NotImplementedError("Writing netCDF files with the %s backend "
                                  "is not currently supported with dask's %s "
                                  "scheduler" % (engine, scheduler))

    target = path_or_file if path_or_file is not None else BytesIO()
    kwargs = dict(autoclose=True) if autoclose else {}
    store = store_open(target, mode, format, group, **kwargs)

    if unlimited_dims is None:
        unlimited_dims = dataset.encoding.get('unlimited_dims', None)
    if isinstance(unlimited_dims, str):
        unlimited_dims = [unlimited_dims]

    writer = ArrayWriter()

    # TODO: figure out how to refactor this logic (here and in save_mfdataset)
    # to avoid this mess of conditionals
    try:
        # TODO: allow this work (setting up the file for writing array data)
        # to be parallelized with dask
        dump_to_store(dataset, store, writer, encoding=encoding,
                      unlimited_dims=unlimited_dims)
        if autoclose:
            store.close()

        if multifile:
            return writer, store

        writes = writer.sync(compute=compute)

        if path_or_file is None:
            store.sync()
            return target.getvalue()
    finally:
        if not multifile and compute:
            store.close()

    if not compute:
        import dask
        return dask.delayed(_finalize_store)(writes, store)
207941
def shuffle_and_batch_data(dataset,
                           target_names,
                           features_info,
                           training,
                           num_devices,
                           shuffle_buffer_size=1024,
                           preprocess_fun=no_preprocess):
  """"""
  def append_targets(example):
    """Append targets to the example dictionary. Needed for Keras."""
    if len(target_names) == 1:
      return (example, example[target_names[0]])
    targets = {}
    for name in target_names:
      targets[name] = example[name]
    return (example, targets)
  dataset = dataset.map(append_targets)
  if training:
    dataset = dataset.repeat()
    # Skip a random fraction at the beginning of the stream.  The skip is
    # essential for synchronous highly-parallel training to avoid multiple
    # replicas reading the same data in lock-step.
    dataset = dataset.skip(random.randint(0, _MAX_SKIP_EXAMPLES))
  dataset = preprocess_fun(dataset, training)
  shapes = {k: features_info[k].shape for k in features_info}
  shapes = (shapes, shapes[target_names[0]])
  dataset = dataset.shuffle(shuffle_buffer_size)
  dataset = batch_fun(dataset, training, shapes, target_names, num_devices)
  return dataset.prefetch(2)
830431
def split_tracks(lat,lon,*args):
        ''''''
        tracks = []
        lt,ln = [lat[0]],[lon[0]]
        zz = [[z[0]] for z in args]
        for i in range(1,len(lon)):
            lt.append(lat[i])
            for z,a in zip(zz,args):
                z.append(a[i])
            d1 = abs(lon[i] - lon[i-1])
            d2 = abs((lon[i-1] + 360) - lon[i])
            d3 = abs(lon[i-1] - (lon[i] + 360))
            if d2 < d1:
                ln.append(lon[i]-360)
                tracks.append([np.array(lt),np.array(ln)] \
                    + [np.array(z) for z in zz])
                lt = [lat[i-1],lat[i]]
                ln = [lon[i-1]+360,lon[i]]
                zz = [[z[i-1]] for z in args]
            elif d3 < d1:
                ln.append(lon[i]+360)
                tracks.append([np.array(lt),np.array(ln)] \
                    + [np.array(z) for z in zz])
                lt = [lat[i-1],lat[i]]
                ln = [lon[i-1]-360,lon[i]]
                zz = [[z[i-1],z[i]] for z in args]
            else:
                ln.append(lon[i])
        if len(lt):
            tracks.append([np.array(lt),np.array(ln)] \
                + [np.array(z) for z in zz])
        return tracks
406271
def gather_data(mypath, key="width"):
    
    filenames = [join(mypath, f) for f in listdir(mypath) if isfile(join(mypath, f))]
    filenames = [f for f in filenames if f.endswith(".p")] 
    W = []
    for filename in filenames:
        with open(filename, 'r') as f:
            data = pickle.load(f)
            W.append(np.expand_dims(data[key], axis=-1) )

    W = np.concatenate(W, axis=-1)
    return W
938559
def randompaths(request, num_paths=1, size=250, mu=0, sigma=1):
    ''''''
    r = []
    for p in range(num_paths):
        v = 0
        path = [v]
        r.append(path)
        for t in range(size):
            v += normalvariate(mu, sigma)
            path.append(v)
    return r
779530
def writer(datasets, dataset_names, output_dir):
    """"""
    for dataset, dataset_name in zip(datasets, dataset_names):
        if dataset:
            filepath = output_dir + '/' + dataset_name + '.txt'
            with open(filepath, 'w+') as out_file:
                joined = '\n'.join(dataset)
                out_file.write(str(joined.encode('utf-8').decode('utf-8')))
                out_file.write('\n')
946734
def to_dataset_files(self, local_path=None, remote_path=None):
        
        return FrameToGMQL.to_dataset_files(self, path_local=local_path, path_remote=remote_path)
573103
def _calc_concat_over(datasets, dim, data_vars, coords):
    
    # Return values
    concat_over = set()
    equals = {}

    if dim in datasets[0]:
        concat_over.add(dim)
    for ds in datasets:
        concat_over.update(k for k, v in ds.variables.items()
                           if dim in v.dims)

    def process_subset_opt(opt, subset):
        if isinstance(opt, str):
            if opt == 'different':
                # all nonindexes that are not the same in each dataset
                for k in getattr(datasets[0], subset):
                    if k not in concat_over:
                        # Compare the variable of all datasets vs. the one
                        # of the first dataset. Perform the minimum amount of
                        # loads in order to avoid multiple loads from disk
                        # while keeping the RAM footprint low.
                        v_lhs = datasets[0].variables[k].load()
                        # We'll need to know later on if variables are equal.
                        computed = []
                        for ds_rhs in datasets[1:]:
                            v_rhs = ds_rhs.variables[k].compute()
                            computed.append(v_rhs)
                            if not v_lhs.equals(v_rhs):
                                concat_over.add(k)
                                equals[k] = False
                                # computed variables are not to be re-computed
                                # again in the future
                                for ds, v in zip(datasets[1:], computed):
                                    ds.variables[k].data = v.data
                                break
                        else:
                            equals[k] = True

            elif opt == 'all':
                concat_over.update(set(getattr(datasets[0], subset)) -
                                   set(datasets[0].dims))
            elif opt == 'minimal':
                pass
            else:
                raise ValueError("unexpected value for %s: %s" % (subset, opt))
        else:
            invalid_vars = [k for k in opt
                            if k not in getattr(datasets[0], subset)]
            if invalid_vars:
                if subset == 'coords':
                    raise ValueError(
                        'some variables in coords are not coordinates on '
                        'the first dataset: %s' % (invalid_vars,))
                else:
                    raise ValueError(
                        'some variables in data_vars are not data variables '
                        'on the first dataset: %s' % (invalid_vars,))
            concat_over.update(opt)

    process_subset_opt(data_vars, 'data_vars')
    process_subset_opt(coords, 'coords')
    return concat_over, equals
1291822
function mirrorPaths(paths, xScale, yScale) {

            var bound = computeBoundingBox.computePath.apply(null, paths);
            var x = bound.x;
            var y = bound.y;
            var w = bound.width;
            var h = bound.height;

            if (xScale === -1) {
                paths.forEach(function (p) {
                    pathAdjust(p, -1, 1, -x, 0);
                    pathAdjust(p, 1, 1, x + w, 0);
                    p.reverse();
                });

            }

            if (yScale === -1) {
                paths.forEach(function (p) {
                    pathAdjust(p, 1, -1, 0, -y);
                    pathAdjust(p, 1, 1, 0, y + h);
                    p.reverse();
                });
            }

            return paths;
        }
1176360
def genslices_ndim(ndim, shape):
    """"""
    iterables = [genslices(shape[n]) for n in range(ndim)]
    yield from product(*iterables)
162352
def create_preprocessed_dataset(path_to_data, outputpath, preprocessing_queue):
    """"""
    # Log everything
    logging.info("Data soure %s", path_to_data)
    logging.info("Output will be stored in %s", outputpath)
    tmp = "Preprocessing Queue:\n"
    for preprocessing_class in preprocessing_queue:
        tmp += str(preprocessing_class) + "\n"
    logging.info(tmp)
    # Load from pickled file
    if not os.path.isfile(path_to_data):
        logging.info(("'%s' does not exist. Please either abort this script "
                      "or update the data location."), path_to_data)
        raw_dataset_path = utils.choose_raw_dataset()
        # Get project-relative path
        raw_dataset_path = "raw-datasets" + \
                           raw_dataset_path.split("raw-datasets")[1]
        print(raw_dataset_path)
        sys.exit()  # TODO: Update model!
    logging.info("Start loading data...")
    loaded = pickle.load(open(path_to_data, "rb"))
    raw_datasets = loaded['handwriting_datasets']
    logging.info("Start applying preprocessing methods")
    start_time = time.time()
    for i, raw_dataset in enumerate(raw_datasets):
        if i % 10 == 0 and i > 0:
            utils.print_status(len(raw_datasets), i, start_time)
        # Do the work
        raw_dataset['handwriting'].preprocessing(preprocessing_queue)
    sys.stdout.write("\r%0.2f%% (done)\033[K\n" % (100))
    print("")
    pickle.dump({'handwriting_datasets': raw_datasets,
                 'formula_id2latex': loaded['formula_id2latex'],
                 'preprocessing_queue': preprocessing_queue},
                open(outputpath, "wb"),
                2)