Pulls all available data and concatenates for model training

        :return: 2d array of points
770810
def get_traindata(self) -> np.ndarray:
        
        traindata = None
        for key, value in self.data.items():
            if key not in ['__header__', '__version__', '__globals__']:
                if traindata is None:
                    traindata = value[np.where(value[:, 4] != 0)]
                else:
                    traindata = np.concatenate((traindata, value[np.where(value[:, 4] != 0)]))
        return traindata
758824
def get_normed_points(point_array, norm): # good to go
    
    norm = float(norm)
    #floated_array = []
    #for p in point_array: # need to make sure each point is a float
        #floated_array.append(float(p))
    points = old_div(numpy.array(point_array), norm)
    return points
177642
def _load_multipoint(tokens, string):
    
    open_paren = next(tokens)
    if not open_paren == '(':
        raise ValueError(INVALID_WKT_FMT % string)

    coords = []
    pt = []

    paren_depth = 1
    try:
        for t in tokens:
            if t == '(':
                paren_depth += 1
            elif t == ')':
                paren_depth -= 1
                if paren_depth == 0:
                    break
            elif t == '':
                pass
            elif t == ',':
                # the point is done
                coords.append(pt)
                pt = []
            else:
                pt.append(float(t))
    except tokenize.TokenError:
        raise ValueError(INVALID_WKT_FMT % string)

    # Given the way we're parsing, we'll probably have to deal with the last
    # point after the loop
    if len(pt) > 0:
        coords.append(pt)

    return dict(type='MultiPoint', coordinates=coords)
1124049
def toArray(self):
        
        arr = np.zeros((self.size,), dtype=np.float64)
        arr[self.indices] = self.values
        return arr
1006269
def train(self, data_iterator):
        
        optimizer = get_optimizer(self.master_optimizer)
        self.model = model_from_yaml(self.yaml, self.custom_objects)
        self.model.compile(optimizer=optimizer,
                           loss=self.master_loss, metrics=self.master_metrics)
        self.model.set_weights(self.parameters.value)

        feature_iterator, label_iterator = tee(data_iterator, 2)
        x_train = np.asarray([x for x, y in feature_iterator])
        y_train = np.asarray([y for x, y in label_iterator])

        self.model.compile(optimizer=self.master_optimizer,
                           loss=self.master_loss,
                           metrics=self.master_metrics)

        weights_before_training = self.model.get_weights()
        if x_train.shape[0] > self.train_config.get('batch_size'):
            self.model.fit(x_train, y_train, **self.train_config)
        weights_after_training = self.model.get_weights()
        deltas = subtract_params(
            weights_before_training, weights_after_training)
        yield deltas
147180
def train(ds, ii):
    
    print("Loading model")
    m = model.CannonModel(2)
    print("Training...")
    m.fit(ds)
    np.savez("./ex%s_coeffs.npz" %ii, m.coeffs)
    np.savez("./ex%s_scatters.npz" %ii, m.scatters)
    np.savez("./ex%s_chisqs.npz" %ii, m.chisqs)
    np.savez("./ex%s_pivots.npz" %ii, m.pivots)
    fig = m.diagnostics_leading_coeffs(ds)
    plt.savefig("ex%s_leading_coeffs.png" %ii)
    # m.diagnostics_leading_coeffs_triangle(ds)
    # m.diagnostics_plot_chisq(ds)
    return m
711796
def download_sparse_points():
    """"""
    saved_file, _ = _download_file('sparsePoints.txt')
    points_reader = vtk.vtkDelimitedTextReader()
    points_reader.SetFileName(saved_file)
    points_reader.DetectNumericColumnsOn()
    points_reader.SetFieldDelimiterCharacters('\t')
    points_reader.SetHaveHeaders(True)
    table_points = vtk.vtkTableToPolyData()
    table_points.SetInputConnection(points_reader.GetOutputPort())
    table_points.SetXColumn('x')
    table_points.SetYColumn('y')
    table_points.SetZColumn('z')
    table_points.Update()
    return vtki.wrap(table_points.GetOutput())
130487
def get_mnist():
    

    np.random.seed(1234) # set seed for deterministic ordering
    mnist_data = mx.test_utils.get_mnist()
    X = np.concatenate([mnist_data['train_data'], mnist_data['test_data']])
    Y = np.concatenate([mnist_data['train_label'], mnist_data['test_label']])
    p = np.random.permutation(X.shape[0])
    X = X[p].reshape((X.shape[0], -1)).astype(np.float32)*5
    Y = Y[p]
    return X, Y
215422
def sel_points(self, dim='points', method=None, tolerance=None,
                   **indexers):
        
        ds = self._to_temp_dataset().sel_points(
            dim=dim, method=method, tolerance=tolerance, **indexers)
        return self._from_temp_dataset(ds)
236702
def data(self) -> Sequence[Tuple[float, float]]:
        
        return [(angle, prob) for angle, prob in zip(self._rabi_angles,
                                                     self._excited_state_probs)]
266350
def init_all_objects(self, data, target=None, single_result=True):
        
        if single_result:
            return self.init_target_object(target, data)
        return list(self.expand_models(target, data))
354045
def data():
    
    from keras.datasets import mnist
    from keras.utils import np_utils
    (x_train, y_train), (x_test, y_test) = mnist.load_data()
    x_train = x_train.reshape(60000, 784)
    x_test = x_test.reshape(10000, 784)
    x_train = x_train.astype('float32')
    x_test = x_test.astype('float32')
    x_train /= 255
    x_test /= 255
    nb_classes = 10
    y_train = np_utils.to_categorical(y_train, nb_classes)
    y_test = np_utils.to_categorical(y_test, nb_classes)
    return x_train, y_train, x_test, y_test
1356413
def df(self, list_of_points, force_read=True):
        

        his = []
        for point in list_of_points:
            try:
                his.append(self._findPoint(point, force_read=force_read).history)
            except ValueError as ve:
                self._log.error("{}".format(ve))
                continue
        if not _PANDAS:
            return dict(zip(list_of_points, his))
        return pd.DataFrame(dict(zip(list_of_points, his)))
100378
def recoSurface(points, bins=256):
    

    if isinstance(points, vtk.vtkActor):
        points = points.coordinates()
    N = len(points)
    if N < 50:
        print("recoSurface: Use at least 50 points.")
        return None
    points = np.array(points)

    ptsSource = vtk.vtkPointSource()
    ptsSource.SetNumberOfPoints(N)
    ptsSource.Update()
    vpts = ptsSource.GetOutput().GetPoints()
    for i, p in enumerate(points):
        vpts.SetPoint(i, p)
    polyData = ptsSource.GetOutput()

    distance = vtk.vtkSignedDistance()
    f = 0.1
    x0, x1, y0, y1, z0, z1 = polyData.GetBounds()
    distance.SetBounds(x0-(x1-x0)*f, x1+(x1-x0)*f,
                       y0-(y1-y0)*f, y1+(y1-y0)*f,
                       z0-(z1-z0)*f, z1+(z1-z0)*f)
    if polyData.GetPointData().GetNormals():
        distance.SetInputData(polyData)
    else:
        normals = vtk.vtkPCANormalEstimation()
        normals.SetInputData(polyData)
        normals.SetSampleSize(int(N / 50))
        normals.SetNormalOrientationToGraphTraversal()
        distance.SetInputConnection(normals.GetOutputPort())
        print("Recalculating normals for", N, "Points, sample size=", int(N / 50))

    b = polyData.GetBounds()
    diagsize = np.sqrt((b[1] - b[0]) ** 2 + (b[3] - b[2]) ** 2 + (b[5] - b[4]) ** 2)
    radius = diagsize / bins * 5
    distance.SetRadius(radius)
    distance.SetDimensions(bins, bins, bins)
    distance.Update()

    print("Calculating mesh from points with R =", radius)
    surface = vtk.vtkExtractSurface()
    surface.SetRadius(radius * 0.99)
    surface.HoleFillingOn()
    surface.ComputeNormalsOff()
    surface.ComputeGradientsOff()
    surface.SetInputConnection(distance.GetOutputPort())
    surface.Update()
    return Actor(surface.GetOutput(), "gold", 1, 0, "tomato")
629552
def array(self):
        
        return np.eye(self.M, self.N, self.k)
1292892
def train(self, data_iterator):
        
        feature_iterator, label_iterator = tee(data_iterator, 2)
        x_train = np.asarray([x for x, y in feature_iterator])
        y_train = np.asarray([y for x, y in label_iterator])

        if x_train.size == 0:
            return

        optimizer = get_optimizer(self.master_optimizer)
        self.model = model_from_yaml(self.yaml, self.custom_objects)
        self.model.compile(optimizer=optimizer,
                           loss=self.master_loss, metrics=self.master_metrics)
        self.model.set_weights(self.parameters.value)

        epochs = self.train_config['epochs']
        batch_size = self.train_config.get('batch_size')
        nb_train_sample = x_train.shape[0]
        nb_batch = int(np.ceil(nb_train_sample / float(batch_size)))
        index_array = np.arange(nb_train_sample)
        batches = [
            (i * batch_size, min(nb_train_sample, (i + 1) * batch_size))
            for i in range(0, nb_batch)
        ]

        if self.frequency == 'epoch':
            for epoch in range(epochs):
                weights_before_training = self.client.get_parameters()
                self.model.set_weights(weights_before_training)
                self.train_config['epochs'] = 1
                if x_train.shape[0] > batch_size:
                    self.model.fit(x_train, y_train, **self.train_config)
                self.train_config['epochs'] = epochs
                weights_after_training = self.model.get_weights()
                deltas = subtract_params(
                    weights_before_training, weights_after_training)
                self.client.update_parameters(deltas)
        elif self.frequency == 'batch':
            for epoch in range(epochs):
                if x_train.shape[0] > batch_size:
                    for (batch_start, batch_end) in batches:
                        weights_before_training = self.client.get_parameters()
                        self.model.set_weights(weights_before_training)
                        batch_ids = index_array[batch_start:batch_end]
                        x = slice_arrays(x_train, batch_ids)
                        y = slice_arrays(y_train, batch_ids)
                        self.model.train_on_batch(x, y)
                        weights_after_training = self.model.get_weights()
                        deltas = subtract_params(
                            weights_before_training, weights_after_training)
                        self.client.update_parameters(deltas)
        else:
            raise ValueError(
                'frequency parameter can be `epoch` or `batch, got {}'.format(self.frequency))
        yield []
846252
def _get_points(self):
        
        return tuple([self._getitem__points(i)
                     for i in range(self._len__points())])