Extract the labels into a 1-hot matrix [image index, label index].
18462
def mnist_extract_labels(filename, num_images):
  """"""
  print('Extracting', filename)
  with gzip.open(filename) as bytestream:
    bytestream.read(8)
    buf = bytestream.read(1 * num_images)
    labels = np.frombuffer(buf, dtype=np.uint8)
  # Convert to dense 1-hot representation.
  return (np.arange(10) == labels[:, None]).astype(np.float32)
59849
def relabel_squeeze(data):
    
    palette, index = np.unique(data, return_inverse=True)
    data = index.reshape(data.shape)
    # realy slow solution
    #        unq = np.unique(data)
    #        actual_label = 0
    #        for lab in unq:
    #            data[data == lab] = actual_label
    #            actual_label += 1

    # one another solution probably slower
    # arr = data
    # data = (np.digitize(arr.reshape(-1,),np.unique(arr))-1).reshape(arr.shape)

    return data
1238389
def skeletonize_labels(labels):
    ''''''
    #
    # The trick here is to separate touching labels by coloring the
    # labels matrix and then processing each color separately
    #
    colors = color_labels(labels)
    max_color = np.max(colors)
    if max_color == 0:
        return labels
    result = np.zeros(labels.shape, labels.dtype)
    for i in range(1,max_color+1):
        mask = skeletonize(colors==i)
        result[mask] = labels[mask]
    return result
650896
def _generate_label_matrix(self):
        """"""
        self.L = np.zeros((self.n, self.m))
        self.Y = np.zeros(self.n, dtype=np.int64)
        for i in range(self.n):
            y = choice(self.k, p=self.p) + 1  # Note that y \in {1,...,k}
            self.Y[i] = y
            for j in range(self.m):
                p_j = self.parent.get(j, 0)
                prob_y = self.P_conditional(j, y, p_j, self.L[i, p_j], y)
                prob_0 = self.P_conditional(j, 0, p_j, self.L[i, p_j], y)
                p = np.ones(self.k + 1) * (1 - prob_y - prob_0) / (self.k - 1)
                p[0] = prob_0
                p[y] = prob_y
                self.L[i, j] = choice(self.k + 1, p=p)
1115681
def to_array(self):
        """"""
        dt = np.dtype(list(zip(self.labels, (c.dtype for c in self.columns))))
        arr = np.empty_like(self.columns[0], dt)
        for label in self.labels:
            arr[label] = self[label]
        return arr
854125
def _extract_intensities(image, mask = slice(None)):
    
    return numpy.array(image, copy=True)[mask].ravel()
1240749
def __get_labels(self):
        
        labels = []
        try:
            with self.fs.open(self.fs.join(self.path, self.LABEL_FILE),
                                'r') as file_desc:
                for line in file_desc.readlines():
                    line = line.strip()
                    (label_name, label_color) = line.split(",", 1)
                    labels.append(Label(name=label_name,
                                        color=label_color))
        except IOError:
            pass
        return labels
986845
def get_image_list(self, name, dir_structure='original'):
        
        assert name in ['train', 'val', 'test']
        assert dir_structure in ['original', 'train']
        add_label_to_fname = (name != 'train' and dir_structure != 'original')
        if add_label_to_fname:
            synset = self.get_synset_1000()

        fname = os.path.join(self.dir, name + '.txt')
        assert os.path.isfile(fname), fname
        with open(fname) as f:
            ret = []
            for line in f.readlines():
                name, cls = line.strip().split()
                cls = int(cls)

                if add_label_to_fname:
                    name = os.path.join(synset[cls], name)

                ret.append((name.strip(), cls))
        assert len(ret), fname
        return ret
742352
def smoothing(labels, smoothing_window):
  

  if numpy.sum(labels)< smoothing_window:
    return labels
  segments = []
  for k in range(1,len(labels)-1):
    if labels[k]==0 and labels[k-1]==1 and labels[k+1]==1 :
      labels[k]=1
  for k in range(1,len(labels)-1):
    if labels[k]==1 and labels[k-1]==0 and labels[k+1]==0 :
      labels[k]=0

  seg = numpy.array([0,0,labels[0]])
  for k in range(1,len(labels)):
    if labels[k] != labels[k-1]:
      seg[1]=k-1
      segments.append(seg)
      seg = numpy.array([k,k,labels[k]])
  seg[1]=len(labels)-1
  segments.append(seg)

  if len(segments) < 2:
    return labels

  curr = segments[0]
  next = segments[1]

  # Look at the first segment. If it's short enough, just change its labels
  if (curr[1]-curr[0]+1) < smoothing_window and (next[1]-next[0]+1) > smoothing_window:
    if curr[2]==1:
      labels[curr[0] : (curr[1]+1)] = numpy.zeros(curr[1] - curr[0] + 1)
      curr[2]=0
    else: #curr[2]==0
      labels[curr[0] : (curr[1]+1)] = numpy.ones(curr[1] - curr[0] + 1)
      curr[2]=1

  for k in range(1,len(segments)-1):
    prev = segments[k-1]
    curr = segments[k]
    next = segments[k+1]

    if (curr[1]-curr[0]+1) < smoothing_window and (prev[1]-prev[0]+1) > smoothing_window and (next[1]-next[0]+1) > smoothing_window:
      if curr[2]==1:
        labels[curr[0] : (curr[1]+1)] = numpy.zeros(curr[1] - curr[0] + 1)
        curr[2]=0
      else: #curr[2]==0
        labels[curr[0] : (curr[1]+1)] = numpy.ones(curr[1] - curr[0] + 1)
        curr[2]=1

  prev = segments[-2]
  curr = segments[-1]

  if (curr[1]-curr[0]+1) < smoothing_window and (prev[1]-prev[0]+1) > smoothing_window:
    if curr[2]==1:
      labels[curr[0] : (curr[1]+1)] = numpy.zeros(curr[1] - curr[0] + 1)
      curr[2]=0
    else: #if curr[2]==0
      labels[curr[0] : (curr[1]+1)] = numpy.ones(curr[1] - curr[0] + 1)
      curr[2]=1

  return labels
816766
def read_data(label, image):
    
    base_url = 'http://yann.lecun.com/exdb/mnist/'
    with gzip.open(download_file(base_url+label, os.path.join('data',label))) as flbl:
        magic, num = struct.unpack(">II", flbl.read(8))
        label = np.fromstring(flbl.read(), dtype=np.int8)
    with gzip.open(download_file(base_url+image, os.path.join('data',image)), 'rb') as fimg:
        magic, num, rows, cols = struct.unpack(">IIII", fimg.read(16))
        image = np.fromstring(fimg.read(), dtype=np.uint8).reshape(len(label), rows, cols)
    return (label, image)
466098
def _split_generators(self, dl_manager):
    """"""
    # At data creation time, parse the folder to deduce number of splits,
    # labels, image size,

    # The splits correspond to the high level folders
    split_names = list_folders(dl_manager.manual_dir)

    # Extract all label names and associated images
    split_label_images = {}  # dict[split_name][label_name] = list(img_paths)
    for split_name in split_names:
      split_dir = os.path.join(dl_manager.manual_dir, split_name)
      split_label_images[split_name] = {
          label_name: list_imgs(os.path.join(split_dir, label_name))
          for label_name in list_folders(split_dir)
      }

    # Merge all label names from all splits to get the final list of labels
    # Sorted list for determinism
    labels = [split.keys() for split in split_label_images.values()]
    labels = list(sorted(set(itertools.chain(*labels))))

    # Could improve the automated encoding format detection
    # Extract the list of all image paths
    image_paths = [
        image_paths
        for label_images in split_label_images.values()
        for image_paths in label_images.values()
    ]
    if any(f.lower().endswith(".png") for f in itertools.chain(*image_paths)):
      encoding_format = "png"
    else:
      encoding_format = "jpeg"

    # Update the info.features. Those info will be automatically resored when
    # the dataset is re-created
    self.info.features["image"].set_encoding_format(encoding_format)
    self.info.features["label"].names = labels

    def num_examples(label_images):
      return sum(len(imgs) for imgs in label_images.values())

    # Define the splits
    return [
        tfds.core.SplitGenerator(
            name=split_name,
            # The number of shards is a dynamic function of the total
            # number of images (between 0-10)
            num_shards=min(10, max(num_examples(label_images) // 1000, 1)),
            gen_kwargs=dict(label_images=label_images,),
        ) for split_name, label_images in split_label_images.items()
    ]
186024
def one_hot_encoding(labels, num_classes, scope=None):
  
  with tf.name_scope(scope, 'OneHotEncoding', [labels]):
    batch_size = labels.get_shape()[0]
    indices = tf.expand_dims(tf.range(0, batch_size), 1)
    labels = tf.cast(tf.expand_dims(labels, 1), indices.dtype)
    concated = tf.concat(axis=1, values=[indices, labels])
    onehot_labels = tf.sparse_to_dense(
        concated, tf.stack([batch_size, num_classes]), 1.0, 0.0)
    onehot_labels.set_shape([batch_size, num_classes])
    return onehot_labels
93539
def _get_labels(self, element, locations, mode):
        
        """
        # Parse inputs
        locations = self._parse_indices(locations)
        element = self._parse_element(element=element)
        # Collect list of all pore OR throat labels
        labels = self.keys(mode='labels', element=element)
        labels.sort()
        labels = sp.array(labels)  # Convert to ND-array for following checks
        # Make an 2D array with locations in rows and labels in cols
        arr = sp.vstack([self[item][locations] for item in labels]).T
        num_hits = sp.sum(arr, axis=0)  # Number of locations with each label
        if mode in ['or', 'union', 'any']:
            temp = labels[num_hits > 0]
        elif mode in ['and', 'intersection']:
            temp = labels[num_hits == locations.size]
        elif mode in ['xor', 'exclusive_or']:
            temp = labels[num_hits == 1]
        elif mode in ['nor', 'not', 'none']:
            temp = labels[num_hits == 0]
        elif mode in ['nand']:
            temp = labels[num_hits == (locations.size - 1)]
        elif mode in ['xnor', 'nxor']:
            temp = labels[num_hits > 1]
        else:
            raise Exception('Unrecognized mode:'+str(mode))
        return PrintableList(temp)
298472
def __learn_labels(self, labels):
        
        if self.feature_length > 0:
            result = list(self.labels.classes_)
        else:
            result = []

        for label in labels:
            result.append(label)
        self.labels.fit(result)
576105
def _label_from_list(self, labels:Iterator, label_cls:Callable=None, from_item_lists:bool=False, **kwargs)->'LabelList':
        ""
        if not from_item_lists:
            raise Exception("Your data isn't split, if you don't want a validation set, please use `split_none`.")
        labels = array(labels, dtype=object)
        label_cls = self.get_label_cls(labels, label_cls=label_cls, **kwargs)
        y = label_cls(labels, path=self.path, **kwargs)
        res = self._label_list(x=self, y=y)
        return res
1209038
def _label_desc(self, label, desc, label_color=''):
        '''  '''
        return self.BRIGHT +  label_color + label + self.RESET + desc
104364
def split_data(self, train_images, train_labels):
        
        valid_images = train_images[:self.num_valid_images]
        valid_labels = train_labels[:self.num_valid_images]
        train_images = train_images[self.num_valid_images:]
        train_labels = train_labels[self.num_valid_images:]
        return train_images, train_labels, valid_images, valid_labels