Puts each data field into a tensor.

        :param batch: The input data batch.
        :type batch: list of (features, feature_weights) pair
        :return: Preprocessed data.
        :rtype: list of torch.Tensor with torch.Tensor (Optional)
5648
def _collate(self, batch):
        

        Y_batch = None
        if isinstance(batch[0], tuple):
            batch, Y_batch = list(zip(*batch))
            Y_batch = self._cuda(torch.Tensor(Y_batch))

        f_batch, v_batch = list(zip(*batch))

        f_batch, _ = pad_batch(f_batch, 0)
        v_batch, _ = pad_batch(v_batch, 0, type="float")

        f_batch = self._cuda(f_batch)
        v_batch = self._cuda(v_batch)

        if Y_batch is not None:
            return [f_batch, v_batch], Y_batch
        else:
            return [f_batch, v_batch]
90531
def get_batch_coordinate(x, axis=0):
  """"""
  # Compute the batch coordinate before flattening all batches
  batch_coordinate = tf.expand_dims(
      common_attention.coordinate_tensor(tf.shape(x)[:-1], axis=axis), axis=-1)
  return batch_coordinate
247208
def _getDataBatch(self, inputData):
    
    if not hasattr(self, 'numImages'):
      self._initializeDimensions(inputData)

    batch = np.zeros((self.filterDim, self.batchSize))

    # choose correct patch size
    if self.imageHeight is None:
      patchSize = self.filterDim
    elif self.numChannels is None or self.numChannels == 0:
      patchSize = int(np.sqrt(self.filterDim))
    else:
      patchSize = int(np.sqrt(self.filterDim / self.numChannels))

    minIndex = patchSize / 2

    for i in xrange(self.batchSize):
      # choose random image
      imageIdx = np.random.choice(range(self.numImages))

      # pick random starting row
      rows = self.imageHeight - 2 * patchSize
      rowNumber = minIndex + np.random.choice(range(rows))

      if self.imageHeight is None:
        patch = inputData[rowNumber : rowNumber + patchSize, imageIdx]
      else:
        # pick random starting column
        cols = self.imageWidth - 2 * patchSize
        colNumber = minIndex + np.random.choice(range(cols))

        if self.numChannels is None:
          patch = inputData[rowNumber : rowNumber + patchSize,
                            colNumber : colNumber + patchSize,
                            imageIdx]
        else:
          patch = inputData[rowNumber : rowNumber + patchSize,
                            colNumber : colNumber + patchSize,
                            :, imageIdx]
      patch = np.reshape(patch, newshape=(self.filterDim, ))
      batch[:, i] = patch

    return batch
1339862
def make_preprocessing_fn(output_dir, features, keep_target):
  
  def preprocessing_fn(inputs):
    """Preprocessing function.

    Args:
      inputs: dictionary of raw input tensors

    Returns:
      A dictionary of transformed tensors
    """
    stats = json.loads(
      file_io.read_file_to_string(
          os.path.join(output_dir, STATS_FILE)).decode())

    result = {}
    for name, transform in six.iteritems(features):
      transform_name = transform['transform']
      source_column = transform['source_column']

      if transform_name == KEY_TRANSFORM:
        transform_name = 'identity'
      elif transform_name == TARGET_TRANSFORM:
        if not keep_target:
          continue
        if file_io.file_exists(os.path.join(output_dir, VOCAB_ANALYSIS_FILE % source_column)):
          transform_name = 'one_hot'
        else:
          transform_name = 'identity'

      if transform_name == 'identity':
        result[name] = inputs[source_column]
      elif transform_name == 'scale':
        result[name] = _scale(
            inputs[name],
            min_x_value=stats['column_stats'][source_column]['min'],
            max_x_value=stats['column_stats'][source_column]['max'],
            output_min=transform.get('value', 1) * (-1),
            output_max=transform.get('value', 1))
      elif transform_name in [ONE_HOT_TRANSFORM, EMBEDDING_TRANSFROM, MULTI_HOT_TRANSFORM,
                              TFIDF_TRANSFORM, BOW_TRANSFORM]:
        vocab, ex_count = read_vocab_file(
            os.path.join(output_dir, VOCAB_ANALYSIS_FILE % source_column))

        if transform_name == TFIDF_TRANSFORM:
          separator = transform.get('separator', ' ')
          tokens = tf.string_split(inputs[source_column], separator)
          ids = _string_to_int(tokens, vocab)
          weights = _tfidf(
              x=ids,
              reduced_term_freq=ex_count + [0],
              vocab_size=len(vocab) + 1,
              corpus_size=stats['num_examples'])

          result[name + '_ids'] = ids
          result[name + '_weights'] = weights
        elif transform_name == BOW_TRANSFORM:
          separator = transform.get('separator', ' ')
          tokens = tf.string_split(inputs[source_column], separator)
          ids = _string_to_int(tokens, vocab)
          weights = _bag_of_words(x=ids)

          result[name + '_ids'] = ids
          result[name + '_weights'] = weights
        elif transform_name == MULTI_HOT_TRANSFORM:
          separator = transform.get('separator', ' ')
          tokens = tf.string_split(inputs[source_column], separator)
          result[name] = _string_to_int(tokens, vocab)
        else:
          # ONE_HOT_TRANSFORM: making a dense vector is done at training
          # EMBEDDING_TRANSFROM: embedding vectors have to be done at training
          result[name] = _string_to_int(inputs[source_column], vocab)
      elif transform_name == IMAGE_TRANSFORM:
        make_image_to_vec_fn = _make_image_to_vec_tito(
            name, checkpoint=transform.get('checkpoint', None))
        result[name] = make_image_to_vec_fn(inputs[source_column])
      else:
        raise ValueError('unknown transform %s' % transform_name)
    return result

  return preprocessing_fn
317246
def _initialize(self):
        
        self._graph = tf.Graph()
        with self._graph.as_default():
            self._input_node = tf.placeholder(tf.float32, (self._batch_size, self._im_height, self._im_width, self._num_channels))
            weights = self.build_alexnet_weights()
            self._output_tensor = self.build_alexnet(weights)
            self._feature_tensor = self.build_alexnet(weights, output_layer=self._feature_layer)
            self._initialized = True
857178
def batch_tensor(self, name):
        
        if name in self.transition_tensors:
            return tensor_util.merge_first_two_dims(self.transition_tensors[name])
        else:
            return self.rollout_tensors[name]
1061394
def _batchify(self, batch_data, batch_label, start=0):
        """"""
        i = start
        batch_size = self.batch_size
        try:
            while i < batch_size:
                label, s = self.next_sample()
                data = self.imdecode(s)
                try:
                    self.check_valid_image(data)
                except RuntimeError as e:
                    logging.debug('Invalid image, skipping:  %s', str(e))
                    continue
                data = self.augmentation_transform(data)
                assert i < batch_size, 'Batch size must be multiples of augmenter output length'
                batch_data[i] = self.postprocess_data(data)
                batch_label[i] = label
                i += 1
        except StopIteration:
            if not i:
                raise StopIteration
        return i
753524
def to_image_list(tensors, size_divisible=0):
    
    if isinstance(tensors, torch.Tensor) and size_divisible > 0:
        tensors = [tensors]

    if isinstance(tensors, ImageList):
        return tensors
    elif isinstance(tensors, torch.Tensor):
        # single tensor shape can be inferred
        assert tensors.dim() == 4
        image_sizes = [tensor.shape[-2:] for tensor in tensors]
        return ImageList(tensors, image_sizes)
    elif isinstance(tensors, (tuple, list)):
        max_size = tuple(max(s) for s in zip(*[img.shape for img in tensors]))

        # TODO Ideally, just remove this and let me model handle arbitrary
        # input sizs
        if size_divisible > 0:
            import math

            stride = size_divisible
            max_size = list(max_size)
            max_size[1] = int(math.ceil(max_size[1] / stride) * stride)
            max_size[2] = int(math.ceil(max_size[2] / stride) * stride)
            max_size = tuple(max_size)

        batch_shape = (len(tensors),) + max_size
        batched_imgs = tensors[0].new(*batch_shape).zero_()
        for img, pad_img in zip(tensors, batched_imgs):
            pad_img[: img.shape[0], : img.shape[1], : img.shape[2]].copy_(img)

        image_sizes = [im.shape[-2:] for im in tensors]

        return ImageList(batched_imgs, image_sizes)
    else:
        raise TypeError("Unsupported type for to_image_list: {}".format(type(tensors)))
491374
def train_batch(self, batch_info, data, target):
        
        batch_info.optimizer.zero_grad()
        loss = self.feed_batch(batch_info, data, target)
        loss.backward()

        if self.max_grad_norm is not None:
            batch_info['grad_norm'] = torch.nn.utils.clip_grad_norm_(
                filter(lambda p: p.requires_grad, self.model.parameters()),
                max_norm=self.max_grad_norm
            )

        batch_info.optimizer.step()
88438
def collate_fn(batch, train=True):
    
    premise_batch, _ = pad_batch([row['premise'] for row in batch])
    hypothesis_batch, _ = pad_batch([row['hypothesis'] for row in batch])
    label_batch = torch.stack([row['label'] for row in batch])

    # PyTorch RNN requires batches to be transposed for speed and integration with CUDA
    transpose = (lambda b: b.t_().squeeze(0).contiguous())

    return (transpose(premise_batch), transpose(hypothesis_batch), transpose(label_batch))
1257371
def batch_flatten(x):
    
    shape = x.get_shape().as_list()[1:]
    if None not in shape:
        return tf.reshape(x, [-1, int(np.prod(shape))])
    return tf.reshape(x, tf.stack([tf.shape(x)[0], -1]))
201948
def feed_batch(self, batch_info, data, target):
        
        data, target = data.to(self.device), target.to(self.device)
        output, loss = self.model.loss(data, target)

        # Store extra batch information for calculation of the statistics
        batch_info['data'] = data
        batch_info['target'] = target
        batch_info['output'] = output
        batch_info['loss'] = loss

        return loss
1060941
def get_batch_size(batch: Union[Dict, torch.Tensor]) -> int:
    
    if isinstance(batch, torch.Tensor):
        return batch.size(0) # type: ignore
    elif isinstance(batch, Dict):
        return get_batch_size(next(iter(batch.values())))
    else:
        return 0
1036297
def forward(self, data_batch, is_train=None):
        
        assert self.binded and self.params_initialized

        # make a shallow copy, just to maintain necessary properties (if any) like
        # bucket_key, pad, etc.
        data_batch = copy.copy(data_batch)

        for i_layer, module in enumerate(self._modules):
            module.forward(data_batch, is_train=is_train)

            if i_layer+1 == len(self._modules):
                # the last layer, do not need to do the followings
                break

            data_batch.data = module.get_outputs()
            if hasattr(data_batch, 'provide_data'):
                # need to update this, in case the internal module is using bucketing
                # or whatever
                data_names = [x[0] for x in module.output_shapes]
                assert len(data_names) == len(data_batch.data)
                data_batch.provide_data = [(name, x.shape) for name, x in
                                           zip(data_names, data_batch.data)]
395365
def data_preprocess(data, size=None, min_axis=None, batch=False):
    
    if batch:
        return [data_preprocess(el, size=size, min_axis=min_axis, batch=False) for el in data]

    if isinstance(data, string_types):
        if file_exists(data):
            # probably a path to an image
            preprocessed = Image.open(data)
        else:
            # base 64 encoded image data, a url, or raw content
            # send raw data to the server and let the server infer type
            b64_or_url = re.sub('^data:image/.+;base64,', '', data)
            return b64_or_url

    elif isinstance(data, Image.Image):
        # data is image from PIL
        preprocessed = data

    elif type(data).__name__ == "ndarray":
        # data is likely image from numpy/scipy
        if "float" in str(data.dtype) and data.min() >= 0 and data.max() <= 1:
            data *= 255.
        try:
            preprocessed = Image.fromarray(data.astype("uint8"))
        except TypeError:
            raise IndicoError(
                "Please ensure the numpy array is in a format by PIL. "
                "Values must be between 0 and 1 or between 0 and 255 in greyscale, rgb, or rgba format."
            )

    else:
        # at this point we are unsure of the type -- it could be malformatted text or image data.
        raise IndicoError(
            "Invalid input datatype: `{}`. "
            "Ensure input data is one of the following types: "
            "`str`, `unicode`, `PIL.Image`, `np.ndarray`.".format(
                data.__class__.__name__
            )
        )

    #
    if size or min_axis:
        preprocessed = resize_image(preprocessed, size, min_axis)

    # standardize on b64 encoding for sending image data over the wire
    temp_output = BytesIO()
    preprocessed.save(temp_output, format='PNG')
    temp_output.seek(0)
    output_s = temp_output.read()
    return base64.b64encode(output_s).decode('utf-8') if PY3 else base64.b64encode(output_s)
878765
def cat(tensors, dim=0):
    
    assert isinstance(tensors, (list, tuple))
    if len(tensors) == 1:
        return tensors[0]
    return torch.cat(tensors, dim)
90814
def batchify(data, batch_size):
    """"""
    nbatch = data.shape[0] // batch_size
    data = data[:nbatch * batch_size]
    data = data.reshape((batch_size, nbatch)).T
    return data