Preprocessing the inputs before calling session.run()

    :param X_batch: A dictionary of inputs to the first sub-graph
    :return: A tuple, `(fetches, fd)`, with `fetches` being a list of
             Tensors to be fetches and `fd` the feed dictionary.
511806
def set_input(self, X_batch=None):
    
    inputs = self.inputs
    outputs = self.outputs

    # data for first gpu
    fd = {}
    if X_batch is not None:
      self.next_vals[0] = OrderedDict()
      for i, vname in enumerate(self.inputs[0]):
        if vname in X_batch:
          self.next_vals[0][vname] = X_batch[vname]
        else:
          self.next_vals[0][vname] = None
    else:
      self.next_vals[0] = None

    # Set `feed_dict` for each GPU. If there is something to run for that
    # GPU, collect outputs to be fetched.
    fetches = []
    self.active_gpus = []
    for i in range(len(outputs)):
      if self.next_vals[i] is None:
        self.active_gpus += [False]
        continue
      self.active_gpus += [True]
      for k in inputs[i]:
        if self.next_vals[i][k] is not None:
          fd[inputs[i][k]] = self.next_vals[i][k]
      for k, v in outputs[i].iteritems():
        fetches += [v]

    fd.update(self.feed_dict)

    return fetches, fd
810203
def get_input_tensors(batch_size, tf_records, num_repeats=1,
                      shuffle_records=True, shuffle_examples=True,
                      shuffle_buffer_size=None,
                      filter_amount=0.05, random_rotation=True):
    
    print("Reading tf_records from {} inputs".format(len(tf_records)))
    dataset = read_tf_records(
        batch_size,
        tf_records,
        num_repeats=num_repeats,
        shuffle_records=shuffle_records,
        shuffle_examples=shuffle_examples,
        shuffle_buffer_size=shuffle_buffer_size,
        filter_amount=filter_amount,
        interleave=True)
    dataset = dataset.filter(lambda t: tf.equal(tf.shape(t)[0], batch_size))
    dataset = dataset.map(
        functools.partial(batch_parse_tf_example, batch_size))
    if random_rotation:
        dataset = dataset.map(_random_rotation_pyfunc)

    return dataset.make_one_shot_iterator().get_next()
224123
def _call(self, x):
        """"""
        result = self.sess.run(self.output_tensor,
                               feed_dict={self.input_tensor: np.asarray(x)})

        return result
54853
def run(self, out, model_inputs, X):
        
        feed_dict = dict(zip(model_inputs, X))
        for t in self.learning_phase_flags:
            feed_dict[t] = False
        return self.session.run(out, feed_dict)
763359
def _get_batch(self):
    
    batch_indices = np.mod(
        np.array([
            np.arange(head_index, head_index + self._num_steps + 1) for
            head_index in self._head_indices]),
        self._n_flat_elements)

    obs = np.array([
        self._flat_data[indices[:self._num_steps]]
        for indices in batch_indices]).T
    target = np.array([
        self._flat_data[indices[1:self._num_steps + 1]]
        for indices in batch_indices]).T

    if self._random_sampling:
      self._reset_head_indices()
    else:
      self._head_indices = np.mod(
          self._head_indices + self._num_steps, self._n_flat_elements)
    return obs, target
153257
def start_batch(job, input_args):
    
    samples = parse_sra(input_args['sra'])
    # for analysis_id in samples:
    job.addChildJobFn(download_and_transfer_sample, input_args, samples, cores=1, disk='30')
594293
def _from_dict(cls, _dict):
        """"""
        args = {}
        if 'batches' in _dict:
            args['batches'] = [
                BatchStatus._from_dict(x) for x in (_dict.get('batches'))
            ]
        return cls(**args)
947793
def forward(self, x):
        
        head_outputs = [None] * self.t

        # Execute input layer
        if isinstance(self.input_layer, list):  # One input_module per task
            input_outputs = [mod(x) for mod, x in zip(self.input_layer, x)]
            x = torch.stack(input_outputs, dim=1)

            # Execute level-0 task heads from their respective input modules
            for t in self.task_map[0]:
                head = self.heads[t]
                head_outputs[t] = head(input_outputs[t])
        else:  # One input_module for all tasks
            x = self.input_layer(x)

            # Execute level-0 task heads from the single input module
            for t in self.task_map[0]:
                head = self.heads[t]
                head_outputs[t] = head(x)

        # Execute middle layers
        for i, layer in enumerate(self.middle_layers, start=1):
            x = layer(x)

            # Attach level-i task heads from the ith middle module
            for t in self.task_map[i]:
                head = self.heads[t]
                # Optionally include as input the predictions of parent tasks
                if self.config["pass_predictions"] and bool(self.task_graph.parents[t]):
                    task_input = [x]
                    for p in self.task_graph.parents[t]:
                        task_input.append(head_outputs[p])
                    task_input = torch.stack(task_input, dim=1)
                else:
                    task_input = x
                head_outputs[t] = head(task_input)
        return head_outputs
121822
def _sample_batch_prioritized(self, segment_tree, batch_size, history, forward_steps=1):
        
        p_total = segment_tree.total()
        segment = p_total / batch_size

        # Get batch of valid samples
        batch = [
            self._get_sample_from_segment(segment_tree, segment, i, history, forward_steps)
            for i in range(batch_size)
        ]

        probs, idxs, tree_idxs = zip(*batch)
        return np.array(probs), np.array(idxs), np.array(tree_idxs)
405247
def compute_batch(self, duplicate_manager=None,context_manager=None):
        
        x, _ = self.acquisition.optimize(duplicate_manager=duplicate_manager)
        return x
1151070
def _get_batch(self):
        
        batch_data = mx.nd.zeros((self.batch_size, 3, self._data_shape[0], self._data_shape[1]))
        batch_label = []
        for i in range(self.batch_size):
            if (self._current + i) >= self._size:
                if not self.is_train:
                    continue
                # use padding from middle in each epoch
                idx = (self._current + i + self._size // 2) % self._size
                index = self._index[idx]
            else:
                index = self._index[self._current + i]
            # index = self.debug_index
            im_path = self._imdb.image_path_from_index(index)
            with open(im_path, 'rb') as fp:
                img_content = fp.read()
            img = mx.img.imdecode(img_content)
            gt = self._imdb.label_from_index(index).copy() if self.is_train else None
            data, label = self._data_augmentation(img, gt)
            batch_data[i] = data
            if self.is_train:
                batch_label.append(label)
        self._data = {'data': batch_data}
        if self.is_train:
            self._label = {'label': mx.nd.array(np.array(batch_label))}
        else:
            self._label = {'label': None}
1267331
def _get_batches(self, items):
        """"""
        for i in xrange(0, len(items), self.max_batch_size):
            yield items[i:i + self.max_batch_size]
1109596
def batch(self, methods):
        
        methods = list(methods) # methods can be iterator
        if not methods:
            #accepts also empty list (or generator) - it allows call batch directly with query result (eventually empty)
            return
        queries, callbacks = list(zip(*[m(batch=True) for m in methods]))
        # perform all the operations in one batch
        responses = self.execute("", "POST", requests=queries)
        # perform the callbacks with the response data (updating the existing
        # objets, etc)

        batched_errors = []
        for callback, response in zip(callbacks, responses):
            if "success" in response:
                callback(response["success"])
            else:
                batched_errors.append(response["error"])

        if batched_errors:
            raise core.ParseBatchError(batched_errors)
184952
def process(self, element):
    
    import collections
    import apache_beam as beam

    num_in_batch = 0
    try:
      assert self._session is not None

      feed_dict = collections.defaultdict(list)
      for line in element:

        # Remove trailing newline.
        if line.endswith('\n'):
          line = line[:-1]

        feed_dict[self._input_alias_map.values()[0]].append(line)
        num_in_batch += 1

      # batch_result is list of numpy arrays with batch_size many rows.
      batch_result = self._session.run(fetches=self._tensor_names,
                                       feed_dict=feed_dict)

      # ex batch_result for batch_size > 1:
      # (array([value1, value2, ..., value_batch_size]),
      #  array([[a1, b1, c1]], ..., [a_batch_size, b_batch_size, c_batch_size]]),
      #  ...)
      # ex batch_result for batch_size == 1:
      # (value,
      #  array([a1, b1, c1]),
      #  ...)

      # Convert the results into a dict and unbatch the results.
      if num_in_batch > 1:
        for result in zip(*batch_result):
          predictions = {}
          for name, value in zip(self._aliases, result):
            predictions[name] = (value.tolist() if getattr(value, 'tolist', None) else value)
          yield predictions
      else:
        predictions = {}
        for i in range(len(self._aliases)):
          value = batch_result[i]
          value = (value.tolist() if getattr(value, 'tolist', None)
                   else value)
          predictions[self._aliases[i]] = value
        yield predictions

    except Exception as e:  # pylint: disable=broad-except
      yield beam.pvalue.TaggedOutput('errors', (str(e), element))
554637
def batch_fn(dataset, training, shapes, target_names,
             batch_size=32, eval_batch_size=32, bucket_batch_length=32,
             bucket_max_length=256, bucket_min_length=8,
             bucket_length_step=1.1, buckets=None):
  """"""
  del target_names
  # If bucketing is not specified, check if target shapes are variable.
  cur_batch_size = batch_size if training else eval_batch_size
  if buckets is None:
    variable_target_shapes = False
    target_shape = shapes[1]
    for dim in target_shape:
      if dim is None:
        variable_target_shapes = True
    tf.logging.info("Heuristically setting bucketing to %s based on shapes "
                    "of target tensors." % variable_target_shapes)
    if variable_target_shapes:
      batch_size_per_token = cur_batch_size * bucket_batch_length
      scheme = data_reader.batching_scheme(batch_size_per_token,
                                           bucket_max_length,
                                           bucket_min_length,
                                           bucket_length_step,
                                           drop_long_sequences=training)
      buckets = (scheme["boundaries"], scheme["batch_sizes"])

  if buckets:
    tf.logging.info("Bucketing with buckets %s." % str(buckets))
    def example_length(_, target):
      return tf.shape(target)[0]
    boundaries, batch_sizes = buckets
    dataset = dataset.apply(tf.data.experimental.bucket_by_sequence_length(
        example_length, boundaries, batch_sizes))
  else:
    dataset = dataset.padded_batch(cur_batch_size, shapes)
  return dataset
364153
def get_batch(self):
    
    params = [self._batch_size, self._num_objects, self._num_features]
    inputs, labels = tf.py_func(self._get_batch_data, params,
                                [tf.float32, tf.float32])
    inputs = tf.reshape(inputs, [self._batch_size, self._num_objects,
                                 self._num_features + self._num_objects * 3])
    labels = tf.reshape(labels, [-1])
    return inputs, labels
52641
def _get_new_batch(self, ds_type:DatasetType)->Collection[Tensor]:
        ""
        return self.learn.data.one_batch(ds_type=ds_type, detach=True, denorm=False, cpu=False)