Submit a single or batch tensor to refine the streaming mean.
10421
def submit(self, value):
    """"""
    # Add a batch dimension if necessary.
    if value.shape.ndims == self._sum.shape.ndims:
      value = value[None, ...]
    return tf.group(
        self._sum.assign_add(tf.reduce_sum(value, 0)),
        self._count.assign_add(tf.shape(value)[0]))
512898
def denormalize(x:TensorImage, mean:FloatTensor,std:FloatTensor, do_x:bool=True)->TensorImage:
    ""
    return x.cpu().float()*std[...,None,None] + mean[...,None,None] if do_x else x.cpu()
312880
def stream_tfevents(path, file_api, step=0):
    """"""
    last_step = 0
    row = {}
    buffer = []
    last_row = {}
    for summary in tf.train.summary_iterator(path):
        parsed = tf_summary_to_dict(summary)
        if last_step != parsed["tensorflow_step"]:
            step += 1
            last_step = parsed["tensorflow_step"]
            # TODO: handle time
            if len(row) > 0:
                last_row = to_json(row)
                buffer.append(Chunk("wandb-history.jsonl",
                                    util.json_dumps_safer_history(to_json(row))))
        row.update(parsed)
    file_api._send(buffer)
    return last_row
194448
def _batch_norm_without_layers(self, input_layer, decay, use_scale,
                                   epsilon):
        """"""
        shape = input_layer.shape
        num_channels = shape[3] if self.data_format == "NHWC" else shape[1]
        beta = self.get_variable(
            "beta", [num_channels],
            tf.float32,
            tf.float32,
            initializer=tf.zeros_initializer())
        if use_scale:
            gamma = self.get_variable(
                "gamma", [num_channels],
                tf.float32,
                tf.float32,
                initializer=tf.ones_initializer())
        else:
            gamma = tf.constant(1.0, tf.float32, [num_channels])
        moving_mean = tf.get_variable(
            "moving_mean", [num_channels],
            tf.float32,
            initializer=tf.zeros_initializer(),
            trainable=False)
        moving_variance = tf.get_variable(
            "moving_variance", [num_channels],
            tf.float32,
            initializer=tf.ones_initializer(),
            trainable=False)
        if self.phase_train:
            bn, batch_mean, batch_variance = tf.nn.fused_batch_norm(
                input_layer,
                gamma,
                beta,
                epsilon=epsilon,
                data_format=self.data_format,
                is_training=True)
            mean_update = moving_averages.assign_moving_average(
                moving_mean, batch_mean, decay=decay, zero_debias=False)
            variance_update = moving_averages.assign_moving_average(
                moving_variance,
                batch_variance,
                decay=decay,
                zero_debias=False)
            tf.add_to_collection(tf.GraphKeys.UPDATE_OPS, mean_update)
            tf.add_to_collection(tf.GraphKeys.UPDATE_OPS, variance_update)
        else:
            bn, _, _ = tf.nn.fused_batch_norm(
                input_layer,
                gamma,
                beta,
                mean=moving_mean,
                variance=moving_variance,
                epsilon=epsilon,
                data_format=self.data_format,
                is_training=False)
        return bn
1273926
def write(self, model:nn.Module, tbwriter:SummaryWriter, input_to_model:torch.Tensor)->None:
        ""
        request = GraphTBRequest(model=model, tbwriter=tbwriter, input_to_model=input_to_model)
        asyncTBWriter.request_write(request)
1048714
def _batch(self,
               vectors,
               batch_size,
               num,
               show_progressbar,
               return_names):
        """"""
        vectors = self.normalize(vectors)

        # Single transpose, makes things faster.
        reference_transposed = self.norm_vectors.T

        for i in tqdm(range(0, len(vectors), batch_size),
                      disable=not show_progressbar):

            distances = vectors[i: i+batch_size].dot(reference_transposed)
            # For safety we clip
            distances = np.clip(distances, a_min=.0, a_max=1.0)
            if num == 1:
                sorted_indices = np.argmax(distances, 1)[:, None]
            else:
                sorted_indices = np.argpartition(-distances, kth=num, axis=1)
                sorted_indices = sorted_indices[:, :num]
            for lidx, indices in enumerate(sorted_indices):
                dists = distances[lidx, indices]
                if return_names:
                    dindex = np.argsort(-dists)
                    yield [(self.indices[indices[d]], dists[d])
                           for d in dindex]
                else:
                    yield list(-1 * np.sort(-dists))
639701
def preprocess_data(tokenizer, task, batch_size, dev_batch_size, max_len, pad=False):
    """"""
    # transformation
    trans = BERTDatasetTransform(
        tokenizer,
        max_len,
        labels=task.get_labels(),
        pad=pad,
        pair=task.is_pair,
        label_dtype='float32' if not task.get_labels() else 'int32')

    data_train = task('train').transform(trans, lazy=False)
    data_train_len = data_train.transform(
        lambda input_id, length, segment_id, label_id: length)

    num_samples_train = len(data_train)
    # bucket sampler
    batchify_fn = nlp.data.batchify.Tuple(
        nlp.data.batchify.Pad(axis=0), nlp.data.batchify.Stack(),
        nlp.data.batchify.Pad(axis=0),
        nlp.data.batchify.Stack(
            'float32' if not task.get_labels() else 'int32'))
    batch_sampler = nlp.data.sampler.FixedBucketSampler(
        data_train_len,
        batch_size=batch_size,
        num_buckets=10,
        ratio=0,
        shuffle=True)
    # data loaders
    dataloader_train = gluon.data.DataLoader(
        dataset=data_train,
        num_workers=1,
        batch_sampler=batch_sampler,
        batchify_fn=batchify_fn)
    if task.task_name == 'MNLI':
        data_dev_matched = task('dev_matched').transform(trans, lazy=False)
        data_dev_mismatched = task('dev_mismatched').transform(trans, lazy=False)

        dataloader_dev_matched = mx.gluon.data.DataLoader(
            data_dev_matched, batch_size=dev_batch_size,
            num_workers=1, shuffle=False, batchify_fn=batchify_fn)
        dataloader_dev_mismatched = mx.gluon.data.DataLoader(
            data_dev_mismatched, batch_size=dev_batch_size,
            num_workers=1, shuffle=False, batchify_fn=batchify_fn)
        return dataloader_train, dataloader_dev_matched, \
            dataloader_dev_mismatched, num_samples_train
    else:
        data_dev = task('dev').transform(trans, lazy=False)
        dataloader_dev = mx.gluon.data.DataLoader(
            data_dev,
            batch_size=dev_batch_size,
            num_workers=1,
            shuffle=False,
            batchify_fn=batchify_fn)
        return dataloader_train, dataloader_dev, num_samples_train
554637
def batch_fn(dataset, training, shapes, target_names,
             batch_size=32, eval_batch_size=32, bucket_batch_length=32,
             bucket_max_length=256, bucket_min_length=8,
             bucket_length_step=1.1, buckets=None):
  """"""
  del target_names
  # If bucketing is not specified, check if target shapes are variable.
  cur_batch_size = batch_size if training else eval_batch_size
  if buckets is None:
    variable_target_shapes = False
    target_shape = shapes[1]
    for dim in target_shape:
      if dim is None:
        variable_target_shapes = True
    tf.logging.info("Heuristically setting bucketing to %s based on shapes "
                    "of target tensors." % variable_target_shapes)
    if variable_target_shapes:
      batch_size_per_token = cur_batch_size * bucket_batch_length
      scheme = data_reader.batching_scheme(batch_size_per_token,
                                           bucket_max_length,
                                           bucket_min_length,
                                           bucket_length_step,
                                           drop_long_sequences=training)
      buckets = (scheme["boundaries"], scheme["batch_sizes"])

  if buckets:
    tf.logging.info("Bucketing with buckets %s." % str(buckets))
    def example_length(_, target):
      return tf.shape(target)[0]
    boundaries, batch_sizes = buckets
    dataset = dataset.apply(tf.data.experimental.bucket_by_sequence_length(
        example_length, boundaries, batch_sizes))
  else:
    dataset = dataset.padded_batch(cur_batch_size, shapes)
  return dataset
668012
def _tf_load_model(sess, model_dir):
  """"""

  meta_graph_pb = tf.saved_model.loader.load(
      sess=sess,
      tags=[tf.saved_model.tag_constants.SERVING],
      export_dir=model_dir)

  signature = meta_graph_pb.signature_def[signature_constants.DEFAULT_SERVING_SIGNATURE_DEF_KEY]
  input_alias_map = {friendly_name: tensor_info_proto.name
                     for (friendly_name, tensor_info_proto) in signature.inputs.items()}
  output_alias_map = {friendly_name: tensor_info_proto.name
                      for (friendly_name, tensor_info_proto) in signature.outputs.items()}

  return input_alias_map, output_alias_map
88438
def collate_fn(batch, train=True):
    
    premise_batch, _ = pad_batch([row['premise'] for row in batch])
    hypothesis_batch, _ = pad_batch([row['hypothesis'] for row in batch])
    label_batch = torch.stack([row['label'] for row in batch])

    # PyTorch RNN requires batches to be transposed for speed and integration with CUDA
    transpose = (lambda b: b.t_().squeeze(0).contiguous())

    return (transpose(premise_batch), transpose(hypothesis_batch), transpose(label_batch))
1055592
def batch_stats(self, funcs:Collection[Callable]=None, ds_type:DatasetType=DatasetType.Train)->Tensor:
        ""
        funcs = ifnone(funcs, [torch.mean,torch.std])
        x = self.one_batch(ds_type=ds_type, denorm=False)[0].cpu()
        return [func(channel_view(x), 1) for func in funcs]
1244404
def update(self, mini_batch, num_sequences):
        

        feed_dict = {self.model.dropout_rate: self.update_rate,
                     self.model.batch_size: num_sequences,
                     self.model.sequence_length: self.sequence_length}
        if self.use_continuous_act:
            feed_dict[self.model.true_action] = mini_batch['actions']. \
                reshape([-1, self.brain.vector_action_space_size[0]])
        else:
            feed_dict[self.model.true_action] = mini_batch['actions'].reshape(
                [-1, len(self.brain.vector_action_space_size)])
            feed_dict[self.model.action_masks] = np.ones(
                (num_sequences, sum(self.brain.vector_action_space_size)))
        if self.use_vec_obs:
            apparent_obs_size = self.brain.vector_observation_space_size * \
                                self.brain.num_stacked_vector_observations
            feed_dict[self.model.vector_in] = mini_batch['vector_obs'] \
                .reshape([-1,apparent_obs_size])
        for i, _ in enumerate(self.model.visual_in):
            visual_obs = mini_batch['visual_obs%d' % i]
            feed_dict[self.model.visual_in[i]] = visual_obs
        if self.use_recurrent:
            feed_dict[self.model.memory_in] = np.zeros([num_sequences, self.m_size])
        run_out = self._execute_model(feed_dict, self.update_dict)
        return run_out
212996
def batch_norm(attrs, inputs, proto_obj):
    """"""
    new_attrs = translation_utils._fix_attribute_names(attrs, {'epsilon': 'eps',
                                                               'is_test': 'fix_gamma'})
    new_attrs = translation_utils._remove_attributes(new_attrs,
                                                     ['spatial', 'consumed_inputs'])
    # Disable cuDNN BN only if epsilon from model is < than minimum cuDNN eps (1e-5)
    cudnn_min_eps = 1e-5
    cudnn_off = 0 if attrs.get('epsilon', cudnn_min_eps) >= cudnn_min_eps else 1
    new_attrs = translation_utils._add_extra_attributes(new_attrs, {'cudnn_off': cudnn_off})

    # in test mode "fix_gamma" should be unset.
    new_attrs['fix_gamma'] = not attrs.get('is_test', 1)
    return 'BatchNorm', new_attrs, inputs
235382
def createrawtransaction(self, inputs, outputs):
        """"""
        if not isinstance(outputs, dict):
            raise TypeError("outputs variable must be a dictionary")
        if not isinstance(inputs, list):
            raise TypeError("inputs variable must be a list")
        return self.req("createrawtransaction", [inputs, outputs])
469535
def _batchify(self, batch_data, batch_label, start=0):
        """"""
        i = start
        batch_size = self.batch_size
        try:
            while i < batch_size:
                label, s = self.next_sample()
                data = self.imdecode(s)
                try:
                    self.check_valid_image([data])
                    label = self._parse_label(label)
                    data, label = self.augmentation_transform(data, label)
                    self._check_valid_label(label)
                except RuntimeError as e:
                    logging.debug('Invalid image, skipping:  %s', str(e))
                    continue
                for datum in [data]:
                    assert i < batch_size, 'Batch size must be multiples of augmenter output length'
                    batch_data[i] = self.postprocess_data(datum)
                    num_object = label.shape[0]
                    batch_label[i][0:num_object] = nd.array(label)
                    if num_object < batch_label[i].shape[0]:
                        batch_label[i][num_object:] = -1
                    i += 1
        except StopIteration:
            if not i:
                raise StopIteration

        return i
639318
def _make_eval_func(self, tensors, session, feed_dict, fetches,
                      callback=None):
    """"""
    if not isinstance(tensors, list):
      tensors = [tensors]
    num_tensors = len(tensors)

    def eval_func(x):
      """Function to evaluate a `Tensor`."""
      shapes = dict(zip(self._vars, self._var_shapes))
      augmented_feed_dict = {
          var: x[packing_slice].reshape(shapes[var])
          for var, packing_slice in zip(self._vars, self._packing_slices)
      }
      augmented_feed_dict.update(feed_dict)
      augmented_fetches = tensors + fetches

      augmented_fetch_vals = session.run(
          augmented_fetches, feed_dict=augmented_feed_dict)

      if callable(callback):
        callback(*augmented_fetch_vals[num_tensors:])

      return augmented_fetch_vals[:num_tensors]

    return eval_func
618608
def dummy_batch(m: nn.Module, size:tuple=(64,64))->Tensor:
    ""
    ch_in = in_channels(m)
    return one_param(m).new(1, ch_in, *size).requires_grad_(False).uniform_(-1.,1.)