Batch normalization.
508718
def _batch_norm(self, name, x):
        """"""
        with tf.variable_scope(name):
            params_shape = [x.get_shape()[-1]]

            beta = tf.get_variable(
                "beta",
                params_shape,
                tf.float32,
                initializer=tf.constant_initializer(0.0, tf.float32))
            gamma = tf.get_variable(
                "gamma",
                params_shape,
                tf.float32,
                initializer=tf.constant_initializer(1.0, tf.float32))

            if self.mode == "train":
                mean, variance = tf.nn.moments(x, [0, 1, 2], name="moments")

                moving_mean = tf.get_variable(
                    "moving_mean",
                    params_shape,
                    tf.float32,
                    initializer=tf.constant_initializer(0.0, tf.float32),
                    trainable=False)
                moving_variance = tf.get_variable(
                    "moving_variance",
                    params_shape,
                    tf.float32,
                    initializer=tf.constant_initializer(1.0, tf.float32),
                    trainable=False)

                self._extra_train_ops.append(
                    moving_averages.assign_moving_average(
                        moving_mean, mean, 0.9))
                self._extra_train_ops.append(
                    moving_averages.assign_moving_average(
                        moving_variance, variance, 0.9))
            else:
                mean = tf.get_variable(
                    "moving_mean",
                    params_shape,
                    tf.float32,
                    initializer=tf.constant_initializer(0.0, tf.float32),
                    trainable=False)
                variance = tf.get_variable(
                    "moving_variance",
                    params_shape,
                    tf.float32,
                    initializer=tf.constant_initializer(1.0, tf.float32),
                    trainable=False)
                tf.summary.histogram(mean.op.name, mean)
                tf.summary.histogram(variance.op.name, variance)
            # elipson used to be 1e-5. Maybe 0.001 solves NaN problem in deeper
            # net.
            y = tf.nn.batch_normalization(x, mean, variance, beta, gamma,
                                          0.001)
            y.set_shape(x.get_shape())
            return y
552948
def normalize(self, text, cleaned=False, **kwargs):
        """"""
        if not cleaned:
            text = self.clean(text, **kwargs)
        return ensure_list(text)
1171181
def _normalize_seqs(s, t):
    """"""
    for ids in s:
        obj = s[ids]
        [obj.norm_freq.update({sample: 1.0 * obj.freq[sample] / (t[sample]+1) * 1000000}) for sample in obj.norm_freq]
        s[ids] = obj
    return s
350472
def normalize(self, stats:Collection[Tensor]=None, do_x:bool=True, do_y:bool=False)->None:
        ""
        if getattr(self,'norm',False): raise Exception('Can not call normalize twice')
        if stats is None: self.stats = self.batch_stats()
        else:             self.stats = stats
        self.norm,self.denorm = normalize_funcs(*self.stats, do_x=do_x, do_y=do_y)
        self.add_tfm(self.norm)
        return self
556752
def skipgram_batch(centers, contexts, num_tokens, dtype, index_dtype):
    """"""
    contexts = mx.nd.array(contexts[2], dtype=index_dtype)
    indptr = mx.nd.arange(len(centers) + 1)
    centers = mx.nd.array(centers, dtype=index_dtype)
    centers_csr = mx.nd.sparse.csr_matrix(
        (mx.nd.ones(centers.shape), centers, indptr), dtype=dtype,
        shape=(len(centers), num_tokens))
    return centers_csr, contexts, centers
896575
@SuppressWarnings("all")
  public void normalize() {
    // Start on samples
    int samplesPerFrame = (int) ((float) getSamplesPerSecond() / (float) getFramesPerSecond());
    setFrames(getFrames() + (int) (getSamples() * (1.0f / samplesPerFrame)));
    setSamples(getSamples() % samplesPerFrame);

    setSeconds(getSeconds() + (int) ((float) getFrames() / getFramesPerSecond()));
    setFrames((int) (getFrames() % getFramesPerSecond()));

    while (getSeconds() >= 60) {
      setSeconds(getSeconds() - 60);
      setMinutes(getMinutes() + 1);
    }
    while (getMinutes() >= 60) {
      setMinutes(getMinutes() - 60);
      setHours(getHours() + 1);
    }
  }
922793
def image_embedding(images,
                    model_fn=resnet_v1_152,
                    trainable=True,
                    is_training=True,
                    weight_decay=0.0001,
                    batch_norm_decay=0.997,
                    batch_norm_epsilon=1e-5,
                    batch_norm_scale=True,
                    add_summaries=False,
                    reuse=False):
  """"""

  is_resnet_training = trainable and is_training

  batch_norm_params = {
      "is_training": is_resnet_training,
      "trainable": trainable,
      "decay": batch_norm_decay,
      "epsilon": batch_norm_epsilon,
      "scale": batch_norm_scale,
  }

  if trainable:
    weights_regularizer = tf.contrib.layers.l2_regularizer(weight_decay)
  else:
    weights_regularizer = None

  with tf.variable_scope(model_fn.__name__, [images], reuse=reuse) as scope:
    with slim.arg_scope(
        [slim.conv2d],
        weights_regularizer=weights_regularizer,
        trainable=trainable):
      with slim.arg_scope(
          [slim.conv2d],
          weights_initializer=slim.variance_scaling_initializer(),
          activation_fn=tf.nn.relu,
          normalizer_fn=slim.batch_norm,
          normalizer_params=batch_norm_params):
        with slim.arg_scope([slim.batch_norm],
                            is_training=is_resnet_training,
                            trainable=trainable):
          with slim.arg_scope([slim.max_pool2d], padding="SAME"):
            net, end_points = model_fn(
                images, num_classes=None, global_pool=False,
                is_training=is_resnet_training,
                reuse=reuse, scope=scope)

  if add_summaries:
    for v in end_points.values():
      tf.contrib.layers.summaries.summarize_activation(v)

  return net
26448
def normalize(v, axis=None, eps=1e-10):
  """"""
  return v / max(anorm(v, axis=axis, keepdims=True), eps)
436515
def _batch_norm(name, x):
  """"""
  with tf.name_scope(name):
    return tf.contrib.layers.batch_norm(
        inputs=x,
        decay=.9,
        center=True,
        scale=True,
        activation_fn=None,
        updates_collections=None,
        is_training=False)
762163
def _normalize_missing(g):
        """"""
        g = g.astype(float)
        g[g == -1.0] = np.nan
        return g
86371
def _create_batch_norm_variables(self, dtype):
    """"""
    # The paper recommends a value of 0.1 for good gradient flow through the
    # tanh nonlinearity (although doesn't say whether this is for all gammas,
    # or just some).
    gamma_initializer = tf.constant_initializer(0.1)

    if self._use_batch_norm_h:
      self._gamma_h = tf.get_variable(
          self.GAMMA_H,
          shape=[4 * self._hidden_size],
          dtype=dtype,
          initializer=self._initializers.get(self.GAMMA_H, gamma_initializer),
          partitioner=self._partitioners.get(self.GAMMA_H),
          regularizer=self._regularizers.get(self.GAMMA_H))
    if self._use_batch_norm_x:
      self._gamma_x = tf.get_variable(
          self.GAMMA_X,
          shape=[4 * self._hidden_size],
          dtype=dtype,
          initializer=self._initializers.get(self.GAMMA_X, gamma_initializer),
          partitioner=self._partitioners.get(self.GAMMA_X),
          regularizer=self._regularizers.get(self.GAMMA_X))
    if self._use_batch_norm_c:
      self._gamma_c = tf.get_variable(
          self.GAMMA_C,
          shape=[self._hidden_size],
          dtype=dtype,
          initializer=self._initializers.get(self.GAMMA_C, gamma_initializer),
          partitioner=self._partitioners.get(self.GAMMA_C),
          regularizer=self._regularizers.get(self.GAMMA_C))
      self._beta_c = tf.get_variable(
          self.BETA_C,
          shape=[self._hidden_size],
          dtype=dtype,
          initializer=self._initializers.get(self.BETA_C),
          partitioner=self._partitioners.get(self.BETA_C),
          regularizer=self._regularizers.get(self.BETA_C))
400590
def normalize_funcs(mean:FloatTensor, std:FloatTensor, do_x:bool=True, do_y:bool=False)->Tuple[Callable,Callable]:
    ""
    mean,std = tensor(mean),tensor(std)
    return (partial(_normalize_batch, mean=mean, std=std, do_x=do_x, do_y=do_y),
            partial(denormalize,      mean=mean, std=std, do_x=do_x))
61098
def rsdl_rn(self, AX, Y):
        """"""

        # The primal residual normalisation term is
        # max( ||A x^(k)||_2, ||B y^(k)||_2 ) and B = -(I I I ...)^T.
        # The scaling by sqrt(Nb) of the l2 norm of Y accounts for the
        # block replication introduced by multiplication by B
        return max((np.linalg.norm(AX), np.sqrt(self.Nb) * np.linalg.norm(Y)))
978878
def _build_scale_offset(self, dtype):
    """"""

    # tf.nn.fused_batch_norm accepts float16 batch data, but not scale/offset.
    if self._fused and dtype == tf.float16:
      dtype = tf.float32

    # The fused batch norm operation needs the beta, gamma variables,
    # so in this case we build them and set the trainable option according
    # to the values of _offset and _scale.
    self._beta = None
    if self._offset or self._fused:
      if self.BETA not in self._initializers:
        self._initializers[self.BETA] = create_beta_initializer()
      self._beta = tf.get_variable(
          self.BETA,
          dtype=dtype,
          shape=self._mean_shape,
          initializer=self._initializers[self.BETA],
          partitioner=self._partitioners.get(self.BETA, None),
          regularizer=self._regularizers.get(self.BETA, None),
          trainable=self._offset)

    self._gamma = None
    if self._scale or self._fused:
      if self.GAMMA not in self._initializers:
        self._initializers[self.GAMMA] = create_gamma_initializer()
      self._gamma = tf.get_variable(
          self.GAMMA,
          dtype=dtype,
          shape=self._mean_shape,
          initializer=self._initializers[self.GAMMA],
          partitioner=self._partitioners.get(self.GAMMA, None),
          regularizer=self._regularizers.get(self.GAMMA, None),
          trainable=self._scale)
1044115
def _stack_batches(X):
        """"""
        X = [Classifier._to_numpy(Xb) for Xb in X]
        if len(X[0].shape) == 1:
            return np.hstack(X)
        elif len(X[0].shape) == 2:
            return np.vstack(X)
        else:
            raise ValueError(f"Can't stack {len(X[0].shape)}-dim batches.")
1014489
private void normalize() {
		for(int i = 0; i < vocab.size(); ++i) {
			double len = 0;
			for(int j = i * layerSize; j < (i + 1) * layerSize; ++j)
				len += vectors.get(j) * vectors.get(j);
			len = Math.sqrt(len);

			for(int j = i * layerSize; j < (i + 1) * layerSize; ++j)
				vectors.put(j, vectors.get(j) / len);
		}
	}
1353914
def batch(items, size):
    """"""
    return [items[x:x + size] for x in xrange(0, len(items), size)]