Convolutional GRU in 1 dimension.
170517
def conv_gru(x,
             kernel_size,
             filters,
             padding="SAME",
             dilation_rate=(1, 1),
             name=None,
             reuse=None):
  """"""

  # Let's make a shorthand for conv call first.
  def do_conv(args, name, bias_start, padding):
    return conv(
        args,
        filters,
        kernel_size,
        padding=padding,
        dilation_rate=dilation_rate,
        bias_initializer=tf.constant_initializer(bias_start),
        name=name)

  # Here comes the GRU gate.
  with tf.variable_scope(
      name, default_name="conv_gru", values=[x], reuse=reuse):
    reset = saturating_sigmoid(do_conv(x, "reset", 1.0, padding))
    gate = saturating_sigmoid(do_conv(x, "gate", 1.0, padding))
    candidate = tf.tanh(do_conv(reset * x, "candidate", 0.0, padding))
    return gate * x + (1 - gate) * candidate
642846
def double_discriminator(x, filters1=128, filters2=None,
                         kernel_size=8, strides=4, pure_mean=False):
  """"""
  if filters2 is None:
    filters2 = 4 * filters1
  with tf.variable_scope("discriminator"):
    batch_size = shape_list(x)[0]
    net = layers().Conv2D(
        filters1, kernel_size, strides=strides, padding="SAME", name="conv1")(x)
    if pure_mean:
      net1 = tf.reduce_mean(net, [1, 2])
    else:
      net1 = mean_with_attention(net, "mean_with_attention1")
      tf.reshape(net, [batch_size, -1])
    net = tf.nn.relu(net)
    net = layers().Conv2D(
        filters2, kernel_size, strides=strides, padding="SAME", name="conv2")(x)
    if pure_mean:
      net2 = tf.reduce_mean(net, [1, 2])
    else:
      net2 = mean_with_attention(net, "mean_with_attention2")
    return tf.concat([net1, net2], axis=-1)
1211934
def dueling_model(img_in, num_actions, scope, noisy=False, reuse=False,
                  concat_softmax=False):
  """"""
  with tf.variable_scope(scope, reuse=reuse):
    out = img_in
    with tf.variable_scope("convnet"):
      # original architecture
      out = layers.convolution2d(out, num_outputs=32, kernel_size=8,
                                 stride=4, activation_fn=tf.nn.relu)
      out = layers.convolution2d(out, num_outputs=64, kernel_size=4,
                                 stride=2, activation_fn=tf.nn.relu)
      out = layers.convolution2d(out, num_outputs=64, kernel_size=3,
                                 stride=1, activation_fn=tf.nn.relu)
    out = layers.flatten(out)

    with tf.variable_scope("state_value"):
      if noisy:
        # Apply noisy network on fully connected layers
        # ref: https://arxiv.org/abs/1706.10295
        state_hidden = noisy_dense(out, name='noisy_fc1', size=512,
                                   activation_fn=tf.nn.relu)
        state_score = noisy_dense(state_hidden, name='noisy_fc2',
                                  size=1)
      else:
        state_hidden = layers.fully_connected(
            out,
            num_outputs=512,
            activation_fn=tf.nn.relu
        )
        state_score = layers.fully_connected(state_hidden,
                                             num_outputs=1,
                                             activation_fn=None)
    with tf.variable_scope("action_value"):
      if noisy:
        # Apply noisy network on fully connected layers
        # ref: https://arxiv.org/abs/1706.10295
        actions_hidden = noisy_dense(out, name='noisy_fc1', size=512,
                                     activation_fn=tf.nn.relu)
        action_scores = noisy_dense(actions_hidden, name='noisy_fc2',
                                    size=num_actions)
      else:
        actions_hidden = layers.fully_connected(
            out,
            num_outputs=512,
            activation_fn=tf.nn.relu
        )
        action_scores = layers.fully_connected(
            actions_hidden,
            num_outputs=num_actions,
            activation_fn=None
        )
      action_scores_mean = tf.reduce_mean(action_scores, 1)
      action_scores = action_scores - tf.expand_dims(
          action_scores_mean,
          1
      )

    return state_score + action_scores
813061
def unit(w, sparsity):
  """"""
  w_shape = common_layers.shape_list(w)
  count = tf.to_int32(w_shape[-1] * sparsity)
  mask = common_layers.unit_targeting(w, count)
  return (1 - mask) * w
992782
def conv_lstm_2d(inputs, state, output_channels,
                 kernel_size=5, name=None, spatial_dims=None):
  """"""
  input_shape = common_layers.shape_list(inputs)
  batch_size, input_channels = input_shape[0], input_shape[-1]
  if spatial_dims is None:
    input_shape = input_shape[1:]
  else:
    input_shape = spatial_dims + [input_channels]

  cell = tf.contrib.rnn.ConvLSTMCell(
      2, input_shape, output_channels,
      [kernel_size, kernel_size], name=name)
  if state is None:
    state = cell.zero_state(batch_size, tf.float32)
  outputs, new_state = cell(inputs, state)
  return outputs, new_state
552374
def convolutional_layer_series(initial_size, layer_sequence):
    
    size = initial_size

    for filter_size, padding, stride in layer_sequence:
        size = convolution_size_equation(size, filter_size, padding, stride)

    return size
1370960
private static void convolve1D(GrayU8 gray) {
		ImageBorder<GrayU8> border = FactoryImageBorder.wrap(BorderType.EXTENDED, gray);
		Kernel1D_S32 kernel = new Kernel1D_S32(2);
		kernel.offset = 1; // specify the kernel's origin
		kernel.data[0] = 1;
		kernel.data[1] = -1;

		GrayS16 output = new GrayS16(gray.width,gray.height);

		GConvolveImageOps.horizontal(kernel, gray, output, border);
		panel.addImage(VisualizeImageData.standard(output, null), "1D Horizontal");

		GConvolveImageOps.vertical(kernel, gray, output, border);
		panel.addImage(VisualizeImageData.standard(output, null), "1D Vertical");
	}
474798
function (inData, outData, width, height) {
        var i, n = width * height * 4,
            gr1, gr2, gg1, gg2, gb1, gb2,
            data1 = [],
            data2 = [];

        convolve3x3(inData, data1, width, height,
            [
                [-1, 0, 1],
                [-2, 0, 2],
                [-1, 0, 1]
            ]);

        convolve3x3(inData, data2, width, height,
            [
                [-1, -2, -1],
                [ 0, 0, 0],
                [ 1, 2, 1]
            ]);

        for (i = 0; i < n; i += 4) {
            gr1 = data1[i];
            gr2 = data2[i];
            gg1 = data1[i + 1];
            gg2 = data2[i + 1];
            gb1 = data1[i + 2];
            gb2 = data2[i + 2];

            if (gr1 < 0) {
                gr1 = -gr1;
            }
            if (gr2 < 0) {
                gr2 = -gr2;
            }
            if (gg1 < 0) {
                gg1 = -gg1;
            }
            if (gg2 < 0) {
                gg2 = -gg2;
            }
            if (gb1 < 0) {
                gb1 = -gb1;
            }
            if (gb2 < 0) {
                gb2 = -gb2;
            }

            outData[i] = 255 - (gr1 + gr2) * 0.8;
            outData[i + 1] = 255 - (gg1 + gg2) * 0.8;
            outData[i + 2] = 255 - (gb1 + gb2) * 0.8;
            outData[i + 3] = inData[i + 3];
        }
    }
787987
def local_attention1d_masked_decoder(x, kv_dim, heads_dim,
                                     feedforward_dim, hparams):
  """"""
  print(x)
  _, length_dim, model_dim = x.shape.dims
  for layer in range(hparams.num_decoder_layers):
    layer_name = "decoder_layer_%d" % layer
    with tf.variable_scope(layer_name):
      # Self attention layer
      length_per_split = mtf.tensor_dim_to_size_per_split(
          hparams.layout, hparams.mesh_shape, length_dim)
      x += layer_prepostprocess_dropout(
          mtf.layers.masked_local_attention_1d(
              mtf.layers.layer_norm(x, model_dim, name="layer_norm_att"),
              kv_dim,
              heads_dim,
              window_size=hparams.block_length,
              length_per_split=length_per_split,
              name="self_att"), hparams)
      # ffn layer
      x += layer_prepostprocess_dropout(
          mtf.layers.dense_relu_dense(
              mtf.layers.layer_norm(x, model_dim, name="layer_norm_ffn"),
              feedforward_dim,
              hparams.dropout,
              dropout_broadcast_dims=[length_dim]), hparams)

  output = mtf.layers.layer_norm(x, model_dim, name="final_layer_norm")
  return output
1114339
def conv_layer(ni:int, nf:int, ks:int=3, stride:int=1, padding:int=None, bias:bool=None, is_1d:bool=False,
               norm_type:Optional[NormType]=NormType.Batch,  use_activ:bool=True, leaky:float=None,
               transpose:bool=False, init:Callable=nn.init.kaiming_normal_, self_attention:bool=False):
    ""
    if padding is None: padding = (ks-1)//2 if not transpose else 0
    bn = norm_type in (NormType.Batch, NormType.BatchZero)
    if bias is None: bias = not bn
    conv_func = nn.ConvTranspose2d if transpose else nn.Conv1d if is_1d else nn.Conv2d
    conv = init_default(conv_func(ni, nf, kernel_size=ks, bias=bias, stride=stride, padding=padding), init)
    if   norm_type==NormType.Weight:   conv = weight_norm(conv)
    elif norm_type==NormType.Spectral: conv = spectral_norm(conv)
    layers = [conv]
    if use_activ: layers.append(relu(True, leaky=leaky))
    if bn: layers.append((nn.BatchNorm1d if is_1d else nn.BatchNorm2d)(nf))
    if self_attention: layers.append(SelfAttention(nf))
    return nn.Sequential(*layers)
250236
def separable_conv_block(inputs, filters, dilation_rates_and_kernel_sizes,
                         **kwargs):
  """"""
  return conv_block_internal(separable_conv, inputs, filters,
                             dilation_rates_and_kernel_sizes, **kwargs)
755164
def conv(input, kernel, biases, k_h, k_w, c_o, s_h, s_w,  padding="VALID", group=1):
    
    c_i = input.get_shape()[-1]
    assert c_i%group==0
    assert c_o%group==0
    convolve = lambda i, k: tf.nn.conv2d(i, k, [1, s_h, s_w, 1], padding=padding)
    
    if group==1:
        conv = convolve(input, kernel)
    else:
        input_groups = tf.split(input, group, axis=3)
        kernel_groups = tf.split(kernel, group, axis=3)
        output_groups = [convolve(i, k) for i,k in zip(input_groups, kernel_groups)]
        conv = tf.concat(output_groups, axis=3)
    return  tf.reshape(tf.nn.bias_add(conv, biases), [-1]+conv.get_shape().as_list()[1:])
999906
def conv1d(ni:int, no:int, ks:int=1, stride:int=1, padding:int=0, bias:bool=False):
    ""
    conv = nn.Conv1d(ni, no, ks, stride=stride, padding=padding, bias=bias)
    nn.init.kaiming_normal_(conv.weight)
    if bias: conv.bias.data.zero_()
    return spectral_norm(conv)
1136767
def gan_critic(n_channels:int=3, nf:int=128, n_blocks:int=3, p:int=0.15):
    ""
    layers = [
        _conv(n_channels, nf, ks=4, stride=2),
        nn.Dropout2d(p/2),
        res_block(nf, dense=True,**_conv_args)]
    nf *= 2 # after dense block
    for i in range(n_blocks):
        layers += [
            nn.Dropout2d(p),
            _conv(nf, nf*2, ks=4, stride=2, self_attention=(i==0))]
        nf *= 2
    layers += [
        _conv(nf, 1, ks=4, bias=False, padding=0, use_activ=False),
        Flatten()]
    return nn.Sequential(*layers)
481413
def subseparable_conv(inputs, filters, kernel_size, **kwargs):
  """"""

  def conv_fn(inputs, filters, kernel_size, **kwargs):
    """Sub-separable convolution, splits into separability-many blocks."""
    separability = None
    if "separability" in kwargs:
      separability = kwargs.pop("separability")
    if separability:
      parts = []
      abs_sep = separability if separability > 0 else -1 * separability
      for split_idx, split in enumerate(tf.split(inputs, abs_sep, axis=3)):
        with tf.variable_scope("part_%d" % split_idx):
          if separability > 0:
            parts.append(
                layers().Conv2D(filters // separability, kernel_size,
                                **kwargs)(split))
          else:
            parts.append(
                layers().SeparableConv2D(filters // abs_sep,
                                         kernel_size, **kwargs)(split))
      if separability > 1:
        result = layers().Conv2D(filters, (1, 1))(tf.concat(parts, axis=3))
      elif abs_sep == 1:  # If we have just one block, return it.
        assert len(parts) == 1
        result = parts[0]
      else:
        result = tf.concat(parts, axis=3)
    else:
      result = layers().SeparableConv2D(filters, kernel_size,
                                        **kwargs)(inputs)
    if separability is not None:
      kwargs["separability"] = separability
    return result

  return conv_internal(conv_fn, inputs, filters, kernel_size, **kwargs)
1057506
def _compute_gradients(self, loss_fn, x, unused_optim_state):
    """"""
    # Assumes `x` is a list, containing a [1, H, W, C] image
    # If static batch dimension is None, tf.reshape to batch size 1
    # so that static shape can be inferred
    assert len(x) == 1
    static_x_shape = x[0].get_shape().as_list()
    if static_x_shape[0] is None:
      x[0] = tf.reshape(x[0], [1] + static_x_shape[1:])
    assert x[0].get_shape().as_list()[0] == 1
    x = x[0]
    x_shape = x.get_shape().as_list()

    def body(i, grad_array):
      delta = self._delta
      delta_x = self._get_delta(x, delta)
      delta_x = tf.concat([delta_x, -delta_x], axis=0)
      loss_vals = tf.reshape(
          loss_fn(x + delta_x),
          [2 * self._num_samples] + [1] * (len(x_shape) - 1))
      avg_grad = reduce_mean(loss_vals * delta_x, axis=0) / delta
      avg_grad = tf.expand_dims(avg_grad, axis=0)
      new_grad_array = grad_array.write(i, avg_grad)
      return i + 1, new_grad_array

    def cond(i, _):
      return i < self._num_iters

    _, all_grads = tf.while_loop(
        cond,
        body,
        loop_vars=[
            0, tf.TensorArray(size=self._num_iters, dtype=tf_dtype)
        ],
        back_prop=False,
        parallel_iterations=1)
    avg_grad = reduce_sum(all_grads.stack(), axis=0)
    return [avg_grad]
992985
public static INDArray convn(INDArray input, INDArray kernel, Type type, int[] axes) {
        return Nd4j.getConvolution().convn(input, kernel, type, axes);
    }