PolyNet architecture from the paper
    'PolyNet: A Pursuit of Structural Diversity in Very Deep Networks'
    https://arxiv.org/abs/1611.05725
774783
def polynet(num_classes=1000, pretrained='imagenet'):
    
    if pretrained:
        settings = pretrained_settings['polynet'][pretrained]
        assert num_classes == settings['num_classes'], \
            'num_classes should be {}, but is {}'.format(
                settings['num_classes'], num_classes)
        model = PolyNet(num_classes=num_classes)
        model.load_state_dict(model_zoo.load_url(settings['url']))
        model.input_space = settings['input_space']
        model.input_size = settings['input_size']
        model.input_range = settings['input_range']
        model.mean = settings['mean']
        model.std = settings['std']
    else:
        model = PolyNet(num_classes=num_classes)
    return model
1036976
def transformer_decoder_layers(inputs,
                               encoder_output,
                               num_layers,
                               hparams,
                               self_attention_bias=None,
                               encoder_decoder_attention_bias=None,
                               attention_type=AttentionType.LOCAL_2D,
                               losses=None,
                               name="transformer"):
  """"""
  x = inputs
  x = tf.nn.dropout(x, 1.0 - hparams.layer_prepostprocess_dropout)
  if attention_type == AttentionType.DILATED:
    assert len(hparams.gap_sizes) == num_layers
  for layer in range(num_layers):
    with tf.variable_scope("%s_layer_%d" % (name, layer)):
      # self-attention + skip connections
      if attention_type == AttentionType.LOCAL_2D:
        y = local_attention_2d(common_layers.layer_preprocess(x, hparams),
                               hparams,
                               attention_type="masked_local_attention_2d")
      elif attention_type == AttentionType.LOCAL_1D:
        y = local_attention_1d(common_layers.layer_preprocess(x, hparams),
                               hparams,
                               attention_type="local_mask_right",
                               q_padding="LEFT", kv_padding="LEFT")
      elif attention_type == AttentionType.RELATIVE_LOCAL_1D:
        y = local_attention_1d(
            common_layers.layer_preprocess(x, hparams),
            hparams,
            attention_type="local_relative_mask_right",
            q_padding="LEFT",
            kv_padding="LEFT")
      elif attention_type == AttentionType.NON_CAUSAL_1D:
        y = local_attention_1d(common_layers.layer_preprocess(x, hparams),
                               hparams,
                               attention_type="local_unmasked",
                               q_padding="VALID", kv_padding="VALID")
      elif attention_type == AttentionType.LOCAL_BLOCK:
        y = local_within_block_attention(
            common_layers.layer_preprocess(x, hparams),
            self_attention_bias, hparams,
            attention_type="local_within_block_mask_right",
            q_padding="LEFT", kv_padding="LEFT")
      elif attention_type == AttentionType.GLOCAL:
        y = local_global_attention(common_layers.layer_preprocess(x, hparams),
                                   self_attention_bias, hparams,
                                   q_padding="LEFT", kv_padding="LEFT")
      elif attention_type == AttentionType.DILATED:
        y = dilated_attention_1d(common_layers.layer_preprocess(x, hparams),
                                 hparams, q_padding="LEFT",
                                 kv_padding="LEFT",
                                 gap_size=hparams.gap_sizes[layer])
      elif attention_type == AttentionType.GLOBAL:
        y = full_self_attention(common_layers.layer_preprocess(x, hparams),
                                self_attention_bias, hparams,
                                q_padding="LEFT", kv_padding="LEFT")
      x = common_layers.layer_postprocess(x, y, hparams)
      # enc-dec attention + skip connections
      if encoder_output is not None:
        y = encdec_attention_1d(common_layers.layer_preprocess(x, hparams),
                                encoder_output,
                                encoder_decoder_attention_bias,
                                hparams)
        x = common_layers.layer_postprocess(x, y, hparams)
      # feed-fwd layers + skip connections
      y = ffn_layer(common_layers.layer_preprocess(x, hparams), hparams,
                    losses=losses)
      x = common_layers.layer_postprocess(x, y, hparams)
  return common_layers.layer_preprocess(x, hparams)
922793
def image_embedding(images,
                    model_fn=resnet_v1_152,
                    trainable=True,
                    is_training=True,
                    weight_decay=0.0001,
                    batch_norm_decay=0.997,
                    batch_norm_epsilon=1e-5,
                    batch_norm_scale=True,
                    add_summaries=False,
                    reuse=False):
  """"""

  is_resnet_training = trainable and is_training

  batch_norm_params = {
      "is_training": is_resnet_training,
      "trainable": trainable,
      "decay": batch_norm_decay,
      "epsilon": batch_norm_epsilon,
      "scale": batch_norm_scale,
  }

  if trainable:
    weights_regularizer = tf.contrib.layers.l2_regularizer(weight_decay)
  else:
    weights_regularizer = None

  with tf.variable_scope(model_fn.__name__, [images], reuse=reuse) as scope:
    with slim.arg_scope(
        [slim.conv2d],
        weights_regularizer=weights_regularizer,
        trainable=trainable):
      with slim.arg_scope(
          [slim.conv2d],
          weights_initializer=slim.variance_scaling_initializer(),
          activation_fn=tf.nn.relu,
          normalizer_fn=slim.batch_norm,
          normalizer_params=batch_norm_params):
        with slim.arg_scope([slim.batch_norm],
                            is_training=is_resnet_training,
                            trainable=trainable):
          with slim.arg_scope([slim.max_pool2d], padding="SAME"):
            net, end_points = model_fn(
                images, num_classes=None, global_pool=False,
                is_training=is_resnet_training,
                reuse=reuse, scope=scope)

  if add_summaries:
    for v in end_points.values():
      tf.contrib.layers.summaries.summarize_activation(v)

  return net
584072
def dpcnn(embedding_matrix, embedding_size, trainable_embedding, maxlen, max_features,
          filter_nr, kernel_size, repeat_block, dense_size, repeat_dense, output_size, output_activation,
          max_pooling, mean_pooling, weighted_average_attention, concat_mode,
          dropout_embedding, conv_dropout, dense_dropout, dropout_mode,
          conv_kernel_reg_l2, conv_bias_reg_l2,
          dense_kernel_reg_l2, dense_bias_reg_l2,
          use_prelu, use_batch_norm, batch_norm_first):
    

    input_text = Input(shape=(maxlen,))
    if embedding_matrix is not None:
        embedding = Embedding(max_features, embedding_size,
                              weights=[embedding_matrix], trainable=trainable_embedding)(input_text)
    else:
        embedding = Embedding(max_features, embedding_size)(input_text)

    embedding = dropout_block(dropout_embedding, dropout_mode)(embedding)

    x = convolutional_block(filter_nr, kernel_size, use_batch_norm, use_prelu, conv_dropout, dropout_mode,
                            conv_kernel_reg_l2, conv_bias_reg_l2, batch_norm_first)(embedding)
    x = convolutional_block(filter_nr, kernel_size, conv_bias_reg_l2, use_prelu, conv_dropout, dropout_mode,
                            conv_kernel_reg_l2, conv_bias_reg_l2, batch_norm_first)(x)
    if embedding_size == filter_nr:
        x = add([embedding, x])
    else:
        embedding_resized = shape_matching_layer(filter_nr, use_prelu, conv_kernel_reg_l2, conv_bias_reg_l2)(embedding)
        x = add([embedding_resized, x])
    for _ in range(repeat_block):
        x = dpcnn_block(filter_nr, kernel_size, use_batch_norm, use_prelu, conv_dropout, dropout_mode,
                        conv_kernel_reg_l2, conv_bias_reg_l2, batch_norm_first)(x)

    predictions = classification_block(dense_size=dense_size, repeat_dense=repeat_dense,
                                       output_size=output_size, output_activation=output_activation,
                                       max_pooling=max_pooling,
                                       mean_pooling=mean_pooling,
                                       weighted_average_attention=weighted_average_attention,
                                       concat_mode=concat_mode,
                                       dropout=dense_dropout,
                                       kernel_reg_l2=dense_kernel_reg_l2, bias_reg_l2=dense_bias_reg_l2,
                                       use_prelu=use_prelu, use_batch_norm=use_batch_norm,
                                       batch_norm_first=batch_norm_first)(x)
    model = Model(inputs=input_text, outputs=predictions)
    return model
1156057
def build_network(self):
        '''
        
        '''

        # Reshape to use within a convolutional neural net.
        # Last dimension is for "features" - there is only one here, since images are
        # grayscale -- it would be 3 for an RGB image, 4 for RGBA, etc.
        with tf.name_scope('reshape'):
            try:
                input_dim = int(math.sqrt(self.x_dim))
            except:
                print(
                    'input dim cannot be sqrt and reshape. input dim: ' + str(self.x_dim))
                logger.debug(
                    'input dim cannot be sqrt and reshape. input dim: %s', str(self.x_dim))
                raise
            x_image = tf.reshape(self.images, [-1, input_dim, input_dim, 1])

        # First convolutional layer - maps one grayscale image to 32 feature maps.
        with tf.name_scope('conv1'):
            w_conv1 = weight_variable(
                [self.conv_size, self.conv_size, 1, self.channel_1_num])
            b_conv1 = bias_variable([self.channel_1_num])
            h_conv1 = tf.nn.relu(conv2d(x_image, w_conv1) + b_conv1)

        # Pooling layer - downsamples by 2X.
        with tf.name_scope('pool1'):
            h_pool1 = max_pool(h_conv1, self.pool_size)

        # Second convolutional layer -- maps 32 feature maps to 64.
        with tf.name_scope('conv2'):
            w_conv2 = weight_variable([self.conv_size, self.conv_size,
                                       self.channel_1_num, self.channel_2_num])
            b_conv2 = bias_variable([self.channel_2_num])
            h_conv2 = tf.nn.relu(conv2d(h_pool1, w_conv2) + b_conv2)

        # Second pooling layer.
        with tf.name_scope('pool2'):
            h_pool2 = max_pool(h_conv2, self.pool_size)

        # Fully connected layer 1 -- after 2 round of downsampling, our 28x28 image
        # is down to 7x7x64 feature maps -- maps this to 1024 features.
        last_dim = int(input_dim / (self.pool_size * self.pool_size))
        with tf.name_scope('fc1'):
            w_fc1 = weight_variable(
                [last_dim * last_dim * self.channel_2_num, self.hidden_size])
            b_fc1 = bias_variable([self.hidden_size])

        h_pool2_flat = tf.reshape(
            h_pool2, [-1, last_dim * last_dim * self.channel_2_num])
        h_fc1 = tf.nn.relu(tf.matmul(h_pool2_flat, w_fc1) + b_fc1)

        # Dropout - controls the complexity of the model, prevents co-adaptation of features.
        with tf.name_scope('dropout'):
            h_fc1_drop = tf.nn.dropout(h_fc1, self.keep_prob)

        # Map the 1024 features to 10 classes, one for each digit
        with tf.name_scope('fc2'):
            w_fc2 = weight_variable([self.hidden_size, self.y_dim])
            b_fc2 = bias_variable([self.y_dim])
            y_conv = tf.matmul(h_fc1_drop, w_fc2) + b_fc2

        with tf.name_scope('loss'):
            cross_entropy = tf.reduce_mean(
                tf.nn.softmax_cross_entropy_with_logits(labels=self.labels, logits=y_conv))
        with tf.name_scope('adam_optimizer'):
            self.train_step = tf.train.AdamOptimizer(
                self.learning_rate).minimize(cross_entropy)

        with tf.name_scope('accuracy'):
            correct_prediction = tf.equal(
                tf.argmax(y_conv, 1), tf.argmax(self.labels, 1))
            self.accuracy = tf.reduce_mean(
                tf.cast(correct_prediction, tf.float32))
1063894
def conv2d(self, x_in: Connection, w_in: Connection, receptive_field_size, filters_number, stride=1, padding=1,
               name=""):
        
        x_cols = self.tensor_3d_to_cols(x_in, receptive_field_size, stride=stride, padding=padding)
        mul = self.transpose(self.matrix_multiply(x_cols, w_in), 0, 2, 1)

        #output_width = self.sum(self.div(self.sum(self.sum(self.shape(x_in, 2), self.constant(-1 * receptive_field_size)),
        #                        self.constant(2 * padding)), self.constant(stride)), self.constant(1))
        # output_height = (h - f + 2 * p) / s + 1

        output = self.reshape(mul, (-1, filters_number, receptive_field_size, receptive_field_size))

        output.name = name
        return output
905208
def cnn_model_fn(features, labels, mode):
  """"""
  # Input Layer
  # Reshape X to 4-D tensor: [batch_size, width, height, channels]
  # MNIST images are 28x28 pixels, and have one color channel
  input_layer = tf.reshape(features["x"], [-1, 28, 28, 1])

  # Convolutional Layer #1
  # Computes 32 features using a 5x5 filter with ReLU activation.
  # Padding is added to preserve width and height.
  # Input Tensor Shape: [batch_size, 28, 28, 1]
  # Output Tensor Shape: [batch_size, 28, 28, 32]
  conv1 = tf.layers.conv2d(
      inputs=input_layer,
      filters=32,
      kernel_size=[5, 5],
      padding="same",
      activation=tf.nn.relu)

  # Pooling Layer #1
  # First max pooling layer with a 2x2 filter and stride of 2
  # Input Tensor Shape: [batch_size, 28, 28, 32]
  # Output Tensor Shape: [batch_size, 14, 14, 32]
  pool1 = tf.layers.max_pooling2d(inputs=conv1, pool_size=[2, 2], strides=2)

  # Convolutional Layer #2
  # Computes 64 features using a 5x5 filter.
  # Padding is added to preserve width and height.
  # Input Tensor Shape: [batch_size, 14, 14, 32]
  # Output Tensor Shape: [batch_size, 14, 14, 64]
  conv2 = tf.layers.conv2d(
      inputs=pool1,
      filters=64,
      kernel_size=[5, 5],
      padding="same",
      activation=tf.nn.relu)

  # Pooling Layer #2
  # Second max pooling layer with a 2x2 filter and stride of 2
  # Input Tensor Shape: [batch_size, 14, 14, 64]
  # Output Tensor Shape: [batch_size, 7, 7, 64]
  pool2 = tf.layers.max_pooling2d(inputs=conv2, pool_size=[2, 2], strides=2)

  # Flatten tensor into a batch of vectors
  # Input Tensor Shape: [batch_size, 7, 7, 64]
  # Output Tensor Shape: [batch_size, 7 * 7 * 64]
  pool2_flat = tf.reshape(pool2, [-1, 7 * 7 * 64])

  # Dense Layer
  # Densely connected layer with 1024 neurons
  # Input Tensor Shape: [batch_size, 7 * 7 * 64]
  # Output Tensor Shape: [batch_size, 1024]
  dense = tf.layers.dense(inputs=pool2_flat, units=1024, activation=tf.nn.relu)

  # Add dropout operation; 0.6 probability that element will be kept
  dropout = tf.layers.dropout(
      inputs=dense, rate=0.4, training=mode == tf.estimator.ModeKeys.TRAIN)

  # Logits layer
  # Input Tensor Shape: [batch_size, 1024]
  # Output Tensor Shape: [batch_size, 10]
  logits = tf.layers.dense(inputs=dropout, units=10)

  predictions = {
      # Generate predictions (for PREDICT and EVAL mode)
      "classes": tf.argmax(input=logits, axis=1),
      # Add `softmax_tensor` to the graph. It is used for PREDICT and by the
      # `logging_hook`.
      "probabilities": tf.nn.softmax(logits, name="softmax_tensor")
  }
  if mode == tf.estimator.ModeKeys.PREDICT:
    return tf.estimator.EstimatorSpec(mode=mode, predictions=predictions)

  # Calculate Loss (for both TRAIN and EVAL modes)
  loss = tf.losses.sparse_softmax_cross_entropy(labels=labels, logits=logits)

  # Configure the Training Op (for TRAIN mode)
  if mode == tf.estimator.ModeKeys.TRAIN:
    optimizer = tf.train.GradientDescentOptimizer(learning_rate=0.001)
    train_op = optimizer.minimize(
        loss=loss,
        global_step=tf.train.get_global_step())
    return tf.estimator.EstimatorSpec(mode=mode, loss=loss, train_op=train_op)

  # Add evaluation metrics (for EVAL mode)
  eval_metric_ops = {
      "accuracy": tf.metrics.accuracy(
          labels=labels, predictions=predictions["classes"])}
  return tf.estimator.EstimatorSpec(
      mode=mode, loss=loss, eval_metric_ops=eval_metric_ops)
1279045
def forward(self, inputs, lengths):
        
        x = self.embedder(inputs)

        # bidirectional layer
        x = self.dropout(x)
        x = pack_padded_sequence(x, lengths.cpu().numpy(),
                                 batch_first=self.batch_first)
        x, _ = self.rnn_layers[0](x)
        x, _ = pad_packed_sequence(x, batch_first=self.batch_first)

        # 1st unidirectional layer
        x = self.dropout(x)
        x, _ = self.rnn_layers[1](x)

        # the rest of unidirectional layers,
        # with residual connections starting from 3rd layer
        for i in range(2, len(self.rnn_layers)):
            residual = x
            x = self.dropout(x)
            x, _ = self.rnn_layers[i](x)
            x = x + residual

        return x
582445
def analogy_computation_2d(f_first_enc,
                           f_first_frame,
                           f_current_enc,
                           first_depth):
  """"""
  with tf.variable_scope('analogy_computation'):

    frame_enc_diff = f_first_frame - f_first_enc

    frame_enc_diff_enc = tf.layers.conv2d(
        frame_enc_diff,
        first_depth * 4,
        3,
        padding='same',
        activation=tf.nn.relu,
        strides=1)
    f_current_enc_enc = tf.layers.conv2d(
        f_current_enc,
        first_depth * 4,
        3,
        padding='same',
        activation=tf.nn.relu,
        strides=1)

    analogy = tf.concat([frame_enc_diff_enc, f_current_enc_enc], 3)
    analogy = tf.layers.conv2d(
        analogy,
        first_depth * 4,
        3,
        padding='same',
        activation=tf.nn.relu,
        strides=1)
    analogy = tf.contrib.layers.layer_norm(analogy)
    analogy = tf.layers.conv2d(
        analogy,
        first_depth * 4,
        3,
        padding='same',
        activation=tf.nn.relu,
        strides=1)
    return tf.layers.conv2d(
        analogy,
        first_depth * 4,
        3,
        padding='same',
        activation=tf.nn.relu,
        strides=1)
508951
public static ComputationGraphConfiguration.GraphBuilder inceptionV1ResC(
                    ComputationGraphConfiguration.GraphBuilder graph, String blockName, int scale,
                    double activationScale, String input) {
        // loop and add each subsequent resnet blocks
        String previousBlock = input;
        for (int i = 1; i <= scale; i++) {
            graph
                            // 1x1
                            .addLayer(nameLayer(blockName, "cnn1", i),
                                            new ConvolutionLayer.Builder(new int[] {1, 1})
                                                            .convolutionMode(ConvolutionMode.Same).nIn(1344).nOut(192)
                                                            .cudnnAlgoMode(ConvolutionLayer.AlgoMode.NO_WORKSPACE)
                                                            .build(),
                                            previousBlock)
                            .addLayer(nameLayer(blockName, "batch1", i),
                                            new BatchNormalization.Builder(false).decay(0.995).eps(0.001).nIn(192)
                                                            .nOut(192).build(),
                                            nameLayer(blockName, "cnn1", i))
                            // 1x1 -> 1x3 -> 3x1
                            .addLayer(nameLayer(blockName, "cnn2", i),
                                            new ConvolutionLayer.Builder(new int[] {1, 1})
                                                            .convolutionMode(ConvolutionMode.Same).nIn(1344).nOut(192)
                                                            .cudnnAlgoMode(ConvolutionLayer.AlgoMode.NO_WORKSPACE)
                                                            .build(),
                                            previousBlock)
                            .addLayer(nameLayer(blockName, "batch2", i),
                                            new BatchNormalization.Builder(false).decay(0.995).eps(0.001).nIn(192)
                                                            .nOut(192).build(),
                                            nameLayer(blockName, "cnn2", i))
                            .addLayer(nameLayer(blockName, "cnn3", i),
                                            new ConvolutionLayer.Builder(new int[] {1, 3})
                                                            .convolutionMode(ConvolutionMode.Same).nIn(192).nOut(192)
                                                            .cudnnAlgoMode(ConvolutionLayer.AlgoMode.NO_WORKSPACE)
                                                            .build(),
                                            nameLayer(blockName, "batch2", i))
                            .addLayer(nameLayer(blockName, "batch3", i),
                                            new BatchNormalization.Builder(false).decay(0.995).eps(0.001).nIn(192)
                                                            .nOut(192).build(),
                                            nameLayer(blockName, "cnn3", i))
                            .addLayer(nameLayer(blockName, "cnn4", i),
                                            new ConvolutionLayer.Builder(new int[] {3, 1})
                                                            .convolutionMode(ConvolutionMode.Same).nIn(192).nOut(192)
                                                            .cudnnAlgoMode(ConvolutionLayer.AlgoMode.NO_WORKSPACE)
                                                            .build(),
                                            nameLayer(blockName, "batch3", i))
                            .addLayer(nameLayer(blockName, "batch4", i),
                                            new BatchNormalization.Builder(false).decay(0.995).eps(0.001)
                                                            .activation(Activation.TANH).nIn(192).nOut(192).build(),
                                            nameLayer(blockName, "cnn4", i))
                            // --> 1x1 --> scale -->
                            .addVertex(nameLayer(blockName, "merge1", i), new MergeVertex(),
                                            nameLayer(blockName, "batch1", i), nameLayer(blockName, "batch4", i))
                            .addLayer(nameLayer(blockName, "cnn5", i),
                                            new ConvolutionLayer.Builder(new int[] {1, 1})
                                                            .convolutionMode(ConvolutionMode.Same).nIn(384).nOut(1344)
                                                            .cudnnAlgoMode(ConvolutionLayer.AlgoMode.NO_WORKSPACE)
                                                            .build(),
                                            nameLayer(blockName, "merge1", i))
                            .addLayer(nameLayer(blockName, "batch5", i),
                                            new BatchNormalization.Builder(false).decay(0.995).eps(0.001)
                                                            .activation(Activation.TANH).nIn(1344).nOut(1344).build(),
                                            nameLayer(blockName, "cnn5", i))
                            .addVertex(nameLayer(blockName, "scaling", i), new ScaleVertex(activationScale),
                                            nameLayer(blockName, "batch5", i))
                            // -->
                            .addLayer(nameLayer(blockName, "shortcut-identity", i),
                                            new ActivationLayer.Builder().activation(Activation.IDENTITY).build(),
                                            previousBlock)
                            .addVertex(nameLayer(blockName, "shortcut", i),
                                            new ElementWiseVertex(ElementWiseVertex.Op.Add),
                                            nameLayer(blockName, "scaling", i),
                                            nameLayer(blockName, "shortcut-identity", i));

            // leave the last vertex as the block name for convenience
            if (i == scale)
                graph.addLayer(blockName, new ActivationLayer.Builder().activation(Activation.TANH).build(),
                                nameLayer(blockName, "shortcut", i));
            else
                graph.addLayer(nameLayer(blockName, "activation", i),
                                new ActivationLayer.Builder().activation(Activation.TANH).build(),
                                nameLayer(blockName, "shortcut", i));

            previousBlock = nameLayer(blockName, "activation", i);
        }
        return graph;
    }
606573
def deep_discriminator(x,
                       batch_norm,
                       is_training,
                       filters=64,
                       filter_size=4,
                       stride=2,
                       output_size=1024):
  """"""
  with tf.variable_scope(
      "discriminator", initializer=tf.random_normal_initializer(stddev=0.02)):
    batch_size, height, width = shape_list(x)[:3]  # pylint: disable=unbalanced-tuple-unpacking
    net = layers().Conv2D(
        filters, filter_size, strides=stride, padding="SAME", name="conv1")(x)
    net = lrelu(net)
    net = layers().Conv2D(
        2 * filters,
        filter_size,
        strides=stride,
        padding="SAME",
        name="conv2")(net)
    # [bs, h/4, w/4, 128]
    if batch_norm:
      net = layers().BatchNormalization(
          training=is_training, momentum=0.999, name="d_bn2")(net)
    net = lrelu(net)
    size = height * width
    x_shape = x.get_shape().as_list()
    if x_shape[1] is None or x_shape[2] is None:
      net = tf.reduce_mean(net, axis=[1, 2])
    else:
      net = tf.reshape(net, [batch_size, size * 8])
    net = layers().Dense(output_size, name="d_fc3")(net)
    if batch_norm:
      net = layers().BatchNormalization(
          training=is_training, momentum=0.999, name="d_bn3")(net)
    net = lrelu(net)
    return net
254078
def slicenet_internal(inputs, targets, target_space, hparams, run_decoder=True):
  """"""
  with tf.variable_scope("slicenet"):
    # Project to hidden size if necessary
    if inputs.get_shape().as_list()[-1] != hparams.hidden_size:
      inputs = common_layers.conv_block(
          inputs,
          hparams.hidden_size, [((1, 1), (3, 3))],
          first_relu=False,
          padding="SAME",
          force2d=True)

    # Flatten inputs and encode.
    inputs = tf.expand_dims(common_layers.flatten4d3d(inputs), axis=2)
    inputs_mask = 1.0 - embedding_to_padding(inputs)
    inputs = common_layers.add_timing_signal(inputs)  # Add position info.
    target_space_emb = embed_target_space(target_space, hparams.hidden_size)
    extra_layers = int(hparams.num_hidden_layers * 1.5)
    inputs_encoded = multi_conv_res(
        inputs, "SAME", "encoder", extra_layers, hparams, mask=inputs_mask)
    if not run_decoder:
      return inputs_encoded
    # Do the middle part.
    decoder_start, similarity_loss = slicenet_middle(
        inputs_encoded, targets, target_space_emb, inputs_mask, hparams)
    # Decode.
    decoder_final = multi_conv_res(
        decoder_start,
        "LEFT",
        "decoder",
        hparams.num_hidden_layers,
        hparams,
        mask=inputs_mask,
        source=inputs_encoded)
    return decoder_final, tf.reduce_mean(similarity_loss)
1298642
def _init_evolved_transformer_cache(cache, hparams, batch_size,
                                    attention_init_length, encoder_output,
                                    encoder_decoder_attention_bias,
                                    scope_prefix):
  """"""
  key_channels = hparams.attention_key_channels or hparams.hidden_size
  value_channels = hparams.attention_value_channels or hparams.hidden_size
  num_layers = hparams.num_decoder_layers or hparams.num_hidden_layers
  vars_3d_num_heads = (
      hparams.num_heads if hparams.get("attention_variables_3d") else 0)

  # Add self-attentions.
  if cache is None:
    cache = {}
  cache.update({
      "layer_%d" % layer: {  # pylint: disable=g-complex-comprehension
          _SIXTEEN_HEAD_ATTENTION_NAME: {
              "k":
                  common_attention.split_heads(
                      tf.zeros(
                          [batch_size, attention_init_length, key_channels]),
                      _capped_double_heads(hparams.num_heads)),
              "v":
                  common_attention.split_heads(
                      tf.zeros(
                          [batch_size, attention_init_length, value_channels]),
                      _capped_double_heads(hparams.num_heads)),
          },
          _VANILLA_ATTENTION_NAME: {
              "k":
                  common_attention.split_heads(
                      tf.zeros(
                          [batch_size, attention_init_length, key_channels]),
                      hparams.num_heads),
              "v":
                  common_attention.split_heads(
                      tf.zeros(
                          [batch_size, attention_init_length, value_channels]),
                      hparams.num_heads),
          }
      } for layer in range(num_layers)
  })

  # Add branched layers. Pad with additional zeros for causal convolution.
  for layer in range(num_layers):
    cache["layer_%d" % layer][_CONV_BRANCHES_FIRST_LAYER_NAME] = tf.zeros([
        batch_size, attention_init_length + _DECODER_LEFT_CONV_PADDING,
        hparams.hidden_size
    ])
    cache["layer_%d" % layer][_CONV_BRANCHES_SECOND_LAYER_NAME] = tf.zeros([
        batch_size, attention_init_length + _DECODER_FINAL_CONV_PADDING,
        hparams.hidden_size * 2
    ])

  # Add encoder embedding attentions.
  if encoder_output is not None:
    cache = _add_attend_to_encoder_cache(
        cache=cache,
        attention_name=_FIRST_ATTEND_TO_ENCODER_NAME,
        hparams=hparams,
        num_layers=num_layers,
        key_channels=key_channels,
        value_channels=value_channels,
        vars_3d_num_heads=vars_3d_num_heads,
        scope_prefix=scope_prefix,
        encoder_output=encoder_output)
    cache = _add_attend_to_encoder_cache(
        cache=cache,
        attention_name=_SECOND_ATTEND_TO_ENCODER_NAME,
        hparams=hparams,
        num_layers=num_layers,
        key_channels=key_channels,
        value_channels=value_channels,
        vars_3d_num_heads=vars_3d_num_heads,
        scope_prefix=scope_prefix,
        encoder_output=encoder_output)

    cache["encoder_output"] = encoder_output
    cache["encoder_decoder_attention_bias"] = encoder_decoder_attention_bias

  return cache
1031806
def gru(name, input, state, kernel_r, kernel_u, kernel_c, bias_r, bias_u, bias_c, new_state, number_of_gates = 2):
    ''' 
    '''
    nn = Build(name)
    inputs = nn.concat(input, state)

    u = nn.sigmoid(nn.mad(inputs, kernel_u, bias_u))
    r = nn.sigmoid(nn.mad(inputs, kernel_r, bias_r))
    r_state = nn.mul(r, state)

    c = nn.tanh(nn.mad(kernel=kernel_c, bias=bias_c,
        x=nn.concat(input, r_state)))

    # new_h = u' * state + (1 - u') * c'
    #       = u' * state + c' - u' * c'

    # u' * state + c'
    nn.add(nn.mul(u, state), c)
    # - u' * c'
    nn.sub(nn._, nn.mul(u, c),
        out=new_state)

    return nn.layers;
191540
def conv(self,
             num_out_channels,
             k_height,
             k_width,
             d_height=1,
             d_width=1,
             mode="SAME",
             input_layer=None,
             num_channels_in=None,
             use_batch_norm=None,
             stddev=None,
             activation="relu",
             bias=0.0):
        """"""
        if input_layer is None:
            input_layer = self.top_layer
        if num_channels_in is None:
            num_channels_in = self.top_size
        kernel_initializer = None
        if stddev is not None:
            kernel_initializer = tf.truncated_normal_initializer(stddev=stddev)
        name = "conv" + str(self.counts["conv"])
        self.counts["conv"] += 1
        with tf.variable_scope(name):
            strides = [1, d_height, d_width, 1]
            if self.data_format == "NCHW":
                strides = [strides[0], strides[3], strides[1], strides[2]]
            if mode != "SAME_RESNET":
                conv = self._conv2d_impl(
                    input_layer,
                    num_channels_in,
                    num_out_channels,
                    kernel_size=[k_height, k_width],
                    strides=[d_height, d_width],
                    padding=mode,
                    kernel_initializer=kernel_initializer)
            else:  # Special padding mode for ResNet models
                if d_height == 1 and d_width == 1:
                    conv = self._conv2d_impl(
                        input_layer,
                        num_channels_in,
                        num_out_channels,
                        kernel_size=[k_height, k_width],
                        strides=[d_height, d_width],
                        padding="SAME",
                        kernel_initializer=kernel_initializer)
                else:
                    rate = 1  # Unused (for 'a trous' convolutions)
                    kernel_height_effective = k_height + (k_height - 1) * (
                        rate - 1)
                    pad_h_beg = (kernel_height_effective - 1) // 2
                    pad_h_end = kernel_height_effective - 1 - pad_h_beg
                    kernel_width_effective = k_width + (k_width - 1) * (
                        rate - 1)
                    pad_w_beg = (kernel_width_effective - 1) // 2
                    pad_w_end = kernel_width_effective - 1 - pad_w_beg
                    padding = [[0, 0], [pad_h_beg, pad_h_end],
                               [pad_w_beg, pad_w_end], [0, 0]]
                    if self.data_format == "NCHW":
                        padding = [
                            padding[0], padding[3], padding[1], padding[2]
                        ]
                    input_layer = tf.pad(input_layer, padding)
                    conv = self._conv2d_impl(
                        input_layer,
                        num_channels_in,
                        num_out_channels,
                        kernel_size=[k_height, k_width],
                        strides=[d_height, d_width],
                        padding="VALID",
                        kernel_initializer=kernel_initializer)
            if use_batch_norm is None:
                use_batch_norm = self.use_batch_norm
            if not use_batch_norm:
                if bias is not None:
                    biases = self.get_variable(
                        "biases", [num_out_channels],
                        self.variable_dtype,
                        self.dtype,
                        initializer=tf.constant_initializer(bias))
                    biased = tf.reshape(
                        tf.nn.bias_add(
                            conv, biases, data_format=self.data_format),
                        conv.get_shape())
                else:
                    biased = conv
            else:
                self.top_layer = conv
                self.top_size = num_out_channels
                biased = self.batch_norm(**self.batch_norm_config)
            if activation == "relu":
                conv1 = tf.nn.relu(biased)
            elif activation == "linear" or activation is None:
                conv1 = biased
            elif activation == "tanh":
                conv1 = tf.nn.tanh(biased)
            else:
                raise KeyError("Invalid activation type \"%s\"" % activation)
            self.top_layer = conv1
            self.top_size = num_out_channels
            return conv1
668497
def fig_network_input_structure(fig, params, bottom=0.1, top=0.9, transient=200, T=[800, 1000], Df= 0., mlab= True, NFFT=256, srate=1000,
             window=plt.mlab.window_hanning, noverlap=256*3/4, letters='abcde', flim=(4, 400),
             show_titles=True, show_xlabels=True, show_CSD=False):
    '''
    
    
    '''
    #load spike as database
    networkSim = CachedNetwork(**params.networkSimParams)
    if analysis_params.bw:
        networkSim.colors = phlp.get_colors(len(networkSim.X))


    # ana_params.set_PLOS_2column_fig_style(ratio=ratio)
    # fig = plt.figure()
    # fig.subplots_adjust(left=0.06, right=0.94, bottom=0.09, top=0.92, wspace=0.5, hspace=0.2)
    
    #use gridspec to get nicely aligned subplots througout panel
    gs1 = gridspec.GridSpec(5, 5, bottom=bottom, top=top)
    
    
    ############################################################################ 
    # A part, full dot display
    ############################################################################
    
    ax0 = fig.add_subplot(gs1[:, 0])
    phlp.remove_axis_junk(ax0)
    phlp.annotate_subplot(ax0, ncols=5, nrows=1, letter=letters[0],
                     linear_offset=0.065)
   
    x, y = networkSim.get_xy(T, fraction=1)
    networkSim.plot_raster(ax0, T, x, y,
                           markersize=0.2, marker='_',
                           alpha=1.,
                           legend=False, pop_names=True,
                           rasterized=False)
    ax0.set_ylabel('population', labelpad=0.)
    ax0.set_xticks([800,900,1000])
   
    if show_titles:
        ax0.set_title('spiking activity',va='center')
    if show_xlabels:
        ax0.set_xlabel(r'$t$ (ms)', labelpad=0.)
    else:
        ax0.set_xlabel('')
      
    ############################################################################
    # B part, firing rate spectra
    ############################################################################

  
    # Get the firing rate from Potjan Diesmann et al network activity
    #collect the spikes x is the times, y is the id of the cell.
    T_all=[transient, networkSim.simtime]
    bins = np.arange(transient, networkSim.simtime+1)
        
    x, y = networkSim.get_xy(T_all, fraction=1)

    # create invisible axes to position labels correctly
    ax_ = fig.add_subplot(gs1[:, 1])
    phlp.annotate_subplot(ax_, ncols=5, nrows=1, letter=letters[1],
                                 linear_offset=0.065)
    if show_titles:
        ax_.set_title('firing rate PSD', va='center')
    
    ax_.axis('off')

    colors = phlp.get_colors(len(params.Y))+['k']
    
    COUNTER = 0
    label_set = False
    
    tits = ['L23E/I', 'L4E/I', 'L5E/I', 'L6E/I', 'TC']

    if x['TC'].size > 0:
        TC = True
    else:
        TC = False

    BAxes = []
    for i, X in enumerate(networkSim.X):

        if i % 2 == 0:
            ax1 = fig.add_subplot(gs1[COUNTER, 1])
            phlp.remove_axis_junk(ax1)

            if x[X].size > 0:
                ax1.text(0.05, 0.85, tits[COUNTER],
                    horizontalalignment='left',
                    verticalalignment='bottom',
                    transform=ax1.transAxes)
            BAxes.append(ax1)


        #firing rate histogram
        hist = np.histogram(x[X], bins=bins)[0].astype(float)
        hist -= hist.mean()
        
        if mlab:
            Pxx, freqs=plt.mlab.psd(hist, NFFT=NFFT,
                                    Fs=srate, noverlap=noverlap, window=window)
        else:
            [freqs, Pxx] = hlp.powerspec([hist], tbin= 1.,
                                        Df=Df, pointProcess=False)
            mask = np.where(freqs >= 0.)
            freqs = freqs[mask]
            Pxx = Pxx.flatten()
            Pxx = Pxx[mask]
            Pxx = Pxx/(T_all[1]-T_all[0])**2
        
        if x[X].size > 0:
            ax1.loglog(freqs[1:], Pxx[1:],
                       label=X, color=colors[i],
                       clip_on=True)
            ax1.axis(ax1.axis('tight'))
            ax1.set_ylim([5E-4,5E2])
            ax1.set_yticks([1E-3,1E-1,1E1])
            if label_set == False:
                ax1.set_ylabel(r'(s$^{-2}$/Hz)', labelpad=0.)
                label_set = True
            if i > 1:
                ax1.set_yticklabels([])
            if i >= 6 and not TC and show_xlabels or X == 'TC' and TC and show_xlabels:
                ax1.set_xlabel('$f$ (Hz)', labelpad=0.)
            if TC and i < 8 or not TC and i < 6:
                ax1.set_xticklabels([])    

        else:
            ax1.axis('off')
                       
        ax1.set_xlim(flim)
           
        
        if i % 2 == 0:
            COUNTER += 1
        
        ax1.yaxis.set_minor_locator(plt.NullLocator())
        



    ############################################################################
    # c part, LFP traces and CSD color plots
    ############################################################################
   
    ax2 = fig.add_subplot(gs1[:, 2])
    
    phlp.annotate_subplot(ax2, ncols=5, nrows=1, letter=letters[2],
                     linear_offset=0.065)


    phlp.remove_axis_junk(ax2)
    plot_signal_sum(ax2, params,
                    fname=os.path.join(params.savefolder, 'LFPsum.h5'),
                    unit='mV', T=T, ylim=[-1600, 40],
                    rasterized=False)
    
    # CSD background colorplot
    if show_CSD:
        im = plot_signal_sum_colorplot(ax2, params, os.path.join(params.savefolder, 'CSDsum.h5'),
                                  unit=r'($\mu$Amm$^{-3}$)', T=[800, 1000],
                                  colorbar=False,
                                  ylim=[-1600, 40], fancy=False, cmap=plt.cm.get_cmap('bwr_r', 21),
                                  rasterized=False)
        cb = phlp.colorbar(fig, ax2, im,
                           width=0.05, height=0.4,
                           hoffset=-0.05, voffset=0.3)
        cb.set_label('($\mu$Amm$^{-3}$)', labelpad=0.1)

    ax2.set_xticks([800,900,1000])
    ax2.axis(ax2.axis('tight'))
     
    if show_titles:
        if show_CSD:
            ax2.set_title('LFP & CSD', va='center')
        else:
            ax2.set_title('LFP', va='center')
    if show_xlabels:
        ax2.set_xlabel(r'$t$ (ms)', labelpad=0.)
    else:
        ax2.set_xlabel('')
  
 
    ############################################################################
    # d part, LFP power trace for each layer
    ############################################################################

    freqs, PSD = calc_signal_power(params, fname=os.path.join(params.savefolder,
                                                           'LFPsum.h5'),
                                        transient=transient, Df=Df, mlab=mlab,
                                        NFFT=NFFT, noverlap=noverlap,
                                        window=window)

    channels = [0, 3, 7, 11, 13]
  
    # create invisible axes to position labels correctly
    ax_ = fig.add_subplot(gs1[:, 3])
    phlp.annotate_subplot(ax_, ncols=5, nrows=1, letter=letters[3],
                                 linear_offset=0.065)

    if show_titles:
        ax_.set_title('LFP PSD',va='center')
    
    ax_.axis('off')

    for i, ch in enumerate(channels):

        ax = fig.add_subplot(gs1[i, 3])
        phlp.remove_axis_junk(ax)

        if i == 0:
            ax.set_ylabel('(mV$^2$/Hz)', labelpad=0)

        ax.loglog(freqs[1:],PSD[ch][1:], color='k')
        ax.xaxis.set_ticks_position('bottom')
        ax.yaxis.set_ticks_position('left')
        if i < 4:
            ax.set_xticklabels([])
        ax.text(0.75, 0.85,'ch. %i' %(channels[i]+1),
                horizontalalignment='left',
                verticalalignment='bottom',
                fontsize=6,
                transform=ax.transAxes)
        ax.tick_params(axis='y', which='minor', bottom='off')
        ax.axis(ax.axis('tight'))
        ax.yaxis.set_minor_locator(plt.NullLocator())

        ax.set_xlim(flim)
        ax.set_ylim(1E-7,2E-4)
        if i != 0 :
            ax.set_yticklabels([])


    if show_xlabels:
        ax.set_xlabel('$f$ (Hz)', labelpad=0.)
    
    ############################################################################
    # e part signal power
    ############################################################################
    
    ax4 = fig.add_subplot(gs1[:, 4])

    phlp.annotate_subplot(ax4, ncols=5, nrows=1, letter=letters[4],
                     linear_offset=0.065)
  
    fname=os.path.join(params.savefolder, 'LFPsum.h5')
    im = plot_signal_power_colorplot(ax4, params, fname=fname, transient=transient, Df=Df,
                                mlab=mlab, NFFT=NFFT, window=window,
                                cmap=plt.cm.get_cmap('gray_r', 12),
                                vmin=1E-7, vmax=1E-4)
    phlp.remove_axis_junk(ax4)

    ax4.set_xlim(flim)

    cb = phlp.colorbar(fig, ax4, im,
                       width=0.05, height=0.5,
                       hoffset=-0.05, voffset=0.5)
    cb.set_label('(mV$^2$/Hz)', labelpad=0.1)


    if show_titles:
        ax4.set_title('LFP PSD', va='center')
    if show_xlabels:
        ax4.set_xlabel(r'$f$ (Hz)', labelpad=0.)
    else:
        ax4.set_xlabel('')
       
    return fig
632712
def attn(image_feat, query, hparams, name="attn"):
  """"""
  with tf.variable_scope(name, "attn", values=[image_feat, query]):
    attn_dim = hparams.attn_dim
    num_glimps = hparams.num_glimps
    num_channels = common_layers.shape_list(image_feat)[-1]
    if len(common_layers.shape_list(image_feat)) == 4:
      image_feat = common_layers.flatten4d3d(image_feat)
    query = tf.expand_dims(query, 1)
    image_proj = common_attention.compute_attention_component(
        image_feat, attn_dim, name="image_proj")
    query_proj = common_attention.compute_attention_component(
        query, attn_dim, name="query_proj")
    h = tf.nn.relu(image_proj + query_proj)
    h_proj = common_attention.compute_attention_component(
        h, num_glimps, name="h_proj")
    p = tf.nn.softmax(h_proj, axis=1)
    image_ave = tf.matmul(image_feat, p, transpose_a=True)
    image_ave = tf.reshape(image_ave, [-1, num_channels*num_glimps])

    return image_ave