Create a step model for an RNN, given weights and gates functions.
760637
def RNN_step(weights, gates):
    """"""

    def rnn_step_fwd(prevstate_inputs, drop=0.0):
        prevstate, inputs = prevstate_inputs
        cell_tm1, hidden_tm1 = prevstate

        acts, bp_acts = weights.begin_update((inputs, hidden_tm1), drop=drop)
        (cells, hiddens), bp_gates = gates.begin_update((acts, cell_tm1), drop=drop)

        def rnn_step_bwd(d_state_d_hiddens, sgd=None):
            (d_cells, d_hiddens), d_hiddens = d_state_d_hiddens
            d_acts, d_cell_tm1 = bp_gates((d_cells, d_hiddens), sgd=sgd)
            d_inputs, d_hidden_tm1 = bp_acts(d_acts, sgd=sgd)
            return (d_cell_tm1, d_hidden_tm1), d_inputs

        return ((cells, hiddens), hiddens), rnn_step_bwd

    model = wrap(rnn_step_fwd, weights, gates)
    model.nO = weights.nO
    model.nI = weights.nI
    model.weights = weights
    model.gates = gates
    return model
1091467
def convert_lstm(builder, layer, input_names, output_names, keras_layer):
    

    hidden_size = keras_layer.output_dim
    input_size = keras_layer.input_shape[-1]
    if keras_layer.consume_less not in ['cpu', 'gpu']:
        raise ValueError('Cannot convert Keras layer with consume_less = %s' % keras_layer.consume_less)

    output_all = keras_layer.return_sequences
    reverse_input = keras_layer.go_backwards

    # Keras: I C F O; W_x, W_h, b
    # CoreML: I F O G; W_h and W_x are separated
    W_h, W_x, b = ([], [], [])
    if keras_layer.consume_less == 'cpu':
        W_h.append(keras_layer.get_weights()[1].T)
        W_h.append(keras_layer.get_weights()[7].T)
        W_h.append(keras_layer.get_weights()[10].T)
        W_h.append(keras_layer.get_weights()[4].T)

        W_x.append(keras_layer.get_weights()[0].T)
        W_x.append(keras_layer.get_weights()[6].T)
        W_x.append(keras_layer.get_weights()[9].T)
        W_x.append(keras_layer.get_weights()[3].T)

        b.append(keras_layer.get_weights()[2])
        b.append(keras_layer.get_weights()[8])
        b.append(keras_layer.get_weights()[11])
        b.append(keras_layer.get_weights()[5])
    else:
        keras_W_h = keras_layer.get_weights()[1].T
        W_h.append(keras_W_h[0 * hidden_size:][:hidden_size])
        W_h.append(keras_W_h[1 * hidden_size:][:hidden_size])
        W_h.append(keras_W_h[3 * hidden_size:][:hidden_size])
        W_h.append(keras_W_h[2 * hidden_size:][:hidden_size])

        keras_W_x = keras_layer.get_weights()[0].T
        W_x.append(keras_W_x[0 * hidden_size:][:hidden_size])
        W_x.append(keras_W_x[1 * hidden_size:][:hidden_size])
        W_x.append(keras_W_x[3 * hidden_size:][:hidden_size])
        W_x.append(keras_W_x[2 * hidden_size:][:hidden_size])

        keras_b = keras_layer.get_weights()[2]
        b.append(keras_b[0 * hidden_size:][:hidden_size])
        b.append(keras_b[1 * hidden_size:][:hidden_size])
        b.append(keras_b[3 * hidden_size:][:hidden_size])
        b.append(keras_b[2 * hidden_size:][:hidden_size])

    # Set activation type
    inner_activation_str = _get_recurrent_activation_name_from_keras(keras_layer.inner_activation)
    activation_str = _get_recurrent_activation_name_from_keras(keras_layer.activation)

    # Add to the network
    builder.add_unilstm(
        name = layer,
        W_h = W_h, W_x = W_x, b = b,
        hidden_size = hidden_size,
        input_size = input_size,
        input_names = input_names,
        output_names = output_names,
        inner_activation = inner_activation_str,
        cell_state_update_activation = activation_str,
        output_activation = activation_str,
        output_all = output_all,
        reverse_input = reverse_input)
1243333
def create_mnist_model(hyper_params, input_shape=(H, W, 1), num_classes=NUM_CLASSES):
    '''
    
    '''
    layers = [
        Conv2D(32, kernel_size=(3, 3), activation='relu', input_shape=input_shape),
        Conv2D(64, (3, 3), activation='relu'),
        MaxPooling2D(pool_size=(2, 2)),
        Flatten(),
        Dense(100, activation='relu'),
        Dense(num_classes, activation='softmax')
    ]

    model = Sequential(layers)

    if hyper_params['optimizer'] == 'Adam':
        optimizer = keras.optimizers.Adam(lr=hyper_params['learning_rate'])
    else:
        optimizer = keras.optimizers.SGD(lr=hyper_params['learning_rate'], momentum=0.9)
    model.compile(loss=keras.losses.categorical_crossentropy, optimizer=optimizer, metrics=['accuracy'])

    return model
1035153
public static void step(long seed, Neurons[] neurons, DeepLearningModel.DeepLearningModelInfo minfo, boolean training, double[] responses) {
    try {
      for (int i=1; i<neurons.length-1; ++i) {
        neurons[i].fprop(seed, training);
      }
      if (minfo.get_params().autoencoder) {
        neurons[neurons.length - 1].fprop(seed, training);
        if (training) {
          for (int i=neurons.length-1; i>0; --i) {
            neurons[i].bprop();
          }
        }
      } else {
        if (minfo.get_params().classification) {
          ((Neurons.Softmax) neurons[neurons.length - 1]).fprop();
          if (training) {
            for (int i = 1; i < neurons.length - 1; i++)
              Arrays.fill(neurons[i]._e.raw(), 0);
            int target_label;
            if (Double.isNaN(responses[0])) { //missing response
              target_label = Neurons.missing_int_value;
            } else {
              assert ((double) (int) responses[0] == responses[0]); //classification -> integer labels expected
              target_label = (int) responses[0];
            }
            ((Neurons.Softmax) neurons[neurons.length - 1]).bprop(target_label);
          }
        } else {
          ((Neurons.Linear) neurons[neurons.length - 1]).fprop();
          if (training) {
            for (int i = 1; i < neurons.length - 1; i++)
              Arrays.fill(neurons[i]._e.raw(), 0);
            float target_value;
            if (Double.isNaN(responses[0])) { //missing response
              target_value = Neurons.missing_real_value;
            } else {
              target_value = (float) responses[0];
            }
            ((Neurons.Linear) neurons[neurons.length - 1]).bprop(target_value);
          }
        }
        if (training) {
          for (int i=neurons.length-2; i>0; --i)
            neurons[i].bprop();
        }
      }
    }
    catch(RuntimeException ex) {
      Log.warn(ex.getMessage());
      minfo.set_unstable();
      throw new Job.JobCancelledException("Canceling job due to numerical instability.");
    }
  }
1307860
def mnist_model(image, labels, mesh):
  
  batch_dim = mtf.Dimension("batch", FLAGS.batch_size)
  row_blocks_dim = mtf.Dimension("row_blocks", 4)
  col_blocks_dim = mtf.Dimension("col_blocks", 4)
  rows_dim = mtf.Dimension("rows_size", 7)
  cols_dim = mtf.Dimension("cols_size", 7)

  classes_dim = mtf.Dimension("classes", 10)
  one_channel_dim = mtf.Dimension("one_channel", 1)

  x = mtf.import_tf_tensor(
      mesh, tf.reshape(image, [FLAGS.batch_size, 4, 7, 4, 7, 1]),
      mtf.Shape(
          [batch_dim, row_blocks_dim, rows_dim,
           col_blocks_dim, cols_dim, one_channel_dim]))
  x = mtf.transpose(x, [
      batch_dim, row_blocks_dim, col_blocks_dim,
      rows_dim, cols_dim, one_channel_dim])

  # add some convolutional layers to demonstrate that convolution works.
  fh_dim = mtf.Dimension("fh", 9)
  fw_dim = mtf.Dimension("fw", 9)
  filters1_dim = mtf.Dimension("filters1", 16)
  filters2_dim = mtf.Dimension("filters2", 16)
  kernel1 = mtf.get_variable(
      mesh, "kernel1", [fh_dim, fw_dim, one_channel_dim, filters1_dim])
  kernel2 = mtf.get_variable(
      mesh, "kernel2", [fh_dim, fw_dim, filters1_dim, filters2_dim])

  f1 = mtf.relu(mtf.conv2d_with_blocks(
      x, kernel1, strides=[1, 1, 1, 1], padding="SAME",
      h_blocks_dim=row_blocks_dim, w_blocks_dim=col_blocks_dim))
  f2 = mtf.relu(mtf.conv2d_with_blocks(
      f1, kernel2, strides=[1, 1, 1, 1], padding="SAME",
      h_blocks_dim=row_blocks_dim, w_blocks_dim=col_blocks_dim))
  x = mtf.reduce_mean(f2, reduced_dim=filters2_dim)

  # add some fully-connected dense layers.
  hidden_dim1 = mtf.Dimension("hidden1", FLAGS.hidden_size)
  hidden_dim2 = mtf.Dimension("hidden2", FLAGS.hidden_size)

  h1 = mtf.layers.dense(
      x, hidden_dim1,
      reduced_dims=x.shape.dims[-4:],
      activation=mtf.relu, name="hidden1")
  h2 = mtf.layers.dense(
      h1, hidden_dim2,
      activation=mtf.relu, name="hidden2")
  logits = mtf.layers.dense(h2, classes_dim, name="logits")
  if labels is None:
    loss = None
  else:
    labels = mtf.import_tf_tensor(
        mesh, tf.reshape(labels, [FLAGS.batch_size]), mtf.Shape([batch_dim]))
    loss = mtf.layers.softmax_cross_entropy_with_logits(
        logits, mtf.one_hot(labels, classes_dim), classes_dim)
    loss = mtf.reduce_mean(loss)
  return logits, loss
1299321
def _create_gate_variables(self, input_shape, dtype):
    """"""
    if len(input_shape) != 2:
      raise ValueError(
          "Rank of shape must be {} not: {}".format(2, len(input_shape)))

    equiv_input_size = self._hidden_state_size + input_shape.dims[1].value
    initializer = basic.create_linear_initializer(equiv_input_size)

    self._w_xh = tf.get_variable(
        self.W_GATES,
        shape=[equiv_input_size, 4 * self._hidden_size],
        dtype=dtype,
        initializer=self._initializers.get(self.W_GATES, initializer),
        partitioner=self._partitioners.get(self.W_GATES),
        regularizer=self._regularizers.get(self.W_GATES))
    self._b = tf.get_variable(
        self.B_GATES,
        shape=[4 * self._hidden_size],
        dtype=dtype,
        initializer=self._initializers.get(self.B_GATES, initializer),
        partitioner=self._partitioners.get(self.B_GATES),
        regularizer=self._regularizers.get(self.B_GATES))
    if self._use_projection:
      w_h_initializer = basic.create_linear_initializer(self._hidden_size)
      self._w_h_projection = tf.get_variable(
          self.W_H_PROJECTION,
          shape=[self._hidden_size, self._hidden_state_size],
          dtype=dtype,
          initializer=self._initializers.get(self.W_H_PROJECTION,
                                             w_h_initializer),
          partitioner=self._partitioners.get(self.W_H_PROJECTION),
          regularizer=self._regularizers.get(self.W_H_PROJECTION))
932270
def make_step(
    net,
    step_size = 1.5,
    end       = "inception_4c/output", 
    jitter    = 32,
    clip      = True,
    objective = objective_L2
    ):
    '''
    
    '''

    src = net.blobs["data"]
    dst = net.blobs[end]

    ox, oy = np.random.randint(- jitter, jitter + 1, 2)
    # Apply jitter shift.
    src.data[0] = np.roll(np.roll(src.data[0], ox, -1), oy, -2)

    # Specify the optimisation objective.
    net.forward(end = end)
    objective(dst)
    net.backward(start = end)
    g = src.diff[0]
    # Apply normalised ascent step to the input image.
    src.data[:] += step_size / np.abs(g).mean() * g
    # Unshift the image.
    src.data[0] = np.roll(np.roll(src.data[0], - ox, - 1), -oy, -2)
            
    if clip:
        bias = net.transformer.mean["data"]
        src.data[:] = np.clip(src.data, -bias, 255-bias)
183353
def build_model(self, n_features, n_classes):
        
        self._create_placeholders(n_features, n_classes)
        self._create_variables(n_features, n_classes)

        self.mod_y = tf.nn.softmax(
            tf.add(tf.matmul(self.input_data, self.W_), self.b_))

        self.cost = self.loss.compile(self.mod_y, self.input_labels)
        self.train_step = tf.train.GradientDescentOptimizer(
            self.learning_rate).minimize(self.cost)
        self.accuracy = Evaluation.accuracy(self.mod_y, self.input_labels)
123426
def create(input_width, input_height, input_channels=1, output_dim=512, initial_std_dev=0.4, factorized_noise=True):
    
    def instantiate(**_):
        return DoubleNoisyNatureCnn(
            input_width=input_width, input_height=input_height, input_channels=input_channels,
            output_dim=output_dim, initial_std_dev=initial_std_dev, factorized_noise=factorized_noise
        )

    return ModelFactory.generic(instantiate)
290892
def conv_lstm(x,
              kernel_size,
              filters,
              padding="SAME",
              dilation_rate=(1, 1),
              name=None,
              reuse=None):
  """"""
  with tf.variable_scope(
      name, default_name="conv_lstm", values=[x], reuse=reuse):
    gates = conv(
        x,
        4 * filters,
        kernel_size,
        padding=padding,
        dilation_rate=dilation_rate)
    g = tf.split(layer_norm(gates, 4 * filters), 4, axis=3)
    new_cell = tf.sigmoid(g[0]) * x + tf.sigmoid(g[1]) * tf.tanh(g[3])
    return tf.sigmoid(g[2]) * tf.tanh(new_cell)
981932
def train(num_training_iterations, report_interval,
          reduce_learning_rate_interval):
  """"""

  # Build the computation graph.
  graph_tensors, dataset_train = build_graph(
      lstm_depth=FLAGS.lstm_depth, batch_size=FLAGS.batch_size,
      num_embedding=FLAGS.num_embedding, num_hidden=FLAGS.num_hidden,
      truncation_length=FLAGS.truncation_length,
      sample_length=FLAGS.sample_length, max_grad_norm=FLAGS.max_grad_norm,
      initial_learning_rate=FLAGS.learning_rate,
      reduce_learning_rate_multiplier=FLAGS.reduce_learning_rate_multiplier,
      optimizer_epsilon=FLAGS.optimizer_epsilon)

  # Configure a checkpoint saver.
  saver_hook = _configure_saver(FLAGS.checkpoint_dir,
                                FLAGS.checkpoint_interval)

  # Train the network.
  with tf.train.SingularMonitoredSession(
      hooks=[saver_hook], checkpoint_dir=FLAGS.checkpoint_dir) as sess:

    start_iteration = sess.run(graph_tensors["global_step"])

    for train_iteration in range(start_iteration, num_training_iterations):
      if (train_iteration + 1) % report_interval == 0:
        train_loss_v, valid_loss_v, _ = sess.run(
            (graph_tensors["train_loss"],
             graph_tensors["valid_loss"],
             graph_tensors["train_step"]))

        train_generated_string_v = sess.run(
            graph_tensors["train_generated_string"])

        train_generated_string_human = dataset_train.to_human_readable(
            (train_generated_string_v, 0), indices=[0])

        tf.logging.info("%d: Training loss %f. Validation loss %f. Sample = %s",
                        train_iteration,
                        train_loss_v,
                        valid_loss_v,
                        train_generated_string_human)
      else:
        train_loss_v, _ = sess.run((graph_tensors["train_loss"],
                                    graph_tensors["train_step"]))
        tf.logging.info("%d: Training loss %f.", train_iteration, train_loss_v)

      if (train_iteration + 1) % reduce_learning_rate_interval == 0:
        sess.run(graph_tensors["reduce_learning_rate"])
        tf.logging.info("Reducing learning rate.")

    test_loss = sess.run(graph_tensors["test_loss"])
    tf.logging.info("Test loss %f", test_loss)
315222
function() {
      var input_depth = this.data[0].w.length;
      var num_classes = this.unique_labels.length;

      // sample network topology and hyperparameters
      var layer_defs = [];
      layer_defs.push({type:'input', out_sx:1, out_sy:1, out_depth: input_depth});
      var nl = weightedSample([0,1,2,3], [0.2, 0.3, 0.3, 0.2]); // prefer nets with 1,2 hidden layers
      for(var q=0;q<nl;q++) {
        var ni = randi(this.neurons_min, this.neurons_max);
        var act = ['tanh','maxout','relu'][randi(0,3)];
        if(randf(0,1)<0.5) {
          var dp = Math.random();
          layer_defs.push({type:'fc', num_neurons: ni, activation: act, drop_prob: dp});
        } else {
          layer_defs.push({type:'fc', num_neurons: ni, activation: act});
        }
      }
      layer_defs.push({type:'softmax', num_classes: num_classes});
      var net = new Net();
      net.makeLayers(layer_defs);

      // sample training hyperparameters
      var bs = randi(this.batch_size_min, this.batch_size_max); // batch size
      var l2 = Math.pow(10, randf(this.l2_decay_min, this.l2_decay_max)); // l2 weight decay
      var lr = Math.pow(10, randf(this.learning_rate_min, this.learning_rate_max)); // learning rate
      var mom = randf(this.momentum_min, this.momentum_max); // momentum. Lets just use 0.9, works okay usually ;p
      var tp = randf(0,1); // trainer type
      var trainer_def;
      if(tp<0.33) {
        trainer_def = {method:'adadelta', batch_size:bs, l2_decay:l2};
      } else if(tp<0.66) {
        trainer_def = {method:'adagrad', learning_rate: lr, batch_size:bs, l2_decay:l2};
      } else {
        trainer_def = {method:'sgd', learning_rate: lr, momentum: mom, batch_size:bs, l2_decay:l2};
      }
      
      var trainer = new Trainer(net, trainer_def);

      var cand = {};
      cand.acc = [];
      cand.accv = 0; // this will maintained as sum(acc) for convenience
      cand.layer_defs = layer_defs;
      cand.trainer_def = trainer_def;
      cand.net = net;
      cand.trainer = trainer;
      return cand;
    }
1279045
def forward(self, inputs, lengths):
        
        x = self.embedder(inputs)

        # bidirectional layer
        x = self.dropout(x)
        x = pack_padded_sequence(x, lengths.cpu().numpy(),
                                 batch_first=self.batch_first)
        x, _ = self.rnn_layers[0](x)
        x, _ = pad_packed_sequence(x, batch_first=self.batch_first)

        # 1st unidirectional layer
        x = self.dropout(x)
        x, _ = self.rnn_layers[1](x)

        # the rest of unidirectional layers,
        # with residual connections starting from 3rd layer
        for i in range(2, len(self.rnn_layers)):
            residual = x
            x = self.dropout(x)
            x, _ = self.rnn_layers[i](x)
            x = x + residual

        return x
596858
def _init_model(self):
        
        model = Sequential()
        model.add(Conv2D(input_shape=list(self.image_size) + [self.channels], filters=self.filters,
                         kernel_size=self.kernel_size, activation='relu', data_format='channels_last'))
        model.add(
            Conv2D(filters=self.filters, kernel_size=self.kernel_size, activation='relu', data_format='channels_last'))
        model.add(MaxPooling2D())
        model.add(
            Conv2D(filters=self.filters, kernel_size=self.kernel_size, activation='relu', data_format='channels_last'))
        model.add(
            Conv2D(filters=self.filters, kernel_size=self.kernel_size, activation='relu', data_format='channels_last'))
        model.add(MaxPooling2D())

        model.add(Flatten())
        model.add(Dense(units=len(self.emotion_map.keys()), activation="relu"))
        if self.verbose:
            model.summary()
        self.model = model
131174
def next_frame_basic_recurrent():
  """"""
  hparams = basic_stochastic.next_frame_basic_stochastic_discrete()
  hparams.filter_double_steps = 2
  hparams.hidden_size = 64
  hparams.video_num_input_frames = 4
  hparams.video_num_target_frames = 4
  hparams.concat_internal_states = False
  hparams.add_hparam("num_lstm_layers", 2)
  hparams.add_hparam("num_lstm_filters", 256)
  return hparams
344833
def create_model(model_name: Optional[str], params: ModelParams) -> 'Sequential':
    
    if model_name and isfile(model_name):
        print('Loading from ' + model_name + '...')
        model = load_precise_model(model_name)
    else:
        from keras.layers.core import Dense
        from keras.layers.recurrent import GRU
        from keras.models import Sequential

        model = Sequential()
        model.add(GRU(
            params.recurrent_units, activation='linear',
            input_shape=(pr.n_features, pr.feature_size), dropout=params.dropout, name='net'
        ))
        model.add(Dense(1, activation='sigmoid'))

    load_keras()
    metrics = ['accuracy'] + params.extra_metrics * [false_pos, false_neg]
    set_loss_bias(params.loss_bias)
    for i in model.layers[:params.freeze_till]:
        i.trainable = False
    model.compile('rmsprop', weighted_log_loss, metrics=(not params.skip_acc) * metrics)
    return model
999588
def step_preprocess(x, step, hparams):
  
  original_channel_size = common_layers.shape_list(x)[-1]

  if hparams.add_position_timing_signal:
    x = add_position_timing_signal(x, step, hparams)

  if hparams.add_step_timing_signal:
    x = add_step_timing_signal(x, step, hparams)

  if ((hparams.add_position_timing_signal or hparams.add_position_timing_signal)
      and hparams.add_or_concat_timing_signal == "concat"):
    # linear projection to the original dimension of x
    x = common_layers.dense(
        x, original_channel_size, activation=None, use_bias=False)

  if hparams.add_sru:
    x = common_layers.sru(x)

  return x