The basic LSTM seq2seq model with bidirectional encoder.
764818
def lstm_seq2seq_internal_bid_encoder(inputs, targets, hparams, train):
  """"""
  with tf.variable_scope("lstm_seq2seq_bid_encoder"):
    if inputs is not None:
      inputs_length = common_layers.length_from_embedding(inputs)
      # Flatten inputs.
      inputs = common_layers.flatten4d3d(inputs)
      # LSTM encoder.
      _, final_encoder_state = lstm_bid_encoder(
          inputs, inputs_length, hparams, train, "encoder")
    else:
      inputs_length = None
      final_encoder_state = None
    # LSTM decoder.
    shifted_targets = common_layers.shift_right(targets)
    # Add 1 to account for the padding added to the left from shift_right
    targets_length = common_layers.length_from_embedding(shifted_targets) + 1
    hparams_decoder = copy.copy(hparams)
    hparams_decoder.hidden_size = 2 * hparams.hidden_size
    decoder_outputs, _ = lstm(
        common_layers.flatten4d3d(shifted_targets),
        targets_length,
        hparams_decoder,
        train,
        "decoder",
        initial_state=final_encoder_state)
    return tf.expand_dims(decoder_outputs, axis=2)
547595
def _forward_kernel(self, F, inputs, states, **kwargs):
        
        if self._layout == 'NTC':
            inputs = F.swapaxes(inputs, dim1=0, dim2=1)
        if self._projection_size is None:
            params = (kwargs['{}{}_{}_{}'.format(d, l, g, t)].reshape(-1)
                      for t in ['weight', 'bias']
                      for l in range(self._num_layers)
                      for d in ['l', 'r'][:self._dir]
                      for g in ['i2h', 'h2h'])
        else:
            params = (kwargs['{}{}_{}_{}'.format(d, l, g, t)].reshape(-1)
                      for t in ['weight', 'bias']
                      for l in range(self._num_layers)
                      for d in ['l', 'r'][:self._dir]
                      for g in ['i2h', 'h2h', 'h2r']
                      if g != 'h2r' or t != 'bias')

        params = F._internal._rnn_param_concat(*params, dim=0)

        rnn = F.RNN(inputs, params, *states, state_size=self._hidden_size,
                    projection_size=self._projection_size,
                    num_layers=self._num_layers, bidirectional=self._dir == 2,
                    p=self._dropout, state_outputs=True, mode=self._mode,
                    lstm_state_clip_min=self._lstm_state_clip_min,
                    lstm_state_clip_max=self._lstm_state_clip_max,
                    lstm_state_clip_nan=self._lstm_state_clip_nan)

        if self._mode == 'lstm':
            outputs, states = rnn[0], [rnn[1], rnn[2]]
        else:
            outputs, states = rnn[0], [rnn[1]]

        if self._layout == 'NTC':
            outputs = F.swapaxes(outputs, dim1=0, dim2=1)

        return outputs, states
1171227
def construct_model(images,
                    actions=None,
                    context_frames=2,
                    hparams=None,
                    is_training=True):
  """"""

  pred_depth = 20

  enc_out_all, pred_out_all, van_out_all, van_on_enc_all = [], [], [], []

  lstm_states = [None] * (pred_depth + 2)

  enc_out = encoder_vgg(
      images[0], hparams.enc_size, False, scope_prefix='timestep/',
      hparams=hparams, is_training=is_training)
  enc_out = tf.identity(enc_out, 'enc_out')
  enc_out_all.append(enc_out)

  num_timesteps = len(actions) - 1
  sum_freq = int(num_timesteps / 4 + 1)

  reuse = False
  for timestep, action in zip(range(len(actions) - 1), actions[:-1]):
    done_warm_start = timestep > context_frames - 1

    with tf.variable_scope('timestep', reuse=reuse):
      if done_warm_start:
        pred_input = pred_out_all[-1]
      else:
        pred_input = enc_out_all[-1]
      pred_out = predictor(
          pred_input, action, lstm_states, pred_depth, False, hparams=hparams)
      pred_out = tf.identity(pred_out, 'pred_out')
      if timestep % sum_freq == 0:  # and not hparams.use_tpu:
        tf.summary.histogram('pred_out', pred_out)
      pred_out_all.append(pred_out)

      if timestep % sum_freq == 0:  # and not hparams.use_tpu:
        tf.summary.histogram('lstm_state', lstm_states[0])
      van_out, _, _ = van(
          enc_out_all[0],
          images[0],
          pred_out,
          images[timestep + 1],
          tf.AUTO_REUSE,
          hparams=hparams)
      van_out = tf.identity(van_out, 'van_out')
      van_out_all.append(van_out)

      enc_out = encoder_vgg(
          images[timestep + 1], hparams.enc_size, True, hparams=hparams,
          is_training=is_training)
      enc_out = tf.identity(enc_out, 'enc_out')
      if timestep % sum_freq == 0:  # and not hparams.use_tpu:
        tf.summary.histogram('enc_out', enc_out)
      enc_out_all.append(enc_out)

      van_input = images[0]
      enc_noise = tf.zeros_like(enc_out)
      if timestep % sum_freq == 0:  # and not hparams.use_tpu:
        tf.summary.histogram('enc_noise', enc_noise)
      van_on_enc, _, _ = van(
          enc_out_all[0],
          van_input,
          enc_out + enc_noise,
          images[timestep + 1],
          tf.AUTO_REUSE,
          hparams=hparams)
      van_on_enc = tf.identity(van_on_enc, 'van_on_enc')
      van_on_enc_all.append(van_on_enc)

      reuse = True

  return enc_out_all, pred_out_all, van_out_all, van_on_enc_all
1032896
def recurrent_transformer_decoder(
    decoder_input,
    encoder_output,
    decoder_self_attention_bias,
    encoder_decoder_attention_bias,
    hparams,
    name="decoder",
    nonpadding=None,
    save_weights_to=None,
    make_image_summary=True):
  """"""
  x = decoder_input
  attention_dropout_broadcast_dims = (
      common_layers.comma_separated_string_to_integer_list(
          getattr(hparams, "attention_dropout_broadcast_dims", "")))
  with tf.variable_scope(name):
    ffn_unit = functools.partial(
        # use encoder ffn, since decoder ffn use left padding
        universal_transformer_util.transformer_encoder_ffn_unit,
        hparams=hparams,
        nonpadding_mask=nonpadding)

    attention_unit = functools.partial(
        universal_transformer_util.transformer_decoder_attention_unit,
        hparams=hparams,
        encoder_output=encoder_output,
        decoder_self_attention_bias=decoder_self_attention_bias,
        encoder_decoder_attention_bias=encoder_decoder_attention_bias,
        attention_dropout_broadcast_dims=attention_dropout_broadcast_dims,
        save_weights_to=save_weights_to,
        make_image_summary=make_image_summary)

    x, extra_output = universal_transformer_util.universal_transformer_layer(
        x, hparams, ffn_unit, attention_unit)

    return common_layers.layer_preprocess(x, hparams), extra_output
562500
def awd_lstm_lm_split(model:nn.Module) -> List[nn.Module]:
    ""
    groups = [[rnn, dp] for rnn, dp in zip(model[0].rnns, model[0].hidden_dps)]
    return groups + [[model[0].encoder, model[0].encoder_dp, model[1]]]
258909
def convert_simple_rnn(builder, layer, input_names, output_names, keras_layer):
    
    # Get input and output names
    hidden_size = keras_layer.units
    input_size = keras_layer.input_shape[-1]

    output_all = keras_layer.return_sequences
    reverse_input = keras_layer.go_backwards

    W_h = _np.zeros((hidden_size, hidden_size))
    W_x = _np.zeros((hidden_size, input_size))
    b = None

    implementation = keras_layer.implementation if hasattr(keras_layer,
            'implementation') else 0
    if implementation == 0:
        W_h = keras_layer.get_weights()[1].T
        W_x = keras_layer.get_weights()[0].T
        if keras_layer.use_bias:
            b = keras_layer.get_weights()[2]

    # Set actication type
    activation_str = _get_recurrent_activation_name_from_keras(keras_layer.activation)

    # Add to the network
    builder.add_simple_rnn(
        name = layer,
        W_h = W_h, W_x = W_x, b = b,
        hidden_size = hidden_size,
        input_size = input_size,
        activation = activation_str,
        input_names = input_names,
        output_names = output_names,
        output_all=output_all,
        reverse_input=reverse_input)
462340
Word2VecModel train(Log log, TrainingProgressListener listener, Iterable<List<String>> sentences) throws InterruptedException {
		try (ProfilingTimer timer = ProfilingTimer.createLoggingSubtasks(log, "Training word2vec")) {
			final Multiset<String> counts;
			
			try (AC ac = timer.start("Acquiring word frequencies")) {
				listener.update(Stage.ACQUIRE_VOCAB, 0.0);
				counts = (vocab.isPresent())
							? vocab.get()
							: count(Iterables.concat(sentences));
			}
			
			final ImmutableMultiset<String> vocab;
			try (AC ac = timer.start("Filtering and sorting vocabulary")) {
				listener.update(Stage.FILTER_SORT_VOCAB, 0.0);
				vocab = filterAndSort(counts);
			}
			
			final Map<String, HuffmanNode> huffmanNodes;
			try (AC task = timer.start("Create Huffman encoding")) {
				huffmanNodes = new HuffmanCoding(vocab, listener).encode();
			}
			
			final NeuralNetworkModel model;
			try (AC task = timer.start("Training model %s", neuralNetworkConfig)) {
				model = neuralNetworkConfig.createTrainer(vocab, huffmanNodes, listener).train(sentences);
			}
			
			return new Word2VecModel(vocab.elementSet(), model.layerSize(), Doubles.concat(model.vectors()));
		}
	}
795392
def convert_examples_to_features(examples, label_list, max_seq_length,
                                 tokenizer, output_mode):
    """"""

    label_map = {label : i for i, label in enumerate(label_list)}

    features = []
    for (ex_index, example) in enumerate(examples):
        if ex_index % 10000 == 0:
            logger.info("Writing example %d of %d" % (ex_index, len(examples)))

        tokens_a = tokenizer.tokenize(example.text_a)

        tokens_b = None
        if example.text_b:
            tokens_b = tokenizer.tokenize(example.text_b)
            # Modifies `tokens_a` and `tokens_b` in place so that the total
            # length is less than the specified length.
            # Account for [CLS], [SEP], [SEP] with "- 3"
            _truncate_seq_pair(tokens_a, tokens_b, max_seq_length - 3)
        else:
            # Account for [CLS] and [SEP] with "- 2"
            if len(tokens_a) > max_seq_length - 2:
                tokens_a = tokens_a[:(max_seq_length - 2)]

        # The convention in BERT is:
        # (a) For sequence pairs:
        #  tokens:   [CLS] is this jack ##son ##ville ? [SEP] no it is not . [SEP]
        #  type_ids: 0   0  0    0    0     0       0 0    1  1  1  1   1 1
        # (b) For single sequences:
        #  tokens:   [CLS] the dog is hairy . [SEP]
        #  type_ids: 0   0   0   0  0     0 0
        #
        # Where "type_ids" are used to indicate whether this is the first
        # sequence or the second sequence. The embedding vectors for `type=0` and
        # `type=1` were learned during pre-training and are added to the wordpiece
        # embedding vector (and position vector). This is not *strictly* necessary
        # since the [SEP] token unambiguously separates the sequences, but it makes
        # it easier for the model to learn the concept of sequences.
        #
        # For classification tasks, the first vector (corresponding to [CLS]) is
        # used as as the "sentence vector". Note that this only makes sense because
        # the entire model is fine-tuned.
        tokens = ["[CLS]"] + tokens_a + ["[SEP]"]
        segment_ids = [0] * len(tokens)

        if tokens_b:
            tokens += tokens_b + ["[SEP]"]
            segment_ids += [1] * (len(tokens_b) + 1)

        input_ids = tokenizer.convert_tokens_to_ids(tokens)

        # The mask has 1 for real tokens and 0 for padding tokens. Only real
        # tokens are attended to.
        input_mask = [1] * len(input_ids)

        # Zero-pad up to the sequence length.
        padding = [0] * (max_seq_length - len(input_ids))
        input_ids += padding
        input_mask += padding
        segment_ids += padding

        assert len(input_ids) == max_seq_length
        assert len(input_mask) == max_seq_length
        assert len(segment_ids) == max_seq_length

        if output_mode == "classification":
            label_id = label_map[example.label]
        elif output_mode == "regression":
            label_id = float(example.label)
        else:
            raise KeyError(output_mode)

        if ex_index < 5:
            logger.info("*** Example ***")
            logger.info("guid: %s" % (example.guid))
            logger.info("tokens: %s" % " ".join(
                    [str(x) for x in tokens]))
            logger.info("input_ids: %s" % " ".join([str(x) for x in input_ids]))
            logger.info("input_mask: %s" % " ".join([str(x) for x in input_mask]))
            logger.info(
                    "segment_ids: %s" % " ".join([str(x) for x in segment_ids]))
            logger.info("label: %s (id = %d)" % (example.label, label_id))

        features.append(
                InputFeatures(input_ids=input_ids,
                              input_mask=input_mask,
                              segment_ids=segment_ids,
                              label_id=label_id))
    return features
429780
def encoder(nef, z_dim, batch_size, no_bias=True, fix_gamma=True, eps=1e-5 + 1e-12):
    '''
    '''
    BatchNorm = mx.sym.BatchNorm

    data = mx.sym.Variable('data')

    e1 = mx.sym.Convolution(data, name='enc1', kernel=(5,5), stride=(2,2), pad=(2,2), num_filter=nef, no_bias=no_bias)
    ebn1 = BatchNorm(e1, name='encbn1', fix_gamma=fix_gamma, eps=eps)
    eact1 = mx.sym.LeakyReLU(ebn1, name='encact1', act_type='leaky', slope=0.2)

    e2 = mx.sym.Convolution(eact1, name='enc2', kernel=(5,5), stride=(2,2), pad=(2,2), num_filter=nef*2, no_bias=no_bias)
    ebn2 = BatchNorm(e2, name='encbn2', fix_gamma=fix_gamma, eps=eps)
    eact2 = mx.sym.LeakyReLU(ebn2, name='encact2', act_type='leaky', slope=0.2)

    e3 = mx.sym.Convolution(eact2, name='enc3', kernel=(5,5), stride=(2,2), pad=(2,2), num_filter=nef*4, no_bias=no_bias)
    ebn3 = BatchNorm(e3, name='encbn3', fix_gamma=fix_gamma, eps=eps)
    eact3 = mx.sym.LeakyReLU(ebn3, name='encact3', act_type='leaky', slope=0.2)

    e4 = mx.sym.Convolution(eact3, name='enc4', kernel=(5,5), stride=(2,2), pad=(2,2), num_filter=nef*8, no_bias=no_bias)
    ebn4 = BatchNorm(e4, name='encbn4', fix_gamma=fix_gamma, eps=eps)
    eact4 = mx.sym.LeakyReLU(ebn4, name='encact4', act_type='leaky', slope=0.2)

    eact4 = mx.sym.Flatten(eact4)

    z_mu = mx.sym.FullyConnected(eact4, num_hidden=z_dim, name="enc_mu")
    z_lv = mx.sym.FullyConnected(eact4, num_hidden=z_dim, name="enc_lv")

    z = z_mu + mx.symbol.broadcast_mul(mx.symbol.exp(0.5*z_lv),mx.symbol.random_normal(loc=0, scale=1,shape=(batch_size,z_dim)))

    return z_mu, z_lv, z
227194
def LSTM(nO, nI):
    """"""
    weights = LSTM_weights(nO, nI)
    gates = LSTM_gates(weights.ops)
    return Recurrent(RNN_step(weights, gates))
1265898
def _encode_sequence(self, inputs, token_types, valid_length=None):
        
        # embedding
        word_embedding = self.word_embed(inputs)
        type_embedding = self.token_type_embed(token_types)
        embedding = word_embedding + type_embedding
        # encoding
        outputs, additional_outputs = self.encoder(embedding, None, valid_length)
        return outputs, additional_outputs
482847
def lstm_asr_v1():
  """"""
  hparams = lstm_bahdanau_attention()
  hparams.num_hidden_layers = 2
  hparams.hidden_size = 256
  hparams.batch_size = 36
  hparams.max_input_seq_length = 600000
  hparams.max_target_seq_length = 350
  hparams.max_length = hparams.max_input_seq_length
  hparams.min_length_bucket = hparams.max_input_seq_length // 2
  hparams.learning_rate = 0.05
  return hparams
342241
def _create_rnn_architecture(self):
        """"""
        self.inputs = [tf.squeeze(i, [1]) for i in tf.split(
            axis=1, num_or_size_splits=self.num_steps, value=self.inputs)]
        outputs, state = tf.nn.rnn(
            self.cell, self.inputs, initial_state=self._init_state)

        output = tf.reshape(tf.concat(axis=1, values=outputs), [-1, self.num_hidden])
        softmax_w = tf.get_variable(
            "softmax_w", [self.num_hidden, self.vocab_size])
        softmax_b = tf.get_variable("softmax_b", [self.vocab_size])
        logits = tf.add(tf.matmul(output, softmax_w), softmax_b)
        loss = tf.nn.seq2seq.sequence_loss_by_example(
            [logits],
            [tf.reshape(self.input_labels, [-1])],
            [tf.ones([self.batch_size * self.num_steps])])

        self.cost = tf.div(tf.reduce_sum(loss), self.batch_size)
        self.final_state = state
484498
def calculate_lstm_output_shapes(operator):
    '''
    
    '''
    check_input_and_output_numbers(operator, input_count_range=[1, 3], output_count_range=[1, 3])
    check_input_and_output_types(operator, good_input_types=[FloatTensorType])

    input_shape = operator.inputs[0].type.shape

    if len(input_shape) not in [2, 4]:
        raise RuntimeError('Input must be a 2-D tensor')

    params = operator.raw_operator.uniDirectionalLSTM

    # The following line is more accurate but it may break some tests
    # output_shape = ['None', params.outputVectorSize] if params.params.sequenceOutput else [1, params.outputVectorSize]
    output_shape = ['None', params.outputVectorSize]
    state_shape = [1, params.outputVectorSize]

    # TODO: Changing input shapes of an operator is dangerous, this should be move to Topology's _fix_shapes function
    if len(operator.inputs) > 1:
        Y_h_in = operator.inputs[1]  # The initial hidden state of a single sequence
        Y_h_in.type.shape = state_shape
    if len(operator.inputs) > 2:
        Y_c_in = operator.inputs[2]  # The initial cell state of a single sequence
        Y_c_in.type.shape = state_shape

    operator.outputs[0].type.shape = output_shape
    if len(operator.outputs) > 1:
        operator.outputs[1].type.shape = state_shape
    if len(operator.outputs) > 2:
        operator.outputs[2].type.shape = state_shape
1069007
def _build_model(self):
        

        # Set up LSTM modules
        self.lstms = nn.ModuleList(
            [
                RNN(
                    num_classes=0,
                    num_tokens=self.word_dict.s,
                    emb_size=self.settings["emb_dim"],
                    lstm_hidden=self.settings["hidden_dim"],
                    attention=self.settings["attention"],
                    dropout=self.settings["dropout"],
                    bidirectional=self.settings["bidirectional"],
                    use_cuda=self.settings["host_device"] in self._gpu,
                )
            ]
            * self.settings["relation_arity"]
        )

        if "input_dim" not in self.settings:
            raise ValueError("Model parameter input_dim cannot be None.")

        # Set up final linear layer
        self.sparse_linear = SparseLinear(
            self.settings["input_dim"], self.cardinality, self.settings["bias"]
        )
16343
def predictor(enc_flat,
              action,
              lstm_states,
              pred_depth,
              reuse=False,
              scope_prefix='',
              hparams=None):
  """"""
  with tf.variable_scope(scope_prefix + 'predict', reuse=reuse):

    enc_final_size = enc_flat.get_shape().as_list()[1]
    action_size = action.get_shape().as_list()[1]
    initial_size = (enc_final_size + action_size)

    batch_size = tf.shape(enc_flat)[0]

    init_stddev = 1e-2

    pre_pred = tf.concat([enc_flat, action], 1)
    pre_pred = tf.layers.dense(
        pre_pred,
        initial_size,
        kernel_initializer=tf.truncated_normal_initializer(stddev=init_stddev))

    # This is only needed or the GAN version.
    if hparams.pred_noise_std > 0:
      # Add the noise like this so a pretrained model can be used.
      pred_noise = tf.random_normal(
          shape=[batch_size, 100], stddev=hparams.pred_noise_std)
      pre_pred += tf.layers.dense(
          pred_noise,
          initial_size,
          kernel_initializer=tf.truncated_normal_initializer(
              stddev=init_stddev),
          name='noise_dense')

    pre_pred = tf.nn.relu(pre_pred)

    if lstm_states[pred_depth - 2] is None:
      back_connect = tf.tile(
          tf.get_variable(
              'back_connect_init',
              shape=[1, initial_size * 2],
              initializer=tf.truncated_normal_initializer(stddev=init_stddev))
          , (batch_size, 1))
    else:
      back_connect = lstm_states[pred_depth - 2]

    lstm_init_stddev = 1e-4

    part_pred, lstm_states[0] = common_video.lstm_cell(
        tf.concat([pre_pred, back_connect], 1),
        lstm_states[0],
        initial_size,
        use_peepholes=True,
        initializer=tf.truncated_normal_initializer(stddev=lstm_init_stddev),
        num_proj=initial_size)
    part_pred = tf.contrib.layers.layer_norm(part_pred)
    pred = part_pred

    for pred_layer_num in range(1, pred_depth, 2):
      part_pred, lstm_states[pred_layer_num] = common_video.lstm_cell(
          pred,
          lstm_states[pred_layer_num],
          initial_size,
          use_peepholes=True,
          initializer=tf.truncated_normal_initializer(stddev=lstm_init_stddev),
          num_proj=initial_size)
      pred += part_pred

      part_pred, lstm_states[pred_layer_num + 1] = common_video.lstm_cell(
          tf.concat([pred, pre_pred], 1),
          lstm_states[pred_layer_num + 1],
          initial_size,
          use_peepholes=True,
          initializer=tf.truncated_normal_initializer(stddev=lstm_init_stddev),
          num_proj=initial_size)
      part_pred = tf.contrib.layers.layer_norm(part_pred)
      pred += part_pred

    pred = tf.layers.dense(
        pred,
        enc_final_size,
        kernel_initializer=tf.truncated_normal_initializer(stddev=init_stddev))

    if hparams.enc_pred_use_l2norm:
      pred = tf.nn.l2_normalize(pred, 1)

    return pred
737574
def predict(self, encoder_outputs, encoder_decoder_attention_bias):
    """"""
    batch_size = tf.shape(encoder_outputs)[0]
    input_length = tf.shape(encoder_outputs)[1]
    max_decode_length = input_length + self.params.extra_decode_length

    symbols_to_logits_fn = self._get_symbols_to_logits_fn(max_decode_length)

    # Create initial set of IDs that will be passed into symbols_to_logits_fn.
    initial_ids = tf.zeros([batch_size], dtype=tf.int32)

    # Create cache storing decoder attention values for each layer.
    cache = {
        "layer_%d" % layer: {
            "k": tf.zeros([batch_size, 0, self.params.hidden_size]),
            "v": tf.zeros([batch_size, 0, self.params.hidden_size]),
        } for layer in range(self.params.num_hidden_layers)}

    # Add encoder output and attention bias to the cache.
    cache["encoder_outputs"] = encoder_outputs
    cache["encoder_decoder_attention_bias"] = encoder_decoder_attention_bias

    # Use beam search to find the top beam_size sequences and scores.
    mlperf_log.transformer_print(
      key=mlperf_log.MODEL_HP_SEQ_BEAM_SEARCH,
      value={
        "vocab_size": self.params.vocab_size,
        "beam_size": self.params.beam_size,
        "alpha": self.params.alpha,
        "extra_decode_length": self.params.extra_decode_length})
    decoded_ids, scores = beam_search.sequence_beam_search(
        symbols_to_logits_fn=symbols_to_logits_fn,
        initial_ids=initial_ids,
        initial_cache=cache,
        vocab_size=self.params.vocab_size,
        beam_size=self.params.beam_size,
        alpha=self.params.alpha,
        max_decode_length=max_decode_length,
        eos_id=EOS_ID)

    # Get the top sequence for each batch element
    top_decoded_ids = decoded_ids[:, 0, 1:]
    top_scores = scores[:, 0]

    return {"outputs": top_decoded_ids, "scores": top_scores}