This method does training on previously unseen paragraph, and returns inferred vector

@param sequence
@param nr
@param learningRate
@return
162261
@Override
    public INDArray inferSequence(Sequence<T> sequence, long nr, double learningRate, double minLearningRate,
                    int iterations) {
        AtomicLong nextRandom = new AtomicLong(nr);

        // we probably don't want subsampling here
        // Sequence<T> seq = cbow.applySubsampling(sequence, nextRandom);
        // if (sequence.getSequenceLabel() == null) throw new IllegalStateException("Label is NULL");

        if (sequence.isEmpty())
            return null;

        Random random = Nd4j.getRandomFactory().getNewRandomInstance(configuration.getSeed() * sequence.hashCode(),
                        lookupTable.layerSize() + 1);
        INDArray ret = Nd4j.rand(new int[] {1, lookupTable.layerSize()}, random).subi(0.5)
                        .divi(lookupTable.layerSize());

        log.info("Inf before: {}", ret);

        for (int iter = 0; iter < iterations; iter++) {
            for (int i = 0; i < sequence.size(); i++) {
                nextRandom.set(Math.abs(nextRandom.get() * 25214903917L + 11));
                dm(i, sequence, (int) nextRandom.get() % window, nextRandom, learningRate, null, true, ret, null);
            }
            learningRate = ((learningRate - minLearningRate) / (iterations - iter)) + minLearningRate;
        }

        finish();

        return ret;
    }
608561
public FastqBuilder withSequence(final String sequence)
	{
		if (sequence == null)
		{
			throw new IllegalArgumentException("sequence must not be null");
		}
		if (this.sequence == null)
		{
			this.sequence = new StringBuilder(sequence.length());
		}
		this.sequence.replace(0, this.sequence.length(), sequence);
		return this;
	}
645719
public void trainOnInstanceImpl(Instance inst) {
		accumulatedError= Math.abs(this.prediction(inst)-inst.classValue()) + fadingFactor*accumulatedError;
		nError=1+fadingFactor*nError;
		// Initialise Perceptron if necessary   
		if (this.initialisePerceptron) {
			//this.fadingFactor=this.fadingFactorOption.getValue();
			//this.classifierRandom.setSeed(randomSeedOption.getValue());
			this.classifierRandom.setSeed(randomSeed);
			this.initialisePerceptron = false; // not in resetLearningImpl() because it needs Instance!
			this.weightAttribute = new double[inst.numAttributes()];
			for (int j = 0; j < inst.numAttributes(); j++) {
				weightAttribute[j] = 2 * this.classifierRandom.nextDouble() - 1;
			}
			// Update Learning Rate
			learningRatio = originalLearningRatio;
			//this.learningRateDecay = learningRateDecayOption.getValue();

		}

		// Update attribute statistics
		this.perceptronInstancesSeen++;
		this.perceptronYSeen++;


		for(int j = 0; j < inst.numAttributes() -1; j++)
		{
			perceptronattributeStatistics.addToValue(j, inst.value(j));     
			squaredperceptronattributeStatistics.addToValue(j, inst.value(j)*inst.value(j));
		}
		this.perceptronsumY += inst.classValue();
		this.squaredperceptronsumY += inst.classValue() * inst.classValue();

		if(!constantLearningRatioDecay){
			learningRatio = originalLearningRatio / (1+ perceptronInstancesSeen*learningRateDecay); 
		}

		this.updateWeights(inst,learningRatio);
		//this.printPerceptron();
	}
518775
@Override
    public void trainOnInstanceImpl(Instance instance) {

        if (m_weights == null) {
            m_weights = new DoubleVector();
            m_velocity = new DoubleVector();
            m_bias = 0;

            m_weights.setValue(instance.numAttributes(), 0);
            m_velocity.setValue(instance.numAttributes(), 0);
        }

        if (instance.classIsMissing()) {
            return;
        }
   
        double z = dotProd(instance, m_weights, instance.classIndex()) + m_bias;

        double y;
        double dldz;

        if (instance.classAttribute().isNominal()) {
            y = (instance.classValue() == 0) ? 0 : 1;
            
            if (m_loss == LOGLOSS) {
                double yhat = 1.0 / (1.0 + Math.exp(-z));
                dldz = (yhat - y);
            }
            else {
                y = y * 2 - 1;
                
                if(y * z < 1.0)
                {
                    dldz = -y;
                }
                else
                {
                    dldz = 0;
                }
            }
        }
        else {
            y = instance.classValue();
            dldz = z - y;
        }

        int n = instance.numValues();
        DoubleVector gradients = new DoubleVector();
        gradients.setValue(instance.numAttributes(), 0);

        for(int i = 0; i < n; i++)
        {
            int idx = instance.index(i);
            gradients.setValue(idx, instance.valueSparse(i) * dldz + (m_lambda / (m_t + m_epsilon)) * m_weights.getValue(idx));
        }

        //Weight update for the bias
        double biasGradient = dldz;
        m_biasVelocity += biasGradient * biasGradient;
        m_bias -= (m_learningRate / (Math.sqrt(m_biasVelocity) + m_epsilon)) * biasGradient;

        for(int i = 0; i < m_weights.numValues(); i++) {
            //Weight update
            double g = gradients.getValue(i);
            m_velocity.addToValue(i, g * g);
            m_weights.addToValue(i, -(m_learningRate / (Math.sqrt(m_velocity.getValue(i)) + m_epsilon)) * g);
        }

        m_t += 1.0;
    }
1167657
public final List<SequenceLabel> getSequencesFromSpans(final String[] tokens,
      final Span[] seqSpans) {
    final List<SequenceLabel> sequences = new ArrayList<SequenceLabel>();
    for (final Span seqSpan : seqSpans) {
      final String seqString = seqSpan.getCoveredText(tokens);
      final String seqType = seqSpan.getType();
      final SequenceLabel sequence = this.sequenceFactory
          .createSequence(seqString, seqType, seqSpan);
      sequences.add(sequence);
    }
    return sequences;
  }
398923
def runExperiment():
  
  trainSeqN = [5, 10, 20, 50, 100, 200]
  rptPerCondition = 5
  correctRateAll = np.zeros((len(trainSeqN), rptPerCondition))
  missRateAll = np.zeros((len(trainSeqN), rptPerCondition))
  fpRateAll = np.zeros((len(trainSeqN), rptPerCondition))
  for i in xrange(len(trainSeqN)):
    for rpt in xrange(rptPerCondition):
      tm = initializeTM()
      train_seed = 1
      numTrainSequence = trainSeqN[i]
      trainTMonReberSeq(tm, numTrainSequence, seedSeq=train_seed)
      (correctRate, missRate, fpRate) = testTMOnReberSeq(tm, numTestSequence, seedSeq=train_seed+rpt)
      correctRateAll[i, rpt] = correctRate
      missRateAll[i, rpt] = missRate
      fpRateAll[i, rpt] = fpRate

  np.savez('result/reberSequenceTM.npz',
           correctRateAll=correctRateAll, missRateAll=missRateAll,
           fpRateAll=fpRateAll, trainSeqN=trainSeqN)

  plt.figure()
  plt.subplot(2,2,1)
  plt.semilogx(trainSeqN, 100*np.mean(correctRateAll,1),'-*')
  plt.xlabel(' Training Sequence Number')
  plt.ylabel(' Hit Rate - Best Match (%)')
  plt.subplot(2,2,2)
  plt.semilogx(trainSeqN, 100*np.mean(missRateAll,1),'-*')
  plt.xlabel(' Training Sequence Number')
  plt.ylabel(' Miss Rate (%)')
  plt.subplot(2,2,3)
  plt.semilogx(trainSeqN, 100*np.mean(fpRateAll,1),'-*')
  plt.xlabel(' Training Sequence Number')
  plt.ylabel(' False Positive Rate (%)')
  plt.savefig('result/ReberSequence_TMperformance.pdf')
1294505
@Override
	public <C extends Compound> Sequence<C> getSubSequence(Sequence<C> sequence) {
		if(isCircular()) {
			List<Sequence<C>> sequences = new ArrayList<Sequence<C>>();
			for(Location l: this) {
				sequences.add(l.getSubSequence(sequence));
			}
			return new JoiningSequenceReader<C>(sequence.getCompoundSet(), sequences);
		}
		return reverseSequence(sequence.getSubSequence(
				getStart().getPosition(), getEnd().getPosition()));
	}
1185798
public double learn(double[] x) {
        if (x.length != n) {
            throw new IllegalArgumentException(String.format("Invalid input vector size: %d, expected: %d", x.length, n));
        }

        projection.ax(x, y);

        for (int j = 0; j < p; j++) {
            for (int i = 0; i < n; i++) {
                double delta = x[i];
                for (int l = 0; l <= j; l++) {
                    delta -= projection.get(l, i) * y[l];
                }
                projection.add(j, i, r * y[j] * delta);

                if (Double.isInfinite(projection.get(j, i))) {
                    throw new IllegalStateException("GHA lost convergence. Lower learning rate?");
                }
            }
        }

        projection.ax(x, y);
        projection.atx(y, wy);
        return Math.squaredDistance(x, wy);
    }
523498
def learning_rate_factor(name, step_num, hparams):
  """"""
  if name == "constant":
    tf.logging.info("Base learning rate: %f", hparams.learning_rate_constant)
    return hparams.learning_rate_constant
  elif name == "linear_warmup":
    return tf.minimum(1.0, step_num / hparams.learning_rate_warmup_steps)
  elif name == "linear_decay":
    ret = (hparams.train_steps - step_num) / hparams.learning_rate_decay_steps
    return tf.minimum(1.0, tf.maximum(0.0, ret))
  elif name == "cosdecay":  # openai gpt
    in_warmup = tf.cast(step_num <= hparams.learning_rate_warmup_steps,
                        dtype=tf.float32)
    ret = 0.5 * (1 + tf.cos(
        np.pi * step_num / hparams.learning_rate_decay_steps))
    # if in warmup stage return 1 else return the decayed value
    return in_warmup * 1 + (1 - in_warmup) * ret
  elif name == "single_cycle_cos_decay":
    # Cosine decay to zero with a single cycle. This is different from
    # "cosdecay" because it starts at 1 when the warmup steps end.
    x = tf.maximum(step_num, hparams.learning_rate_warmup_steps)
    step = x - hparams.learning_rate_warmup_steps
    return tf.math.cos(
        step * np.pi / hparams.learning_rate_decay_steps) / 2.0 + 0.5
  elif name == "rsqrt_decay":
    return tf.rsqrt(tf.maximum(step_num, hparams.learning_rate_warmup_steps))
  elif name == "rsqrt_normalized_decay":
    scale = tf.sqrt(tf.to_float(hparams.learning_rate_warmup_steps))
    return scale * tf.rsqrt(tf.maximum(
        step_num, hparams.learning_rate_warmup_steps))
  elif name == "exp_decay":
    decay_steps = hparams.learning_rate_decay_steps
    warmup_steps = hparams.learning_rate_warmup_steps
    p = (step_num - warmup_steps) / decay_steps
    p = tf.maximum(p, 0.)
    if hparams.learning_rate_decay_staircase:
      p = tf.floor(p)
    return tf.pow(hparams.learning_rate_decay_rate, p)
  elif name == "rsqrt_hidden_size":
    return hparams.hidden_size ** -0.5
  elif name == "legacy":
    return legacy_learning_rate_schedule(hparams)
  else:
    raise ValueError("unknown learning rate factor %s" % name)
356602
def legacy_learning_rate_schedule(hparams):
  """"""
  step_num = _global_step(hparams)
  warmup_steps = tf.to_float(hparams.learning_rate_warmup_steps)
  if hparams.learning_rate_decay_scheme == "noam":
    ret = 5000.0 * hparams.hidden_size**-0.5 * tf.minimum(
        (step_num + 1) * warmup_steps**-1.5, (step_num + 1)**-0.5)
  else:
    warmup_steps = hparams.learning_rate_warmup_steps
    warmup = _learning_rate_warmup(warmup_steps, hparams=hparams)
    decay = _learning_rate_decay(hparams, warmup_steps)
    ret = tf.where(step_num < warmup_steps, warmup, decay)
  optimizer_correction = 0.002 if "adam" in hparams.optimizer else 1.0
  tf.logging.info("Base learning rate: %f", hparams.learning_rate)
  return ret * optimizer_correction * hparams.learning_rate
428195
public static SpanFeatureAnnotation annotate(AnnotatedSentence sentence,
      FeatureVectorGenerator<StringContext> featureGenerator) {
    int numTerminals = sentence.size();
    Tensor[] featureVectors = new Tensor[numTerminals * (numTerminals + 1) / 2];
    for (int i = 0; i < numTerminals; i++) {
      for (int j = i; j < numTerminals; j++) {
        int spanIndex = getSpanIndex(i, j, numTerminals);
        featureVectors[spanIndex] = featureGenerator.apply(
            new StringContext(i, j, sentence));
      }
    }
    return new SpanFeatureAnnotation(featureVectors, numTerminals);
  }
650062
protected void train() throws Exception {
      final SequenceFeaturizer<Annotation> featurizer = getFeaturizer();
      Dataset<Sequence> train = getDataset(featurizer);
      PreprocessorList<Sequence> preprocessors = getPreprocessors();
      if (preprocessors != null && preprocessors.size() > 0) {
         train = train.preprocess(preprocessors);
      }
      train.encode();
      SequenceLabelerLearner learner = getLearner();
      learner.setValidator(getValidator());
      SequenceLabeler labeler = learner.train(train);
      BIOTagger tagger = new BIOTagger(featurizer, annotationType, labeler);
      tagger.write(model);
   }
26651
public void learnFern(boolean positive, ImageRectangle r) {

		float rectWidth = r.getWidth();
		float rectHeight = r.getHeight();

		float c_x = r.x0+(rectWidth-1)/2f;
		float c_y = r.y0+(rectHeight-1)/2f;

		for( int i = 0; i < ferns.length; i++ ) {

			// first learn it with no noise
			int value = computeFernValue(c_x, c_y, rectWidth, rectHeight,ferns[i]);
			TldFernFeature f = managers[i].lookupFern(value);
			increment(f,positive);
		}
	}
813956
def train(args):
    
    train_file = args.input
    test_file = args.validation
    ngram_range = args.ngrams
    logging.info('Ngrams range for the training run : %s', ngram_range)
    logging.info('Loading Training data')
    train_labels, train_data = read_input_data(train_file)
    logging.info('Loading Test data')
    test_labels, test_data = read_input_data(test_file)
    tokens_list = []
    for line in train_data:
        tokens_list.extend(line.split())

    cntr = Counter(tokens_list)
    train_vocab = gluonnlp.Vocab(cntr)
    logging.info('Vocabulary size: %s', len(train_vocab))
    logging.info('Training data converting to sequences...')
    embedding_matrix_len = len(train_vocab)
    # Preprocess the dataset
    train_sequences = convert_to_sequences(train_data, train_vocab)
    test_sequences = convert_to_sequences(test_data, train_vocab)

    if ngram_range >= 2:
        logging.info('Adding %s-gram features', ngram_range)
        # Create set of unique n-gram from the training set.
        ngram_set = set()
        for input_list in train_sequences:
            for i in range(2, ngram_range + 1):
                set_of_ngram = create_ngram_set(input_list, ngram_value=i)
                ngram_set.update(set_of_ngram)
        start_index = len(cntr)
        token_indices = {v: k + start_index for k, v in enumerate(ngram_set)}
        embedding_matrix_len = embedding_matrix_len + len(token_indices)
        train_sequences = add_ngram(train_sequences, token_indices,
                                    ngram_range)
        test_sequences = add_ngram(test_sequences, token_indices, ngram_range)
        logging.info('Added n-gram features to train and test datasets!! ')
    logging.info('Encoding labels')

    label_mapping = get_label_mapping(train_labels)
    y_train_final = list(map(lambda x: label_mapping[x], train_labels))
    y_test_final = list(map(lambda x: label_mapping[x], test_labels))

    train_sequences, train_data_lengths = preprocess_dataset(
        train_sequences, y_train_final)
    test_sequences, _ = preprocess_dataset(
        test_sequences, y_test_final)
    train_dataloader, test_dataloader = get_dataloader(train_sequences, train_data_lengths,
                                                       test_sequences, args.batch_size)

    num_classes = len(np.unique(train_labels))
    logging.info('Number of labels: %s', num_classes)
    logging.info('Initializing network')
    ctx = get_context(args)
    logging.info('Running Training on ctx:%s', ctx)
    embedding_dim = args.emsize
    logging.info('Embedding Matrix Length:%s', embedding_matrix_len)
    net = FastTextClassificationModel(
        embedding_matrix_len, embedding_dim, num_classes)
    net.hybridize()
    net.collect_params().initialize(mx.init.Xavier(), ctx=ctx)
    logging.info('Network initialized')

    softmax_cross_entropy = gluon.loss.SoftmaxCrossEntropyLoss()
    sigmoid_loss_fn = gluon.loss.SigmoidBinaryCrossEntropyLoss()

    loss_function = softmax_cross_entropy
    if num_classes == 2:
        logging.info(
            'Changing the loss function to sigmoid since its Binary Classification'
        )
        loss_function = sigmoid_loss_fn
    logging.info('Loss function for training:%s', loss_function)
    num_epochs = args.epochs
    batch_size = args.batch_size
    logging.info('Starting Training!')
    learning_rate = args.lr
    trainer = gluon.Trainer(net.collect_params(), 'adam',
                            {'learning_rate': learning_rate})
    num_batches = len(train_data) / batch_size
    display_batch_cadence = int(math.ceil(num_batches / 10))
    logging.info('Training on %s samples and testing on %s samples',
                 len(train_data), len(test_data))
    logging.info('Number of batches for each epoch : %s, Display cadence: %s',
                 num_batches, display_batch_cadence)
    for epoch in range(num_epochs):
        for batch, ((data, length), label) in enumerate(train_dataloader):
            data = data.as_in_context(ctx)
            label = label.as_in_context(ctx)
            length = length.astype('float32').as_in_context(ctx)
            with autograd.record():
                output = net(data, length)
                loss = loss_function(output, label)
            loss.backward()
            trainer.step(data.shape[0])
            if batch % display_batch_cadence == 0:
                logging.info('Epoch : %s, Batches complete :%s', epoch, batch)
        logging.info('Epoch complete :%s, Computing Accuracy', epoch)
        test_accuracy, test_loss = evaluate_accuracy(
            test_dataloader, net, ctx, loss_function, num_classes)
        logging.info('Epochs completed : %s Test Accuracy: %s, Test Loss: %s',
                     epoch, test_accuracy, test_loss)
        learning_rate = learning_rate * 0.5
        trainer.set_learning_rate(learning_rate)
    save_model(net, args.output)
1005566
@Override
    public void pretrain(@NonNull SequenceIterator<T> iterator) {
        // CoOccurence table should be built here
        coOccurrences = new AbstractCoOccurrences.Builder<T>()
                        // TODO: symmetric should be handled via VectorsConfiguration
                        .symmetric(this.symmetric).windowSize(configuration.getWindow()).iterate(iterator)
                        .workers(workers).vocabCache(vocabCache).maxMemory(maxmemory).build();

        coOccurrences.fit();
    }
36661
private void trainNN(DataSet dataSet)
    {
        //batchSize
        
        List<List<Vec>> activations = new ArrayList<List<Vec>>(batchSize);
        List<List<Vec>> derivatives = new ArrayList<List<Vec>>(batchSize);
        List<List<Vec>> deltas = new ArrayList<List<Vec>>(batchSize);
        
        List<Matrix> updates = new ArrayList<Matrix>(Ws.size());
        
        List<Vec> cur_x = new ArrayList<Vec>(batchSize);
        List<Vec> prev_x = new ArrayList<Vec>(batchSize);
        
        for(int i = 0; i < batchSize; i++)
        {
            activations.add(new ArrayList<Vec>(Ws.size()));
            derivatives.add(new ArrayList<Vec>(Ws.size()));
            deltas.add(new ArrayList<Vec>(Ws.size()));
            
            for(Matrix w : Ws)
            {
                int L = w.rows();
                activations.get(i).add(new DenseVector(L));
                derivatives.get(i).add(new DenseVector(L));
                deltas.get(i).add(new DenseVector(L));
                if(i == 0)
                    updates.add(new DenseMatrix(w.rows(), w.cols()));
            }
        }
        
        IntList iterOrder = new IntList(dataSet.size());
        ListUtils.addRange(iterOrder, 0, dataSet.size(), 1);
        
        final double bSizeInv = 1.0/batchSize;
        
        for(int epoch = 0; epoch < epochs; epoch++)
        {
            Collections.shuffle(iterOrder);
            final double eta = learningRateDecay.rate(epoch, epochs, initialLearningRate);//learningRate;
            double error = 0.0;
            for(int iter = 0; iter < dataSet.size(); iter+=batchSize)
            {
                if(dataSet.size() - iter < batchSize)
                    continue;//we have run out of enough sampels to do an update
                
                cur_x.clear();
                
                //Feed batches thought network and get final mistakes
                for(int bi = 0; bi < batchSize; bi++)
                {
                    final int idx = iterOrder.get(iter+bi);
                    Vec x = dataSet.getDataPoint(idx).getNumericalValues();
                    cur_x.add(x);
                    feedForward(x, activations.get(bi), derivatives.get(bi));

                    
                    //Compution of Deltas
                    Vec delta_out = deltas.get(bi).get(npl.length);

                    Vec a_i = activations.get(bi).get(npl.length);
                    Vec d_i = derivatives.get(bi).get(npl.length);
                    
                    error += computeOutputDelta(dataSet, idx, delta_out, a_i, d_i);
                }
                
                //Propigate the collected errors back
                for(int bi = 0; bi < batchSize; bi++)
                {
                    for(int i = Ws.size()-2; i >= 0; i--)
                    {
                        Vec delta = deltas.get(bi).get(i);
                        delta.zeroOut();
                        Matrix W = Ws.get(i+1);
                        W.transposeMultiply(1, deltas.get(bi).get(i+1), delta);
                        delta.mutablePairwiseMultiply(derivatives.get(bi).get(i));
                    }

                    //Apply weight changes
                    for(int i = 1; i < Ws.size(); i++)
                    {
                        Matrix W = Ws.get(i);
                        Vec b = bs.get(i);
                        W.mutableSubtract(eta*weightDecay, W);
                        
                        if(momentum != 0)
                        {
                            Matrix update = updates.get(i);
                            update.mutableMultiply(momentum);
                            Matrix.OuterProductUpdate(update, deltas.get(bi).get(i), activations.get(bi).get(i-1), -eta*bSizeInv);
                            W.mutableAdd(update);
                        }
                        else//update directly
                        {
                            Matrix.OuterProductUpdate(W, deltas.get(bi).get(i), activations.get(bi).get(i-1), -eta*bSizeInv);
                        }
                        
                        b.mutableAdd(-eta*bSizeInv, deltas.get(bi).get(i));
                    }

                    //input layer
                    Matrix W = Ws.get(0);
                    W.mutableSubtract(eta*weightDecay, W);
                    Vec b = bs.get(0);
                    
                    if(momentum != 0)
                    {
                        Matrix update = updates.get(0);
                        update.mutableMultiply(momentum);

                        Matrix.OuterProductUpdate(update, deltas.get(bi).get(0), cur_x.get(bi), -eta*bSizeInv);
                        W.mutableAdd(update);
                    }
                    else//update directly
                    {
                        Matrix.OuterProductUpdate(W, deltas.get(bi).get(0), cur_x.get(bi), -eta*bSizeInv);
                    }
                        
                    b.mutableAdd(-eta*bSizeInv, deltas.get(bi).get(0));
                }
                
            }
        }
    }
520851
def vqa_recurrent_self_attention_base():
  """"""
  hparams = universal_transformer.universal_transformer_base()
  hparams.batch_size = 1024
  hparams.use_fixed_batch_size = True
  hparams.weight_decay = 0.
  hparams.clip_grad_norm = 0.
  # use default initializer
  # hparams.initializer = "xavier"
  hparams.learning_rate_schedule = (
      "constant*linear_warmup*rsqrt_normalized_decay")
  hparams.learning_rate_warmup_steps = 8000
  hparams.learning_rate_constant = 7e-4
  hparams.learning_rate_decay_rate = 0.5
  hparams.learning_rate_decay_steps = 50000
  # hparams.dropout = 0.5
  hparams.summarize_grads = True
  hparams.summarize_vars = True

  # not used hparams
  hparams.label_smoothing = 0.1
  hparams.multiply_embedding_mode = "sqrt_depth"

  # add new hparams
  # use raw image as input
  hparams.add_hparam("image_input_type", "feature")
  hparams.add_hparam("image_model_fn", "resnet_v1_152")
  hparams.add_hparam("resize_side", 512)
  hparams.add_hparam("height", 448)
  hparams.add_hparam("width", 448)
  hparams.add_hparam("distort", True)
  hparams.add_hparam("train_resnet", False)

  # question hidden size
  # hparams.hidden_size = 512
  # hparams.filter_size = 1024
  # hparams.num_hidden_layers = 4

  # self attention parts
  # hparams.norm_type = "layer"
  # hparams.layer_preprocess_sequence = "n"
  # hparams.layer_postprocess_sequence = "da"
  # hparams.layer_prepostprocess_dropout = 0.1
  # hparams.attention_dropout = 0.1
  # hparams.relu_dropout = 0.1
  # hparams.add_hparam("pos", "timing")
  # hparams.add_hparam("num_encoder_layers", 0)
  # hparams.add_hparam("num_decoder_layers", 0)
  # hparams.add_hparam("num_heads", 8)
  # hparams.add_hparam("attention_key_channels", 0)
  # hparams.add_hparam("attention_value_channels", 0)
  # hparams.add_hparam("self_attention_type", "dot_product")

  # iterative part
  hparams.transformer_ffn_type = "fc"

  return hparams