Compute activations of all layers from input (inclusive) to output of the final/output layer.
Equivalent to calling {@link #feedForward(INDArray, boolean)} with train = false

@return the list of activations for each layer, including the input
163296
public List<INDArray> feedForward(INDArray input) {
        if (input == null)
            throw new IllegalStateException("Unable to perform feed forward; no input found");
        setInput(input);
        return feedForward();
    }
921105
private void backpropagate(Layer upper, Layer lower) {
        for (int i = 0; i <= lower.units; i++) {
            double out = lower.output[i];
            double err = 0;
            for (int j = 0; j < upper.units; j++) {
                err += upper.weight[j][i] * upper.error[j];
            }
            if (activationFunction==ActivationFunction.LOGISTIC_SIGMOID) {
                lower.error[i] = out * (1.0 - out) * err;
            }
            else if (activationFunction==ActivationFunction.TANH){
                lower.error[i] = (1-(out*out))*err;
            }
        }
    }
774009
def fc(self, output_nodes, keep_prob=1, activation_fn=tf.nn.relu, b_value=0.0, s_value=1.0, bn=True,
           trainable=True):
        
        self.count['fc'] += 1
        scope = 'fc_' + str(self.count['fc'])
        with tf.variable_scope(scope):

            # Flatten if necessary
            if len(self.input.get_shape()) == 4:
                input_nodes = tf.Dimension(
                    self.input.get_shape()[1] * self.input.get_shape()[2] * self.input.get_shape()[3])
                output_shape = tf.stack([-1, input_nodes])
                self.input = tf.reshape(self.input, output_shape)

            # Matrix Multiplication Function
            input_nodes = self.input.get_shape()[1]
            output_shape = [input_nodes, output_nodes]
            w = self.weight_variable(name='weights', shape=output_shape, trainable=trainable)
            self.input = tf.matmul(self.input, w)

            if bn is True:  # batch normalization
                self.input = self.batch_norm(self.input, 'fc')
            if b_value is not None:  # bias value
                b = self.const_variable(name='bias', shape=[output_nodes], value=b_value, trainable=trainable)
                self.input = tf.add(self.input, b)
            if s_value is not None:  # scale value
                s = self.const_variable(name='scale', shape=[output_nodes], value=s_value, trainable=trainable)
                self.input = tf.multiply(self.input, s)
            if activation_fn is not None:  # activation function
                self.input = activation_fn(self.input)
            if keep_prob != 1:  # dropout function
                self.input = tf.nn.dropout(self.input, keep_prob=keep_prob)
        print(scope + ' output: ' + str(self.input.get_shape()))
1092524
public java.util.List<Activation> getActivationList() {
        if (activationList == null) {
            activationList = new com.amazonaws.internal.SdkInternalList<Activation>();
        }
        return activationList;
    }
564418
private Pair<Gradient, INDArray> getGradientsAndDelta(INDArray preOut, LayerWorkspaceMgr workspaceMgr) {
        // delta calculation
        ILossFunction lossFunction = layerConf().getLossFn();
        INDArray delta = lossFunction.computeGradient(getLabels2d(), preOut, layerConf().getActivationFn(), maskArray);

        // grab the empty gradient
        Gradient gradient = new DefaultGradient();

        delta = workspaceMgr.leverageTo(ArrayType.ACTIVATION_GRAD, delta);
        return new Pair<>(gradient, delta);
    }
821674
def study_inputs(self):
        
        return set(chain(
            (i for i in self.inputs if not i.derived),
            *(self.study.pipeline(p, required_outputs=r).study_inputs
              for p, r in self.prerequisites.items())))
708683
def convert_activation(builder, layer, input_names, output_names, keras_layer):
    
    # Get input and output names
    input_name, output_name = (input_names[0], output_names[0])
    non_linearity = _get_activation_name_from_keras_layer(keras_layer)

    # Add a non-linearity layer
    if non_linearity == 'SOFTMAX':
        builder.add_softmax(name = layer, input_name = input_name,
                output_name = output_name)
        return
    if non_linearity == 'RELU6':
        # No direct support of RELU with max-activation value - use negate and
        # clip layers
        relu_output_name = output_name + '_relu'
        builder.add_activation(layer, 'RELU', input_name, relu_output_name)
        # negate it
        neg_output_name = relu_output_name + '_neg'
        builder.add_activation(layer+'__neg__', 'LINEAR', relu_output_name,
                neg_output_name,[-1.0, 0])
        # apply threshold
        clip_output_name = relu_output_name + '_clip'
        builder.add_unary(layer+'__clip__', neg_output_name, clip_output_name,
                'threshold', alpha = -6.0)
        # negate it back
        builder.add_activation(layer+'_neg2', 'LINEAR', clip_output_name,
                output_name,[-1.0, 0])
        return

    if non_linearity == 'SELU':
        elu_output_name = output_name + '_elu'
        builder.add_activation(layer+'__elu__', 'ELU', input_name, elu_output_name,
                             params=1.6732)
        builder.add_elementwise(layer,
                          input_names=elu_output_name,
                          output_name=output_name,
                          mode='MULTIPLY',
                          alpha=1.0507)
        return

    params = None
    if non_linearity == 'UNIT_ELU':
        params = 1.0
        non_linearity = 'ELU'
    elif non_linearity == 'LEAKYRELU':
        params = [keras_layer.alpha]
    elif non_linearity == 'PRELU':
        shared_axes = list(keras_layer.shared_axes)
        if not (shared_axes == [1,2,3] or shared_axes == [1,2]):
            _utils.raise_error_unsupported_scenario(
                    "Shared axis not being [1,2,3] or [1,2]",
                    'parametric_relu', layer)
        params = _keras.backend.eval(keras_layer.weights[0])
    elif non_linearity == 'ELU':
        params = keras_layer.alpha
    elif non_linearity == 'THRESHOLDEDRELU':
        params = keras_layer.theta
    else:
        pass # do nothing to parameters

    builder.add_activation(name = layer,
            non_linearity = non_linearity,
            input_name = input_name, output_name = output_name,
            params = params)
949832
public INDArray activate(INDArray input, TrainingMode training) {
        return output(input, training == TrainingMode.TRAIN);
    }
73865
public java.util.List<? extends org.tensorflow.framework.OpDef.ArgDefOrBuilder> 
      getOutputArgOrBuilderList() {
    return outputArg_;
  }
65948
public List<Next<Flow<T>>> getAllNext()
   {
      List<Next<Flow<T>>> list = new ArrayList<Next<Flow<T>>>();
      List<Node> nodeList = childNode.get("next");
      for(Node node: nodeList)
      {
         Next<Flow<T>>  type = new NextImpl<Flow<T>>(this, "next", childNode, node);
         list.add(type);
      }
      return list;
   }
36661
private void trainNN(DataSet dataSet)
    {
        //batchSize
        
        List<List<Vec>> activations = new ArrayList<List<Vec>>(batchSize);
        List<List<Vec>> derivatives = new ArrayList<List<Vec>>(batchSize);
        List<List<Vec>> deltas = new ArrayList<List<Vec>>(batchSize);
        
        List<Matrix> updates = new ArrayList<Matrix>(Ws.size());
        
        List<Vec> cur_x = new ArrayList<Vec>(batchSize);
        List<Vec> prev_x = new ArrayList<Vec>(batchSize);
        
        for(int i = 0; i < batchSize; i++)
        {
            activations.add(new ArrayList<Vec>(Ws.size()));
            derivatives.add(new ArrayList<Vec>(Ws.size()));
            deltas.add(new ArrayList<Vec>(Ws.size()));
            
            for(Matrix w : Ws)
            {
                int L = w.rows();
                activations.get(i).add(new DenseVector(L));
                derivatives.get(i).add(new DenseVector(L));
                deltas.get(i).add(new DenseVector(L));
                if(i == 0)
                    updates.add(new DenseMatrix(w.rows(), w.cols()));
            }
        }
        
        IntList iterOrder = new IntList(dataSet.size());
        ListUtils.addRange(iterOrder, 0, dataSet.size(), 1);
        
        final double bSizeInv = 1.0/batchSize;
        
        for(int epoch = 0; epoch < epochs; epoch++)
        {
            Collections.shuffle(iterOrder);
            final double eta = learningRateDecay.rate(epoch, epochs, initialLearningRate);//learningRate;
            double error = 0.0;
            for(int iter = 0; iter < dataSet.size(); iter+=batchSize)
            {
                if(dataSet.size() - iter < batchSize)
                    continue;//we have run out of enough sampels to do an update
                
                cur_x.clear();
                
                //Feed batches thought network and get final mistakes
                for(int bi = 0; bi < batchSize; bi++)
                {
                    final int idx = iterOrder.get(iter+bi);
                    Vec x = dataSet.getDataPoint(idx).getNumericalValues();
                    cur_x.add(x);
                    feedForward(x, activations.get(bi), derivatives.get(bi));

                    
                    //Compution of Deltas
                    Vec delta_out = deltas.get(bi).get(npl.length);

                    Vec a_i = activations.get(bi).get(npl.length);
                    Vec d_i = derivatives.get(bi).get(npl.length);
                    
                    error += computeOutputDelta(dataSet, idx, delta_out, a_i, d_i);
                }
                
                //Propigate the collected errors back
                for(int bi = 0; bi < batchSize; bi++)
                {
                    for(int i = Ws.size()-2; i >= 0; i--)
                    {
                        Vec delta = deltas.get(bi).get(i);
                        delta.zeroOut();
                        Matrix W = Ws.get(i+1);
                        W.transposeMultiply(1, deltas.get(bi).get(i+1), delta);
                        delta.mutablePairwiseMultiply(derivatives.get(bi).get(i));
                    }

                    //Apply weight changes
                    for(int i = 1; i < Ws.size(); i++)
                    {
                        Matrix W = Ws.get(i);
                        Vec b = bs.get(i);
                        W.mutableSubtract(eta*weightDecay, W);
                        
                        if(momentum != 0)
                        {
                            Matrix update = updates.get(i);
                            update.mutableMultiply(momentum);
                            Matrix.OuterProductUpdate(update, deltas.get(bi).get(i), activations.get(bi).get(i-1), -eta*bSizeInv);
                            W.mutableAdd(update);
                        }
                        else//update directly
                        {
                            Matrix.OuterProductUpdate(W, deltas.get(bi).get(i), activations.get(bi).get(i-1), -eta*bSizeInv);
                        }
                        
                        b.mutableAdd(-eta*bSizeInv, deltas.get(bi).get(i));
                    }

                    //input layer
                    Matrix W = Ws.get(0);
                    W.mutableSubtract(eta*weightDecay, W);
                    Vec b = bs.get(0);
                    
                    if(momentum != 0)
                    {
                        Matrix update = updates.get(0);
                        update.mutableMultiply(momentum);

                        Matrix.OuterProductUpdate(update, deltas.get(bi).get(0), cur_x.get(bi), -eta*bSizeInv);
                        W.mutableAdd(update);
                    }
                    else//update directly
                    {
                        Matrix.OuterProductUpdate(W, deltas.get(bi).get(0), cur_x.get(bi), -eta*bSizeInv);
                    }
                        
                    b.mutableAdd(-eta*bSizeInv, deltas.get(bi).get(0));
                }
                
            }
        }
    }
92625
public java.util.List<ConditionalForwarder> getConditionalForwarders() {
        if (conditionalForwarders == null) {
            conditionalForwarders = new com.amazonaws.internal.SdkInternalList<ConditionalForwarder>();
        }
        return conditionalForwarders;
    }
1282276
public static int cudnnConvolutionBiasActivationForward(
        cudnnHandle handle, 
        Pointer alpha1, 
        cudnnTensorDescriptor xDesc, 
        Pointer x, 
        cudnnFilterDescriptor wDesc, 
        Pointer w, 
        cudnnConvolutionDescriptor convDesc, 
        int algo, 
        Pointer workSpace, 
        long workSpaceSizeInBytes, 
        Pointer alpha2, 
        cudnnTensorDescriptor zDesc, 
        Pointer z, 
        cudnnTensorDescriptor biasDesc, 
        Pointer bias, 
        cudnnActivationDescriptor activationDesc, 
        cudnnTensorDescriptor yDesc, 
        Pointer y)
    {
        return checkResult(cudnnConvolutionBiasActivationForwardNative(handle, alpha1, xDesc, x, wDesc, w, convDesc, algo, workSpace, workSpaceSizeInBytes, alpha2, zDesc, z, biasDesc, bias, activationDesc, yDesc, y));
    }
1128297
public List<InputColumn<?>> getOutputColumns() {
        final List<InputColumn<?>> result = new ArrayList<>();
        for (final RowProcessingConsumer consumer : _consumers) {
            final InputColumn<?>[] outputColumns = consumer.getOutputColumns();
            for (final InputColumn<?> outputColumn : outputColumns) {
                result.add(outputColumn);
            }
        }
        return result;
    }
826442
@Override
	public List<IN> classify(List<IN> tokens) {
		if (baseClassifiers.isEmpty()) {
			return tokens;
		}
		List<List<IN>> baseOutputs = new ArrayList<List<IN>>();

		// the first base model works in place, modifying the original tokens
		List<IN> output = baseClassifiers.get(0).classifySentence(tokens);
		// classify(List<IN>) is supposed to work in place, so add AnswerAnnotation to tokens!
		for (int i = 0, sz = output.size(); i < sz; i++) {
			tokens.get(i).set(AnswerAnnotation.class, output.get(i).get(AnswerAnnotation.class));
		}
		baseOutputs.add(tokens);

		for (int i = 1, sz = baseClassifiers.size(); i < sz; i ++) {
			//List<CoreLabel> copy = deepCopy(tokens);
			// no need for deep copy: classifySentence creates a copy of the input anyway
			// List<CoreLabel> copy = tokens;
			output = baseClassifiers.get(i).classifySentence(tokens);
			baseOutputs.add(output);
		}
		assert(baseOutputs.size() == baseClassifiers.size());
		List<IN> finalAnswer = mergeDocuments(baseOutputs);

		return finalAnswer;
	}
490965
def defuse_activation(self):
        
        idx, nb_layers = 0, len(self.layer_list)
        while idx < nb_layers:
            layer = self.layer_list[idx]
            k_layer = self.keras_layer_map[layer]
            if (isinstance(k_layer, _keras.layers.TimeDistributed)):
                k_layer = k_layer.layer
            if (isinstance(k_layer, _keras.layers.Conv2D) or
                isinstance(k_layer, _keras.layers.Conv1D) or
                isinstance(k_layer, _keras.layers.SeparableConv2D) or
                isinstance(k_layer, _keras.layers.SeparableConv1D) or
                isinstance(k_layer, _keras.layers.Dense)):

                func_name = k_layer.activation.func_name if six.PY2 else \
                        k_layer.activation.__name__

                if func_name != 'linear':
                    # Create new layer
                    new_layer = layer + '__activation__'
                    new_keras_layer = _keras.layers.core.Activation(func_name)
                    # insert new layer after it
                    self._insert_layer_after(idx, new_layer, new_keras_layer)
                    idx += 1
                    nb_layers += 1
            idx += 1
805153
public java.util.List<? extends org.tensorflow.framework.OpDef.ArgDefOrBuilder> 
      getInputArgOrBuilderList() {
    return inputArg_;
  }