Fully Connected Layer
        :param output_nodes: int
        :param keep_prob: int. set to 1 for no dropout
        :param activation_fn: tf.nn function
        :param b_value: float or None
        :param s_value: float or None
        :param bn: bool
774009
def fc(self, output_nodes, keep_prob=1, activation_fn=tf.nn.relu, b_value=0.0, s_value=1.0, bn=True,
           trainable=True):
        
        self.count['fc'] += 1
        scope = 'fc_' + str(self.count['fc'])
        with tf.variable_scope(scope):

            # Flatten if necessary
            if len(self.input.get_shape()) == 4:
                input_nodes = tf.Dimension(
                    self.input.get_shape()[1] * self.input.get_shape()[2] * self.input.get_shape()[3])
                output_shape = tf.stack([-1, input_nodes])
                self.input = tf.reshape(self.input, output_shape)

            # Matrix Multiplication Function
            input_nodes = self.input.get_shape()[1]
            output_shape = [input_nodes, output_nodes]
            w = self.weight_variable(name='weights', shape=output_shape, trainable=trainable)
            self.input = tf.matmul(self.input, w)

            if bn is True:  # batch normalization
                self.input = self.batch_norm(self.input, 'fc')
            if b_value is not None:  # bias value
                b = self.const_variable(name='bias', shape=[output_nodes], value=b_value, trainable=trainable)
                self.input = tf.add(self.input, b)
            if s_value is not None:  # scale value
                s = self.const_variable(name='scale', shape=[output_nodes], value=s_value, trainable=trainable)
                self.input = tf.multiply(self.input, s)
            if activation_fn is not None:  # activation function
                self.input = activation_fn(self.input)
            if keep_prob != 1:  # dropout function
                self.input = tf.nn.dropout(self.input, keep_prob=keep_prob)
        print(scope + ' output: ' + str(self.input.get_shape()))
1285428
def convert_fully_connected(node, **kwargs):
    
    name, input_nodes, attrs = get_inputs(node, kwargs)

    initializer = kwargs["initializer"]

    no_bias = get_boolean_attribute_value(attrs, "no_bias")

    fcnode = []

    op_name = "flatten_" + str(kwargs["idx"])
    flatten_node = onnx.helper.make_node(
        'Flatten',
        inputs=[input_nodes[0]],
        outputs=[op_name],
        name=op_name
    )

    input_nodes[0] = op_name
    fcnode.append(flatten_node)

    if no_bias:
        data_type = onnx.mapping.NP_TYPE_TO_TENSOR_TYPE[np.dtype('int64')]
        bias_name = "bias" + str(kwargs["idx"])
        tensor_node = onnx.helper.make_tensor_value_info(bias_name, data_type, (1,))
        initializer.append(
            onnx.helper.make_tensor(
                name=bias_name,
                data_type=data_type,
                dims=(1,),
                vals=[0],
                raw=False,
            )
        )
        input_nodes.append(bias_name)
        fcnode.append(tensor_node)

    node = onnx.helper.make_node(
        "Gemm",
        input_nodes,  # input (A, B, C) - C can be in place
        [name],  # output
        alpha=1.0,
        beta=1.0,
        transA=False,
        transB=True,
        name=name
    )

    fcnode.append(node)

    return fcnode
159297
def compute_output(t0, t1):
    ''''''
    if t0 is None or t1 is None:
        # If neither of the output neurons fired within the allotted time,
        # give a response which produces a large error.
        return -1.0
    else:
        # If the output neurons fire within 1.0 milliseconds of each other,
        # the output is 1, and if they fire more than 11 milliseconds apart,
        # the output is 0, with linear interpolation between 1 and 11 milliseconds.
        response = 1.1 - 0.1 * abs(t0 - t1)
        return max(0.0, min(1.0, response))
718537
def dropout(input_layer, keep_prob, phase=Phase.train, name=PROVIDED):
  """"""
  if phase == Phase.train:
    return tf.nn.dropout(input_layer, keep_prob, name=name)
  else:
    return input_layer
429463
def verifyInputs(self):
        
        for layer in self.layers:
            if (layer.verify and
                layer.type == 'Input' and
                layer.kind != 'Context' and
                layer.active and
                not layer.activationSet):
                raise LayerError("Inputs are not set and verifyInputs() was called on layer '%s'." % layer.name)
            else:
                layer.resetActivationFlag()
490965
def defuse_activation(self):
        
        idx, nb_layers = 0, len(self.layer_list)
        while idx < nb_layers:
            layer = self.layer_list[idx]
            k_layer = self.keras_layer_map[layer]
            if (isinstance(k_layer, _keras.layers.TimeDistributed)):
                k_layer = k_layer.layer
            if (isinstance(k_layer, _keras.layers.Conv2D) or
                isinstance(k_layer, _keras.layers.Conv1D) or
                isinstance(k_layer, _keras.layers.SeparableConv2D) or
                isinstance(k_layer, _keras.layers.SeparableConv1D) or
                isinstance(k_layer, _keras.layers.Dense)):

                func_name = k_layer.activation.func_name if six.PY2 else \
                        k_layer.activation.__name__

                if func_name != 'linear':
                    # Create new layer
                    new_layer = layer + '__activation__'
                    new_keras_layer = _keras.layers.core.Activation(func_name)
                    # insert new layer after it
                    self._insert_layer_after(idx, new_layer, new_keras_layer)
                    idx += 1
                    nb_layers += 1
            idx += 1
1342146
def connectAt(self, fromName, toName, position = None):
        
        fromLayer = self.getLayer(fromName)
        toLayer   = self.getLayer(toName)
        if self.getLayerIndex(fromLayer) >= self.getLayerIndex(toLayer):
            raise NetworkError('Layers out of order.', (fromLayer.name, toLayer.name))
        if (fromLayer.type == 'Output'):
            fromLayer.type = 'Hidden'
            fromLayer.patternReport = 0 # automatically turned off for hidden layers
            if fromLayer.kind == 'Output':
                fromLayer.kind = 'Hidden'
        elif (fromLayer.type == 'Undefined'):
            fromLayer.type = 'Input'
            fromLayer.patternReport = 0 # automatically turned off for input layers
            if fromLayer.kind == 'Undefined':
                fromLayer.kind = 'Input'
        if (toLayer.type == 'Input'):
            raise NetworkError('Connections out of order', (fromLayer.name, toLayer.name))
        elif (toLayer.type == 'Undefined'):
            toLayer.type = 'Output'
            toLayer.patternReport = 1 # automatically turned on for output layers
            if toLayer.kind == 'Undefined':
                toLayer.kind = 'Output'
        if position == None:
            self.connections.append(Connection(fromLayer, toLayer))
        else:
            self.connections.insert(position, Connection(fromLayer, toLayer))
1013033
def step(self, **args):
        
        if self.verbosity > 0:
            print("Network.step() called with:", args)
        # First, copy the values into either activations or targets:
        retargs = self.preStep(**args)
        if retargs: args = retargs # replace the args
        # Propagate activation through network:
        self.propagate(**args)
        retargs = self.postPropagate(**args)
        if retargs: args = retargs # replace the args
        # Next, take care of any Auto-association, and copy
        # activations to targets
        for aa in self.association:
            (inName, outName) = aa
            inLayer = self.getLayer(inName)
            if inLayer.type not in ['Input', "Hidden"]:
                raise LayerError('Associated input layer not type \'Input\' or \'Hidden\'.', \
                                   inLayer.type)
            outLayer = self.getLayer(outName)
            if not outLayer.type == 'Output':
                raise LayerError('Associated output layer not type \'Output\'.', \
                                   outLayer.type)
            outLayer.copyTargets(inLayer.activation)
        # Compute error, and back prop it:
        retargs = self.preBackprop(**args)
        if retargs: args = retargs # replace the args
        (error, correct, total, pcorrect) = self.backprop(**args) # compute_error()
        if self.verbosity > 2 or self.interactive:
            self.display()
            if self.interactive:
                self.prompt()
        retargs = self.postBackprop(**args)
        if retargs: args = retargs # replace the args
        # if learning is true, and need to update weights here:
        if self.learning and not self.batch:
            self.change_weights() # else change weights in sweep
        retargs = self.postStep(**args)
        if retargs: args = retargs # replace the args
        self.reportPattern()
        return (error, correct, total, pcorrect)
570624
def flownet2_fusion(self, x):
        
        with argscope([tf.layers.conv2d], activation=lambda x: tf.nn.leaky_relu(x, 0.1),
                      padding='valid', strides=2, kernel_size=3,
                      data_format='channels_first'), \
            argscope([tf.layers.conv2d_transpose], padding='same', activation=tf.identity,
                     data_format='channels_first', strides=2, kernel_size=4):
            conv0 = tf.layers.conv2d(pad(x, 1), 64, name='conv0', strides=1)

            x = tf.layers.conv2d(pad(conv0, 1), 64, name='conv1')
            conv1 = tf.layers.conv2d(pad(x, 1), 128, name='conv1_1', strides=1)
            x = tf.layers.conv2d(pad(conv1, 1), 128, name='conv2')
            conv2 = tf.layers.conv2d(pad(x, 1), 128, name='conv2_1', strides=1)

            flow2 = tf.layers.conv2d(pad(conv2, 1), 2, name='predict_flow2', strides=1, activation=tf.identity)
            flow2_up = tf.layers.conv2d_transpose(flow2, 2, name='upsampled_flow2_to_1')
            x = tf.layers.conv2d_transpose(conv2, 32, name='deconv1', activation=lambda x: tf.nn.leaky_relu(x, 0.1))

            concat1 = tf.concat([conv1, x, flow2_up], axis=1, name='concat1')
            interconv1 = tf.layers.conv2d(pad(concat1, 1), 32, strides=1, name='inter_conv1', activation=tf.identity)

            flow1 = tf.layers.conv2d(pad(interconv1, 1), 2, name='predict_flow1', strides=1, activation=tf.identity)
            flow1_up = tf.layers.conv2d_transpose(flow1, 2, name='upsampled_flow1_to_0')
            x = tf.layers.conv2d_transpose(concat1, 16, name='deconv0', activation=lambda x: tf.nn.leaky_relu(x, 0.1))

            concat0 = tf.concat([conv0, x, flow1_up], axis=1, name='concat0')
            interconv0 = tf.layers.conv2d(pad(concat0, 1), 16, strides=1, name='inter_conv0', activation=tf.identity)
            flow0 = tf.layers.conv2d(pad(interconv0, 1), 2, name='predict_flow0', strides=1, activation=tf.identity)

            return tf.identity(flow0, name='flow2')
201247
def _get_bmu(self, activations):
        """"""
        # If the neural gas is a recursive neural gas, we need reverse argsort.
        if self.argfunc == 'argmax':
            activations = -activations
        sort = np.argsort(activations, 1)
        return sort.argsort()
579803
def calculateHiddenLayerActivation(self, features):
    
    if self.activationFunction is "sig":
      H = sigmoidActFunc(features, self.inputWeights, self.bias)
    else:
      print " Unknown activation function type"
      raise NotImplementedError
    return H
1005407
def useFahlmanActivationFunction(self):
        
        self.activationFunction = self.activationFunctionFahlman
        self.ACTPRIME = self.ACTPRIME_Fahlman
        self.actDeriv = self.actDerivFahlman
        for layer in self:
            layer.minTarget, layer.minActivation = -0.5, -0.5 
            layer.maxTarget, layer.maxActivation = 0.5, 0.5
781093
def _residual(self, x, in_filter, out_filter, stride,
                activate_before_residual=False):
    """"""
    if activate_before_residual:
      with tf.variable_scope('shared_activation'):
        x = self._layer_norm('init_bn', x)
        x = self._relu(x, self.hps.relu_leakiness)
        orig_x = x
    else:
      with tf.variable_scope('residual_only_activation'):
        orig_x = x
        x = self._layer_norm('init_bn', x)
        x = self._relu(x, self.hps.relu_leakiness)

    with tf.variable_scope('sub1'):
      x = self._conv('conv1', x, 3, in_filter, out_filter, stride)

    with tf.variable_scope('sub2'):
      x = self._layer_norm('bn2', x)
      x = self._relu(x, self.hps.relu_leakiness)
      x = self._conv('conv2', x, 3, out_filter, out_filter, [1, 1, 1, 1])

    with tf.variable_scope('sub_add'):
      if in_filter != out_filter:
        orig_x = tf.nn.avg_pool(orig_x, stride, stride, 'VALID')
        orig_x = tf.pad(
            orig_x, [[0, 0], [0, 0], [0, 0],
                     [(out_filter - in_filter) // 2,
                      (out_filter - in_filter) // 2]])
      x += orig_x

    return x
365279
def get_layers_output(self, dataset):
        
        layers_out = []

        with self.tf_graph.as_default():
            with tf.Session() as self.tf_session:
                self.tf_saver.restore(self.tf_session, self.model_path)
                for l in self.layer_nodes:
                    layers_out.append(l.eval({self.input_data: dataset,
                                              self.keep_prob: 1}))

        if layers_out == []:
            raise Exception("This method is not implemented for this model")
        else:
            return layers_out
830120
def ce_init(self):
        
        retval = 0.0; correct = 0; totalCount = 0
        for layer in self.layers:
            if layer.active:
                if layer.type == 'Output':
                    layer.error = self.errorFunction(layer.target, layer.activation) 
                    totalCount += layer.size
                    retval += Numeric.add.reduce((layer.target - layer.activation) ** 2)
                    correct += Numeric.add.reduce(Numeric.fabs(layer.target - layer.activation) < self.tolerance)
                elif (layer.type == 'Hidden'):
                    for i in range(layer.size): # do it this way so you don't break reference links
                        layer.error[i] = 0.0
        return (retval, correct, totalCount)
412765
def delete_node_nto1(node_list, begin, node, end):  # type: ([],LinkedNode, LinkedNode, LinkedNode)->[]
        
        if begin is None:
            assert node is not None
            begin = node.precedence
        elif not isinstance(begin, list):
            begin = [begin]

        if end.in_or_out:
            # if the end is output node, the output name will be kept to avoid the model output name updating.
            for nb_ in begin:
                nb_.out_redirect(node.single_input, node.single_output)
        else:
            for nb_ in begin:
                target_var_name = node.single_input
                assert target_var_name in nb_.output.values()  # since the output info never be updated, except the final.
                end.in_redirect(node.single_output, target_var_name)

        for nb_ in begin:
            nb_.successor = [end if v_ == node else v_ for v_ in nb_.successor]
        end.precedence = [v_ for v_ in end.precedence if v_ != node] + node.precedence

        node_list.remove(node)
        return node_list
652113
def convert_advanced_relu(builder, layer, input_names, output_names, keras_layer):
    
    # Get input and output names
    input_name, output_name = (input_names[0], output_names[0])

    if keras_layer.max_value is None:
        builder.add_activation(layer, 'RELU', input_name, output_name)
        return

    # No direct support of RELU with max-activation value - use negate and
    # clip layers
    relu_output_name = output_name + '_relu'
    builder.add_activation(layer, 'RELU', input_name, relu_output_name)
    # negate it
    neg_output_name = relu_output_name + '_neg'
    builder.add_activation(layer+'__neg__', 'LINEAR', relu_output_name,
            neg_output_name,[-1.0, 0])
    # apply threshold
    clip_output_name = relu_output_name + '_clip'
    builder.add_unary(layer+'__clip__', neg_output_name, clip_output_name,
            'threshold', alpha = -keras_layer.max_value)
    # negate it back
    builder.add_activation(layer+'_neg2', 'LINEAR', clip_output_name,
            output_name,[-1.0, 0])