This function serves as interface between model input vectors and
            objective input vectors
347560
def model_to_objective(self, x_model):
        ''' 
        '''
        idx_model = 0
        x_objective = []

        for idx_obj in range(self.objective_dimensionality):
            variable = self.space_expanded[idx_obj]
            new_entry = variable.model_to_objective(x_model, idx_model)
            x_objective += new_entry
            idx_model += variable.dimensionality_in_model

        return x_objective
774009
def fc(self, output_nodes, keep_prob=1, activation_fn=tf.nn.relu, b_value=0.0, s_value=1.0, bn=True,
           trainable=True):
        
        self.count['fc'] += 1
        scope = 'fc_' + str(self.count['fc'])
        with tf.variable_scope(scope):

            # Flatten if necessary
            if len(self.input.get_shape()) == 4:
                input_nodes = tf.Dimension(
                    self.input.get_shape()[1] * self.input.get_shape()[2] * self.input.get_shape()[3])
                output_shape = tf.stack([-1, input_nodes])
                self.input = tf.reshape(self.input, output_shape)

            # Matrix Multiplication Function
            input_nodes = self.input.get_shape()[1]
            output_shape = [input_nodes, output_nodes]
            w = self.weight_variable(name='weights', shape=output_shape, trainable=trainable)
            self.input = tf.matmul(self.input, w)

            if bn is True:  # batch normalization
                self.input = self.batch_norm(self.input, 'fc')
            if b_value is not None:  # bias value
                b = self.const_variable(name='bias', shape=[output_nodes], value=b_value, trainable=trainable)
                self.input = tf.add(self.input, b)
            if s_value is not None:  # scale value
                s = self.const_variable(name='scale', shape=[output_nodes], value=s_value, trainable=trainable)
                self.input = tf.multiply(self.input, s)
            if activation_fn is not None:  # activation function
                self.input = activation_fn(self.input)
            if keep_prob != 1:  # dropout function
                self.input = tf.nn.dropout(self.input, keep_prob=keep_prob)
        print(scope + ' output: ' + str(self.input.get_shape()))
198258
def calculate_batch_normalization_output_shapes(operator):
    '''
    
    '''
    check_input_and_output_numbers(operator, input_count_range=1, output_count_range=1)
    check_input_and_output_types(operator, good_input_types=[FloatTensorType])

    input_shape = operator.inputs[0].type.shape
    if len(input_shape) not in [2, 4]:
        raise RuntimeError('Input must be a 2-D or a 4-D tensor')

    operator.outputs[0].type.shape = copy.deepcopy(operator.inputs[0].type.shape)
1362943
def function v
    Error.call "Line::function : unable to execute function : the vector v_dir is null" if self.nil?
    Error.call "Line::functionx : invalid lambda ( #{v} ) value" if !v.is_a?Numeric

    lp = parametric()
    Point.new(lp[:x][:p] + v * lp[:x][:v], lp[:y][:p] + v * lp[:y][:v], lp[:z][:p] + v * lp[:z][:v])
  end
856584
def main(dimension, iterations):
    
    objective_function = minimize(functions.sphere)
    stopping_condition = max_iterations(iterations)
    (solution, metrics) = optimize(objective_function=objective_function,
                                   domain=Domain(-5.12, 5.12, dimension),
                                   stopping_condition=stopping_condition,
                                   parameters={'seed': 3758117674,
                                               'rho': 1.0, 'e_s': 15, 'e_f': 5},
                                   velocity_update=gc_velocity_update,
                                   parameter_update=update_rho,
                                   measurements=[fitness_measurement])
    return solution
704130
def build_model_input(cls, name='input'):
        
        return cls(name, PortDirection.INPUT, type=PortType.MODEL)
1118344
def _get_numdiff_hessian_element_func(objective_func, nmr_steps, step_ratio):
    """"""
    return SimpleCLFunction.from_string('''
        /**
         * Compute the Hessian using (possibly) multiple steps with various interpolations. 
         */ 
        void _numdiff_hessian_element(
                void* data, local mot_float_type* x_tmp, mot_float_type f_x_input,
                uint px, uint py, global float* initial_step, global double* derivative, 
                global double* error, local double* scratch){

            const uint nmr_steps = ''' + str(nmr_steps) + ''';
            uint nmr_steps_remaining = nmr_steps;

            local double* scratch_ind = scratch;
            local double* steps = scratch_ind;      scratch_ind += nmr_steps; 
            local double* errors = scratch_ind;     scratch_ind += nmr_steps - 1;
            local double* steps_tmp = scratch_ind;  scratch_ind += nmr_steps;
            
            if(get_local_id(0) == 0){
                for(int i = 0; i < nmr_steps - 1; i++){
                    errors[i] = 0;
                }
            }
            barrier(CLK_LOCAL_MEM_FENCE);
            
            _numdiff_hessian_steps(data, x_tmp, f_x_input, px, py, steps, initial_step);

            if(nmr_steps_remaining > 1){
                nmr_steps_remaining = _numdiff_hessian_richardson_extrapolation(steps); 
                barrier(CLK_LOCAL_MEM_FENCE);
            }
            
            if(nmr_steps_remaining >= 3){
                nmr_steps_remaining = _numdiff_wynn_extrapolation(steps, errors, nmr_steps_remaining);
                barrier(CLK_LOCAL_MEM_FENCE);                
            }
            
            if(nmr_steps_remaining > 1){
                _numdiff_find_best_step(steps, errors, steps_tmp, nmr_steps_remaining);
                barrier(CLK_LOCAL_MEM_FENCE);
            }
            
            if(get_local_id(0) == 0){
                *derivative = steps[0];
                *error = errors[0];
            }
        }
    ''', dependencies=[
        _get_numdiff_hessian_steps_func(objective_func, nmr_steps, step_ratio),
        _get_numdiff_hessian_richardson_extrapolation_func(nmr_steps, step_ratio),
        _get_numdiff_wynn_extrapolation_func(),
        _get_numdiff_find_best_step_func()
    ])
1348104
def scalar_input_map(func, input_):
    
    if util_iter.isiterable(input_):
        return list(map(func, input_))
    else:
        return func(input_)
1332388
def _scalar_to_vector(self, m):
        
        if not isinstance(m.y0, numbers.Number):
            return m
        else:
            m = copy.deepcopy(m)
            t0 = 0.0
            if isinstance(m.y0, numbers.Integral):
                numtype = np.float64
            else:
                numtype = type(m.y0)
            y0_orig = m.y0
            m.y0 = np.array([m.y0], dtype=numtype)
            def make_vector_fn(fn):
                def newfn(y, t):
                    return np.array([fn(y[0], t)], dtype=numtype)
                newfn.__name__ = fn.__name__
                return newfn
            def make_matrix_fn(fn):
                def newfn(y, t):
                    return np.array([[fn(y[0], t)]], dtype=numtype)
                newfn.__name__ = fn.__name__
                return newfn
            def make_coupling_fn(fn):
                def newfn(source_y, target_y, weight):
                    return np.array([fn(source_y[0], target_y[0], weight)])
                newfn.__name__ = fn.__name__
                return newfn
            if isinstance(m.f(y0_orig, t0), numbers.Number):
                m.f = make_vector_fn(m.f)
            if hasattr(m, 'G') and isinstance(m.G(y0_orig,t0), numbers.Number):
                m.G = make_matrix_fn(m.G)
            if (hasattr(m, 'coupling') and
                    isinstance(m.coupling(y0_orig, y0_orig, 0.5),
                               numbers.Number)):
                m.coupling = make_coupling_fn(m.coupling)
            return m
1372949
def sg_layer_func(func):
    
    """
    @wraps(func)
    def wrapper(tensor, **kwargs):
        r"""Manages arguments of `tf.sg_opt`.

        Args:
          tensor: A `tensor` (automatically passed by decorator).
          kwargs:
            shape:  A list of integers. The shape of `tensor`. Inferred if not specified.
            in_dim: An integer. The size of input dimension, which is set to the last one by default.
            dim: An integer. The size of output dimension. Has the same value as in_dim by default.
            bn: Boolean. If True, batch normalization is applied.
            ln: Boolean. If True, layer normalization is applied.
            scale: If true, multiple by a trainable gamma variable. When the activation is
              linear (relu included), this can be disabled because it can be implicitly
              learned by the next layer. The default is True.
            dout: A float of range [0, 100). A dropout rate. Set to 0 by default.
            bias: Boolean. If True, biases are added. As a default, it is set to True
            name: A name for the layer. As a default, the function name is assigned.
            act: A name of activation function. e.g., `sigmoid`, `tanh`, etc.
            reuse: `True` or `None`; if `True`, we go into reuse mode for this `layer` scope
              as well as all sub-scopes; if `None`, we just inherit the parent scope reuse.
            regularizer:  A string. None, 'l1' or 'l2'. The default is None
            summary: If True, summaries are added. The default is True.
        """

        from . import sg_initializer as init
        from . import sg_activation

        # kwargs parsing
        opt = tf.sg_opt(kwargs) + sg_get_context()

        # set default argument
        try:
            shape = tensor.get_shape().as_list()
            # batch normalization off, layer normalization off, dropout off
            opt += tf.sg_opt(shape=shape, in_dim=shape[-1], dim=shape[-1],
                             bn=False, ln=False, dout=0, summary=True, scale=True)
            if opt.regularizer == 'l1':
                opt.regularizer = lambda x: tf.reduce_mean(tf.abs(x))
            elif opt.regularizer == 'l2':
                opt.regularizer = lambda x: tf.square(tf.reduce_mean(tf.square(x)))
            else:
                opt.regularizer = None

            assert not (opt.bn and opt.ln), 'one of batch normalization and layer normalization is available.'

            # disable bias when normalization on
            opt += tf.sg_opt(bias=not (opt.bn or opt.ln))
        finally:
            pass

        # automatic layer naming
        if opt.name is None:

            # layer function name will be used as layer name
            opt.name = func.__name__.replace('sg_', '')

            # find existing layer names
            exist_layers = []
            for t in tf.global_variables():
                scope_name = tf.get_variable_scope().name
                prefix = scope_name + '/' if len(scope_name) > 0 else ''
                i = t.name.rfind(prefix + opt.name)
                if i >= 0:
                    exist_layers.append(t.name[i:].split('/')[-2])
            exist_layers = list(set(exist_layers))

            # layer name numbering
            if len(exist_layers) == 0:
                opt.name += '_1'
            else:
                opt.name += '_%d' % (max([int(n.split('_')[-1]) for n in exist_layers]) + 1)

        with tf.variable_scope(opt.name, reuse=opt.reuse) as scope:

            # call layer function
            out = func(tensor, opt)
            out_shape = out.get_shape()

            # apply batch normalization
            if opt.bn:
                beta = init.constant('beta', opt.dim, summary=opt.summary)
                gamma = init.constant('gamma', opt.dim, value=1, summary=opt.summary, trainable=opt.scale)

                # offset, scale parameter ( for inference )
                mean_running = init.constant('mean', opt.dim, trainable=False, summary=opt.summary)
                variance_running = init.constant('variance', opt.dim, value=1, trainable=False, summary=opt.summary)

                # use fused batch norm if ndims in [2, 3, 4]
                if out_shape.ndims in [2, 3, 4]:
                    # add HW dims if necessary, fused_batch_norm requires shape to be NHWC
                    if out_shape.ndims == 2:
                        out = tf.expand_dims(out, axis=1)
                        out = tf.expand_dims(out, axis=2)
                    elif out_shape.ndims == 3:
                        out = tf.expand_dims(out, axis=2)

                    fused_eps = tf.sg_eps if tf.sg_eps > 1e-5 else 1e-5
                    out, mean, variance = tf.cond(
                        _phase,
                        lambda: tf.nn.fused_batch_norm(out, gamma, beta, epsilon=fused_eps),
                        lambda: tf.nn.fused_batch_norm(out, gamma, beta, mean=mean_running, variance=variance_running, epsilon=fused_eps, is_training=False),
                    )

                    # restore original shape if HW dims was added
                    if out_shape.ndims == 2:
                        out = tf.squeeze(out, axis=[1, 2])
                    elif out_shape.ndims == 3:
                        out = tf.squeeze(out, axis=2)

                # fallback to naive batch norm
                else:
                    mean, variance = tf.nn.moments(out, axes=list(range(len(out.get_shape()) - 1)))
                    out = tf.cond(
                        _phase,
                        lambda: tf.nn.batch_normalization(out, mean, variance, beta, gamma, tf.sg_eps),
                        lambda: tf.nn.batch_normalization(out, mean_running, variance_running, beta, gamma, tf.sg_eps)
                    )

                decay = 0.99
                tf.add_to_collection(tf.GraphKeys.UPDATE_OPS, mean_running.assign(mean_running * decay + mean * (1 - decay)))
                tf.add_to_collection(tf.GraphKeys.UPDATE_OPS, variance_running.assign(variance_running * decay + variance * (1 - decay)))

            # apply layer normalization
            if opt.ln:
                # offset, scale parameter
                beta = init.constant('beta', opt.dim, summary=opt.summary)
                if opt.scale:
                    gamma = init.constant('gamma', opt.dim, value=1, summary=opt.summary)

                # calc layer mean, variance for final axis
                mean, variance = tf.nn.moments(out, axes=[len(out.get_shape()) - 1], keep_dims=True)

                # apply normalization
                out = (out - mean) / tf.sqrt(variance + tf.sg_eps)
                # apply parameter
                if opt.scale:
                    out = gamma * out + beta
                else:
                    out = out + beta

            # apply activation
            if opt.act:
                out = getattr(sg_activation, 'sg_' + opt.act.lower())(out)

            # apply dropout
            if opt.dout:
                out = tf.cond(_phase,
                              lambda: tf.nn.dropout(out, 1 - opt.dout),
                              lambda: out)

            # rename tensor
            out = tf.identity(out, 'out')

            # add final output summary
            if opt.summary:
                tf.sg_summary_activation(out)

            # save node info for reuse
            out._sugar = tf.sg_opt(func=func, arg=tf.sg_opt(kwargs) + sg_get_context(),
                                   prev=tensor, is_layer=True, name=opt.name)
            # inject reuse function
            out.sg_reuse = types.MethodType(sg_reuse, out)

        return out

    return wrapper
1238912
def get_reader_input_fn(train_config, preprocess_output_dir, model_type,
                        data_paths, batch_size, shuffle, num_epochs=None):
  """"""

  def get_input_features():
    """Read the input features from the given data paths."""
    _, examples = util.read_examples(
        input_files=data_paths,
        batch_size=batch_size,
        shuffle=shuffle,
        num_epochs=num_epochs)
    features = util.parse_example_tensor(examples=examples,
                                         train_config=train_config,
                                         keep_target=True)

    target_name = train_config['target_column']
    target = features.pop(target_name)
    features, target = util.preprocess_input(
        features=features,
        target=target,
        train_config=train_config,
        preprocess_output_dir=preprocess_output_dir,
        model_type=model_type)

    return features, target

  # Return a function to input the feaures into the model from a data path.
  return get_input_features
35901
def set_differentiable_objective(self):
    """"""
    # Checking if graphs are already created
    if self.vector_g is not None:
      return

    # Computing the scalar term
    bias_sum = 0
    for i in range(0, self.nn_params.num_hidden_layers):
      bias_sum = bias_sum + tf.reduce_sum(
          tf.multiply(self.nn_params.biases[i], self.lambda_pos[i + 1]))
    lu_sum = 0
    for i in range(0, self.nn_params.num_hidden_layers + 1):
      lu_sum = lu_sum + tf.reduce_sum(
          tf.multiply(tf.multiply(self.lower[i], self.upper[i]),
                      self.lambda_lu[i]))

    self.scalar_f = -bias_sum - lu_sum + self.final_constant

    # Computing the vector term
    g_rows = []
    for i in range(0, self.nn_params.num_hidden_layers):
      if i > 0:
        current_row = (self.lambda_neg[i] + self.lambda_pos[i] -
                       self.nn_params.forward_pass(self.lambda_pos[i+1],
                                                   i, is_transpose=True) +
                       tf.multiply(self.lower[i]+self.upper[i],
                                   self.lambda_lu[i]) +
                       tf.multiply(self.lambda_quad[i],
                                   self.nn_params.biases[i-1]))
      else:
        current_row = (-self.nn_params.forward_pass(self.lambda_pos[i+1],
                                                    i, is_transpose=True)
                       + tf.multiply(self.lower[i]+self.upper[i],
                                     self.lambda_lu[i]))
      g_rows.append(current_row)

    # Term for final linear term
    g_rows.append((self.lambda_pos[self.nn_params.num_hidden_layers] +
                   self.lambda_neg[self.nn_params.num_hidden_layers] +
                   self.final_linear +
                   tf.multiply((self.lower[self.nn_params.num_hidden_layers]+
                                self.upper[self.nn_params.num_hidden_layers]),
                               self.lambda_lu[self.nn_params.num_hidden_layers])
                   + tf.multiply(
                       self.lambda_quad[self.nn_params.num_hidden_layers],
                       self.nn_params.biases[
                           self.nn_params.num_hidden_layers-1])))
    self.vector_g = tf.concat(g_rows, axis=0)
    self.unconstrained_objective = self.scalar_f + 0.5 * self.nu
647199
def init_nn_params(scale, layer_sizes, rs=npr.RandomState(0)):
    """"""
    return [(rs.randn(insize, outsize) * scale,   # weight matrix
             rs.randn(outsize) * scale)           # bias vector
            for insize, outsize in zip(layer_sizes[:-1], layer_sizes[1:])]
941881
def find_signature_inputs_from_multivalued_ops(inputs):
  """"""
  dense_inputs = []  # List of (str, Tensor), with SparseTensors decomposed.
  for name, tensor in sorted(inputs.items()):
    if isinstance(tensor, tf.SparseTensor):
      dense_inputs.extend(("%s.%s" % (name, attr), getattr(tensor, attr))
                          for attr in ("indices", "values", "dense_shape"))
    else:
      dense_inputs.append((name, tensor))
  warnings = [(name, tensor.name) for name, tensor in dense_inputs
              if len(tensor.op.outputs) != 1]
  if warnings:
    return (
        "WARNING: The inputs declared in hub.add_signature() should be tensors "
        "from ops with a single output, or else uses of tf.colocate_with() on "
        "that op can trigger fatal errors when the module is applied and "
        "colocation constraints have to be rewritten.\nAffected inputs: %s" %
        ", ".join("%s='%s'" % pair for pair in warnings))
  return None
1066614
def create(input_length, hidden_layers, activation='tanh', normalization=None):
    
    def instantiate(**_):
        return MLP(
            input_length=input_length,
            hidden_layers=hidden_layers,
            activation=activation,
            normalization=normalization
        )

    return ModelFactory.generic(instantiate)
997453
def create(input_block: ModelFactory, rnn_type: str, hidden_layers: typing.List[int],
           output_dim: int, dropout=0.0):
    
    def instantiate(**_):
        return MultilayerRnnSequenceModel(
            input_block.instantiate(), rnn_type=rnn_type, hidden_layers=hidden_layers, output_dim=output_dim, dropout=dropout
        )

    return ModelFactory.generic(instantiate)
8241
def _interactive_input_tensor_to_features_dict(feature_map, hparams):
  
  inputs = tf.convert_to_tensor(feature_map["inputs"])
  input_is_image = False if len(inputs.get_shape()) < 3 else True

  x = inputs
  if input_is_image:
    x = tf.image.resize_images(x, [299, 299])
    x = tf.reshape(x, [1, 299, 299, -1])
    x = tf.to_int32(x)
  else:
    # Remove the batch dimension.
    num_samples = x[0]
    length = x[2]
    x = tf.slice(x, [3], tf.to_int32([length]))
    x = tf.reshape(x, [1, -1, 1, 1])
    # Transform into a batch of size num_samples to get that many random
    # decodes.
    x = tf.tile(x, tf.to_int32([num_samples, 1, 1, 1]))

  p_hparams = hparams.problem_hparams
  input_space_id = tf.constant(p_hparams.input_space_id)
  target_space_id = tf.constant(p_hparams.target_space_id)

  features = {}
  features["input_space_id"] = input_space_id
  features["target_space_id"] = target_space_id
  features["decode_length"] = (
      IMAGE_DECODE_LENGTH if input_is_image else inputs[1])
  features["inputs"] = x
  return features