Computes bag of words weights

  Note the return type is a float sparse tensor, not a int sparse tensor. This
  is so that the output types batch tfidf, and any downstream transformation
  in tf layers during training can be applied to both.
517433
def _bag_of_words(x):
  
  def _bow(x):
    """Comptue BOW weights.

    As tf layer's sum combiner is used, the weights can be just ones. Tokens are
    not summed together here.
    """
    return tf.SparseTensor(
      indices=x.indices,
      values=tf.to_float(tf.ones_like(x.values)),
      dense_shape=x.dense_shape)

  return _bow(x)
878765
def cat(tensors, dim=0):
    
    assert isinstance(tensors, (list, tuple))
    if len(tensors) == 1:
        return tensors[0]
    return torch.cat(tensors, dim)
986026
def is_layer(layer, layer_type):
    '''
    '''

    if layer_type == "Input":
        return isinstance(layer, StubInput)
    elif layer_type == "Conv":
        return isinstance(layer, StubConv)
    elif layer_type == "Dense":
        return isinstance(layer, (StubDense,))
    elif layer_type == "BatchNormalization":
        return isinstance(layer, (StubBatchNormalization,))
    elif layer_type == "Concatenate":
        return isinstance(layer, (StubConcatenate,))
    elif layer_type == "Add":
        return isinstance(layer, (StubAdd,))
    elif layer_type == "Pooling":
        return isinstance(layer, StubPooling)
    elif layer_type == "Dropout":
        return isinstance(layer, (StubDropout,))
    elif layer_type == "Softmax":
        return isinstance(layer, (StubSoftmax,))
    elif layer_type == "ReLU":
        return isinstance(layer, (StubReLU,))
    elif layer_type == "Flatten":
        return isinstance(layer, (StubFlatten,))
    elif layer_type == "GlobalAveragePooling":
        return isinstance(layer, StubGlobalPooling)
1010522
def _item_to_tf_feature(item, key_name):
  """"""
  v = item
  if isinstance(v, (list, tuple)) and not v:
    raise ValueError(
        "Feature {} received an empty list value, so is unable to infer the "
        "feature type to record. To support empty value, the corresponding "
        "FeatureConnector should return a numpy array with the correct dtype "
        "instead of a Python list.".format(key_name)
    )

  # Handle strings/bytes first
  if isinstance(v, (six.binary_type, six.string_types)):
    v = [tf.compat.as_bytes(v)]
    return tf.train.Feature(bytes_list=tf.train.BytesList(value=v))
  elif (isinstance(v, (tuple, list)) and
        all(isinstance(x, (six.binary_type, six.string_types)) for x in v)):
    v = [tf.compat.as_bytes(x) for x in v]
    return tf.train.Feature(bytes_list=tf.train.BytesList(value=v))
  elif (isinstance(v, np.ndarray) and
        (v.dtype.kind in ("U", "S") or v.dtype == object)):  # binary or unicode
    v = [tf.compat.as_bytes(x) for x in v.flatten()]
    return tf.train.Feature(bytes_list=tf.train.BytesList(value=v))

  # Use NumPy for numeric types
  v = np.array(v).flatten()  # Convert v into a 1-d array

  if np.issubdtype(v.dtype, np.integer):
    return tf.train.Feature(int64_list=tf.train.Int64List(value=v))
  elif np.issubdtype(v.dtype, np.floating):
    return tf.train.Feature(float_list=tf.train.FloatList(value=v))
  else:
    raise ValueError(
        "Value received: {}.\n"
        "tf.train.Feature does not support type {} for feature key {}. "
        "This may indicate that one of the FeatureConnectors received an "
        "unsupported value as input.".format(repr(v), repr(type(v)), key_name)
    )
962936
def l1_l2_regularizer(weight_l1=1.0, weight_l2=1.0, scope=None):
  
  def regularizer(tensor):
    with tf.name_scope(scope, 'L1L2Regularizer', [tensor]):
      weight_l1_t = tf.convert_to_tensor(weight_l1,
                                         dtype=tensor.dtype.base_dtype,
                                         name='weight_l1')
      weight_l2_t = tf.convert_to_tensor(weight_l2,
                                         dtype=tensor.dtype.base_dtype,
                                         name='weight_l2')
      reg_l1 = tf.multiply(weight_l1_t, tf.reduce_sum(tf.abs(tensor)),
                      name='value_l1')
      reg_l2 = tf.multiply(weight_l2_t, tf.nn.l2_loss(tensor),
                      name='value_l2')
      return tf.add(reg_l1, reg_l2, name='value')
  return regularizer
1236149
def load_tf_weights_in_gpt2(model, gpt2_checkpoint_path):
    
    try:
        import re
        import numpy as np
        import tensorflow as tf
    except ImportError:
        print("Loading a TensorFlow models in PyTorch, requires TensorFlow to be installed. Please see "
            "https://www.tensorflow.org/install/ for installation instructions.")
        raise
    tf_path = os.path.abspath(gpt2_checkpoint_path)
    print("Converting TensorFlow checkpoint from {}".format(tf_path))
    # Load weights from TF model
    init_vars = tf.train.list_variables(tf_path)
    names = []
    arrays = []
    for name, shape in init_vars:
        print("Loading TF weight {} with shape {}".format(name, shape))
        array = tf.train.load_variable(tf_path, name)
        names.append(name)
        arrays.append(array.squeeze())

    for name, array in zip(names, arrays):
        name = name[6:]  # skip "model/"
        name = name.split('/')
        pointer = model
        for m_name in name:
            if re.fullmatch(r'[A-Za-z]+\d+', m_name):
                l = re.split(r'(\d+)', m_name)
            else:
                l = [m_name]
            if l[0] == 'w' or l[0] == 'g':
                pointer = getattr(pointer, 'weight')
            elif l[0] == 'b':
                pointer = getattr(pointer, 'bias')
            elif l[0] == 'wpe' or l[0] == 'wte':
                pointer = getattr(pointer, l[0])
                pointer = getattr(pointer, 'weight')
            else:
                pointer = getattr(pointer, l[0])
            if len(l) >= 2:
                num = int(l[1])
                pointer = pointer[num]
        try:
            assert pointer.shape == array.shape
        except AssertionError as e:
            e.args += (pointer.shape, array.shape)
            raise
        print("Initialize PyTorch weight {}".format(name))
        pointer.data = torch.from_numpy(array)
    return model
1204290
def convert_weights_and_inputs(node, **kwargs):
    
    name, _, _ = get_inputs(node, kwargs)

    if kwargs["is_input"] is False:
        weights = kwargs["weights"]
        initializer = kwargs["initializer"]
        np_arr = weights[name]
        data_type = onnx.mapping.NP_TYPE_TO_TENSOR_TYPE[np_arr.dtype]
        dims = np.shape(np_arr)

        tensor_node = onnx.helper.make_tensor_value_info(name, data_type, dims)

        initializer.append(
            onnx.helper.make_tensor(
                name=name,
                data_type=data_type,
                dims=dims,
                vals=np_arr.flatten().tolist(),
                raw=False,
            )
        )

        return [tensor_node]
    else:
        tval_node = onnx.helper.make_tensor_value_info(name, kwargs["in_type"], kwargs["in_shape"])
        return [tval_node]
326428
public INDArray ndArrayFromTensor(TF_Tensor tensor) {
        int rank = TF_NumDims(tensor);

        int[] ndShape;
        if (rank == 0) {
            // scalar
            ndShape = new int[] { 1 };
        } else {
            ndShape = new int[rank];
            for (int i = 0; i < ndShape.length; i++) {
                ndShape[i] = (int) TF_Dim(tensor,i);
            }
        }

        int tfType = TF_TensorType(tensor);
        DataType nd4jType = typeFor(tfType);

        int length = ArrayUtil.prod(ndShape);
        INDArray array;
        if (nd4jType == DataType.UTF8) {
            String[] strings = new String[length];
            BytePointer data = new BytePointer(TF_TensorData(tensor)).capacity(TF_TensorByteSize(tensor));
            BytePointer str = new BytePointer((Pointer)null);
            SizeTPointer size = new SizeTPointer(1);
            TF_Status status = TF_NewStatus();
            for (int i = 0; i < length; i++) {
                long offset = data.position(8 * i).getLong();
                TF_StringDecode(data.position(8 * length + offset), data.capacity() - data.position(), str, size, status);
                if (TF_GetCode(status) != TF_OK) {
                    throw new IllegalStateException("ERROR: Unable to convert tensor " + TF_Message(status).getString());
                }
                strings[i] = str.position(0).capacity(size.get()).getString();
            }
            TF_DeleteStatus(status);
            array = Nd4j.create(strings);
        } else {
            Pointer pointer = TF_TensorData(tensor).capacity(length);
            Indexer indexer = indexerForType(nd4jType,pointer);
            DataBuffer d = Nd4j.createBuffer(indexer.pointer(),nd4jType,length,indexer);
            array = Nd4j.create(d,ndShape);
        }
        Nd4j.getAffinityManager().tagLocation(array, AffinityManager.Location.HOST);
        return array;
    }
1177672
def _result_type_many(*arrays_and_dtypes):
    
    try:
        return np.result_type(*arrays_and_dtypes)
    except ValueError:
        # we have > NPY_MAXARGS terms in our expression
        return reduce(np.result_type, arrays_and_dtypes)
65330
def calculate_weights(self, measure='tfidf'):
        
        from math import log

        # TODO replace with spipy matrices (and calculate with scikit)

        if measure == 'tfidf':
            self.calculate_idf()

        for doc_idx, document in enumerate(self.resulting_documents):
            train_word_count = defaultdict(int)
            self.tf_idfs[doc_idx] = {}
            for word in document:
                train_word_count[word] += 1

            for word in document:
                if measure == "binary":
                    tf = 1
                    idf = 1
                else:
                    tf = train_word_count[word]
                    idf = 1 if measure == "tf" else (self.idf[word] if word in self.idf else None)

                if idf != None:
                    self.tf_idfs[doc_idx][word] = tf * idf
208561
async def _get(self):
        
        input_bag = await self.input.get()

        # Store or check input type
        if self._input_type is None:
            self._input_type = type(input_bag)
        elif type(input_bag) != self._input_type:
            try:
                if self._input_type == tuple:
                    input_bag = self._input_type(input_bag)
                else:
                    input_bag = self._input_type(*input_bag)
            except Exception as exc:
                raise UnrecoverableTypeError(
                    "Input type changed to incompatible type between calls to {!r}.\nGot {!r} which is not of type {!r}.".format(
                        self.wrapped, input_bag, self._input_type
                    )
                ) from exc

        # Store or check input length, which is a soft fallback in case we're just using tuples
        if self._input_length is None:
            self._input_length = len(input_bag)
        elif len(input_bag) != self._input_length:
            raise UnrecoverableTypeError(
                "Input length changed between calls to {!r}.\nExpected {} but got {}: {!r}.".format(
                    self.wrapped, self._input_length, len(input_bag), input_bag
                )
            )

        self.increment("in")  # XXX should that go before type check ?

        return input_bag
1179319
def init_conv_weight(layer):
    '''
    '''
    n_filters = layer.filters
    filter_shape = (layer.kernel_size,) * get_n_dim(layer)
    weight = np.zeros((n_filters, n_filters) + filter_shape)

    center = tuple(map(lambda x: int((x - 1) / 2), filter_shape))
    for i in range(n_filters):
        filter_weight = np.zeros((n_filters,) + filter_shape)
        index = (i,) + center
        filter_weight[index] = 1
        weight[i, ...] = filter_weight
    bias = np.zeros(n_filters)

    layer.set_weights(
        (add_noise(weight, np.array([0, 1])), add_noise(bias, np.array([0, 1])))
    )
1365574
def tensor_vstack(tensor_list, pad=0):
    
    if len(tensor_list) == 1:
        return tensor_list[0][np.newaxis, :]

    ndim = len(tensor_list[0].shape)
    dimensions = [len(tensor_list)]  # first dim is batch size
    for dim in range(ndim):
        dimensions.append(max([tensor.shape[dim] for tensor in tensor_list]))

    dtype = tensor_list[0].dtype
    if pad == 0:
        all_tensor = np.zeros(tuple(dimensions), dtype=dtype)
    elif pad == 1:
        all_tensor = np.ones(tuple(dimensions), dtype=dtype)
    else:
        all_tensor = np.full(tuple(dimensions), pad, dtype=dtype)
    if ndim == 1:
        for ind, tensor in enumerate(tensor_list):
            all_tensor[ind, :tensor.shape[0]] = tensor
    elif ndim == 2:
        for ind, tensor in enumerate(tensor_list):
            all_tensor[ind, :tensor.shape[0], :tensor.shape[1]] = tensor
    elif ndim == 3:
        for ind, tensor in enumerate(tensor_list):
            all_tensor[ind, :tensor.shape[0], :tensor.shape[1], :tensor.shape[2]] = tensor
    else:
        raise Exception('Sorry, unimplemented.')
    return all_tensor
25262
def _compute_sparse_average_correct(
    input_, labels, per_example_weights, topk=1):
  """"""
  labels = tf.to_int64(labels)
  labels.get_shape().assert_is_compatible_with(
      [input_.get_shape()[0], None])
  if topk == 1:
    predictions = tf.reshape(tf.argmax(input_, 1), [-1, 1])
    in_topk = tf.reduce_any(tf.equal(labels, predictions),
                            reduction_indices=[1])
  else:
    # Use broadcasting to check if ANY of the predictions are in the top k.
    # TODO(eiderman): For a multi-label top k, what does accuracy mean?
    predictions = tf.reshape(tf.nn.top_k(input_, topk)[1], [-1, 1, topk])
    labels = tf.expand_dims(labels, [-1])

    in_topk = tf.reduce_any(tf.equal(tf.cast(labels, predictions.dtype),
                                     predictions),
                            reduction_indices=[1, 2])
  correct_predictions = tf.to_float(in_topk)

  # If individual examples are weighted, then we want to normalize by that.
  if per_example_weights is not None:
    per_example_weights = _convert_and_assert_per_example_weights_compatible(
        input_,
        per_example_weights,
        dtype=None)
    float_weights = tf.to_float(per_example_weights)
    # TODO(eiderman): This should use an op that doesn't support broadcasting.
    correct_predictions *= float_weights
    num_examples = tf.reduce_sum(float_weights)
  else:
    # shape only holds ints, but we want to always return the same type
    # for num_examples to make everything compatible.
    num_examples = tf.to_float(tf.gather(tf.shape(input_), 0))
  return tf.reduce_sum(correct_predictions), num_examples
371106
def listify(p:OptListOrItem=None, q:OptListOrItem=None):
    ""
    if p is None: p=[]
    elif isinstance(p, str):          p = [p]
    elif not isinstance(p, Iterable): p = [p]
    #Rank 0 tensors in PyTorch are Iterable but don't have a length.
    else:
        try: a = len(p)
        except: p = [p]
    n = q if type(q)==int else len(p) if q is None else len(q)
    if len(p)==1: p = p * n
    assert len(p)==n, f'List len mismatch ({len(p)} vs {n})'
    return list(p)
1153332
def _parse_xgboost_simple_model(scope, model, inputs):
    '''
    
    '''
    this_operator = scope.declare_local_operator(_get_xgboost_operator_name(type(model)), model)
    this_operator.inputs = inputs

    if type(model) in xgboost_classifier_list:
        # For classifiers, we may have two outputs, one for label and the other one for probabilities of all classes.
        # Notice that their types here are not necessarily correct and they will be fixed in shape inference phase
        label_variable = scope.declare_local_variable('label', FloatTensorType())
        probability_map_variable = scope.declare_local_variable('probabilities', FloatTensorType())
        this_operator.outputs.append(label_variable)
        this_operator.outputs.append(probability_map_variable)
    else:
        # We assume that all scikit-learn operator can only produce a single float tensor.
        variable = scope.declare_local_variable('variable', FloatTensorType())
        this_operator.outputs.append(variable)
    return this_operator.outputs
992216
def calculate_flatten_output_shapes(operator):
    '''
    
    '''
    check_input_and_output_numbers(operator, input_count_range=1, output_count_range=1)
    check_input_and_output_types(operator, good_input_types=[FloatTensorType])

    input = operator.inputs[0]
    output = operator.outputs[0]

    if len(input.type.shape) not in [2, 4]:
        raise RuntimeError('Input must be 2-D or 4-D float tensor')

    input_shape = input.type.shape
    output_shape = [input_shape[0], 1]

    # Calculate the multiplication of C, H, and W.
    for i in input_shape[1:]:
        if i != 'None':
            output_shape[1] *= i
        else:
            # If any of C, H, W-dimensions is unknown, the flatten C-dimension is unknown
            output_shape[1] = 'None'
            break

    output.type.shape = output_shape