Maps RDDL range types to TensorFlow dtypes.
357902
def range_type_to_dtype(range_type: str) -> Optional[tf.DType]:
    ''''''
    range2dtype = {
        'real': tf.float32,
        'int': tf.int32,
        'bool': tf.bool
    }
    return range2dtype[range_type]
705185
def reshape_range(tensor, i, j, shape):
  """"""
  t_shape = common_layers.shape_list(tensor)
  target_shape = t_shape[:i] + shape + t_shape[j:]
  return tf.reshape(tensor, target_shape)
1141717
function generateTypeDefs(models) {

    types = Object.assign({}, types, getCustomTypeDefs());

    _.forEach(models, (model) => {
        mapType(model);
        mapInputType(model);
    });

    return types;
}
1061649
def mapPartitionsWithIndex(self, f, preservesPartitioning=False):
        
        return self.transform(lambda rdd: rdd.mapPartitionsWithIndex(f, preservesPartitioning))
1339857
def _output(cls,
            tensors: Sequence[tf.Tensor],
            dtypes: Sequence[tf.DType]) -> Sequence[tf.Tensor]:
        ''''''
        outputs = []
        for tensor, dtype in zip(tensors, dtypes):
            tensor = tensor[0]
            if tensor.dtype != dtype:
                tensor = tf.cast(tensor, dtype)
            outputs.append(tensor)
        return tuple(outputs)
1132616
def get_allowed_shape_ranges(spec):
    

    shaper = NeuralNetworkShaper(spec, False)
    inputs = _get_input_names(spec)
    output = {}

    for input in inputs:
        output[input] = shaper.shape(input)

    return output
1343971
def to_pandas_series_rdd(self):
        
        pd_index = self.index().to_pandas_index()
        return self.map(lambda x: (x[0], pd.Series(x[1], pd_index)))
1211065
def suits_with_dtype(mn, mx, dtype):
    
    type_info = np.iinfo(dtype)
    if mx <= type_info.max and mn >= type_info.min:
        return True
    else:
        return False
802501
def calculate_one_hot_encoder_output_shapes(operator):
    '''
    
    '''
    check_input_and_output_numbers(operator, input_count_range=1, output_count_range=1)

    if operator.inputs[0].type.shape[1] != 1 or len(operator.inputs[0].type.shape) > 2:
        raise RuntimeError('Input must be [N, 1]-tensor')

    int_categories = operator.raw_operator.oneHotEncoder.int64Categories.vector
    str_categories = operator.raw_operator.oneHotEncoder.stringCategories.vector

    N = operator.inputs[0].type.shape[0]

    if len(int_categories) > 0:
        operator.outputs[0].type = FloatTensorType([N, len(int_categories)],
                                                   doc_string=operator.outputs[0].type.doc_string)
    elif len(str_categories) > 0 and type(operator.inputs[0].type) == StringTensorType:
        operator.outputs[0].type = FloatTensorType([N, len(str_categories)],
                                                   doc_string=operator.outputs[0].type.doc_string)
    else:
        raise ValueError('Categorical indexes are missing')
198258
def calculate_batch_normalization_output_shapes(operator):
    '''
    
    '''
    check_input_and_output_numbers(operator, input_count_range=1, output_count_range=1)
    check_input_and_output_types(operator, good_input_types=[FloatTensorType])

    input_shape = operator.inputs[0].type.shape
    if len(input_shape) not in [2, 4]:
        raise RuntimeError('Input must be a 2-D or a 4-D tensor')

    operator.outputs[0].type.shape = copy.deepcopy(operator.inputs[0].type.shape)
80231
def _convert_bin_to_numeric_type(bins, dtype):
    
    bins_dtype = infer_dtype(bins, skipna=False)
    if is_timedelta64_dtype(dtype):
        if bins_dtype in ['timedelta', 'timedelta64']:
            bins = to_timedelta(bins).view(np.int64)
        else:
            raise ValueError("bins must be of timedelta64 dtype")
    elif is_datetime64_dtype(dtype) or is_datetime64tz_dtype(dtype):
        if bins_dtype in ['datetime', 'datetime64']:
            bins = to_datetime(bins).view(np.int64)
        else:
            raise ValueError("bins must be of datetime64 dtype")

    return bins
405197
def dtyping(*args):
    
    args = list(args)

    for idx, arg in enumerate(args):

        if isinstance(arg, Poly):
            args[idx] = Poly

        elif isinstance(arg, numpy.generic):
            args[idx] = numpy.asarray(arg).dtype

        elif isinstance(arg, (float, int)):
            args[idx] = type(arg)

    for type_ in DATATYPES:
        if type_ in args:
            return type_

    raise ValueError(
        "dtypes not recognised " + str([str(_) for _ in args]))
796283
def _transform(self, dataset):
    
    spark = SparkSession.builder.getOrCreate()

    # set a deterministic order for input/output columns (lexicographic by key)
    input_cols = [col for col, tensor in sorted(self.getInputMapping().items())]      # input col => input tensor
    output_cols = [col for tensor, col in sorted(self.getOutputMapping().items())]    # output tensor => output col

    # run single-node inferencing on each executor
    logging.info("input_cols: {}".format(input_cols))
    logging.info("output_cols: {}".format(output_cols))

    # merge args + params
    logging.info("===== 1. inference args: {0}".format(self.args))
    logging.info("===== 2. inference params: {0}".format(self._paramMap))
    local_args = self.merge_args_params()
    logging.info("===== 3. inference args + params: {0}".format(local_args))

    tf_args = self.args.argv if self.args.argv else local_args
    rdd_out = dataset.select(input_cols).rdd.mapPartitions(lambda it: _run_model(it, local_args, tf_args))

    # convert to a DataFrame-friendly format
    rows_out = rdd_out.map(lambda x: Row(*x))
    return spark.createDataFrame(rows_out, output_cols)
921409
def imt_dt(self, dtype=F64):
        
        return numpy.dtype([(imt, dtype) for imt in self.imtls])
427448
def calculate_traditional_regressor_output_shapes(operator):
    '''
    
    '''
    check_input_and_output_types(operator, good_input_types=[FloatTensorType, Int64TensorType, FloatType, Int64Type])

    if any(len(variable.type.shape) != 2 for variable in operator.inputs):
        raise RuntimeError('Input(s) must be 2-D tensor(s)')

    model_type = operator.raw_operator.WhichOneof('Type')
    if model_type == 'glmRegressor':
        glm = operator.raw_operator.glmRegressor
        C = len(glm.weights)
    elif model_type == 'treeEnsembleRegressor':
        tree = operator.raw_operator.treeEnsembleRegressor.treeEnsemble
        C = len(tree.basePredictionValue)
    elif model_type == 'supportVectorRegressor':
        C = 1
    else:
        raise ValueError('Model should be one of linear model, tree-based model, and support vector machine')

    N = operator.inputs[0].type.shape[0]
    operator.outputs[0].type = FloatTensorType([N, C], doc_string=operator.outputs[0].type.doc_string)
689062
def _get_tbl_numpy_dtype(self, colnum, include_endianness=True):
        
        table_type = self._info['hdutype']
        table_type_string = _hdu_type_map[table_type]
        try:
            ftype = self._info['colinfo'][colnum]['eqtype']
            if table_type == ASCII_TBL:
                npy_type = _table_fits2npy_ascii[abs(ftype)]
            else:
                npy_type = _table_fits2npy[abs(ftype)]
        except KeyError:
            raise KeyError("unsupported %s fits data "
                           "type: %d" % (table_type_string, ftype))

        istbit = False
        if (ftype == 1):
            istbit = True

        isvar = False
        if ftype < 0:
            isvar = True
        if include_endianness:
            # if binary we will read the big endian bytes directly,
            # if ascii we read into native byte order
            if table_type == ASCII_TBL:
                addstr = ''
            else:
                addstr = '>'
            if npy_type not in ['u1', 'i1', 'S', 'U']:
                npy_type = addstr+npy_type

        if npy_type == 'S':
            width = self._info['colinfo'][colnum]['width']
            npy_type = 'S%d' % width
        elif npy_type == 'U':
            width = self._info['colinfo'][colnum]['width']
            npy_type = 'U%d' % width

        return npy_type, isvar, istbit
861167
def batch_fun(dataset, training, shapes, target_names, num_devices,
              batch_size_per_device=32, batch_size=None, eval_batch_size=32,
              bucket_length=32, buckets=None,
              batch_shuffle_size=128, max_eval_length=None):
  """"""
  del target_names
  # Batch size is batch_size_per_device * num_devices unless given directly.
  batch_size = batch_size or batch_size_per_device * num_devices
  # If bucketing is not specified, check if target shapes are variable.
  cur_batch_size = batch_size if training else eval_batch_size
  # Make cur_batch_size divisible by num_devices.
  cur_batch_size = max(cur_batch_size // num_devices, 1) * num_devices
  # Create heuristic buckets is none are specified.
  if buckets is None:
    variable_target_shapes = False
    target_shape = shapes[1]
    for dim in target_shape:
      if dim is None:
        variable_target_shapes = True
    tf.logging.info("Heuristically setting bucketing to %s based on shapes "
                    "of target tensors." % variable_target_shapes)
    if variable_target_shapes:
      bucket_boundaries = [bucket_length // 4, bucket_length // 2,
                           bucket_length, bucket_length * 2,
                           bucket_length * 4, bucket_length * 8,
                           bucket_length * 16]
      # We will pad to boundaries which pads to bucket_boundary - 1: add 1 here.
      bucket_boundaries = [b + 1 for b in bucket_boundaries]
      if not training:
        max_eval_length = max_eval_length or bucket_length * 32
        bucket_boundaries[-1] = max_eval_length
      bucket_batch_sizes = [cur_batch_size * 4, cur_batch_size * 2,
                            cur_batch_size, cur_batch_size // 2,
                            cur_batch_size // 4, cur_batch_size // 8,
                            cur_batch_size // 16, 1]
      if not training:
        bucket_batch_sizes[-2] = cur_batch_size // max_eval_length
      # Make batch sizes divisible by num_devices.
      bucket_batch_sizes = [max(b // num_devices, 1) * num_devices
                            for b in bucket_batch_sizes]
      buckets = (bucket_boundaries, bucket_batch_sizes)

  if buckets:
    tf.logging.info("Bucketing with buckets %s." % str(buckets))
    def example_length(_, target):
      return tf.shape(target)[0]
    boundaries, batch_sizes = buckets
    dataset = dataset.apply(tf.data.experimental.bucket_by_sequence_length(
        example_length, boundaries, batch_sizes,
        pad_to_bucket_boundary=True))
  else:
    dataset = dataset.padded_batch(cur_batch_size, shapes)
  if training:
    return dataset.shuffle(batch_shuffle_size)
  return dataset