method to validate and prepare data-dependent parameters

        Parameters
        ---------
        X : array-like
            Input dataset

        verbose : bool
            whether to show warnings

        Returns
        -------
        None
4029
def compile(self, X, verbose=False):
        
        super(FactorTerm, self).compile(X)

        self.n_splines = len(np.unique(X[:, self.feature]))
        self.edge_knots_ = gen_edge_knots(X[:, self.feature],
                                          self.dtype,
                                          verbose=verbose)
        return self
1110306
def compile(self, X, verbose=False):
        
        for term in self._terms:
            term.compile(X, verbose=False)

        if self.by is not None and self.by >= X.shape[1]:
            raise ValueError('by variable requires feature {}, '\
                             'but X has only {} dimensions'\
                             .format(self.by, X.shape[1]))
        return self
408865
def validate(self, value):
        
        try:
            # trap blank fields here
            if not self.blank or value:
                v = float(value)
                if (self.allowzero and v != 0 and v < self.fmin) or \
                        (not self.allowzero and v < self.fmin) or v > self.fmax:
                    return None
            return value
        except ValueError:
            return None
629114
def reqs(ctx, dataset, kwargs):
    ""

    kwargs = parse_kwargs(kwargs)
    (print)(data(dataset, **ctx.obj).reqs(**kwargs))
1065178
def __validate(data, classes, labels):
        ""

        if not isinstance(data, dict):
            raise TypeError(
                'data must be a dict! keys: sample ID or any unique identifier')
        if not isinstance(labels, dict):
            raise TypeError(
                'labels must be a dict! keys: sample ID or any unique identifier')
        if classes is not None:
            if not isinstance(classes, dict):
                raise TypeError(
                    'labels must be a dict! keys: sample ID or any unique identifier')

        if not len(data) == len(labels) == len(classes):
            raise ValueError('Lengths of data, labels and classes do not match!')
        if not set(list(data)) == set(list(labels)) == set(list(classes)):
            raise ValueError(
                'data, classes and labels dictionaries must have the same keys!')

        num_features_in_elements = np.unique([sample.size for sample in data.values()])
        if len(num_features_in_elements) > 1:
            raise ValueError(
                'different samples have different number of features - invalid!')

        return True
672785
def _validate_features(self, data):
        
        if self.feature_names is None:
            self.feature_names = data.feature_names
            self.feature_types = data.feature_types
        else:
            # Booster can't accept data with different feature names
            if self.feature_names != data.feature_names:
                dat_missing = set(self.feature_names) - set(data.feature_names)
                my_missing = set(data.feature_names) - set(self.feature_names)

                msg = 'feature_names mismatch: {0} {1}'

                if dat_missing:
                    msg += ('\nexpected ' + ', '.join(str(s) for s in dat_missing) +
                            ' in input data')

                if my_missing:
                    msg += ('\ntraining data did not have the following fields: ' +
                            ', '.join(str(s) for s in my_missing))

                raise ValueError(msg.format(self.feature_names,
                                            data.feature_names))
193844
def _preprocess_inputs(x, weights):
    
    if weights is None:
        w_arr = np.ones(len(x))
    else:
        w_arr = np.array(weights)
    x_arr = np.array(x)
    if x_arr.ndim == 2:
        if w_arr.ndim == 1:
            w_arr = w_arr[:, np.newaxis]
    return x_arr, w_arr
323760
def _validate_dataset_names(dataset):
    """"""
    def check_name(name):
        if isinstance(name, str):
            if not name:
                raise ValueError('Invalid name for DataArray or Dataset key: '
                                 'string must be length 1 or greater for '
                                 'serialization to netCDF files')
        elif name is not None:
            raise TypeError('DataArray.name or Dataset key must be either a '
                            'string or None for serialization to netCDF files')

    for k in dataset.variables:
        check_name(k)
115049
def _validate(self):
        
        variable_names, _unused = getExprNames(self._expr, {})
        expr_indices = []
        for name in variable_names:
            if name == 'inf':
                continue
            match = _VARIABLE_NAME_RE.match(name)
            if not match:
                raise ValueError("%r is not a valid variable name" % name)
            expr_indices.append(int(match.group(2)))

        expr_indices.sort()
        expected_indices = list(range(len(self.inputs)))
        if expr_indices != expected_indices:
            raise ValueError(
                "Expected %s for variable indices, but got %s" % (
                    expected_indices, expr_indices,
                )
            )
        super(NumericalExpression, self)._validate()
746584
def prepare_data(problem, hparams, params, config):
  """"""
  input_fn = problem.make_estimator_input_fn(
      tf.estimator.ModeKeys.EVAL, hparams, force_repeat=True)
  dataset = input_fn(params, config)
  features, _ = dataset.make_one_shot_iterator().get_next()
  inputs, labels = features["targets"], features["inputs"]
  inputs = tf.to_float(inputs)
  input_shape = inputs.shape.as_list()
  inputs = tf.reshape(inputs, [hparams.batch_size] + input_shape[1:])
  labels = tf.reshape(labels, [hparams.batch_size])
  return inputs, labels, features
452457
def __prepare_dataset_parameter(self, dataset):
        

        # Translate the dataset argument into the proper type
        if not isinstance(dataset, _SFrame):
            def raise_dataset_type_exception():
                raise TypeError("The dataset parameter must be either an SFrame, "
                                "or a dictionary of (str : list) or (str : value).")

            if type(dataset) is dict:
                if not all(type(k) is str for k in _six.iterkeys(dataset)):
                    raise_dataset_type_exception()

                if all(type(v) in (list, tuple, _array.array) for v in _six.itervalues(dataset)):
                    dataset = _SFrame(dataset)
                else:
                    dataset = _SFrame({k : [v] for k, v in _six.iteritems(dataset)})
            else:
                raise_dataset_type_exception()

        return dataset
1102636
def validate(self, data=None):
        

        errors = {}

        data = self._getData(data)

        # validate each field, one by one
        for name, field in self.fields.items():
            try:
                field.clean(data.get(name))
            except ValidationError, e:
                errors[name] = e.messages
            except AttributeError, e:
                raise ValidationError('data should be of type dict but is %s' % (type(data),))

        # check for extra fields
        extras = set(data.keys()) - set(self.fields.keys())
        if extras:
            errors[', '.join(extras)] = ['field(s) not allowed']

        # if errors, raise ValidationError
        if errors:
            raise ValidationError(errors)
557599
def data_x_range(self):
        
        try:
            lower = min([min(self._filter_none(s))
                         for type, s in self.annotated_data()
                         if type == 'x'])
            upper = max([max(self._filter_none(s))
                         for type, s in self.annotated_data()
                         if type == 'x'])
            return (lower, upper)
        except ValueError:
            return None
251489
def generate(self, x, **kwargs):
    
    # Parse and save attack-specific parameters
    assert self.parse_params(**kwargs)

    asserts = []

    # If a data range was specified, check that the input was in that range
    if self.clip_min is not None:
      asserts.append(utils_tf.assert_greater_equal(x,
                                                   tf.cast(self.clip_min,
                                                           x.dtype)))

    if self.clip_max is not None:
      asserts.append(utils_tf.assert_less_equal(x,
                                                tf.cast(self.clip_max,
                                                        x.dtype)))

    # Initialize loop variables
    if self.rand_init:
      eta = tf.random_uniform(tf.shape(x),
                              tf.cast(-self.rand_minmax, x.dtype),
                              tf.cast(self.rand_minmax, x.dtype),
                              dtype=x.dtype)
    else:
      eta = tf.zeros(tf.shape(x))

    # Clip eta
    eta = clip_eta(eta, self.ord, self.eps)
    adv_x = x + eta
    if self.clip_min is not None or self.clip_max is not None:
      adv_x = utils_tf.clip_by_value(adv_x, self.clip_min, self.clip_max)

    if self.y_target is not None:
      y = self.y_target
      targeted = True
    elif self.y is not None:
      y = self.y
      targeted = False
    else:
      model_preds = self.model.get_probs(x)
      preds_max = tf.reduce_max(model_preds, 1, keepdims=True)
      y = tf.to_float(tf.equal(model_preds, preds_max))
      y = tf.stop_gradient(y)
      targeted = False
      del model_preds

    y_kwarg = 'y_target' if targeted else 'y'
    fgm_params = {
        'eps': self.eps_iter,
        y_kwarg: y,
        'ord': self.ord,
        'clip_min': self.clip_min,
        'clip_max': self.clip_max
    }
    if self.ord == 1:
      raise NotImplementedError("It's not clear that FGM is a good inner loop"
                                " step for PGD when ord=1, because ord=1 FGM "
                                " changes only one pixel at a time. We need "
                                " to rigorously test a strong ord=1 PGD "
                                "before enabling this feature.")

    # Use getattr() to avoid errors in eager execution attacks
    FGM = self.FGM_CLASS(
        self.model,
        sess=getattr(self, 'sess', None),
        dtypestr=self.dtypestr)

    def cond(i, _):
      """Iterate until requested number of iterations is completed"""
      return tf.less(i, self.nb_iter)

    def body(i, adv_x):
      """Do a projected gradient step"""
      adv_x = FGM.generate(adv_x, **fgm_params)

      # Clipping perturbation eta to self.ord norm ball
      eta = adv_x - x
      eta = clip_eta(eta, self.ord, self.eps)
      adv_x = x + eta

      # Redo the clipping.
      # FGM already did it, but subtracting and re-adding eta can add some
      # small numerical error.
      if self.clip_min is not None or self.clip_max is not None:
        adv_x = utils_tf.clip_by_value(adv_x, self.clip_min, self.clip_max)

      return i + 1, adv_x

    _, adv_x = tf.while_loop(cond, body, (tf.zeros([]), adv_x), back_prop=True,
                             maximum_iterations=self.nb_iter)

    # Asserts run only on CPU.
    # When multi-GPU eval code tries to force all PGD ops onto GPU, this
    # can cause an error.
    common_dtype = tf.float32
    asserts.append(utils_tf.assert_less_equal(tf.cast(self.eps_iter,
                                                      dtype=common_dtype),
                                              tf.cast(self.eps, dtype=common_dtype)))
    if self.ord == np.inf and self.clip_min is not None:
      # The 1e-6 is needed to compensate for numerical error.
      # Without the 1e-6 this fails when e.g. eps=.2, clip_min=.5,
      # clip_max=.7
      asserts.append(utils_tf.assert_less_equal(tf.cast(self.eps, x.dtype),
                                                1e-6 + tf.cast(self.clip_max,
                                                               x.dtype)
                                                - tf.cast(self.clip_min,
                                                          x.dtype)))

    if self.sanity_checks:
      with tf.control_dependencies(asserts):
        adv_x = tf.identity(adv_x)

    return adv_x
649019
def Validate(self, expression):
    
    parsed = self._Load(expression)

    if not parsed:
      raise DefinitionError("Empty StatFilter expression.")

    bad_keys = set(parsed) - self._KEYS
    if bad_keys:
      raise DefinitionError("Invalid parameters: %s" % ",".join(bad_keys))

    if self.cfg.mask and not self.cfg.mode:
      raise DefinitionError("mode can only be set when mask is also defined.")

    if self.cfg.mask:
      if len(self.cfg.mask) > 1:
        raise DefinitionError("Too many mask values defined.")
      if not self._PERM_RE.match(self.cfg.mask[0]):
        raise DefinitionError("mask=%s is not octal, e.g. 0600" % self.cfg.mask)

    if self.cfg.mode:
      if len(self.cfg.mode) > 1:
        raise DefinitionError("Too many mode values defined.")
      if not self._PERM_RE.match(self.cfg.mode[0]):
        raise DefinitionError("mode=%s is not octal, e.g. 0600" % self.cfg.mode)

    if self.cfg.gid:
      for gid in self.cfg.gid:
        matched = self._UID_GID_RE.match(gid)
        if not matched:
          raise DefinitionError("gid: %s is not an integer preceded by "
                                "!, >, < or =." % gid)

    if self.cfg.uid:
      for uid in self.cfg.uid:
        matched = self._UID_GID_RE.match(uid)
        if not matched:
          raise DefinitionError("uid: %s is not an integer preceded by "
                                "!, >, < or =." % uid)

    if self.cfg.file_re:
      if len(self.cfg.file_re) > 1:
        raise DefinitionError("Too many regexes defined: %s" % self.cfg.file_re)
      try:
        self.file_re = re.compile(self.cfg.file_re[0])
      except (re.error, TypeError) as e:
        raise DefinitionError("Invalid file regex: %s" % e)

    if self.cfg.path_re:
      if len(self.cfg.path_re) > 1:
        raise DefinitionError("Too many regexes defined: %s" % self.cfg.path_re)
      try:
        self.path_re = re.compile(self.cfg.path_re[0])
      except (re.error, TypeError) as e:
        raise DefinitionError("Invalid path regex: %s" % e)

    if self.cfg.file_type:
      if len(self.cfg.file_type) > 1:
        raise DefinitionError(
            "Too many file types defined: %s" % self.cfg.file_type)
      file_type = self.cfg.file_type[0].upper()
      if file_type not in self._TYPES:
        raise DefinitionError("Unsupported file type %s" % file_type)

    self._Initialize()
    if not self.matchers:
      raise DefinitionError("StatFilter has no actions: %s" % expression)
    return True
773406
def generate(self, x, **kwargs):
    

    assert self.parse_params(**kwargs)
    labels, _nb_classes = self.get_or_guess_labels(x, kwargs)
    adv_x = self.attack(x, labels)

    return adv_x
194842
def predict_withGradients(self, X):
        
        if X.ndim==1: X = X[None,:]
        ps = self.model.param_array.copy()
        means = []
        stds = []
        dmdxs = []
        dsdxs = []
        for s in self.hmc_samples:
            if self.model._fixes_ is None:
                self.model[:] = s
            else:
                self.model[self.model._fixes_] = s
            self.model._trigger_params_changed()
            m, v = self.model.predict(X)
            std = np.sqrt(np.clip(v, 1e-10, np.inf))
            dmdx, dvdx = self.model.predictive_gradients(X)
            dmdx = dmdx[:,:,0]
            dsdx = dvdx / (2*std)
            means.append(m)
            stds.append(std)
            dmdxs.append(dmdx)
            dsdxs.append(dsdx)
        self.model.param_array[:] = ps
        self.model._trigger_params_changed()
        return means, stds, dmdxs, dsdxs