Combine multiple batch validations into validation outputs.
176380
def combine_validations(items, vkey="validate"):
    
    csvs = set([])
    pngs = set([])
    for v in [x.get(vkey) for x in items]:
        if v and v.get("grading_summary"):
            csvs.add(v.get("grading_summary"))
        if v and v.get("grading_plots"):
            pngs |= set(v.get("grading_plots"))
    if len(csvs) == 1:
        grading_summary = csvs.pop()
    else:
        grading_summary = os.path.join(utils.safe_makedir(os.path.join(dd.get_work_dir(items[0]), vkey)),
                                       "grading-summary-combined.csv")
        with open(grading_summary, "w") as out_handle:
            for i, csv in enumerate(sorted(list(csvs))):
                with open(csv) as in_handle:
                    h = in_handle.readline()
                    if i == 0:
                        out_handle.write(h)
                    for l in in_handle:
                        out_handle.write(l)
    return {"grading_plots": sorted(list(pngs)), "grading_summary": grading_summary}
1268131
def get_validation_image_text_data_iter(data_loader: RawParallelDatasetLoader,
                                        validation_source_root: str,
                                        validation_source: str,
                                        validation_target: str,
                                        buckets: List[Tuple[int, int]],
                                        bucket_batch_sizes: List[BucketBatchSize],
                                        source_image_size: tuple,
                                        vocab_target: vocab.Vocab,
                                        max_seq_len_target: int,
                                        batch_size: int,
                                        use_feature_loader: bool = False,
                                        preload_features: bool = False) -> 'ParallelSampleIter':
    
    logger.info("=================================")
    logger.info("Creating validation data iterator")
    logger.info("=================================")

    validation_source_images = [FileListReader(validation_source, validation_source_root)]
    validation_target_sentences = SequenceReader(validation_target, vocab_target, add_bos=True, limit=None)

    validation_data_statistics = get_data_statistics(source_readers=None,
                                                     target_reader=validation_target_sentences,
                                                     buckets=buckets,
                                                     length_ratio_mean=1.0,
                                                     length_ratio_std=1.0,
                                                     source_vocabs=None,
                                                     target_vocab=vocab_target)
    validation_data_statistics.log(bucket_batch_sizes)

    validation_data = data_loader.load(validation_source_images[0],
                                       validation_target_sentences,
                                       validation_data_statistics.num_sents_per_bucket).fill_up(bucket_batch_sizes)
    return ImageTextSampleIter(data=validation_data,
                               buckets=buckets,
                               batch_size=batch_size,
                               bucket_batch_sizes=bucket_batch_sizes,
                               image_size=source_image_size,
                               use_feature_loader=use_feature_loader,
                               preload_features=preload_features)
1340805
def _get_validate(data):
    
    if data.get("vrn_file") and tz.get_in(["config", "algorithm", "validate"], data):
        return utils.deepish_copy(data)
    elif "group_orig" in data:
        for sub in multi.get_orig_items(data):
            if "validate" in sub["config"]["algorithm"]:
                sub_val = utils.deepish_copy(sub)
                sub_val["vrn_file"] = data["vrn_file"]
                return sub_val
    return None
45887
def validatefeatures(self,features):
        """"""
        validatedfeatures = []
        for feature in features:
            if isinstance(feature, int) or isinstance(feature, float):
                validatedfeatures.append( str(feature) )
            elif self.delimiter in feature and not self.sklearn:
                raise ValueError("Feature contains delimiter: " + feature)
            elif self.sklearn and isinstance(feature, str): #then is sparse added together
                validatedfeatures.append(feature)
            else:
                validatedfeatures.append(feature)
        return validatedfeatures
1265839
def validate_items(self, input_list):
        """"""
        output_list = []
        for item in input_list:
            valid = self.list_item_type.validate(item, self.item_name)
            output_list.append(valid)

            # this might lead to confusing error messages.  tbh, we need to
            # figure out a better way to do validation and error handling here,
            # but i'm brute forcing this a bit so that we have something
            # workable
        return output_list
924684
def workflow_validate_batch(object_id, input_params={}, always_retry=True, **kwargs):
    
    return DXHTTPRequest('/%s/validateBatch' % object_id, input_params, always_retry=always_retry, **kwargs)
42751
function prepValidations(validations) {
  validations = _.cloneDeep(validations);
  for (var key in validations) {
    var validation = validations[key];
    if (!_.isArray(validation)) validations[key] = validation = [validation];
    for (var i = 0, l = validation.length; i < l; i++) {
      validation[i] = assembleValidation(validation[i]);
    }
  }
  return validations;
}
590017
function addVerifiedConditional(validations, conditional) {
  _.each(conditional[0], function(val, key) {
    validations[key] = validations[key] || [];
    validations[key] = validations[key].concat(val);
  })
}
204543
def _run_validators(self, value):
        
        errors = []
        for v in self.validators:
            try:
                v(value)
            except ValidationError, e:
                errors.extend(e.messages)
        if errors:
            raise ValidationError(errors)
1179075
def _validate_entries(self, processes, callback=None):
        
        errors = list()

        if os.name == 'posix':
            worker_init = posix_multiprocessing_worker_initializer
        else:
            worker_init = None

        args = ((self.path,
                 self.normalized_filesystem_names.get(rel_path, rel_path),
                 hashes,
                 self.algorithms) for rel_path, hashes in self.entries.items())

        try:
            if processes == 1:
                count = 0
                hash_results = []
                totalHashes = len(self.entries.items())
                for i in args:
                    hash_results.append(_calc_hashes(i))
                    count += 1
                    if callback:
                        if not callback(count, totalHashes):
                            raise BaggingInterruptedError("Bag validation interrupted!")

            else:
                pool = None
                try:
                    pool = multiprocessing.Pool(processes if processes else None, initializer=worker_init)
                    hash_results = pool.map(_calc_hashes, args)
                finally:
                    if pool:
                        pool.terminate()
        except BaggingInterruptedError:
            raise
        # Any unhandled exceptions are probably fatal
        except:
            LOGGER.error(_("Unable to calculate file hashes for %s"), self)
            raise

        for rel_path, f_hashes, hashes in hash_results:
            for alg, computed_hash in f_hashes.items():
                stored_hash = hashes[alg]
                if stored_hash.lower() != computed_hash:
                    e = ChecksumMismatch(rel_path, alg, stored_hash.lower(), computed_hash)
                    LOGGER.warning(force_unicode(e))
                    errors.append(e)

        if errors:
            raise BagValidationError(_("Bag validation failed"), errors)
17445
def add_validate(subparsers):
    """"""
    validate_parser = subparsers.add_parser(
        'validate', help=add_validate.__doc__, formatter_class=argparse.ArgumentDefaultsHelpFormatter)
    validate_parser.set_defaults(func=validate_parser.print_help)

    validate_subparsers = validate_parser.add_subparsers(title='Testers')

    validate_all_parser = validate_subparsers.add_parser(
        'all', help=validate.validate_all.__doc__, formatter_class=argparse.ArgumentDefaultsHelpFormatter)
    validate_all_parser.set_defaults(func=validate.validate_all)

    validate_gate_parser = validate_subparsers.add_parser(
        'gate', help=validate.validate_gate.__doc__, formatter_class=argparse.ArgumentDefaultsHelpFormatter)
    validate_gate_parser.set_defaults(func=validate.validate_gate)
161371
def get_validation_data_iter(data_loader: RawParallelDatasetLoader,
                             validation_sources: List[str],
                             validation_target: str,
                             buckets: List[Tuple[int, int]],
                             bucket_batch_sizes: List[BucketBatchSize],
                             source_vocabs: List[vocab.Vocab],
                             target_vocab: vocab.Vocab,
                             max_seq_len_source: int,
                             max_seq_len_target: int,
                             batch_size: int) -> 'ParallelSampleIter':
    
    logger.info("=================================")
    logger.info("Creating validation data iterator")
    logger.info("=================================")
    validation_length_statistics = analyze_sequence_lengths(validation_sources, validation_target,
                                                            source_vocabs, target_vocab,
                                                            max_seq_len_source, max_seq_len_target)

    check_condition(validation_length_statistics.num_sents > 0,
                    "No validation sequences found with length smaller or equal than the maximum sequence length."
                    "Consider increasing %s" % C.TRAINING_ARG_MAX_SEQ_LEN)

    validation_sources_sentences, validation_target_sentences = create_sequence_readers(validation_sources,
                                                                                        validation_target,
                                                                                        source_vocabs, target_vocab)

    validation_data_statistics = get_data_statistics(validation_sources_sentences,
                                                     validation_target_sentences,
                                                     buckets,
                                                     validation_length_statistics.length_ratio_mean,
                                                     validation_length_statistics.length_ratio_std,
                                                     source_vocabs, target_vocab)

    validation_data_statistics.log(bucket_batch_sizes)

    validation_data = data_loader.load(validation_sources_sentences, validation_target_sentences,
                                       validation_data_statistics.num_sents_per_bucket).fill_up(bucket_batch_sizes)

    return ParallelSampleIter(data=validation_data,
                              buckets=buckets,
                              batch_size=batch_size,
                              bucket_batch_sizes=bucket_batch_sizes,
                              num_factors=len(validation_sources))
593431
def validate(self, data):
        """"""
        validated = self._validated(data)
        errors = []
        for validator in self.additional_validators:
            if not validator(validated):
                errors.append(
                    "%s invalidated by '%s'" % (
                        validated, _get_repr(validator)))
        if errors:
            raise NotValid(*errors)

        if self.default is UNSPECIFIED:
            return validated

        if self.null_values is not UNSPECIFIED\
                and validated in self.null_values:
            return self.default

        if validated is None:
            return self.default

        return validated
871911
def _merge_multi_context(outputs, major_axis):
    
    rets = []
    for tensors, axis in zip(outputs, major_axis):
        if axis >= 0:
            # pylint: disable=no-member,protected-access
            if len(tensors) == 1:
                rets.append(tensors[0])
            else:
                # Concatenate if necessary
                rets.append(nd.concat(*[tensor.as_in_context(tensors[0].context)
                                        for tensor in tensors],
                                      dim=axis))
            # pylint: enable=no-member,protected-access
        else:
            # negative axis means the there is no batch_size axis, and all the
            # results should be the same on each device. We simply take the
            # first one, without checking they are actually the same
            rets.append(tensors[0])
    return rets
1166900
def validate_mixture(supports_mixture: SupportsMixture):
    """"""
    mixture_tuple = mixture(supports_mixture, None)
    if mixture_tuple is None:
        raise TypeError('{}_mixture did not have a _mixture_ method'.format(
            supports_mixture))

    def validate_probability(p, p_str):
        if p < 0:
            raise ValueError('{} was less than 0.'.format(p_str))
        elif p > 1:
            raise ValueError('{} was greater than 1.'.format(p_str))

    total = 0.0
    for p, val in mixture_tuple:
        validate_probability(p, '{}\'s probability'.format(str(val)))
        total += p
    if not np.isclose(total, 1.0):
        raise ValueError('Sum of probabilities of a mixture was not 1.0')
1048714
def _batch(self,
               vectors,
               batch_size,
               num,
               show_progressbar,
               return_names):
        """"""
        vectors = self.normalize(vectors)

        # Single transpose, makes things faster.
        reference_transposed = self.norm_vectors.T

        for i in tqdm(range(0, len(vectors), batch_size),
                      disable=not show_progressbar):

            distances = vectors[i: i+batch_size].dot(reference_transposed)
            # For safety we clip
            distances = np.clip(distances, a_min=.0, a_max=1.0)
            if num == 1:
                sorted_indices = np.argmax(distances, 1)[:, None]
            else:
                sorted_indices = np.argpartition(-distances, kth=num, axis=1)
                sorted_indices = sorted_indices[:, :num]
            for lidx, indices in enumerate(sorted_indices):
                dists = distances[lidx, indices]
                if return_names:
                    dindex = np.argsort(-dists)
                    yield [(self.indices[indices[d]], dists[d])
                           for d in dindex]
                else:
                    yield list(-1 * np.sort(-dists))
206244
def _validated(self, data):
        """"""
        errors = []
        for sub in self.schemas:
            try:
                return sub(data)
            except NotValid as ex:
                errors.extend(ex.args)

        raise NotValid(' and '.join(errors))