Validate ``Samples`` and ``Factors`` identifiers across the file.

    :param mwtabfile: Instance of :class:`~mwtab.mwtab.MWTabFile`.
    :type mwtabfile: :class:`~mwtab.mwtab.MWTabFile`
    :return: None
    :rtype: :py:obj:`None`
505198
def _validate_samples_factors(mwtabfile, validate_samples=True, validate_factors=True):
    
    from_subject_samples = {i["local_sample_id"] for i in mwtabfile["SUBJECT_SAMPLE_FACTORS"]["SUBJECT_SAMPLE_FACTORS"]}
    from_subject_factors = {i["factors"] for i in mwtabfile["SUBJECT_SAMPLE_FACTORS"]["SUBJECT_SAMPLE_FACTORS"]}

    if validate_samples:

        if "MS_METABOLITE_DATA" in mwtabfile:
            from_metabolite_data_samples = set(mwtabfile["MS_METABOLITE_DATA"]["MS_METABOLITE_DATA_START"]["Samples"])
            assert from_subject_samples == from_metabolite_data_samples

        if "NMR_BINNED_DATA" in mwtabfile:
            from_nmr_binned_data_samples = set(mwtabfile["NMR_BINNED_DATA"]["NMR_BINNED_DATA_START"]["Fields"][1:])
            assert from_subject_samples == from_nmr_binned_data_samples

    if validate_factors:

        if "MS_METABOLITE_DATA" in mwtabfile:
            from_metabolite_data_factors = set(mwtabfile["MS_METABOLITE_DATA"]["MS_METABOLITE_DATA_START"]["Factors"])
            assert from_subject_factors == from_metabolite_data_factors
733263
def _change_sample_name(in_file, sample_name, data=None):
    
    out_file = append_stem(in_file, "_fixed")
    with file_transaction(data, out_file) as tx_out:
        with open(tx_out, "w") as out_handle:
            with open(in_file) as in_handle:
                for line in in_handle:
                    if line.startswith("Status"):
                        line = "Status\t%s.bam" % sample_name
                    out_handle.write("%s\n" % line.strip())
    return out_file
100158
def _IsMetadataFile(self, file_entry):
    
    if (file_entry.type_indicator == dfvfs_definitions.TYPE_INDICATOR_TSK and
        file_entry.path_spec.location in self._METADATA_FILE_LOCATIONS_TSK):
      return True

    return False
1286919
def p_factor_item(self, p):
        

        def resolve_item(obj, index):
            if is_ndarray(obj) or isinstance(obj, (list, tuple)):
                if not isinstance(index, int):
                    raise TypeError("Not a valid array index: '{}'".format(index))

            elif isinstance(obj, dict):
                if not isinstance(index, (six.string_types, int)):
                    raise TypeError("Not a valid dictionary index: '{}'".format(index))

            else:
                raise TypeError("Object does not support indexing: '{}'".format(type(obj)))

            return obj[index]

        p[0] = Instruction('resolve_item(obj, index)', context={
            'resolve_item': resolve_item,
            'obj': p[1],
            'index': p[3]
        })
432595
def validate_mutations(vcffile, bams, reffa, chr_reffa, samples, output_format, outfile,
                       mpileup_parameters=mpileup_parser.MPILEUP_DEFAULT_PARAMS):
    """"""
    output_header = "sample chrom pos ref cov A C G T * - + " \
        "val_ref val_alt val_al_type val_al_count val_maf "\
        "most_common_indel most_common_indel_count most_common_indel_maf most_common_indel_type most_common_al " \
        "most_common_al_count most_common_al_maf most_common_count most_common_maf".split()

    # for backwards compatibility
    # if bam or samples is a string, convert to list instead
    if isinstance(samples, six.string_types):
        samples = [samples]
    if isinstance(bams, six.string_types):
        bams = [bams]

    if output_format == 'vcf':
        vcf_reader = vcf.Reader(open(vcffile))
        vcf_reader.samples = samples
        vcf_reader.formats['GT'] = vcf.parser._Format(id='GT', num=1, type='String', desc="Genotype")
        vcf_reader.formats['AD'] = vcf.parser._Format(id='AD', num='R', type='Integer', desc="Allelic depth")
        vcf_reader.formats['DP'] = vcf.parser._Format(id='DP', num=1, type='Integer', desc="Depth")
        vcf_writer = vcf.Writer(outfile, vcf_reader)
    else:
        vcf_reader = open(vcffile)


    if output_format == "sufam":
        outfile.write("\t".join(output_header))
        outfile.write("\n")
    for record in vcf_reader:
        if output_format != 'vcf':
            line = record
            if line.startswith("#CHROM"):
                header = line[1:].rstrip('\n').split("\t")
                # create spoof pyvcf record if vcf_reader is not used
                _Record = namedtuple('Record', header)
            if line.startswith("#"):
                continue
            if len(header) == 0:
                raise(Exception("No header found in vcf file #CHROM not found"))
            # zip all column values, except alt (needs to be list in pyvcf)
            record_args = dict(zip(header, line.rstrip('\n').split("\t")))
            record_args['ALT'] = [record_args['ALT']]
            record = _Record(**record_args)

        # determine type of mutation
        record_type = "snv"
        if len(record.ALT) > 1:
            warnings.warn("Multiple ALT in one record is not implemented - using first")
        if len(record.REF) > len(record.ALT[0]):
            record_type = "deletion"
        elif len(record.ALT[0]) > len(record.REF):
            record_type = "insertion"

        # no coverage results
        no_cov = pd.Series({
            "chrom": str(record.CHROM), "pos": str(record.POS),
            "ref": str(record.REF),
            "cov": 0, "A": 0, "C": 0, "G": 0, "T": 0,
            "val_ref": str(record.REF), "val_alt": str(record.ALT[0]),
            "val_al_type": record_type, "val_al_count": 0, "val_maf": 0})

        # collect mpileup baseparser results per bam
        bps = []
        for i, bam in enumerate(bams):
            sample = samples[i]
            no_cov['sample'] = sample
            bp_lines = mpileup_parser.run_and_parse(bam, str(record.CHROM), str(record.POS), str(record.POS), reffa,
                                                    chr_reffa, mpileup_parameters)
            bpdf = get_baseparser_extended_df(sample, bp_lines, str(record.REF), str(record.ALT[0]))
            if bpdf is None:
                bp = no_cov
            else:
                bp = bpdf.ix[0, :]
            bps += [bp]

        # output call
        if output_format == "vcf":
            _write_bp_vcf(outfile, bps, vcf_writer, record)
        else:
            # only one bam file supported for outputs other than vcf
            _write_bp(outfile, bps[0], output_header, output_format)
274104
def create_ped_file(samples, base_vcf, out_dir=None):
    
    out_file = "%s.ped" % utils.splitext_plus(base_vcf)[0]
    if out_dir:
        out_file = os.path.join(out_dir, os.path.basename(out_file))
    sample_ped_lines = {}
    header = ["#Family_ID", "Individual_ID", "Paternal_ID", "Maternal_ID", "Sex", "Phenotype", "Ethnicity"]
    for md_ped in list(set([x for x in [tz.get_in(["metadata", "ped"], data)
                                        for data in samples] if x is not None])):
        with open(md_ped) as in_handle:
            reader = csv.reader(in_handle, dialect="excel-tab")
            for parts in reader:
                if parts[0].startswith("#") and len(parts) > len(header):
                    header = header + parts[len(header):]
                else:
                    sample_ped_lines[parts[1]] = parts
    if not utils.file_exists(out_file):
        with file_transaction(samples[0], out_file) as tx_out_file:
            with open(tx_out_file, "w") as out_handle:
                want_samples = set(vcfutils.get_samples(base_vcf))
                writer = csv.writer(out_handle, dialect="excel-tab")
                writer.writerow(header)
                for data in samples:
                    ped_info = get_ped_info(data, samples)
                    sname = ped_info["individual_id"]
                    if sname in want_samples:
                        want_samples.remove(sname)
                        if sname in sample_ped_lines:
                            writer.writerow(sample_ped_lines[sname])
                        else:
                            writer.writerow([ped_info["family_id"], ped_info["individual_id"],
                                            ped_info["paternal_id"], ped_info["maternal_id"],
                                            ped_info["gender"], ped_info["affected"],
                                            ped_info["ethnicity"]])
    return out_file
1026220
def _validate_inputs(self, inputdict):
        
        # Check inputdict
        try:
            parameters = inputdict.pop(self.get_linkname('parameters'))
        except KeyError:
            raise InputValidationError("No parameters specified for this "
                                       "calculation")
        if not isinstance(parameters, ParameterData):
            raise InputValidationError("parameters not of type "
                                       "ParameterData")
        # Check code
        try:
            code = inputdict.pop(self.get_linkname('code'))
        except KeyError:
            raise InputValidationError("No code specified for this "
                                       "calculation")

        # Check input files
        try:
            structure = inputdict.pop(self.get_linkname('structure'))
            if not isinstance(structure, SinglefileData):
                raise InputValidationError(
                    "structure not of type SinglefileData")
        except KeyError:
            raise InputValidationError(
                "No input structure specified for calculation")

        try:
            surface_sample = inputdict.pop(self.get_linkname('surface_sample'))
            if not isinstance(surface_sample, SinglefileData):
                raise InputValidationError(
                    "surface_sample not of type SinglefileData")
        except KeyError:
            raise InputValidationError(
                "No surface sample specified for calculation")

        # Check that nothing is left unparsed
        if inputdict:
            raise ValidationError("Unrecognized inputs: {}".format(inputdict))

        return parameters, code, structure, surface_sample
1122384
def _check_sig_dims(sig, read_len, n_sig, samps_per_frame):
    
    if isinstance(sig, np.ndarray):
        if sig.shape != (read_len, n_sig):
            raise ValueError('Samples were not loaded correctly')
    else:
        if len(sig) != n_sig:
            raise ValueError('Samples were not loaded correctly')
        for ch in range(n_sig):
            if len(sig[ch]) != samps_per_frame[ch] * read_len:
                raise ValueError('Samples were not loaded correctly')
757108
def check_infile_and_wp(curinf, curwp):
        
        if not os.path.exists(curinf):
            if curwp is None:
                TauDEM.error('You must specify one of the workspace and the '
                             'full path of input file!')
            curinf = curwp + os.sep + curinf
            curinf = os.path.abspath(curinf)
            if not os.path.exists(curinf):
                TauDEM.error('Input files parameter %s is not existed!' % curinf)
        else:
            curinf = os.path.abspath(curinf)
            if curwp is None:
                curwp = os.path.dirname(curinf)
        return curinf, curwp
1003763
def validate_examples(example_file):
    

    def test_example(raw):
        example = tf.train.Example()
        example.ParseFromString(raw)

        pi = np.frombuffer(example.features.feature['pi'].bytes_list.value[0], np.float32)
        value = example.features.feature['outcome'].float_list.value[0]
        assert abs(pi.sum() - 1) < 1e-4, pi.sum()
        assert value in (-1, 1), value

    opts = tf.python_io.TFRecordOptions(tf.python_io.TFRecordCompressionType.ZLIB)
    for record in tqdm(tf.python_io.tf_record_iterator(example_file, opts)):
        test_example(record)
910293
def validate_samples(self):
        

        bsc = getToolByName(self, 'bika_setup_catalog')
        keywords = bsc.uniqueValuesFor('getKeyword')
        profiles = []
        for p in bsc(portal_type='AnalysisProfile'):
            p = p.getObject()
            profiles.append(p.Title())
            profiles.append(p.getProfileKey())

        row_nr = 0
        for gridrow in self.getSampleData():
            row_nr += 1

            # validate against sample and ar schemas
            for k, v in gridrow.items():
                if k in ['Analysis', 'Profiles']:
                    break
                if k in sample_schema:
                    try:
                        self.validate_against_schema(
                            sample_schema, row_nr, k, v)
                        continue
                    except ValueError as e:
                        self.error(e.message)
                        break
                if k in ar_schema:
                    try:
                        self.validate_against_schema(
                            ar_schema, row_nr, k, v)
                    except ValueError as e:
                        self.error(e.message)

            an_cnt = 0
            for v in gridrow['Analyses']:
                if v and v not in keywords:
                    self.error("Row %s: value is invalid (%s=%s)" %
                               ('Analysis keyword', row_nr, v))
                else:
                    an_cnt += 1
            for v in gridrow['Profiles']:
                if v and v not in profiles:
                    self.error("Row %s: value is invalid (%s=%s)" %
                               ('Profile Title', row_nr, v))
                else:
                    an_cnt += 1
            if not an_cnt:
                self.error("Row %s: No valid analyses or profiles" % row_nr)
1294786
def _validate(self):
        
        try:
            self._resolve_workspace()
            if 'mets_unique_identifier' not in self.skip:
                self._validate_mets_unique_identifier()
            if 'mets_file_group_names' not in self.skip:
                self._validate_mets_file_group_names()
            if 'mets_files' not in self.skip:
                self._validate_mets_files()
            if 'pixel_density' not in self.skip:
                self._validate_pixel_density()
            if 'page' not in self.skip:
                self._validate_page()
        except Exception as e: # pylint: disable=broad-except
            self.report.add_error("Failed to instantiate workspace: %s" % e)
            #  raise e
        return self.report
802544
def make_rfft_factors(axes, resshape, facshape, normshape, norm):
    
    N = 1.0
    for n in normshape: N = N * n

    # inplace modification is fine because we produce a constant
    # which doesn't go into autograd.
    # For same reason could have used numpy rather than anp.
    # but we already imported anp, so use it instead.
    fac = anp.zeros(resshape)
    fac[...] = 2
    index = [slice(None)] * len(resshape)
    if facshape[-1] <= resshape[axes[-1]]:
        index[axes[-1]] = (0, facshape[-1] - 1)
    else:
        index[axes[-1]] = (0,)
    fac[tuple(index)] = 1
    if norm is None:
        fac /= N
    return fac
83686
def verify(path):
        
        valid = True
        fifo = SeriesFolder._search_files(path)
        # dataset size
        if len(fifo) == 0:
            valid = False
        # number of different file formats
        fifmts = [ff[1] for ff in fifo]
        if len(set(fifmts)) != 1:
            valid = False
        return valid
131872
def _create_factor_rule(tok):
        
        if tok[0] == 'IPV4':
            return IPV4Rule(tok[1])
        if tok[0] == 'IPV6':
            return IPV6Rule(tok[1])
        if tok[0] == 'DATETIME':
            return DatetimeRule(tok[1])
        if tok[0] == 'TIMEDELTA':
            return TimedeltaRule(tok[1])
        if tok[0] == 'INTEGER':
            return IntegerRule(tok[1])
        if tok[0] == 'FLOAT':
            return FloatRule(tok[1])
        if tok[0] == 'VARIABLE':
            return VariableRule(tok[1])
        return ConstantRule(tok[1])
533864
def _checkMetadata(self, variantFile):
        
        metadata = self._getMetadataFromVcf(variantFile)
        if self._metadata is not None and self._metadata != metadata:
            raise exceptions.InconsistentMetaDataException(
                variantFile.filename)
1139727
def compare_to_rm(data):
    
    if isinstance(data, (list, tuple)) and cwlutils.is_cwl_run(utils.to_single_data(data[0])):
        data = _normalize_cwl_inputs(data)
    toval_data = _get_validate(data)
    toval_data = cwlutils.unpack_tarballs(toval_data, toval_data)
    if toval_data:
        caller = _get_caller(toval_data)
        sample = dd.get_sample_name(toval_data)
        base_dir = utils.safe_makedir(os.path.join(toval_data["dirs"]["work"], "validate", sample, caller))

        if isinstance(toval_data["vrn_file"], (list, tuple)):
            raise NotImplementedError("Multiple input files for validation: %s" % toval_data["vrn_file"])
        else:
            vrn_file = os.path.abspath(toval_data["vrn_file"])
        rm_file = normalize_input_path(toval_data["config"]["algorithm"]["validate"], toval_data)
        rm_interval_file = _gunzip(normalize_input_path(toval_data["config"]["algorithm"].get("validate_regions"),
                                                        toval_data),
                                   toval_data)
        rm_interval_file = bedutils.clean_file(rm_interval_file, toval_data, prefix="validateregions-",
                                               bedprep_dir=utils.safe_makedir(os.path.join(base_dir, "bedprep")))
        rm_file = naming.handle_synonyms(rm_file, dd.get_ref_file(toval_data), data.get("genome_build"),
                                         base_dir, data)
        rm_interval_file = (naming.handle_synonyms(rm_interval_file, dd.get_ref_file(toval_data),
                                                   data.get("genome_build"), base_dir, data)
                            if rm_interval_file else None)
        vmethod = tz.get_in(["config", "algorithm", "validate_method"], data, "rtg")
        # RTG can fail on totally empty files. Call everything in truth set as false negatives
        if not vcfutils.vcf_has_variants(vrn_file):
            eval_files = _setup_call_false(rm_file, rm_interval_file, base_dir, toval_data, "fn")
            data["validate"] = _rtg_add_summary_file(eval_files, base_dir, toval_data)
        # empty validation file, every call is a false positive
        elif not vcfutils.vcf_has_variants(rm_file):
            eval_files = _setup_call_fps(vrn_file, rm_interval_file, base_dir, toval_data, "fp")
            data["validate"] = _rtg_add_summary_file(eval_files, base_dir, toval_data)
        elif vmethod in ["rtg", "rtg-squash-ploidy"]:
            eval_files = _run_rtg_eval(vrn_file, rm_file, rm_interval_file, base_dir, toval_data, vmethod)
            eval_files = _annotate_validations(eval_files, toval_data)
            data["validate"] = _rtg_add_summary_file(eval_files, base_dir, toval_data)
        elif vmethod == "hap.py":
            data["validate"] = _run_happy_eval(vrn_file, rm_file, rm_interval_file, base_dir, toval_data)
        elif vmethod == "bcbio.variation":
            data["validate"] = _run_bcbio_variation(vrn_file, rm_file, rm_interval_file, base_dir,
                                                    sample, caller, toval_data)
    return [[data]]