Utility function to determine fill values for variants fields with
    missing values.
508087
def _variants_fills(fields, fills, info_types):
    """"""
    if fills is None:
        # no fills specified by user
        fills = dict()
    for f, vcf_type in zip(fields, info_types):
        if f == 'FILTER':
            fills[f] = False
        elif f not in fills:
            if f in config.STANDARD_VARIANT_FIELDS:
                fills[f] = config.DEFAULT_VARIANT_FILL[f]
            else:
                fills[f] = config.DEFAULT_FILL_MAP[vcf_type]
    # convert to tuple for zipping with fields
    fills = tuple(fills[f] for f in fields)
    return fills
272041
def normalize_params(params):
    """"""
    # Collect a set of all fields
    fields = set()
    for p in params:
        fields.update(p)
    fields = sorted(fields)

    params2 = list(pluck(fields, params, MISSING))
    # Non-basic types (including MISSING) are unique to their id
    tokens = [
        tuple(x if isinstance(x, (int, float, str)) else id(x) for x in p)
        for p in params2
    ]

    return fields, tokens, params2
315445
def get_variants_in_region(self, chrom, start, end):
        """"""
        if not self.has_index:
            raise NotImplementedError("Not implemented when IMPUTE2 file is "
                                      "not indexed (see genipe)")

        if not self._index_has_location:
            raise NotImplementedError("Not implemented when index doesn't "
                                      "have location information.")

        # Getting the required variants
        required = self._impute2_index.loc[
            (self._impute2_index.chrom == CHROM_STR_TO_INT[chrom]) &
            (start <= self._impute2_index.pos) &
            (self._impute2_index.pos <= end)
        ]

        for name, variant_info in required.iterrows():
            for genotypes in self.get_variant_by_name(name, variant_info):
                self._fix_genotypes_object(genotypes, variant_info)
                yield genotypes
433325
def _missing_(cls, value):
        """"""
        if not (isinstance(value, int) and 0x0000 <= value <= 0xFFFF):
            raise ValueError('%r is not a valid %s' % (value, cls.__name__))
        if 0x0000 <= value <= 0x05DC:
            # [Neil_Sembower]
            extend_enum(cls, 'IEEE802.3 Length Field [0x%s]' % hex(value)[2:].upper().zfill(4), value)
            return cls(value)
        if 0x0101 <= value <= 0x01FF:
            # [Neil_Sembower]
            extend_enum(cls, 'Experimental [0x%s]' % hex(value)[2:].upper().zfill(4), value)
            return cls(value)
        if 0x0888 <= value <= 0x088A:
            # [Neil_Sembower]
            extend_enum(cls, 'Xyplex [0x%s]' % hex(value)[2:].upper().zfill(4), value)
            return cls(value)
        if 0x1001 <= value <= 0x100F:
            # [Neil_Sembower]
            extend_enum(cls, 'Berkeley Trailer encap/IP [0x%s]' % hex(value)[2:].upper().zfill(4), value)
            return cls(value)
        if 0x6008 <= value <= 0x6009:
            # [Neil_Sembower]
            extend_enum(cls, 'DEC Unassigned [0x%s]' % hex(value)[2:].upper().zfill(4), value)
            return cls(value)
        if 0x6010 <= value <= 0x6014:
            # [Neil_Sembower]
            extend_enum(cls, '3Com Corporation [0x%s]' % hex(value)[2:].upper().zfill(4), value)
            return cls(value)
        if 0x7020 <= value <= 0x7029:
            # [Neil_Sembower]
            extend_enum(cls, 'LRT [0x%s]' % hex(value)[2:].upper().zfill(4), value)
            return cls(value)
        if 0x8039 <= value <= 0x803C:
            # [Neil_Sembower]
            extend_enum(cls, 'DEC Unassigned [0x%s]' % hex(value)[2:].upper().zfill(4), value)
            return cls(value)
        if 0x8040 <= value <= 0x8042:
            # [Neil_Sembower]
            extend_enum(cls, 'DEC Unassigned [0x%s]' % hex(value)[2:].upper().zfill(4), value)
            return cls(value)
        if 0x806E <= value <= 0x8077:
            # [Neil_Sembower]
            extend_enum(cls, 'Landmark Graphics Corp. [0x%s]' % hex(value)[2:].upper().zfill(4), value)
            return cls(value)
        if 0x807D <= value <= 0x807F:
            # [Neil_Sembower]
            extend_enum(cls, 'Vitalink Communications [0x%s]' % hex(value)[2:].upper().zfill(4), value)
            return cls(value)
        if 0x8081 <= value <= 0x8083:
            # [Neil_Sembower]
            extend_enum(cls, 'Counterpoint Computers [0x%s]' % hex(value)[2:].upper().zfill(4), value)
            return cls(value)
        if 0x809C <= value <= 0x809E:
            # [Neil_Sembower]
            extend_enum(cls, 'Datability [0x%s]' % hex(value)[2:].upper().zfill(4), value)
            return cls(value)
        if 0x80A4 <= value <= 0x80B3:
            # [Neil_Sembower]
            extend_enum(cls, 'Siemens Gammasonics Inc. [0x%s]' % hex(value)[2:].upper().zfill(4), value)
            return cls(value)
        if 0x80C0 <= value <= 0x80C3:
            # [Neil_Sembower]
            extend_enum(cls, 'DCA Data Exchange Cluster [0x%s]' % hex(value)[2:].upper().zfill(4), value)
            return cls(value)
        if 0x80C8 <= value <= 0x80CC:
            # [Neil_Sembower]
            extend_enum(cls, 'Intergraph Corporation [0x%s]' % hex(value)[2:].upper().zfill(4), value)
            return cls(value)
        if 0x80CD <= value <= 0x80CE:
            # [Neil_Sembower]
            extend_enum(cls, 'Harris Corporation [0x%s]' % hex(value)[2:].upper().zfill(4), value)
            return cls(value)
        if 0x80CF <= value <= 0x80D2:
            # [Neil_Sembower]
            extend_enum(cls, 'Taylor Instrument [0x%s]' % hex(value)[2:].upper().zfill(4), value)
            return cls(value)
        if 0x80D3 <= value <= 0x80D4:
            # [Neil_Sembower]
            extend_enum(cls, 'Rosemount Corporation [0x%s]' % hex(value)[2:].upper().zfill(4), value)
            return cls(value)
        if 0x80DE <= value <= 0x80DF:
            # [Neil_Sembower]
            extend_enum(cls, 'Integrated Solutions TRFS [0x%s]' % hex(value)[2:].upper().zfill(4), value)
            return cls(value)
        if 0x80E0 <= value <= 0x80E3:
            # [Neil_Sembower]
            extend_enum(cls, 'Allen-Bradley [0x%s]' % hex(value)[2:].upper().zfill(4), value)
            return cls(value)
        if 0x80E4 <= value <= 0x80F0:
            # [Neil_Sembower]
            extend_enum(cls, 'Datability [0x%s]' % hex(value)[2:].upper().zfill(4), value)
            return cls(value)
        if 0x80F4 <= value <= 0x80F5:
            # [Neil_Sembower]
            extend_enum(cls, 'Kinetics [0x%s]' % hex(value)[2:].upper().zfill(4), value)
            return cls(value)
        if 0x8101 <= value <= 0x8103:
            # [Neil_Sembower]
            extend_enum(cls, 'Wellfleet Communications [0x%s]' % hex(value)[2:].upper().zfill(4), value)
            return cls(value)
        if 0x8107 <= value <= 0x8109:
            # [Neil_Sembower]
            extend_enum(cls, 'Symbolics Private [0x%s]' % hex(value)[2:].upper().zfill(4), value)
            return cls(value)
        if 0x8132 <= value <= 0x8136:
            # [Neil_Sembower]
            extend_enum(cls, 'Bridge Communications [0x%s]' % hex(value)[2:].upper().zfill(4), value)
            return cls(value)
        if 0x8137 <= value <= 0x8138:
            # [Neil_Sembower]
            extend_enum(cls, 'Novell, Inc. [0x%s]' % hex(value)[2:].upper().zfill(4), value)
            return cls(value)
        if 0x8139 <= value <= 0x813D:
            # [Neil_Sembower]
            extend_enum(cls, 'KTI [0x%s]' % hex(value)[2:].upper().zfill(4), value)
            return cls(value)
        if 0x8151 <= value <= 0x8153:
            # [Neil_Sembower]
            extend_enum(cls, 'Qualcomm [0x%s]' % hex(value)[2:].upper().zfill(4), value)
            return cls(value)
        if 0x815C <= value <= 0x815E:
            # [Neil_Sembower]
            extend_enum(cls, 'Computer Protocol Pty Ltd [0x%s]' % hex(value)[2:].upper().zfill(4), value)
            return cls(value)
        if 0x8164 <= value <= 0x8166:
            # [Neil_Sembower]
            extend_enum(cls, 'Charles River Data System [0x%s]' % hex(value)[2:].upper().zfill(4), value)
            return cls(value)
        if 0x8184 <= value <= 0x818C:
            # [Neil_Sembower]
            extend_enum(cls, 'Silicon Graphics prop. [0x%s]' % hex(value)[2:].upper().zfill(4), value)
            return cls(value)
        if 0x819A <= value <= 0x81A3:
            # [Neil_Sembower]
            extend_enum(cls, 'Qualcomm [0x%s]' % hex(value)[2:].upper().zfill(4), value)
            return cls(value)
        if 0x81A5 <= value <= 0x81AE:
            # [Neil_Sembower]
            extend_enum(cls, 'RAD Network Devices [0x%s]' % hex(value)[2:].upper().zfill(4), value)
            return cls(value)
        if 0x81B7 <= value <= 0x81B9:
            # [Neil_Sembower]
            extend_enum(cls, 'Xyplex [0x%s]' % hex(value)[2:].upper().zfill(4), value)
            return cls(value)
        if 0x81CC <= value <= 0x81D5:
            # [Neil_Sembower]
            extend_enum(cls, 'Apricot Computers [0x%s]' % hex(value)[2:].upper().zfill(4), value)
            return cls(value)
        if 0x81D6 <= value <= 0x81DD:
            # [Neil_Sembower]
            extend_enum(cls, 'Artisoft [0x%s]' % hex(value)[2:].upper().zfill(4), value)
            return cls(value)
        if 0x81E6 <= value <= 0x81EF:
            # [Neil_Sembower]
            extend_enum(cls, 'Polygon [0x%s]' % hex(value)[2:].upper().zfill(4), value)
            return cls(value)
        if 0x81F0 <= value <= 0x81F2:
            # [Neil_Sembower]
            extend_enum(cls, 'Comsat Labs [0x%s]' % hex(value)[2:].upper().zfill(4), value)
            return cls(value)
        if 0x81F3 <= value <= 0x81F5:
            # [Neil_Sembower]
            extend_enum(cls, 'SAIC [0x%s]' % hex(value)[2:].upper().zfill(4), value)
            return cls(value)
        if 0x81F6 <= value <= 0x81F8:
            # [Neil_Sembower]
            extend_enum(cls, 'VG Analytical [0x%s]' % hex(value)[2:].upper().zfill(4), value)
            return cls(value)
        if 0x8203 <= value <= 0x8205:
            # [Neil_Sembower]
            extend_enum(cls, 'Quantum Software [0x%s]' % hex(value)[2:].upper().zfill(4), value)
            return cls(value)
        if 0x8221 <= value <= 0x8222:
            # [Neil_Sembower]
            extend_enum(cls, 'Ascom Banking Systems [0x%s]' % hex(value)[2:].upper().zfill(4), value)
            return cls(value)
        if 0x823E <= value <= 0x8240:
            # [Neil_Sembower]
            extend_enum(cls, 'Advanced Encryption Syste [0x%s]' % hex(value)[2:].upper().zfill(4), value)
            return cls(value)
        if 0x827F <= value <= 0x8282:
            # [Neil_Sembower]
            extend_enum(cls, 'Athena Programming [0x%s]' % hex(value)[2:].upper().zfill(4), value)
            return cls(value)
        if 0x8263 <= value <= 0x826A:
            # [Neil_Sembower]
            extend_enum(cls, 'Charles River Data System [0x%s]' % hex(value)[2:].upper().zfill(4), value)
            return cls(value)
        if 0x829A <= value <= 0x829B:
            # [Neil_Sembower]
            extend_enum(cls, 'Inst Ind Info Tech [0x%s]' % hex(value)[2:].upper().zfill(4), value)
            return cls(value)
        if 0x829C <= value <= 0x82AB:
            # [Neil_Sembower]
            extend_enum(cls, 'Taurus Controls [0x%s]' % hex(value)[2:].upper().zfill(4), value)
            return cls(value)
        if 0x82AC <= value <= 0x8693:
            # [Neil_Sembower]
            extend_enum(cls, 'Walker Richer & Quinn [0x%s]' % hex(value)[2:].upper().zfill(4), value)
            return cls(value)
        if 0x8694 <= value <= 0x869D:
            # [Neil_Sembower]
            extend_enum(cls, 'Idea Courier [0x%s]' % hex(value)[2:].upper().zfill(4), value)
            return cls(value)
        if 0x869E <= value <= 0x86A1:
            # [Neil_Sembower]
            extend_enum(cls, 'Computer Network Tech [0x%s]' % hex(value)[2:].upper().zfill(4), value)
            return cls(value)
        if 0x86A3 <= value <= 0x86AC:
            # [Neil_Sembower]
            extend_enum(cls, 'Gateway Communications [0x%s]' % hex(value)[2:].upper().zfill(4), value)
            return cls(value)
        if 0x86E0 <= value <= 0x86EF:
            # [Neil_Sembower]
            extend_enum(cls, 'Landis & Gyr Powers [0x%s]' % hex(value)[2:].upper().zfill(4), value)
            return cls(value)
        if 0x8700 <= value <= 0x8710:
            # [Neil_Sembower]
            extend_enum(cls, 'Motorola [0x%s]' % hex(value)[2:].upper().zfill(4), value)
            return cls(value)
        if 0x8A96 <= value <= 0x8A97:
            # [Neil_Sembower]
            extend_enum(cls, 'Invisible Software [0x%s]' % hex(value)[2:].upper().zfill(4), value)
            return cls(value)
        if 0xFF00 <= value <= 0xFF0F:
            # [Neil_Sembower]
            extend_enum(cls, 'ISC Bunker Ramo [0x%s]' % hex(value)[2:].upper().zfill(4), value)
            return cls(value)
        super()._missing_(value)
949405
def validate(self):
        """"""
        missing = []
        for k, v in self._map.items():
            attr = getattr(self, k, False)
            if not attr or attr == CONFIG_PLACEHOLDER:
                missing.append(v)
        if missing:
            return "Missing or invalid config values: {}".format(
                ", ".join(sorted(missing))
            )
126780
def refetch_fields(self, missing_fields):
        
        db_fields = self.mongokat_collection.find_one({"_id": self["_id"]}, fields={k: 1 for k in missing_fields})

        self._fetched_fields += tuple(missing_fields)

        if not db_fields:
            return

        for k, v in db_fields.items():
            self[k] = v
420092
def filter_missing(self):
        """"""

        missing             = None
        locus_count         = 0

        # Filter out individuals according to missingness
        self.genotype_file.seek(0)
        for genotypes in self.genotype_file:
            genotypes = genotypes.split()
            chr, rsid, junk, pos = genotypes[0:4]
            if DataParser.boundary.TestBoundary(chr, pos, rsid):
                locus_count += 1
                allelic_data = numpy.array(genotypes[4:], dtype="S2").reshape(-1, 2)
                if missing is None:
                    missing             = numpy.zeros(allelic_data.shape[0], dtype='int8')
                missing += (numpy.sum(0+(allelic_data==DataParser.missing_representation), axis=1)/2)

        max_missing = DataParser.ind_miss_tol * locus_count
        dropped_individuals = 0+(max_missing<missing)

        self.ind_mask[:,0] = self.ind_mask[:,0]|dropped_individuals
        self.ind_mask[:,1] = self.ind_mask[:,1]|dropped_individuals

        valid_individuals = numpy.sum(self.ind_mask==0)
        max_missing = DataParser.snp_miss_tol * valid_individuals

        self.locus_count = 0
        # We can't merge these two iterations since we need to know which individuals
        # to consider for filtering on MAF
        dropped_snps = []
        self.genotype_file.seek(0)
        for genotypes in self.genotype_file:
            genotypes = genotypes.split()
            chr, rsid, junk, pos = genotypes[0:4]
            chr = int(chr)
            pos = int(pos)
            if DataParser.boundary.TestBoundary(chr, pos, rsid):
                allelic_data = numpy.ma.MaskedArray(numpy.array(genotypes[4:], dtype="S2").reshape(-1, 2), self.ind_mask).compressed()
                missing = numpy.sum(0+(allelic_data==DataParser.missing_representation))
                if missing > max_missing:
                    DataParser.boundary.dropped_snps[int(chr)].add(int(pos))
                    dropped_snps.append(rsid)
                else:
                    self.locus_count += 1
73394
def required_fields(self):
        """"""
        return {f:v for f, v in self.normal_fields.items() if v.required}
508104
static VcfSliceProtos.Fields buildDefaultFields(List<Variant> variants) {
        return buildDefaultFields(variants, null, null);
    }
369837
def replace_missing_value(self, str_in):
        # type: (Text) -> Text
        
        if self.missing_value is None:
            return str_in
        elif self.missing_value.sentinel == str_in:
            return self.missing_value.replace_with
        else:
            return str_in
883316
private static boolean shouldHasUniqueValues(List<Field> dataFields) {
        Set<String> dataValues = new HashSet<>();
        for (Field field : dataFields) {
            String value = field.getAnnotation(Plugin.Data.class).value();
            if (dataValues.contains(value)) {
                return false;
            }
            dataValues.add(value);
        }
        return true;
    }
596686
def missingDataValue(self):
        
        value =  getMissingDataValue(self._array)
        fieldNames = self._array.dtype.names

        # If the missing value attibute is a list with the same length as the number of fields,
        # return the missing value for field that equals the self.nodeName.
        if hasattr(value, '__len__') and len(value) == len(fieldNames):
            idx = fieldNames.index(self.nodeName)
            return value[idx]
        else:
            return value
1359473
def _fill_untouched(idx, ret, fill_value):
    """"""
    untouched = np.ones_like(ret, dtype=bool)
    untouched[idx] = False
    ret[untouched] = fill_value
80522
def check_mutate_fields( op, op_data ):
    

    mutate_fields = op_get_mutate_fields( op )
    assert mutate_fields is not None, "No mutate fields defined for %s" % op

    missing = []
    for field in mutate_fields:
        if not op_data.has_key(field):
            missing.append(field)

    assert len(missing) == 0, "Missing mutation fields for %s: %s" % (op, ",".join(missing))
    return True
476861
def ensure_params(kty, provided, required):
    """"""
    if not required <= provided:
        missing = required - provided
        raise MissingValue('Missing properties for kty={}, {}'.format(kty, str(list(missing))))
847468
def _missing_(cls, value):
        """"""
        if not (isinstance(value, int) and 0x0000 <= value <= 0xFFFF):
            raise ValueError('%r is not a valid %s' % (value, cls.__name__))
        if 0x0001 <= value <= 0x0BB8:
            extend_enum(cls, 'Registered by Xerox [0x%s]' % hex(value)[2:].upper().zfill(4), value)
            return cls(value)
        if 0x0020 <= value <= 0x003F:
            extend_enum(cls, 'Experimental [0x%s]' % hex(value)[2:].upper().zfill(4), value)
            return cls(value)
        if 0x0BB9 <= value <= 0xFFFF:
            extend_enum(cls, 'Dynamically Assigned [0x%s]' % hex(value)[2:].upper().zfill(4), value)
            return cls(value)
        if 0x4000 <= value <= 0x4FFF:
            extend_enum(cls, 'Dynamically Assigned Socket Numbers [0x%s]' % hex(value)[2:].upper().zfill(4), value)
            return cls(value)
        if 0x8000 <= value <= 0xFFFF:
            extend_enum(cls, 'Statically Assigned Socket Numbers [0x%s]' % hex(value)[2:].upper().zfill(4), value)
            return cls(value)
        super()._missing_(value)
132642
def iter_variants(self):
        """"""
        for v in self.get_vcf():
            yield Variant(v.ID, v.CHROM, v.POS, {v.REF} | set(v.ALT))