Reads a dbf header. Xbase-related code borrows heavily from ActiveState Python Cookbook Recipe 362715 by Raymond Hettinger
508513
def __dbfHeader(self):
        """"""
        if not self.dbf:
            raise ShapefileException("Shapefile Reader requires a shapefile or file-like object. (no dbf file found)")
        dbf = self.dbf
        headerLength = self.__dbfHeaderLength()
        numFields = (headerLength - 33) // 32
        for field in range(numFields):
            fieldDesc = list(unpack("<11sc4xBB14x", dbf.read(32)))
            name = 0
            idx = 0
            if b("\x00") in fieldDesc[name]:
                idx = fieldDesc[name].index(b("\x00"))
            else:
                idx = len(fieldDesc[name]) - 1
            fieldDesc[name] = fieldDesc[name][:idx]
            fieldDesc[name] = u(fieldDesc[name])
            fieldDesc[name] = fieldDesc[name].lstrip()
            fieldDesc[1] = u(fieldDesc[1])
            self.fields.append(fieldDesc)
        terminator = dbf.read(1)
        assert terminator == b("\r")
        self.fields.insert(0, ('DeletionFlag', 'C', 1, 0))
389446
def from_genesis_header(cls,
                            base_db: BaseDB,
                            genesis_header: BlockHeader) -> 'BaseHeaderChain':
        
        headerdb = cls.get_headerdb_class()(cast(BaseAtomicDB, base_db))
        headerdb.persist_header(genesis_header)
        return cls(base_db, genesis_header)
251918
def read_data_page(file_obj, schema_helper, page_header, column_metadata,
                   dictionary):
    
    daph = page_header.data_page_header
    raw_bytes = _read_page(file_obj, page_header, column_metadata)
    io_obj = io.BytesIO(raw_bytes)
    vals = []
    debug_logging = logger.isEnabledFor(logging.DEBUG)

    if debug_logging:
        logger.debug("  definition_level_encoding: %s",
                     _get_name(parquet_thrift.Encoding, daph.definition_level_encoding))
        logger.debug("  repetition_level_encoding: %s",
                     _get_name(parquet_thrift.Encoding, daph.repetition_level_encoding))
        logger.debug("  encoding: %s", _get_name(parquet_thrift.Encoding, daph.encoding))

    # definition levels are skipped if data is required.
    definition_levels = None
    num_nulls = 0
    max_definition_level = -1
    if not schema_helper.is_required(column_metadata.path_in_schema[-1]):
        max_definition_level = schema_helper.max_definition_level(
            column_metadata.path_in_schema)
        bit_width = encoding.width_from_max_int(max_definition_level)
        if debug_logging:
            logger.debug("  max def level: %s   bit_width: %s",
                         max_definition_level, bit_width)
        if bit_width == 0:
            definition_levels = [0] * daph.num_values
        else:
            definition_levels = _read_data(io_obj,
                                           daph.definition_level_encoding,
                                           daph.num_values,
                                           bit_width)[:daph.num_values]

        # any thing that isn't at max definition level is a null.
        num_nulls = len(definition_levels) - definition_levels.count(max_definition_level)
        if debug_logging:
            logger.debug("  Definition levels: %s", len(definition_levels))

    # repetition levels are skipped if data is at the first level.
    repetition_levels = None  # pylint: disable=unused-variable
    if len(column_metadata.path_in_schema) > 1:
        max_repetition_level = schema_helper.max_repetition_level(
            column_metadata.path_in_schema)
        bit_width = encoding.width_from_max_int(max_repetition_level)
        repetition_levels = _read_data(io_obj,
                                       daph.repetition_level_encoding,
                                       daph.num_values,
                                       bit_width)

    # NOTE: The repetition levels aren't yet used.
    if daph.encoding == parquet_thrift.Encoding.PLAIN:
        read_values = encoding.read_plain(io_obj, column_metadata.type, daph.num_values - num_nulls)
        schema_element = schema_helper.schema_element(column_metadata.path_in_schema[-1])
        read_values = convert_column(read_values, schema_element) \
            if schema_element.converted_type is not None else read_values
        if definition_levels:
            itr = iter(read_values)
            vals.extend([next(itr) if level == max_definition_level else None for level in definition_levels])
        else:
            vals.extend(read_values)
        if debug_logging:
            logger.debug("  Values: %s, nulls: %s", len(vals), num_nulls)

    elif daph.encoding == parquet_thrift.Encoding.PLAIN_DICTIONARY:
        # bit_width is stored as single byte.
        bit_width = struct.unpack(b"<B", io_obj.read(1))[0]
        if debug_logging:
            logger.debug("bit_width: %d", bit_width)

        dict_values_bytes = io_obj.read()
        dict_values_io_obj = io.BytesIO(dict_values_bytes)
        # read_values stores the bit-packed values. If there are definition levels and the data contains nulls,
        # the size of read_values will be less than daph.num_values
        read_values = []
        while dict_values_io_obj.tell() < len(dict_values_bytes):
            read_values.extend(encoding.read_rle_bit_packed_hybrid(
                dict_values_io_obj, bit_width, len(dict_values_bytes)))

        if definition_levels:
            itr = iter(read_values)
            # add the nulls into a new array, values, but using the definition_levels data.
            values = [dictionary[next(itr)] if level == max_definition_level else None for level in definition_levels]
        else:
            values = [dictionary[v] for v in read_values]

        # there can be extra values on the end of the array because the last bit-packed chunk may be zero-filled.
        if len(values) > daph.num_values:
            values = values[0: daph.num_values]
        vals.extend(values)

        if debug_logging:
            logger.debug("  Read %s values using PLAIN_DICTIONARY encoding and definition levels show %s nulls",
                         len(vals), num_nulls)

    else:
        raise ParquetFormatException("Unsupported encoding: %s",
                                     _get_name(parquet_thrift.Encoding, daph.encoding))
    return vals
452628
def read_header(cls, headers_path, block_height, allow_none=False):
        
        if os.path.exists(headers_path):
    
            header_parser = BlockHeaderSerializer()
            sb = os.stat( headers_path )
            if sb.st_size < BLOCK_HEADER_SIZE * block_height:
                # beyond EOF 
                if allow_none:
                    return None 
                else:
                    raise Exception('EOF on block headers')

            with open( headers_path, "rb" ) as f:
                f.seek( block_height * BLOCK_HEADER_SIZE, os.SEEK_SET )
                hdr = SPVClient.read_header_at( f )

            return hdr
        else:
            if allow_none:
                return None
            else:
                raise Exception('No such file or directory: {}'.format(headers_path))
499247
private function read_header()
    {
        $this->seek(0, IO::SEEK_SET);

        $magic = $this->read(DataIO::magic_size());

        if (strlen($magic) < DataIO::magic_size())
            throw new DataIoException(
                'Not an Avro data file: shorter than the Avro magic block');

        if (DataIO::magic() != $magic) {

            throw new DataIoException(
                sprintf('Not an Avro data file: %s does not match %s',
                    $magic, DataIO::magic()));
        }

        $this->metadata = $this->datum_reader->read_data(DataIO::metadata_schema(),
            DataIO::metadata_schema(),
            $this->decoder);
        $this->sync_marker = $this->read(DataIO::SYNC_SIZE);
    }
360494
private void loadHeader(RandomAccessFile fc) throws IOException {
		dbstats.clear();

		ByteBuffer buf = ByteBuffer.allocateDirect(HEADER_SIZE);

		long n = 0;
		int read;
		do {
			read = fc.getChannel().read(buf);
			n += read;
		} while (read > 0); // fill the buffer

		assert n == HEADER_SIZE;

		Check.state(n == HEADER_SIZE, "The file doesn't have a valid OhmDB format!");

		dbstats.put(DbStats.HEADER_SIZE, n);

		buf.rewind();

		String mark = readStr(buf, MARK.length());
		Check.state(mark.equals(MARK), "The file doesn't have a valid OhmDB format (incorrect mark)!");

		int fileVersion = buf.getInt();
		Check.state(fileVersion == 1, "Unsupported OhmDB file format version: %s", fileVersion);

		dbstats.put(DbStats.FILE_VERSION, fileVersion);

		long oddTx1 = buf.getLong();
		long oddTx2 = buf.getLong();
		long oddTx3 = buf.getLong();

		long evenTx1 = buf.getLong();
		long evenTx2 = buf.getLong();
		long evenTx3 = buf.getLong();

		long latestTx = 0;
		if (oddTx1 == oddTx2 && oddTx2 == oddTx3 && oddTx1 > latestTx) {
			latestTx = oddTx1;
			firstTxCounter = false;
		}
		if (evenTx1 == evenTx2 && evenTx2 == evenTx3 && evenTx1 > latestTx) {
			latestTx = evenTx1;
			firstTxCounter = true;
		}

		Long[] commited = { oddTx1, oddTx2, oddTx3, evenTx1, evenTx2, evenTx3 };
		dbstats.put(DbStats.COMMITED, Arrays.asList(commited));

		dbstats.put(DbStats.LATEST_TRANSACTION, latestTx);
		txCounter.set(latestTx);

		// ...

		dbstats.put(DbStats.BLOCK_SIZE, BLOCK_SIZE);

		buf.position(HEADER_SIZE - MARK2.length());
		String mark2 = readStr(buf, MARK2.length());
		Check.state(mark2.equals(MARK2), "The file doesn't have a valid OhmDB format (incorrect mark2)!");
	}
780296
public void readHeader(FileChannel channel,String forceEncoding) throws IOException {
        if(forceEncoding != null) {
            fileEncoding = forceEncoding;
        }
		// we'll read in chunks of 1K
		ReadBufferManager in = new ReadBufferManager(channel);
		// do this or GO CRAZY
		// ByteBuffers come preset to BIG_ENDIAN !
		in.order(ByteOrder.LITTLE_ENDIAN);

		// type of file.
		byte magic = in.get();
		if (magic != MAGIC) {
			log.warn("Unsupported DBF file Type "
                    + Integer.toHexString(magic));
		}

		// parse the update date information.
		int tempUpdateYear = in.get();
		int tempUpdateMonth = in.get();
		int tempUpdateDay = in.get();
		// ouch Y2K uncompliant
		if (tempUpdateYear > 90) {
			tempUpdateYear += 1900;
		} else {
			tempUpdateYear += 2000;
		}
		Calendar c = Calendar.getInstance();
		c.set(Calendar.YEAR, tempUpdateYear);
		c.set(Calendar.MONTH, tempUpdateMonth - 1);
		c.set(Calendar.DATE, tempUpdateDay);
		date = c.getTime();

		// read the number of records.
		recordCnt = in.getInt();

		// read the length of the header structure.
		// ahhh.. unsigned little-endian shorts
		// mask out the byte and or it with shifted 2nd byte
		headerLength = (in.get() & 0xff) | ((in.get() & 0xff) << 8);

		// read the length of a record
		// ahhh.. unsigned little-endian shorts
		recordLength = (in.get() & 0xff) | ((in.get() & 0xff) << 8);

		// skip / skip thesreserved bytes in the header.
		in.skip(17);
        // read Language driver
        byte lngDriver = in.get();
        String encoding = CODE_PAGE_ENCODING.get(lngDriver);
        if(encoding!=null && forceEncoding == null) {
            this.fileEncoding = encoding;
        }
        // skip reserved
        in.skip(2);

		// calculate the number of Fields in the header
		fieldCnt = (headerLength - FILE_DESCRIPTOR_SIZE - 1)
				/ FILE_DESCRIPTOR_SIZE;

		// read all of the header records
		List<DbaseField> lfields = new ArrayList<DbaseField>();
		for (int i = 0; i < fieldCnt; i++) {
			DbaseField field = new DbaseField();

			// read the field name
			byte[] buffer = new byte[11];
			in.get(buffer);
			String name = new String(buffer, fileEncoding);
			int nullPoint = name.indexOf(0);
			if (nullPoint != -1) {
				name = name.substring(0, nullPoint);
			}
			field.fieldName = name.trim();

			// read the field type
			field.fieldType = (char) in.get();

			// read the field data address, offset from the start of the record.
			field.fieldDataAddress = in.getInt();

			// read the field length in bytes
			int length = (int) in.get();
			if (length < 0) {
				length += 256;
			}
			field.fieldLength = length;

			if (length > largestFieldSize) {
				largestFieldSize = length;
			}

			// read the field decimal count in bytes
			field.decimalCount = (int) in.get();

			// rreservedvededved bytes.
			// in.skipBytes(14);
			in.skip(14);

			// some broken shapefiles have 0-length attributes. The reference
			// implementation
			// (ArcExplorer 2.0, built with MapObjects) just ignores them.
			if (field.fieldLength > 0) {
				lfields.add(field);
			}
		}

		// Last byte is a marker for the end of the field definitions.
		// in.skipBytes(1);
		in.skip(1);


		fields = new DbaseField[lfields.size()];
		fields = lfields.toArray(fields);
	}
1279054
def readddl(db):
    ''''''
    jdoc = []
    for schema in db.contents:
        j = JDOC(None,
                 schema.object_name,
                 schema.comment,
                 schema.object_type)
        jdoc.append(j)
        for table in schema.contents:
            j = JDOC(table.schema_name,
                     table.object_name,
                     table.comment,
                     table.object_type)
            for c in table.contents:
                j.params.append(c)
            jdoc.append(j)
    return jdoc
298509
def get_x():
    '''
    
    '''
    cmd = 'setxkbmap -query | grep layout'
    out = __salt__['cmd.run'](cmd, python_shell=True).split(':')
    return out[1].strip()
1145921
def read_input(buf, has_header = True):
    """"""

    # Use stdin if there is no supplied buffer
    if buf is None:
        buf = sys.stdin

    # Attempt to read a header if necessary
    header = {}
    if has_header:
        # Until we get a blank line, read "attr:val" lines, 
        # setting the values in 'header'
        last_attr = None
        while True:
            line = buf.readline()

            # remove lastcharacter (which is a newline)
            line = line[:-1] 

            # When we encounter a newline, we are done with the header
            if len(line) == 0:
                break

            colon = line.find(':')

            # If we can't find a colon, then it might be that we are
            # on a new line, and it belongs to the previous attribute
            if colon < 0:
                if last_attr:
                    header[last_attr] = header[last_attr] + '\n' + urllib.parse.unquote(line)
                else:
                    continue

            # extract it and set value in settings
            last_attr = attr = line[:colon]
            val  = urllib.parse.unquote(line[colon+1:])
            header[attr] = val

    return buf, header
562554
def _read_header(self):
        

        self._header_pos = self.fp.tell()

        line = self.fp.readline('20sffii')
        modelname, res0, res1, halfpolar, center180 = line
        self._attributes.update({
            "modelname": str(modelname, 'utf-8').strip(),
            "halfpolar": halfpolar,
            "center180": center180,
            "res": (res0, res1)
        })
        self.__setattr__('modelname', modelname)
        self.__setattr__('res', (res0, res1))
        self.__setattr__('halfpolar', halfpolar)
        self.__setattr__('center180', center180)

        # Re-wind the file
        self.fp.seek(self._header_pos)
232032
def get_xf  # :nodoc:

    # Local Variable
    #    record;     # Record identifier
    #    length;     # Number of bytes to follow
    #
    #    ifnt;       # Index to FONT record
    #    ifmt;       # Index to FORMAT record
    #    style;      # Style and other options
    #    align;      # Alignment
    #    indent;     #
    #    icv;        # fg and bg pattern colors
    #    border1;    # Border line options
    #    border2;    # Border line options
    #    border3;    # Border line options

    # Set the type of the XF record and some of the attributes.
    if @type == 0xFFF5 then
      style = 0xFFF5
    else
      style  = @locked
      style |= @hidden << 1
    end

    # Flags to indicate if attributes have been set.
    atr_num  = (@num_format   != 0) ? 1 : 0
    atr_fnt  = (@font_index   != 0) ? 1 : 0
    atr_alc  = (@text_h_align != 0 ||
                @text_v_align != 2 ||
                @shrink       != 0 ||
                @merge_range  != 0 ||
                @text_wrap    != 0 ||
                @indent       != 0) ? 1 : 0
    atr_bdr  = (@bottom       != 0 ||
                @top          != 0 ||
                @left         != 0 ||
                @right        != 0 ||
                @diag_type    != 0) ? 1 : 0
    atr_pat  = (@fg_color     != 0x40 ||
                @bg_color     != 0x41 ||
                @pattern      != 0x00) ? 1 : 0
    atr_prot = (@hidden       != 0 ||
                @locked       != 1) ? 1 : 0

    # Set attribute changed flags for the style formats.
    if @xf_index != 0 and @type == 0xFFF5
      if @xf_index >= 16
        atr_num    = 0
        atr_fnt    = 1
      else
        atr_num    = 1
        atr_fnt    = 0
      end
      atr_alc    = 1
      atr_bdr    = 1
      atr_pat    = 1
      atr_prot   = 1
    end

    # Set a default diagonal border style if none was specified.
    @diag_border = 1 if (@diag_border ==0 and @diag_type != 0)

    # Reset the default colours for the non-font properties
    @fg_color     = 0x40 if @fg_color     == 0x7FFF
    @bg_color     = 0x41 if @bg_color     == 0x7FFF
    @bottom_color = 0x40 if @bottom_color == 0x7FFF
    @top_color    = 0x40 if @top_color    == 0x7FFF
    @left_color   = 0x40 if @left_color   == 0x7FFF
    @right_color  = 0x40 if @right_color  == 0x7FFF
    @diag_color   = 0x40 if @diag_color   == 0x7FFF

    # Zero the default border colour if the border has not been set.
    @bottom_color = 0 if @bottom    == 0
    @top_color    = 0 if @top       == 0
    @right_color  = 0 if @right     == 0
    @left_color   = 0 if @left      == 0
    @diag_color   = 0 if @diag_type == 0

    # The following 2 logical statements take care of special cases in relation
    # to cell colours and patterns:
    # 1. For a solid fill (_pattern == 1) Excel reverses the role of foreground
    #    and background colours.
    # 2. If the user specifies a foreground or background colour without a
    #    pattern they probably wanted a solid fill, so we fill in the defaults.
    #
    if (@pattern  <= 0x01 && @bg_color != 0x41 && @fg_color == 0x40)
      @fg_color = @bg_color
      @bg_color = 0x40
      @pattern  = 1
    end

    if (@pattern <= 0x01 && @bg_color == 0x41 && @fg_color != 0x40)
      @bg_color = 0x40
      @pattern  = 1
    end

    # Set default alignment if indent is set.
    @text_h_align = 1 if @indent != 0 and @text_h_align == 0


    record         = 0x00E0
    length         = 0x0014

    ifnt           = @font_index
    ifmt           = @num_format


    align          = @text_h_align
    align         |= @text_wrap     << 3
    align         |= @text_v_align  << 4
    align         |= @text_justlast << 7
    align         |= @rotation      << 8

    indent         = @indent
    indent        |= @shrink        << 4
    indent        |= @merge_range   << 5
    indent        |= @reading_order << 6
    indent        |= atr_num        << 10
    indent        |= atr_fnt        << 11
    indent        |= atr_alc        << 12
    indent        |= atr_bdr        << 13
    indent        |= atr_pat        << 14
    indent        |= atr_prot       << 15


    border1        = @left
    border1       |= @right         << 4
    border1       |= @top           << 8
    border1       |= @bottom        << 12

    border2        = @left_color
    border2       |= @right_color   << 7
    border2       |= @diag_type     << 14

    border3        = @top_color
    border3       |= @bottom_color  << 7
    border3       |= @diag_color    << 14
    border3       |= @diag_border   << 21
    border3       |= @pattern       << 26

    icv            = @fg_color
    icv           |= @bg_color      << 7

    header = [record, length].pack("vv")
    data   = [ifnt, ifmt, style, align, indent,
              border1, border2, border3, icv].pack("vvvvvvvVv")

    header + data
  end
836335
def _encode_dbref(name, value, check_keys, opts):
    """"""
    buf = bytearray(b"\x03" + name + b"\x00\x00\x00\x00")
    begin = len(buf) - 4

    buf += _name_value_to_bson(b"$ref\x00",
                               value.collection, check_keys, opts)
    buf += _name_value_to_bson(b"$id\x00",
                               value.id, check_keys, opts)
    if value.database is not None:
        buf += _name_value_to_bson(
            b"$db\x00", value.database, check_keys, opts)
    for key, val in iteritems(value._DBRef__kwargs):
        buf += _element_to_bson(key, val, check_keys, opts)

    buf += b"\x00"
    buf[begin:begin + 4] = _PACK_INT(len(buf) - begin)
    return bytes(buf)
398646
def get_db(data):
    
    snpeff_db = utils.get_in(data, ("genome_resources", "aliases", "snpeff"))
    snpeff_base_dir = None
    if snpeff_db:
        snpeff_base_dir = utils.get_in(data, ("reference", "snpeff"))
        if not (isinstance(snpeff_base_dir, six.string_types) and os.path.isdir(snpeff_base_dir)):
            snpeff_base_dir = utils.get_in(data, ("reference", "snpeff", snpeff_db))
        if not snpeff_base_dir:
            # We need to mask '.' characters for CWL/WDL processing, check for them here
            snpeff_base_dir = utils.get_in(data, ("reference", "snpeff", snpeff_db.replace(".", "_")))
            if snpeff_base_dir:
                snpeff_db = snpeff_db.replace("_", ".")
        if isinstance(snpeff_base_dir, dict) and snpeff_base_dir.get("base"):
            snpeff_base_dir = snpeff_base_dir["base"]
        if (snpeff_base_dir and isinstance(snpeff_base_dir, six.string_types) and os.path.isfile(snpeff_base_dir)):
            snpeff_base_dir = os.path.dirname(snpeff_base_dir)
        if (snpeff_base_dir and isinstance(snpeff_base_dir, six.string_types)
              and snpeff_base_dir.endswith("%s%s" % (os.path.sep, snpeff_db))):
            snpeff_base_dir = os.path.dirname(snpeff_base_dir)
        if not snpeff_base_dir:
            ref_file = utils.get_in(data, ("reference", "fasta", "base"))
            snpeff_base_dir = utils.safe_makedir(os.path.normpath(os.path.join(
                os.path.dirname(os.path.dirname(ref_file)), "snpeff")))
            # back compatible retrieval of genome from installation directory
            if "config" in data and not os.path.exists(os.path.join(snpeff_base_dir, snpeff_db)):
                snpeff_base_dir, snpeff_db = _installed_snpeff_genome(snpeff_db, data["config"])
        if snpeff_base_dir.endswith("/%s" % snpeff_db):
            snpeff_base_dir = os.path.dirname(snpeff_base_dir)
    return snpeff_db, snpeff_base_dir
58892
function dbf_to_aoa(buf, opts)/*:AOA*/ {
	var out/*:AOA*/ = [];
	/* TODO: browser based */
	var d/*:Block*/ = (new_raw_buf(1)/*:any*/);
	switch(opts.type) {
		case 'base64': d = s2a(Base64.decode(buf)); break;
		case 'binary': d = s2a(buf); break;
		case 'buffer':
		case 'array': d = buf; break;
	}
	prep_blob(d, 0);
	/* header */
	var ft = d.read_shift(1);
	var memo = false;
	var vfp = false, l7 = false;
	switch(ft) {
		case 0x02: case 0x03: break;
		case 0x30: vfp = true; memo = true; break;
		case 0x31: vfp = true; break;
		case 0x83: memo = true; break;
		case 0x8B: memo = true; break;
		case 0x8C: memo = true; l7 = true; break;
		case 0xF5: memo = true; break;
		default: throw new Error("DBF Unsupported Version: " + ft.toString(16));
	}
	var /*filedate = new Date(),*/ nrow = 0, fpos = 0;
	if(ft == 0x02) nrow = d.read_shift(2);
	/*filedate = new Date(d.read_shift(1) + 1900, d.read_shift(1) - 1, d.read_shift(1));*/d.l += 3;
	if(ft != 0x02) nrow = d.read_shift(4);
	if(ft != 0x02) fpos = d.read_shift(2);
	var rlen = d.read_shift(2);

	var /*flags = 0,*/ current_cp = 1252;
	if(ft != 0x02) {
	d.l+=16;
	/*flags = */d.read_shift(1);
	//if(memo && ((flags & 0x02) === 0)) throw new Error("DBF Flags " + flags.toString(16) + " ft " + ft.toString(16));

	/* codepage present in FoxPro */
	if(d[d.l] !== 0) current_cp = dbf_codepage_map[d[d.l]];
	d.l+=1;

	d.l+=2;
	}
	if(l7) d.l += 36;
/*:: type DBFField = { name:string; len:number; type:string; } */
	var fields/*:Array<DBFField>*/ = [], field/*:DBFField*/ = ({}/*:any*/);
	var hend = fpos - 10 - (vfp ? 264 : 0), ww = l7 ? 32 : 11;
	while(ft == 0x02 ? d.l < d.length && d[d.l] != 0x0d: d.l < hend) {
		field = ({}/*:any*/);
		field.name = cptable.utils.decode(current_cp, d.slice(d.l, d.l+ww)).replace(/[\u0000\r\n].*$/g,"");
		d.l += ww;
		field.type = String.fromCharCode(d.read_shift(1));
		if(ft != 0x02 && !l7) field.offset = d.read_shift(4);
		field.len = d.read_shift(1);
		if(ft == 0x02) field.offset = d.read_shift(2);
		field.dec = d.read_shift(1);
		if(field.name.length) fields.push(field);
		if(ft != 0x02) d.l += l7 ? 13 : 14;
		switch(field.type) {
			case 'B': // VFP Double
				if((!vfp || field.len != 8) && opts.WTF) console.log('Skipping ' + field.name + ':' + field.type);
				break;
			case 'G': // General
			case 'P': // Picture
				if(opts.WTF) console.log('Skipping ' + field.name + ':' + field.type);
				break;
			case 'C': // character
			case 'D': // date
			case 'F': // floating point
			case 'I': // long
			case 'L': // boolean
			case 'M': // memo
			case 'N': // number
			case 'O': // double
			case 'T': // datetime
			case 'Y': // currency
			case '0': // VFP _NullFlags
			case '@': // timestamp
			case '+': // autoincrement
				break;
			default: throw new Error('Unknown Field Type: ' + field.type);
		}
	}
	if(d[d.l] !== 0x0D) d.l = fpos-1;
	else if(ft == 0x02) d.l = 0x209;
	if(ft != 0x02) {
		if(d.read_shift(1) !== 0x0D) throw new Error("DBF Terminator not found " + d.l + " " + d[d.l]);
		d.l = fpos;
	}
	/* data */
	var R = 0, C = 0;
	out[0] = [];
	for(C = 0; C != fields.length; ++C) out[0][C] = fields[C].name;
	while(nrow-- > 0) {
		if(d[d.l] === 0x2A) { d.l+=rlen; continue; }
		++d.l;
		out[++R] = []; C = 0;
		for(C = 0; C != fields.length; ++C) {
			var dd = d.slice(d.l, d.l+fields[C].len); d.l+=fields[C].len;
			prep_blob(dd, 0);
			var s = cptable.utils.decode(current_cp, dd);
			switch(fields[C].type) {
				case 'C':
					out[R][C] = cptable.utils.decode(current_cp, dd);
					out[R][C] = out[R][C].trim();
					break;
				case 'D':
					if(s.length === 8) out[R][C] = new Date(+s.slice(0,4), +s.slice(4,6)-1, +s.slice(6,8));
					else out[R][C] = s;
					break;
				case 'F': out[R][C] = parseFloat(s.trim()); break;
				case '+': case 'I': out[R][C] = l7 ? dd.read_shift(-4, 'i') ^ 0x80000000 : dd.read_shift(4, 'i'); break;
				case 'L': switch(s.toUpperCase()) {
					case 'Y': case 'T': out[R][C] = true; break;
					case 'N': case 'F': out[R][C] = false; break;
					case ' ': case '?': out[R][C] = false; break; /* NOTE: technically uninitialized */
					default: throw new Error("DBF Unrecognized L:|" + s + "|");
					} break;
				case 'M': /* TODO: handle memo files */
					if(!memo) throw new Error("DBF Unexpected MEMO for type " + ft.toString(16));
					out[R][C] = "##MEMO##" + (l7 ? parseInt(s.trim(), 10): dd.read_shift(4));
					break;
				case 'N': out[R][C] = +s.replace(/\u0000/g,"").trim(); break;
				case '@': out[R][C] = new Date(dd.read_shift(-8, 'f') - 0x388317533400); break;
				case 'T': out[R][C] = new Date((dd.read_shift(4) - 0x253D8C) * 0x5265C00 + dd.read_shift(4)); break;
				case 'Y': out[R][C] = dd.read_shift(4,'i')/1e4; break;
				case 'O': out[R][C] = -dd.read_shift(-8, 'f'); break;
				case 'B': if(vfp && fields[C].len == 8) { out[R][C] = dd.read_shift(8,'f'); break; }
					/* falls through */
				case 'G': case 'P': dd.l += fields[C].len; break;
				case '0':
					if(fields[C].name === '_NullFlags') break;
					/* falls through */
				default: throw new Error("DBF Unsupported data type " + fields[C].type);
			}
		}
	}
	if(ft != 0x02) if(d.l < d.length && d[d.l++] != 0x1A) throw new Error("DBF EOF Marker missing " + (d.l-1) + " of " + d.length + " " + d[d.l-1].toString(16));
	if(opts && opts.sheetRows) out = out.slice(0, opts.sheetRows);
	return out;
}
1114679
def decode(self, buf):
        
        index = 0
        if self.length > len(buf):
            raise exc.ParseError("Insufficient data for reading header.") 
        for (name, nbytes, typecode) in self.format:
            string = buf[index:index + nbytes]
            index += nbytes
            value = struct.unpack('<' + typecode, string)[0]
            setattr(self, name, value)
        if const.DB_SIGNATURE1 != self.signature1 or \
                const.DB_SIGNATURE2 != self.signature2:
            msg = 'Bad signatures: {0} {0}'.format(hex(self.signature1),
                                                   hex(self.signature2))
            raise exc.InvalidDatabase(msg)
1084522
def _process_pub_dbxref(self, limit):
        

        if self.test_mode:
            graph = self.testgraph
        else:
            graph = self.graph
        model = Model(graph)
        raw = '/'.join((self.rawdir, 'pub_dbxref'))
        LOG.info("processing pub_dbxref")

        line_counter = 0

        with open(raw, 'r') as f:
            filereader = csv.reader(f, delimiter='\t', quotechar='\"')
            f.readline()  # read the header row; skip
            for line in filereader:
                (pub_dbxref_id, pub_id, dbxref_id, is_current) = line
                # 49648	43222	395730	t

                pub_key = pub_id
                pub_id = self.idhash['publication'][pub_key]

                if self.test_mode and int(pub_key) not in self.test_keys['pub']:
                    continue

                # get any dbxrefs for pubs, including pmids and dois
                dbxref_key = dbxref_id
                if str(dbxref_key) in self.dbxrefs:
                    dbxrefs = self.dbxrefs[str(dbxref_key)]
                    # pub_dbs = [75, 51, 76, 95, 126]
                    pmid_ids = [50, 77, 275, 286, 347]
                    # flybase_ids = [4]  # TODO unused
                    isbn = [75, 51]
                    for d in dbxrefs:
                        dbxref_id = None
                        if int(d) in pmid_ids:
                            if re.match(r'^PMID', dbxrefs[d]):
                                dbxref_id = dbxrefs[d].strip()
                            else:
                                dbxref_id = 'PMID:'+dbxrefs[d].strip()
                            model.makeLeader(dbxref_id)
                        elif int(d) in isbn:
                            dbxref_id = 'ISBN:'+dbxrefs[d].strip()
                        elif int(d) == 161:
                            dbxref_id = 'DOI:'+dbxrefs[d].strip()
                        # elif int(d) == 4:
                        #     dbxref_id = 'FlyBase:'+dbxrefs[d].strip()

                        if dbxref_id is not None:
                            reference = Reference(
                                graph, dbxref_id, self.globaltt['publication'])
                            reference.addRefToGraph()
                            model.addSameIndividual(pub_id, dbxref_id)
                            line_counter += 1

                if not self.test_mode and limit is not None and line_counter > limit:
                    break

        return