// Chunks returns a new ChunkReader against the block data.
353792
func (pb *Block) Chunks() (ChunkReader, error) {
	if err := pb.startRead(); err != nil {
		return nil, err
	}
	return blockChunkReader{ChunkReader: pb.chunkr, b: pb}, nil
}
507842
func (reader *DataFileReader) NextBlock() error {
	blockCount, err := reader.dec.ReadLong()
	if err != nil {
		return err
	}

	blockSize, err := reader.dec.ReadLong()
	if err != nil {
		return err
	}

	if blockSize > math.MaxInt32 || blockSize < 0 {
		return fmt.Errorf("Block size invalid or too large: %d", blockSize)
	}

	block := reader.block
	if block.Data == nil || int64(len(block.Data)) < blockSize {
		block.Data = make([]byte, blockSize)
	}
	block.BlockRemaining = blockCount
	block.NumEntries = blockCount
	block.BlockSize = int(blockSize)
	err = reader.dec.ReadFixedWithBounds(block.Data, 0, int(block.BlockSize))
	if err != nil {
		return err
	}
	syncBuffer := make([]byte, syncSize)
	err = reader.dec.ReadFixed(syncBuffer)
	if err != nil {
		return err
	}
	if !bytes.Equal(syncBuffer, reader.header.Sync) {
		return InvalidSync
	}
	reader.blockDecoder.SetBlock(reader.block)

	return nil
}
472009
func (r *Reader) ReadBytes() (b []byte) {
	tag := r.readByte()
	decoder := sliceDecoders[tag]
	if decoder == nil {
		castError(tag, "[]byte")
	}
	decoder(r, reflect.ValueOf(&b).Elem())
	return
}
596504
func (ef *File) FetchChunk(offset int64) (Chunk, error) {
	ef.Lock()
	defer ef.Unlock()
	c := NewChunk()
	GoToSeeker(ef.file, offset)
	c.Offset = offset
	c.Data = make([]byte, ChunkSize)
	if _, err := ef.file.Read(c.Data); err != nil {
		return c, err
	}
	reader := bytes.NewReader(c.Data)
	c.ParseChunkHeader(reader)
	// Go to after Header
	GoToSeeker(reader, int64(c.Header.SizeHeader))
	c.ParseStringTable(reader)
	if err := c.ParseTemplateTable(reader); err != nil {
		return c, err
	}
	if err := c.ParseEventOffsets(reader); err != nil {
		return c, err
	}
	return c, nil
}
225786
func (ch *Chunk) Read(p []byte) (n int, err error) {
	if ch == nil || ch.R == nil {
		return 0, errors.New("nil chunk/reader pointer")
	}
	n, err = ch.R.Read(p)
	ch.Pos += n
	return n, err
}
1221968
func (c *LeveledCompactor) populateBlock(blocks []BlockReader, meta *BlockMeta, indexw IndexWriter, chunkw ChunkWriter) (err error) {
	if len(blocks) == 0 {
		return errors.New("cannot populate block from no readers")
	}

	var (
		set         ChunkSeriesSet
		allSymbols  = make(map[string]struct{}, 1<<16)
		closers     = []io.Closer{}
		overlapping bool
	)
	defer func() {
		var merr tsdb_errors.MultiError
		merr.Add(err)
		merr.Add(closeAll(closers))
		err = merr.Err()
		c.metrics.populatingBlocks.Set(0)
	}()
	c.metrics.populatingBlocks.Set(1)

	globalMaxt := blocks[0].MaxTime()
	for i, b := range blocks {
		select {
		case <-c.ctx.Done():
			return c.ctx.Err()
		default:
		}

		if !overlapping {
			if i > 0 && b.MinTime() < globalMaxt {
				c.metrics.overlappingBlocks.Inc()
				overlapping = true
				level.Warn(c.logger).Log("msg", "found overlapping blocks during compaction", "ulid", meta.ULID)
			}
			if b.MaxTime() > globalMaxt {
				globalMaxt = b.MaxTime()
			}
		}

		indexr, err := b.Index()
		if err != nil {
			return errors.Wrapf(err, "open index reader for block %s", b)
		}
		closers = append(closers, indexr)

		chunkr, err := b.Chunks()
		if err != nil {
			return errors.Wrapf(err, "open chunk reader for block %s", b)
		}
		closers = append(closers, chunkr)

		tombsr, err := b.Tombstones()
		if err != nil {
			return errors.Wrapf(err, "open tombstone reader for block %s", b)
		}
		closers = append(closers, tombsr)

		symbols, err := indexr.Symbols()
		if err != nil {
			return errors.Wrap(err, "read symbols")
		}
		for s := range symbols {
			allSymbols[s] = struct{}{}
		}

		all, err := indexr.Postings(index.AllPostingsKey())
		if err != nil {
			return err
		}
		all = indexr.SortedPostings(all)

		s := newCompactionSeriesSet(indexr, chunkr, tombsr, all)

		if i == 0 {
			set = s
			continue
		}
		set, err = newCompactionMerger(set, s)
		if err != nil {
			return err
		}
	}

	// We fully rebuild the postings list index from merged series.
	var (
		postings = index.NewMemPostings()
		values   = map[string]stringset{}
		i        = uint64(0)
	)

	if err := indexw.AddSymbols(allSymbols); err != nil {
		return errors.Wrap(err, "add symbols")
	}

	for set.Next() {
		select {
		case <-c.ctx.Done():
			return c.ctx.Err()
		default:
		}

		lset, chks, dranges := set.At() // The chunks here are not fully deleted.
		if overlapping {
			// If blocks are overlapping, it is possible to have unsorted chunks.
			sort.Slice(chks, func(i, j int) bool {
				return chks[i].MinTime < chks[j].MinTime
			})
		}

		// Skip the series with all deleted chunks.
		if len(chks) == 0 {
			continue
		}

		for i, chk := range chks {
			if chk.MinTime < meta.MinTime || chk.MaxTime > meta.MaxTime {
				return errors.Errorf("found chunk with minTime: %d maxTime: %d outside of compacted minTime: %d maxTime: %d",
					chk.MinTime, chk.MaxTime, meta.MinTime, meta.MaxTime)
			}

			if len(dranges) > 0 {
				// Re-encode the chunk to not have deleted values.
				if !chk.OverlapsClosedInterval(dranges[0].Mint, dranges[len(dranges)-1].Maxt) {
					continue
				}
				newChunk := chunkenc.NewXORChunk()
				app, err := newChunk.Appender()
				if err != nil {
					return err
				}

				it := &deletedIterator{it: chk.Chunk.Iterator(), intervals: dranges}
				for it.Next() {
					ts, v := it.At()
					app.Append(ts, v)
				}

				chks[i].Chunk = newChunk
			}
		}

		mergedChks := chks
		if overlapping {
			mergedChks, err = chunks.MergeOverlappingChunks(chks)
			if err != nil {
				return errors.Wrap(err, "merge overlapping chunks")
			}
		}
		if err := chunkw.WriteChunks(mergedChks...); err != nil {
			return errors.Wrap(err, "write chunks")
		}

		if err := indexw.AddSeries(i, lset, mergedChks...); err != nil {
			return errors.Wrap(err, "add series")
		}

		meta.Stats.NumChunks += uint64(len(mergedChks))
		meta.Stats.NumSeries++
		for _, chk := range mergedChks {
			meta.Stats.NumSamples += uint64(chk.Chunk.NumSamples())
		}

		for _, chk := range mergedChks {
			if err := c.chunkPool.Put(chk.Chunk); err != nil {
				return errors.Wrap(err, "put chunk")
			}
		}

		for _, l := range lset {
			valset, ok := values[l.Name]
			if !ok {
				valset = stringset{}
				values[l.Name] = valset
			}
			valset.set(l.Value)
		}
		postings.Add(i, lset)

		i++
	}
	if set.Err() != nil {
		return errors.Wrap(set.Err(), "iterate compaction set")
	}

	s := make([]string, 0, 256)
	for n, v := range values {
		s = s[:0]

		for x := range v {
			s = append(s, x)
		}
		if err := indexw.WriteLabelIndex([]string{n}, s); err != nil {
			return errors.Wrap(err, "write label index")
		}
	}

	for _, l := range postings.SortedKeys() {
		if err := indexw.WritePostings(l.Name, l.Value, postings.Get(l.Name, l.Value)); err != nil {
			return errors.Wrap(err, "write postings")
		}
	}
	return nil
}
478070
func (r *reader) Read(b []byte) (int, error) {
	if len(r.buf) == 0 {
		if r.bufToReturn != nil {
			r.chunkPool.Put(r.bufToReturn)
			r.bufToReturn = nil
		}
		if r.err != nil {
			// Make sure any goroutines are done. Ignore
			// the error since it will just be r.err.
			r.Close()
			return 0, r.err
		}
		resp, ok := <-r.chunkRespc
		if !ok {
			glog.Fatal("chunkRespc channel has been closed without a final error")
		}
		if resp.err != nil {
			r.err = resp.err
		}
		r.buf = resp.chunk
		r.bufToReturn = resp.chunk[:cap(resp.chunk)]
	}
	n := copy(b, r.buf)
	r.buf = r.buf[n:]
	if len(r.buf) == 0 {
		return n, r.err
	}
	return n, nil
}
143446
func (t *Tree) Blocks() map[string]*BlockNode {
	return t.blocks[len(t.blocks)-1]
}
1290039
func (s *StreamingReader) Read(buf []byte) (int, error) {
	switch {
	// After the last chunk is read from underlying reader, we
	// never re-fill s.buf.
	case s.done:

	// s.buf will be (re-)filled with next chunk when has lesser
	// bytes than asked for.
	case s.buf.Len() < len(buf):
		s.chunkBufLen = 0
		for {
			n1, err := s.baseReadCloser.Read(s.chunkBuf[s.chunkBufLen:])
			// Usually we validate `err` first, but in this case
			// we are validating n > 0 for the following reasons.
			//
			// 1. n > 0, err is one of io.EOF, nil (near end of stream)
			// A Reader returning a non-zero number of bytes at the end
			// of the input stream may return either err == EOF or err == nil
			//
			// 2. n == 0, err is io.EOF (actual end of stream)
			//
			// Callers should always process the n > 0 bytes returned
			// before considering the error err.
			if n1 > 0 {
				s.chunkBufLen += n1
				s.bytesRead += int64(n1)

				if s.chunkBufLen == payloadChunkSize ||
					(s.chunkNum == s.totalChunks-1 &&
						s.chunkBufLen == s.lastChunkSize) {
					// Sign the chunk and write it to s.buf.
					s.signChunk(s.chunkBufLen)
					break
				}
			}
			if err != nil {
				if err == io.EOF {
					// No more data left in baseReader - last chunk.
					// Done reading the last chunk from baseReader.
					s.done = true

					// bytes read from baseReader different than
					// content length provided.
					if s.bytesRead != s.contentLen {
						return 0, io.ErrUnexpectedEOF
					}

					// Sign the chunk and write it to s.buf.
					s.signChunk(0)
					break
				}
				return 0, err
			}

		}
	}
	return s.buf.Read(buf)
}
340708
func (i *Index) Chunks(rid, beg, end int) ([]bgzf.Chunk, error) {
	if rid < 0 || rid >= len(i.Refs) {
		return nil, index.ErrNoReference
	}
	i.sort()
	ref := i.Refs[rid]

	iv := beg / TileWidth
	if iv >= len(ref.Intervals) {
		return nil, index.ErrInvalid
	}

	// Collect candidate chunks according to the scheme described in
	// the SAM spec under section 5 Indexing BAM.
	var chunks []bgzf.Chunk
	for _, b := range OverlappingBinsFor(beg, end) {
		c := sort.Search(len(ref.Bins), func(i int) bool { return ref.Bins[i].Bin >= b })
		if c < len(ref.Bins) && ref.Bins[c].Bin == b {
			for _, chunk := range ref.Bins[c].Chunks {
				// Here we check all tiles starting from the left end of the
				// query region until we get a non-zero offset. The spec states
				// that we only need to check tiles that contain beg. That is
				// not correct since we may have no alignments at the left end
				// of the query region.
				chunkEndOffset := vOffset(chunk.End)
				haveNonZero := false
				for j, tile := range ref.Intervals[iv:] {
					// If we have found a non-zero tile, all subsequent active
					// tiles must also be non-zero, so skip zero tiles.
					if haveNonZero && isZero(tile) {
						continue
					}
					haveNonZero = true
					tbeg := (j + iv) * TileWidth
					tend := tbeg + TileWidth
					// We allow adjacent alignment since samtools behaviour here
					// has always irritated me and it is cheap to discard these
					// later if they are not wanted.
					if tend >= beg && tbeg <= end && chunkEndOffset > vOffset(tile) {
						chunks = append(chunks, chunk)
						break
					}
				}
			}
		}
	}

	// Sort and merge overlaps.
	if !sort.IsSorted(byBeginOffset(chunks)) {
		sort.Sort(byBeginOffset(chunks))
	}

	return chunks, nil
}
808409
func (l *State) Load(r io.Reader, chunkName string, mode string) error {
	if chunkName == "" {
		chunkName = "?"
	}

	if err := protectedParser(l, r, chunkName, mode); err != nil {
		return err
	}

	if f := l.stack[l.top-1].(*luaClosure); f.upValueCount() == 1 {
		f.setUpValue(0, l.global.registry.atInt(RegistryIndexGlobals))
	}
	return nil
}
581642
func New(rd io.Reader, pol Pol) *Chunker {
	return NewWithBoundaries(rd, pol, MinSize, MaxSize)
}
734264
func (c *Parser) NextChunk() (*Chunk, error) {
	if c == nil {
		return nil, errors.New("can't calculate the duration of a nil pointer")
	}
	id, size, err := c.IDnSize()
	if err != nil {
		return nil, err
	}

	// all RIFF chunks (including WAVE "data" chunks) must be word aligned.
	// If the data uses an odd number of bytes, a padding byte with a value of zero must be placed at the end of the sample data.
	// The "data" chunk header's size should not include this byte.
	if size%2 == 1 {
		size++
	}

	ch := &Chunk{
		ID:   id,
		Size: int(size),
		R:    c.r,
	}
	return ch, nil
}
267266
func (c *chunkedSeq) ChunkedFirst() iseq.Chunk {
	return newArrayChunk2(c.node, c.offset)
}
1097868
func (bs *BlockStore) Blocks(start, end int64, iter func(*Block) (stop bool)) (stopped bool, err error) {
	if end > 0 && start >= end {
		return false, fmt.Errorf("end height must be strictly greater than start height")
	}
	if start <= 0 {
		// From first block
		start = 1
	}
	if end < 0 {
		// -1 means include the very last block so + 1 for offset
		end = bs.Height() + end + 1
	}

	for height := start; height <= end; height++ {
		block, err := bs.Block(height)
		if err != nil {
			return false, err
		}
		if iter(block) {
			return true, nil
		}
	}

	return false, nil
}
1116025
func (s *state) block(node ast.Node) string {
	var buf bytes.Buffer
	(&state{
	    wr: &buf,
	    scope: s.scope,
	    options: s.options,
	    funcsCalled: s.funcsCalled,
	    funcsInFile: s.funcsInFile,
	}).walk(node)
	return buf.String()
}
1328379
func (d *Decoder) NextChunk() (*chunk.Reader, error) {
	var err error

	if err = d.readHeaders(); err != nil {
		return nil, err
	}

	var (
		id   [4]byte
		size int64
	)

	id, size, d.err = d.iDnSize()
	if d.err != nil {
		if d.err == io.EOF || d.err == io.ErrUnexpectedEOF {
			return nil, io.EOF
		}
		return nil, fmt.Errorf("error reading chunk header - %v", d.err)
	}

	c := &chunk.Reader{
		ID:   id,
		Size: int(size),
		R:    io.LimitReader(d.r, int64(size)),
	}

	return c, d.err
}