Calculates noise properties and saves values to pickle.
    chunk defines window for measurement. at least one measurement always made.
9017
def noisepickle(d, data, u, v, w, chunk=200):
    

    if d['savenoise']:
        noisefile = getnoisefile(d)

        if os.path.exists(noisefile):
            logger.warn('noisefile %s already exists' % noisefile)
        else:
            nints = len(data)
            chunk = min(chunk, nints)  # ensure at least one measurement
            results = []

            rr = range(0, nints, chunk)
            if len(rr) == 1: rr.append(1)   # hack. need to make sure it iterates for nints=1 case
            for i in range(len(rr)-1):
                imid = (rr[i]+rr[i+1])/2
                noiseperbl = estimate_noiseperbl(data[rr[i]:rr[i+1]])
                imstd = sample_image(d, data, u, v, w, imid, verbose=0).std()
                zerofrac = float(len(n.where(data[rr[i]:rr[i+1]] == 0j)[0]))/data[rr[i]:rr[i+1]].size
                results.append( (d['segment'], noiseperbl, zerofrac, imstd) )

            with open(noisefile, 'a') as pkl:
                pickle.dump(results, pkl)
            logger.info('Wrote %d noise measurement%s to %s.' % (len(results), 's'[:len(results)-1], noisefile))
1025853
def get_chunk(self, size=None):
        
        if size is None:
            size = self._chunksize
        return self.read(nrows=size)
1347145
function Stats() {
  var s = this;
  this.metrics = [];
  this.add = function(key, value, ts) {
    s.metrics.push(new Metric(key, value, ts));
  };

  this.toText = function() {
    return s.metrics.map(function(m) { return m.toText(); }).join('\n') + '\n';
  };

  this.toPickle = function() {
    var body = MARK + LIST + s.metrics.map(function(m) { return m.toPickle(); }).join('') + STOP;

    // The first four bytes of the graphite pickle format
    // contain the length of the rest of the payload.
    // We use Buffer because this is binary data.
    var buf = new Buffer(4 + body.length);

    buf.writeUInt32BE(body.length,0);
    buf.write(body,4);

    return buf;
  };
}
152935
def wiggle(self,noiseLevel=.1):
        """"""
        noise=(np.random.rand(*self.data.shape))-.5
        self.data=self.data+noise*noiseLevel
847034
def measureSize(self, diff, chunkSize):
        
        self._fileSystemSync()

        sendContext = self.butter.send(
            self.getSendPath(diff.toVol),
            self.getSendPath(diff.fromVol),
            diff,
            showProgress=self.showProgress is not False,
            allowDryRun=False,
        )

        class _Measure(io.RawIOBase):

            def __init__(self, estimatedSize, showProgress):
                self.totalSize = None
                self.progress = progress.DisplayProgress(estimatedSize) if showProgress else None

            def __enter__(self):
                self.totalSize = 0
                if self.progress:
                    self.progress.__enter__()
                return self

            def __exit__(self, exceptionType, exceptionValue, traceback):
                if self.progress:
                    self.progress.__exit__(exceptionType, exceptionValue, traceback)
                return False  # Don't supress exception

            def writable(self):
                return True

            def write(self, bytes):
                self.totalSize += len(bytes)
                if self.progress:
                    self.progress.update(self.totalSize)

        logger.info("Measuring %s", diff)

        measure = _Measure(diff.size, self.showProgress is not False)
        Store.transfer(sendContext, measure, chunkSize)

        diff.setSize(measure.totalSize, False)

        for path in self.getPaths(diff.toVol):
            path = self._fullPath(path) + Store.theInfoExtension

            with open(path, "a") as infoFile:
                diff.toVol.writeInfoLine(infoFile, diff.fromUUID, measure.totalSize)
388507
def save_beat(
        self,
        output_file_name,
        frequencys,
        play_time,
        sample_rate=44100,
        volume=0.01
    ):
        '''
        
        '''
        left_frequency, right_frequency = frequencys
        left_chunk = self.__create_chunk(left_frequency, play_time, sample_rate)
        right_chunk = self.__create_chunk(right_frequency, play_time, sample_rate)
        frame_list = self.read_stream(left_chunk, right_chunk, volume)

        wf = wave.open(output_file_name, 'wb')
        wf.setparams((2, 2, sample_rate, 0, 'NONE', 'not compressed'))
        wf.writeframes(b''.join(frame_list))
        wf.close()
1307779
def _set_scores(self):
        
        anom_scores = {}
        self._generate_SAX()
        self._construct_all_SAX_chunk_dict()
        length = self.time_series_length
        lws = self.lag_window_size
        fws = self.future_window_size

        for i, timestamp in enumerate(self.time_series.timestamps):
            if i < lws or i > length - fws:
                anom_scores[timestamp] = 0
            else:
                anom_scores[timestamp] = self._compute_anom_score_between_two_windows(i)
        self.anom_scores = TimeSeries(self._denoise_scores(anom_scores))
518553
def analyze(self, chunkSize, *sinks):
        
        measureSize = False
        if self.measureSize:
            for sink in sinks:
                if sink.isRemote:
                    measureSize = True

        # Use destination (already uploaded) edges first
        sinks = list(sinks)
        sinks.reverse()
        self.dest = sinks[0]

        def currentSize():
            return sum([
                n.diffSize
                for n in self.nodes.values()
                if n.diff is not None and n.diff.sink != self.dest
            ])

        while True:
            self._analyzeDontMeasure(chunkSize, measureSize, *sinks)

            if not measureSize:
                return

            estimatedSize = currentSize()

            # logger.info("Measuring any estimated diffs")

            for node in self.nodes.values():
                edge = node.diff
                if edge is not None and edge.sink != self.dest and edge.sizeIsEstimated:
                    edge.sink.measureSize(edge, chunkSize)

            actualSize = currentSize()

            logger.info(
                "measured size (%s), estimated size (%s)",
                humanize(actualSize), humanize(estimatedSize),
            )

            if actualSize <= 1.2 * estimatedSize:
                return
133716
def determine_num_chunks(chunk_size, file_size):
        
        if file_size == 0:
            return 1
        return int(math.ceil(float(file_size) / float(chunk_size)))
556822
def _compute_anom_data_using_window(self):
        
        anom_scores = {}
        values = self.time_series.values
        stdev = numpy.std(values)
        for i, (timestamp, value) in enumerate(self.time_series_items):
            if i < self.lag_window_size:
                anom_score = self._compute_anom_score(values[:i + 1], value)
            else:
                anom_score = self._compute_anom_score(values[i - self.lag_window_size: i + 1], value)
            if stdev:
                anom_scores[timestamp] = anom_score / stdev
            else:
                anom_scores[timestamp] = anom_score
        self.anom_scores = TimeSeries(self._denoise_scores(anom_scores))
1169354
private State handleResetChunk(CellChunk cellChunk) {
    validate(cellChunk.getRowKey().isEmpty(), "Reset chunks can't have row keys");
    validate(!cellChunk.hasFamilyName(), "Reset chunks can't have families");
    validate(!cellChunk.hasQualifier(), "Reset chunks can't have qualifiers");
    validate(cellChunk.getTimestampMicros() == 0, "Reset chunks can't have timestamps");
    validate(cellChunk.getValueSize() == 0, "Reset chunks can't have value sizes");
    validate(cellChunk.getValue().isEmpty(), "Reset chunks can't have values");

    reset();
    return currentState; // AWAITING_NEW_ROW
  }
1044281
def chunks(self):
        
        chunks = {}
        for v in self.variables.values():
            if v.chunks is not None:
                for dim, c in zip(v.dims, v.chunks):
                    if dim in chunks and c != chunks[dim]:
                        raise ValueError('inconsistent chunks')
                    chunks[dim] = c
        return Frozen(SortedKeysDict(chunks))
336108
function findFirstOverlappingChunk(chunks, windowStartIndex, windowEndIndex)
{
  var helper = function(chunks,
                        chunkIndexLeft,
                        chunkIndexRight,
                        windowStartIndex,
                        windowEndIndex)
  {
    var result;
    var chunkIndexMiddle;
    var middleChunk;
    var middleChunkEndIndex;

    // initialize the return value to -1
    result = -1;

    // compute the index of the middle chunk and get the middle chunk
    chunkIndexMiddle = Math.floor((chunkIndexLeft + chunkIndexRight) / 2);
    middleChunk = chunks[chunkIndexMiddle];

    // if we have two or fewer chunks
    if ((chunkIndexMiddle === chunkIndexLeft) ||
        (chunkIndexMiddle === chunkIndexRight))
    {
      // if we have just one chunk, and it overlaps with the specified window,
      // we've found the chunk we were looking for
      if (chunkIndexLeft === chunkIndexRight)
      {
        if (middleChunk.overlapsWithWindow(windowStartIndex, windowEndIndex))
        {
          result = chunkIndexLeft;
        }
      }
      else // we just have two chunks left to check
      {
        // if the first chunk overlaps with the specified window, that's the
        // chunk we were looking for
        if (chunks[chunkIndexLeft].overlapsWithWindow(
                windowStartIndex, windowEndIndex))
        {
          result = chunkIndexLeft;
        }

        // otherwise, if the second chunk overlaps with the specified window,
        // that's the chunk we were looking for
        else if (chunks[chunkIndexRight].overlapsWithWindow(
                windowStartIndex, windowEndIndex))
        {
          result = chunkIndexRight;
        }
      }

      return result;
    }

    // if the middle chunk does not overlap with the specified window
    if (!middleChunk.overlapsWithWindow(windowStartIndex, windowEndIndex))
    {
      middleChunkEndIndex = middleChunk.getEndIndex();

      // if the window is to the right of the middle chunk,
      // recurse on the right half
      if (windowStartIndex > middleChunkEndIndex)
      {
        return helper(
            chunks,
            chunkIndexMiddle,
            chunkIndexRight,
            windowStartIndex,
            windowEndIndex);
      }
      else
      {
        // recurse on the left half
        return helper(
            chunks,
            chunkIndexLeft,
            chunkIndexMiddle,
            windowStartIndex,
            windowEndIndex);
      }
    }
    else
    {
      // if the middle chunk overlaps but the chunk before it does not, the
      // middle chunk is the one we were looking
      if ((chunkIndexMiddle === 0) ||
          !chunks[chunkIndexMiddle - 1].overlapsWithWindow(
              windowStartIndex, windowEndIndex))
      {
        return chunkIndexMiddle;
      }
      else
      {
        // recurse on the left half
        return helper(
            chunks,
            chunkIndexLeft,
            chunkIndexMiddle,
            windowStartIndex,
            windowEndIndex);
      }
    }
  };

  return helper(chunks, 0, chunks.length - 1, windowStartIndex, windowEndIndex);
}
695564
private void checkTopChunk(long p) {
    if (VALIDATING) {
      long sz = head(p) & ~INUSE_BITS; /* third-lowest bit can be set! */
      if (!isAligned(chunkToMem(p)))
        throw new AssertionError("Chunk address [mem:" + p + "=>chunk:" + chunkToMem(p) + "] of top chunk is incorrectly aligned");
      if (!okAddress(p))
        throw new AssertionError("Memory address " + p + " of top chunk is invalid");
      if (sz != topSize)
        throw new AssertionError("Marked size top chunk " + sz + " is not equals to the recorded top size " + topSize);
      if (sz <= 0)
        throw new AssertionError("Top chunk size " + sz + " is not positive");
      if (!previousInUse(p))
        throw new AssertionError("Chunk before top chunk is free - why has it not been merged in to the top chunk?");
    }
  }
511662
def iterchunks(data, chunksize):
    
    offt = 0
    while offt < len(data):
        yield data[offt:offt+chunksize]
        offt += chunksize
1261445
function process(raw) {
  var stats = {
    counters : raw.counters,
    gauges   : raw.gauges,
    sets     : {},
    timers   : {},
    ts       : raw.ts,
    info     : raw.info,
  }

  // process some stats so the backends don't have to
  var timerKeys = Object.keys(raw.timers || {})
  timerKeys.forEach(function(timer) {
    // see if we have any data
    if ( raw.timers[timer].length === 0 ) {
      stats.timers[timer] = {}
    }

    // yes, got some timings, sort them first
    var times = raw.timers[timer].sort(function(a, b) { return a - b })

    // calculate some stats per timer
    var data = {
      sum   : times.reduce(function(a, b) { return a + b }),
      count : times.length,
      min   : times[0],
      max   : times[times.length-1],
    }
    data.mean = data.sum / data.count

    // find the median
    var middle = Math.floor(data.count/2)
    // take the middle value, or the average of the middle two
    var median = (data.count % 2) ? times[middle] : (times[middle-1] + times[middle])/2
    data.median = median

    // find the standard deviation
    var sumOfDiffs = 0
    for (var i = 0; i < data.count; i++) {
      sumOfDiffs += (times[i] - data.mean) * (times[i] - data.mean)
    }
    data.std = Math.sqrt(sumOfDiffs / data.count)

    stats.timers[timer] = data
  })

  var setsKeys = Object.keys(raw.sets || {})
  setsKeys.forEach(function(set) {
    var info = {
      // bottom : 0,
      // top    : 0,
      unique : 0,
      total  : 0,
    }

    // see how many keys this set has
    var keys = Object.keys(raw.sets[set])

    // get the uniques
    info.unique = keys.length

    // get the total
    info.total = keys.reduce(function(sum, key) {
      return sum + raw.sets[set][key]
    }, 0)

    stats.sets[set] = info
  })

  return stats
}
309047
def measureSize(self, diff, chunkSize):
        
        (toUUID, fromUUID) = self.toArg.diff(diff)
        isInteractive = sys.stderr.isatty()
        return self.toObj.diff(self._client.measureSize(
            toUUID,
            fromUUID,
            diff.size,
            chunkSize,
            isInteractive,
        ))