Generate hourly summaries from calibrated data.
773771
def generate_hourly(calib_data, hourly_data, process_from):
    """"""
    start = hourly_data.before(datetime.max)
    if start is None:
        start = datetime.min
    start = calib_data.after(start + SECOND)
    if process_from:
        if start:
            start = min(start, process_from)
        else:
            start = process_from
    if start is None:
        return start
    # set start of hour in local time (not all time offsets are integer hours)
    start += timezone.standard_offset
    start = start.replace(minute=0, second=0)
    start -= timezone.standard_offset
    del hourly_data[start:]
    # preload pressure history, and find last valid rain
    prev = None
    pressure_history = deque()
    last_rain = None
    for data in calib_data[start - HOURx3:start]:
        if data['rel_pressure']:
            pressure_history.append((data['idx'], data['rel_pressure']))
        if data['rain'] is not None:
            last_rain = data['rain']
        prev = data
    # iterate over data in one hour chunks
    stop = calib_data.before(datetime.max)
    acc = HourAcc(last_rain)
    def hourlygen(inputdata, prev):
        """Internal generator function"""
        hour_start = start
        count = 0
        while hour_start <= stop:
            count += 1
            if count % 1008 == 0:
                logger.info("hourly: %s", hour_start.isoformat(' '))
            elif count % 24 == 0:
                logger.debug("hourly: %s", hour_start.isoformat(' '))
            hour_end = hour_start + HOUR
            acc.reset()
            for data in inputdata[hour_start:hour_end]:
                if data['rel_pressure']:
                    pressure_history.append((data['idx'], data['rel_pressure']))
                if prev:
                    err = data['idx'] - prev['idx']
                    if abs(err - timedelta(minutes=data['delay'])) > TIME_ERR:
                        logger.info('unexpected data interval %s %s',
                                    data['idx'].isoformat(' '), str(err))
                acc.add_raw(data)
                prev = data
            new_data = acc.result()
            if new_data and (new_data['idx'] - hour_start) >= timedelta(minutes=9):
                # compute pressure trend
                new_data['pressure_trend'] = None
                if new_data['rel_pressure']:
                    target = new_data['idx'] - HOURx3
                    while (len(pressure_history) >= 2 and
                           abs(pressure_history[0][0] - target) >
                           abs(pressure_history[1][0] - target)):
                        pressure_history.popleft()
                    if (pressure_history and
                            abs(pressure_history[0][0] - target) < HOUR):
                        new_data['pressure_trend'] = (
                            new_data['rel_pressure'] - pressure_history[0][1])
                # store new hourly data
                yield new_data
            hour_start = hour_end
    hourly_data.update(hourlygen(calib_data, prev))
    return start
304304
def _convert_data_to_hourly(self, gssha_data_var):
        
        time_step_hours = np.diff(self.data.time)[0]/np.timedelta64(1, 'h')
        calc_function = self._get_calc_function(gssha_data_var)
        resampled_data = None
        if time_step_hours < 1:
            resampled_data = self.data.resample('1H', dim='time',
                                                how=calc_function,
                                                keep_attrs=True)
        elif time_step_hours > 1:
            resampled_data = self.data.resample('1H', dim='time',
                                                keep_attrs=True)

            for time_idx in range(self.data.dims['time']):
                if time_idx+1 < self.data.dims['time']:
                    # interpolate between time steps
                    start_time = self.data.time[time_idx].values
                    end_time = self.data.time[time_idx+1].values
                    slope_timeslice = slice(str(start_time), str(end_time))
                    slice_size = resampled_data.sel(time=slope_timeslice).dims['time'] - 1
                    first_timestep = resampled_data.sel(time=str(start_time))[gssha_data_var]
                    slope = (resampled_data.sel(time=str(end_time))[gssha_data_var]
                             - first_timestep)/float(slice_size)

                    data_timeslice = slice(str(start_time+np.timedelta64(1, 'm')),
                                            str(end_time-np.timedelta64(1, 'm')))
                    data_subset = resampled_data.sel(time=data_timeslice)
                    for xidx in range(data_subset.dims['time']):
                        data_subset[gssha_data_var][xidx] = first_timestep + slope * (xidx+1)
                else:
                    # just continue to repeat the timestep
                    start_time = self.data.time[time_idx].values
                    end_time = resampled_data.time[-1].values
                    if end_time > start_time:
                        first_timestep = resampled_data.sel(time=str(start_time))[gssha_data_var]

                        data_timeslice = slice(str(start_time), str(end_time))
                        data_subset = resampled_data.sel(time=data_timeslice)
                        slice_size = 1
                        if calc_function == "mean":
                            slice_size = data_subset.dims['time']

                        for xidx in range(data_subset.dims['time']):
                            data_subset[gssha_data_var][xidx] = first_timestep/float(slice_size)

        if resampled_data is not None:
            # make sure coordinates copied
            if self.data.lsm.x_var not in resampled_data.coords:
                resampled_data.coords[self.data.lsm.x_var] = self.data.coords[self.data.lsm.x_var]
            if self.data.lsm.y_var not in resampled_data.coords:
                resampled_data.coords[self.data.lsm.y_var] = self.data.coords[self.data.lsm.y_var]
            self.data = resampled_data
709814
def summary(self, indicator_data):
        """"""
        summary = []
        for v in self._value_fields:
            summary.append(indicator_data.get(v, ''))
        return indicator_data.get('summary', ' : '.join(summary))
569954
def createReport(tm, options, sequenceString, numSegments, numSynapses):
  
  pac = tm.mmGetTracePredictedActiveColumns()
  pic = tm.mmGetTracePredictedInactiveColumns()
  upac = tm.mmGetTraceUnpredictedActiveColumns()

  resultsFilename = os.path.join("results", options.name+"_"+str(int(100*options.noise))+".csv")

  with open(resultsFilename,"wb") as resultsFile:
    csvWriter = csv.writer(resultsFile)

    accuracies = numpy.zeros(len(pac.data))
    smoothedAccuracies = []
    am = 0
    csvWriter.writerow(["time", "element", "pac", "pic", "upac", "a",
                        "am", "accuracy", "sum", "nSegs", "nSyns"])
    for i,j in enumerate(pac.data):
      if i>0:
        # Compute instantaneous and average accuracy.
        a = computePredictionAccuracy(len(j), len(pic.data[i]))

        #  Smooth the curve to get averaged results for the paper.
        am = 0.99*am + 0.01*a
        accuracies[i] = am
        i0 = max(0, i-60+1)
        accuracy = numpy.mean(accuracies[i0:i+1])
        smoothedAccuracies.append(accuracy)

        row=[i, sequenceString[i], len(j), len(pic.data[i]),
                len(upac.data[i]), a, am,
                accuracy,
                numpy.sum(accuracies[i0:i+1]),
                numSegments[i], numSynapses[i]]
        csvWriter.writerow(row)

  return smoothedAccuracies
1069924
def _summarize_losses(self, losses_dict):
    """"""
    if common_layers.should_generate_summaries():
      with tf.name_scope("losses"):
        for loss_name, loss_val in sorted(losses_dict.items()):
          tf.summary.scalar(loss_name, loss_val)
42092
def make_summary_funcs(rows, ids):
    """"""
    return {
        'len': len,
        'list': lambda *x: filter(None, list(x)),
        'max': max,
        'min': min,
        'rows': partial(summary_rows, rows, ids),
        'sum': sum,
        'trace': print_trace
    }
800433
def summarize(group, fs=None, include_source=True):
        
        _line_break = '{0:-<120}\n'.format('')
        tests = sorted(ComparisonBenchmark.groups[group], key=lambda t: getattr(t, 'time_average_seconds'))
        log = StringIO.StringIO()
        log.write('Call statement:\n\n')
        log.write('\t' + tests[0].stmt)
        log.write('\n\n\n')
        fmt = "{0: <8} {1: <35} {2: <12} {3: <15} {4: <15} {5: <14}\n"
        log.write(fmt.format('Rank', 'Function Name', 'Time', '% of Slowest', 'timeit_repeat', 'timeit_number'))
        log.write(_line_break)
        log.write('\n')

        for i, t in enumerate(tests):
            func_name = "{}.{}".format(t.classname, t.callable.__name__) if t.classname else t.callable.__name__
            if i == len(tests)-1:
                time_percent = 'Slowest'
            else:
                time_percent = "{:.1f}".format(t.time_average_seconds / tests[-1].time_average_seconds * 100)
            log.write(fmt.format(i+1,
                                 func_name,
                                 convert_time_units(t.time_average_seconds),
                                 time_percent,
                                 t.timeit_repeat,
                                 t.timeit_number))
        log.write(_line_break)

        if include_source:
            log.write('\n\n\nSource Code:\n')
            log.write(_line_break)
            for test in tests:
                log.write(test.log.getvalue())
                log.write(_line_break)

        if isinstance(fs, str):
            with open(fs, 'w') as f:
                f.write(log.getvalue())

        elif fs is None:
            print(log.getvalue())
        else:
            try:
                fs.write(log.getvalue())
            except AttributeError as e:
                print(e)
16438
def group_by_day(self):
        
        hourly_data_by_day = OrderedDict()
        for d in xrange(1, 366):
            hourly_data_by_day[d] = []
        a_per = self.header.analysis_period
        indx_per_day = 24 * a_per.timestep
        start_doy = sum(a_per._num_of_days_each_month[:a_per.st_time.month-1]) \
            + a_per.st_time.day
        if not a_per.is_reversed:
            for i in range(0, len(self._values), indx_per_day):
                hourly_data_by_day[start_doy] = self._values[i:i + indx_per_day]
                start_doy += 1
        else:
            end_ind = 24 * a_per.timestep * (365 - start_doy)
            for i in range(0, end_ind + 1, indx_per_day):
                hourly_data_by_day[start_doy] = self._values[i:i + indx_per_day]
                start_doy += 1
            start_doy = 1
            for i in range(end_ind, len(self._values), indx_per_day):
                hourly_data_by_day[start_doy] = self._values[i:i + indx_per_day]
                start_doy += 1
        return hourly_data_by_day
1200489
def generate_report(self):
        """"""
        logs_path = self.out_dir / 'logs'

        boilerplate = []
        boiler_idx = 0

        if (logs_path / 'CITATION.html').exists():
            text = (logs_path / 'CITATION.html').read_text(encoding='UTF-8')
            text = '<div class="boiler-html">%s</div>' % re.compile(
                '<body>(.*?)</body>',
                re.DOTALL | re.IGNORECASE).findall(text)[0].strip()
            boilerplate.append((boiler_idx, 'HTML', text))
            boiler_idx += 1

        if (logs_path / 'CITATION.md').exists():
            text = '<pre>%s</pre>\n' % (logs_path / 'CITATION.md').read_text(encoding='UTF-8')
            boilerplate.append((boiler_idx, 'Markdown', text))
            boiler_idx += 1

        if (logs_path / 'CITATION.tex').exists():
            text = (logs_path / 'CITATION.tex').read_text(encoding='UTF-8')
            text = re.compile(
                r'\\begin{document}(.*?)\\end{document}',
                re.DOTALL | re.IGNORECASE).findall(text)[0].strip()
            text = '<pre>%s</pre>\n' % text
            text += '<h3>Bibliography</h3>\n'
            text += '<pre>%s</pre>\n' % Path(
                pkgrf(self.packagename, 'data/boilerplate.bib')).read_text(encoding='UTF-8')
            boilerplate.append((boiler_idx, 'LaTeX', text))
            boiler_idx += 1

        env = jinja2.Environment(
            loader=jinja2.FileSystemLoader(searchpath=str(self.template_path.parent)),
            trim_blocks=True, lstrip_blocks=True
        )
        report_tpl = env.get_template(self.template_path.name)
        report_render = report_tpl.render(sections=self.sections, errors=self.errors,
                                          boilerplate=boilerplate)

        # Write out report
        (self.out_dir / self.out_filename).write_text(report_render, encoding='UTF-8')
        return len(self.errors)
666384
def __generate(self):
        """"""
        base = []
        texted = []
        for pat in ALL_PATTERNS:
            data = pat.copy()
            data['pattern'] = data['pattern']
            data['right'] = True
            data['basekey'] = data['key']
            base.append(data)

            data = pat.copy()
            data['basekey'] = data['key']
            data['key'] += ':time_1'
            data['right'] = True
            data['pattern'] = data['pattern'] + Optional(Literal(",")).suppress() + BASE_TIME_PATTERNS['pat:time:minutes']
            data['time_format'] = '%H:%M'
            data['length'] = {'min' : data['length']['min'] + 5, 'max' : data['length']['max'] + 8}
            base.append(data)

            data = pat.copy()
            data['basekey'] = data['key']
            data['right'] = True
            data['key'] += ':time_2'
            data['pattern'] = data['pattern'] + Optional(oneOf([',', '|'])).suppress() + BASE_TIME_PATTERNS['pat:time:full']
            data['time_format'] = '%H:%M:%S'
            data['length'] = {'min' : data['length']['min'] + 9, 'max' : data['length']['max'] + 9}
            base.append(data)

        for pat in base:
            # Right
            data = pat.copy()
            data['key'] += ':t_right'
            data['pattern'] = lineStart + data['pattern'] +  Optional(oneOf([',', '|', ':', ')'])).suppress() + restOfLine.suppress()
            data['length'] = {'min' : data['length']['min'] + 1, 'max' : data['length']['max'] + 90}
            texted.append(data)

        base.extend(texted)
        self.patterns = base
1136537
def _GenerateSummary(self):
    
    items = []
    if self._notices:
      items.append('notices: %d' % self._notice_count)
    if self._dataset_errors:
      items.append('errors: %d' % self._error_count)
    if self._dataset_warnings:
      items.append('warnings: %d' % self._warning_count)

    if items:
      return '<p><span class="fail">%s</span></p>' % '<br>'.join(items)
    else:
      return '<p><span class="pass">feeds merged successfully</span></p>'
640193
def _generate_time_values(self):
        
        # Populate time values
        log('writing times', 'INFO')
        d1970 = datetime(1970, 1, 1, tzinfo=utc)
        time_array = [[int((self.start_datetime - d1970).total_seconds())]]

        datetime_nc_start_simulation = self.start_datetime
        for raw_nc_index, raw_nc in enumerate(self.raw_nc_list):
            raw_nc_time = raw_nc.get_time_array(
                datetime_simulation_start=datetime_nc_start_simulation,
                simulation_time_step_seconds=self.time_step_array[
                    raw_nc_index])

            time_array.append(raw_nc_time)
            datetime_nc_start_simulation = \
                datetime.utcfromtimestamp(raw_nc_time[-1])

        self.cf_nc.variables['time'][:] = np.concatenate(time_array)
        end_date = datetime.utcfromtimestamp(self.cf_nc.variables['time'][-1])
        self.cf_nc.time_coverage_start = self.start_datetime.isoformat() + 'Z'
        self.cf_nc.time_coverage_end = end_date.isoformat() + 'Z'
1143467
def generate_data(self, *args, **kwargs):
    """"""
    for p in self.problems:
      p.generate_data(*args, **kwargs)
68892
def _summarize_result(result, config):
    
    timing_var = config['scaling_var']
    summary = LIVVDict()
    for size, res in result.items():
        proc_counts = []
        bench_times = []
        model_times = []
        for proc, data in res.items():
            proc_counts.append(int(proc[1:]))
            try:
                bench_times.append(data['bench'][timing_var]['mean'])
            except KeyError:
                pass
            try:
                model_times.append(data['model'][timing_var]['mean'])
            except KeyError:
                pass
        if model_times != [] and bench_times != []:
            time_diff = np.mean(model_times)/np.mean(bench_times)
        else:
            time_diff = 'NA'
        summary[size]['Proc. Counts'] = ", ".join([str(x) for x in sorted(proc_counts)])
        summary[size]['Mean Time Diff (% of benchmark)'] = time_diff
    return summary
1129411
def f_rollup(items, times, freq):
    
    rollup = [np.sum(item for __, item in ts)
              for _, ts in groupby_freq(items, times, freq)]
    return np.array(rollup)
1141326
def _times_to_hours_after_local_midnight(times):
    """"""
    times = times.tz_localize(None)
    hrs = 1 / NS_PER_HR * (
        times.astype(np.int64) - times.normalize().astype(np.int64))
    return np.array(hrs)
854189
def cli(patterns, times, json, csv, rst, md, ref, unit, precision, debug):
    ''''''
    if ref:
        ref = JSON.load(ref)

    filenames = []
    reporters = [CliReporter(ref=ref, debug=debug, unit=unit, precision=precision)]
    kwargs = {}
    for pattern in patterns or ['**/*.bench.py']:
        filenames.extend(resolve_pattern(pattern))
    if json:
        reporters.append(JsonReporter(json, precision=precision))
    if csv:
        reporters.append(CsvReporter(csv, precision=precision))
    if rst:
        reporters.append(RstReporter(rst, precision=precision))
    if md:
        reporters.append(MarkdownReporter(md, precision=precision))
    if times:
        kwargs['times'] = times
    runner = BenchmarkRunner(*filenames, reporters=reporters, debug=debug)
    runner.run(**kwargs)