Load climate data and convert to indicator objects
778379
def run(self):
        
        import numpy as np

        # get all NUTS region IDs, for linking values to region objects
        query = self.session.query(models.NUTS2Region.key,
                                   models.NUTS2Region.id)
        region_ids = self.client.df_query(query).set_index('key')['id'].to_dict()

        # load climate data and NUTS2 polygons
        data = next(self.requires()).load()
        nuts = NUTS2GeoJSONInputFile().load()

        # generated indicator IDs, keyed by season
        indicator_ids = dict()

        # climate data by season
        t_data = dict()

        # create new indicator objects for summer and winter, create averaged climate data
        for season, suffix in CLIMATE_SEASON_SUFFIXES.items():
            # noinspection PyUnresolvedReferences
            indicator = models.ClimateIndicator(description=self.description + suffix)
            self.session.add(indicator)

            # commit, to get indicator ID filled
            self.session.commit()
            indicator_ids[season] = indicator.id

            # select winter or summer data by month index, average over time range
            if season == 'summer':
                t_data[season] = np.ma.average(data['data'][3:9, :, :], axis=0)
            else:
                # noinspection PyTypeChecker
                t_data[season] = np.ma.average(0.5 * (data['data'][0:3, :, :] + data['data'][9:12, :, :]), axis=0)

        # container for output objects, for bulk saving
        objects = []

        # start value for manual object id generation
        current_value_id = models.ClimateValue.get_max_id(self.session)

        # for each region, get a mask, average climate variable over the mask and store the indicator value;
        # loop over features first, then over seasons, because mask generation is expensive
        for feature in nuts:

            # draw region mask (doesn't matter for which season we take the map shape)
            mask = geojson_polygon_to_mask(feature=feature,
                                           shape=t_data['summer'].shape,
                                           lat_idx=data['lat_idx'],
                                           lon_idx=data['lon_idx'])

            # create indicator values for summer and winter
            for season in list(CLIMATE_SEASON_SUFFIXES.keys()):
                # weighted average from region mask
                value = np.ma.average(t_data[season], weights=mask)

                # region ID must be cast to int (DBs don't like numpy dtypes from pandas)
                region_id = region_ids.get(feature.properties['NUTS_ID'], None)
                if region_id is not None:
                    region_id = int(region_id)

                # append an indicator value, manually generate object IDs for bulk saving
                current_value_id += 1
                objects.append(models.ClimateValue(id=current_value_id,
                                                   value=value,
                                                   region_id=region_id,
                                                   indicator_id=indicator_ids[season]))

                # # print some debugging output
                # print self.variable_name + ' ' + season, feature.properties['NUTS_ID'], value

                # # generate some plots for debugging
                # from matplotlib import pyplot as plt
                # plt.subplot(211)
                # plt.imshow(0.02 * t_data + mask * t_data, interpolation='none')
                # plt.subplot(212)
                # plt.imshow(t_data, interpolation='none')
                # plt.savefig('/tmp/' + feature.properties['NUTS_ID'] + '.png')

        # bulk-save all objects
        self.session.bulk_save_objects(objects)
        self.session.commit()

        self.done()
709463
def _parse_data(self):
        
        data = self._cache

        res = dict()
        res[MI_HUMIDITY] = float(data[9:13])
        res[MI_TEMPERATURE] = float(data[2:6])
        return res
89572
def _load(self, scale=0.001):
        """"""
        if self.bandname == 'DNB':
            header_lines_to_skip = N_HEADER_LINES_DNB[self.platform_name]
        else:
            header_lines_to_skip = N_HEADER_LINES[self.platform_name]

        try:
            data = np.genfromtxt(self.filename,
                                 unpack=True, skip_header=header_lines_to_skip,
                                 names=NAMES1[self.platform_name],
                                 dtype=DTYPE1[self.platform_name])
        except ValueError:
            data = np.genfromtxt(self.filename,
                                 unpack=True, skip_header=N_HEADER_LINES[
                                     self.platform_name],
                                 names=NAMES2[self.platform_name],
                                 dtype=DTYPE2[self.platform_name])

        wavelength = data['wavelength'] * scale
        response = data['response']
        det = data['detector']

        detectors = {}
        for idx in range(int(max(det))):
            detectors["det-{0:d}".format(idx + 1)] = {}
            detectors[
                "det-{0:d}".format(idx + 1)]['wavelength'] = np.repeat(wavelength, np.equal(det, idx + 1))
            detectors[
                "det-{0:d}".format(idx + 1)]['response'] = np.repeat(response, np.equal(det, idx + 1))

        self.rsr = detectors
885493
def loadRes(self, resFile):
        
        res = COCO()
        res.dataset['images'] = [img for img in self.dataset['images']]

        print('Loading and preparing results...')
        tic = time.time()
        if type(resFile) == str: #or type(resFile) == unicode:
            anns = json.load(open(resFile))
        elif type(resFile) == np.ndarray:
            anns = self.loadNumpyAnnotations(resFile)
        else:
            anns = resFile
        assert type(anns) == list, 'results in not an array of objects'
        annsImgIds = [ann['image_id'] for ann in anns]
        assert set(annsImgIds) == (set(annsImgIds) & set(self.getImgIds())), \
               'Results do not correspond to current coco set'
        if 'caption' in anns[0]:
            imgIds = set([img['id'] for img in res.dataset['images']]) & set([ann['image_id'] for ann in anns])
            res.dataset['images'] = [img for img in res.dataset['images'] if img['id'] in imgIds]
            for id, ann in enumerate(anns):
                ann['id'] = id+1
        elif 'bbox' in anns[0] and not anns[0]['bbox'] == []:
            res.dataset['categories'] = copy.deepcopy(self.dataset['categories'])
            for id, ann in enumerate(anns):
                bb = ann['bbox']
                x1, x2, y1, y2 = [bb[0], bb[0]+bb[2], bb[1], bb[1]+bb[3]]
                if not 'segmentation' in ann:
                    ann['segmentation'] = [[x1, y1, x1, y2, x2, y2, x2, y1]]
                ann['area'] = bb[2]*bb[3]
                ann['id'] = id+1
                ann['iscrowd'] = 0
        elif 'segmentation' in anns[0]:
            res.dataset['categories'] = copy.deepcopy(self.dataset['categories'])
            for id, ann in enumerate(anns):
                # now only support compressed RLE format as segmentation results
                ann['area'] = maskUtils.area(ann['segmentation'])
                if not 'bbox' in ann:
                    ann['bbox'] = maskUtils.toBbox(ann['segmentation'])
                ann['id'] = id+1
                ann['iscrowd'] = 0
        elif 'keypoints' in anns[0]:
            res.dataset['categories'] = copy.deepcopy(self.dataset['categories'])
            for id, ann in enumerate(anns):
                s = ann['keypoints']
                x = s[0::3]
                y = s[1::3]
                x0,x1,y0,y1 = np.min(x), np.max(x), np.min(y), np.max(y)
                ann['area'] = (x1-x0)*(y1-y0)
                ann['id'] = id + 1
                ann['bbox'] = [x0,y0,x1-x0,y1-y0]
        print('DONE (t={:0.2f}s)'.format(time.time()- tic))

        res.dataset['annotations'] = anns
        res.createIndex()
        return res
1071779
def _load_data(filepath):
  """"""
  with h5py.File(filepath, "r") as h5dataset:
    image_array = np.array(h5dataset["images"])
    # The 'label' data set in the hdf5 file actually contains the float values
    # and not the class labels.
    values_array = np.array(h5dataset["labels"])
  return image_array, values_array
42230
def _parse_supl(content):
        
        col_names = ['year', 'month', 'day', 'hour', 'minute',
                     'hourly_low_pressure', 'hourly_low_pressure_time',
                     'hourly_high_wind', 'hourly_high_wind_direction',
                     'hourly_high_wind_time']

        col_units = {'hourly_low_pressure': 'hPa',
                     'hourly_low_pressure_time': None,
                     'hourly_high_wind': 'meters/second',
                     'hourly_high_wind_direction': 'degrees',
                     'hourly_high_wind_time': None,
                     'time': None}

        df = pd.read_table(StringIO(content), comment='#', na_values='MM',
                           names=col_names, sep=r'\s+')

        df['time'] = pd.to_datetime(df[['year', 'month', 'day', 'hour', 'minute']], utc=True)

        df['hours'] = np.floor(df['hourly_low_pressure_time'] / 100)
        df['minutes'] = df['hourly_low_pressure_time'] - df['hours'] * 100
        df['hours'] = df['hours'].replace(99, np.nan)
        df['minutes'] = df['minutes'].replace(99, np.nan)
        df['hourly_low_pressure_time'] = pd.to_datetime(df[['year', 'month', 'day', 'hours',
                                                            'minutes']], utc=True)

        df['hours'] = np.floor(df['hourly_high_wind_time'] / 100)
        df['minutes'] = df['hourly_high_wind_time'] - df['hours'] * 100
        df['hours'] = df['hours'].replace(99, np.nan)
        df['minutes'] = df['minutes'].replace(99, np.nan)
        df['hourly_high_wind_time'] = pd.to_datetime(df[['year', 'month', 'day',
                                                         'hours', 'minutes']], utc=True)
        df = df.drop(columns=['year', 'month', 'day', 'hour', 'minute', 'hours', 'minutes'])
        df.units = col_units
        return df
547777
def load_forecasts(self):
        
        forecast_path = self.forecast_json_path + "/{0}/{1}/".format(self.run_date.strftime("%Y%m%d"),
                                                                     self.ensemble_member)
        forecast_files = sorted(glob(forecast_path + "*.json"))
        for forecast_file in forecast_files:
            file_obj = open(forecast_file)
            json_obj = json.load(file_obj)
            file_obj.close()
            track_id = json_obj['properties']["id"]
            obs_track_id = json_obj['properties']["obs_track_id"]
            forecast_hours = json_obj['properties']['times']
            duration = json_obj['properties']['duration']
            for f, feature in enumerate(json_obj['features']):
                area = np.sum(feature["properties"]["masks"])
                step_id = track_id + "_{0:02d}".format(f)
                for model_type in self.model_types:
                    for model_name in self.model_names[model_type]:
                        prediction = feature['properties'][model_type + "_" + model_name.replace(" ", "-")]
                        if model_type == "condition":
                            prediction = [prediction]
                        row = [track_id, obs_track_id, self.ensemble_name, self.ensemble_member, forecast_hours[f],
                               f + 1, duration, area] + prediction
                        self.forecasts[model_type][model_name].loc[step_id] = row
1064814
def _load(self, scale=1.0):
        """"""
        data = np.genfromtxt(self.requested_band_filename,
                             unpack=True,
                             names=['wavenumber',
                                    'response'],
                             skip_header=4)

        # Data are wavenumbers in cm-1:
        wavelength = 1. / data['wavenumber'] * 10000.
        response = data['response']

        # The real MetImage has 24 detectors. However, for now we store the
        # single rsr as 'detector-1', indicating that there will be multiple
        # detectors in the future:
        detectors = {}
        detectors['det-1'] = {'wavelength': wavelength, 'response': response}
        self.rsr = detectors
27180
def get_dataset(self, key, info):
        """"""
        if key.name == 'solar_zenith_angle':
            data = self.load('SolarZenith')
        elif key.name == 'solar_azimuth_angle':
            data = self.load('SolarAzimuth')
        elif key.name == 'satellite_zenith_angle':
            data = self.load('SensorZenith')
        elif key.name == 'satellite_azimuth_angle':
            data = self.load('SensorAzimuth')
        elif key.name == 'longitude':
            data = self.load('Longitude')
        elif key.name == 'latitude':
            data = self.load('Latitude')
        else:
            return
        if key.resolution != self.georesolution:
            # let's see if we have something in the cache
            try:
                data = self.cache[key.resolution][key.name]
                data.attrs.update(info)
                data.attrs['standard_name'] = data.attrs['name']
                return data
            except KeyError:
                self.cache.setdefault(key.resolution, {})

            # too bad, now we need to interpolate
            satz = self.load('SensorZenith')
            if key.name in ['longitude', 'latitude']:
                data_a = self.load('Longitude')
                data_b = self.load('Latitude')
            elif key.name in ['satellite_azimuth_angle', 'satellite_zenith_angle']:
                data_a = self.load('SensorAzimuth')
                data_b = self.load('SensorZenith') - 90
            elif key.name in ['solar_azimuth_angle', 'solar_zenith_angle']:
                data_a = self.load('SolarAzimuth')
                data_b = self.load('SolarZenith') - 90

            data_a, data_b = self._interpolate(data_a, data_b, satz,
                                               self.georesolution, key.resolution)

            if key.name in ['longitude', 'latitude']:
                self.cache[key.resolution]['longitude'] = data_a
                self.cache[key.resolution]['latitude'] = data_b
            elif key.name in ['satellite_azimuth_angle', 'satellite_zenith_angle']:
                self.cache[key.resolution]['satellite_azimuth_angle'] = data_a
                self.cache[key.resolution]['satellite_zenith_angle'] = data_b + 90
            elif key.name in ['solar_azimuth_angle', 'solar_zenith_angle']:
                self.cache[key.resolution]['solar_azimuth_angle'] = data_a
                self.cache[key.resolution]['solar_zenith_angle'] = data_b + 90

            data = self.cache[key.resolution][key.name]

        data.attrs.update(info)
        data.attrs['standard_name'] = data.attrs['name']
        return data
1355074
def load_data():
    data_dir = "/Users/annaho/Data/AAOmega"
    out_dir = "%s/%s" %(data_dir, "Run_13_July")

    
    ff, wl, tr_flux, tr_ivar = load_ref_spectra()

    """ pick one that doesn't have extra dead pixels """
    skylines = tr_ivar[4,:] # should be the same across all obj
    np.savez("%s/skylines.npz" %out_dir, skylines)

    contmask = np.load("%s/contmask_regions.npz" %data_dir)['arr_0']
    scatter = estimate_noise(tr_flux, contmask)
    ids, labels = load_labels()
    
    # Select the objects in the catalog corresponding to the files
    inds = []
    ff_short = []
    for fname in ff:
        val = fname.split("/")[-1]
        short = (val.split('.')[0] + '.' + val.split('.')[1])
        ff_short.append(short)
        if short in ids:
            ind = np.where(ids==short)[0][0]
            inds.append(ind)

    # choose the labels
    tr_id = ids[inds]
    tr_label = labels[inds]

    # find the corresponding spectra
    ff_short = np.array(ff_short)
    inds = np.array([np.where(ff_short==val)[0][0] for val in tr_id])
    tr_flux_choose = tr_flux[inds]
    tr_ivar_choose = tr_ivar[inds]
    scatter_choose = scatter[inds]
    np.savez("%s/wl.npz" %out_dir, wl)
    np.savez("%s/ref_id_all.npz" %out_dir, tr_id)
    np.savez("%s/ref_flux_all.npz" %out_dir, tr_flux_choose)
    np.savez("%s/ref_ivar_all.npz" %out_dir, tr_ivar_choose)
    np.savez("%s/ref_label_all.npz" %out_dir, tr_label)
    np.savez("%s/ref_spec_scat_all.npz" %out_dir, scatter_choose)

    # now, the test spectra
    test_id, test_flux = load_test_spectra()
    scatter = estimate_noise(test_flux, contmask) 
    np.savez("%s/test_id.npz" %out_dir, test_id)
    np.savez("%s/test_flux.npz" %out_dir, test_flux)
    np.savez("%s/test_spec_scat.npz" %out_dir, scatter)
1212169
def get_dataset(self, key, info, out=None):
        """"""

        logger.debug("Reading %s.", key.name)
        values = self.file_content[key.name]
        selected = np.array(self.selected)
        if key.name in ("Latitude", "Longitude"):
            values = values / 10000.
        if key.name in ('Tsurf', 'CloudTopPres', 'CloudTopTemp'):
            goods = values > -9998.
            selected = np.array(selected & goods)
            if key.name in ('Tsurf', "Alt_surface", "CloudTopTemp"):
                values = values / 100.
            if key.name in ("CloudTopPres"):
                values = values / 10.
        else:
            selected = self.selected
        info.update(self.finfo)

        fill_value = np.nan

        if key.name == 'ct':
            fill_value = 0
            info['_FillValue'] = 0
        ds = DataArray(values, dims=['y', 'x'], attrs=info).where(selected, fill_value)

        # update dataset info with file_info
        return ds
601037
def items(cls):
        
        return [
            cls.PRECIPITATION,
            cls.WIND,
            cls.TEMPERATURE,
            cls.PRESSURE
        ]
1364843
def get_data(param, data):
    """"""

    try:
        for (_, selected_time_entry) in data:
            loc_data = selected_time_entry['location']
            if param not in loc_data:
                continue
            if param == 'precipitation':
                new_state = loc_data[param]['@value']
            elif param == 'symbol':
                new_state = int(float(loc_data[param]['@number']))
            elif param in ('temperature', 'pressure', 'humidity',
                           'dewpointTemperature'):
                new_state = round(float(loc_data[param]['@value']), 1)
            elif param in ('windSpeed', 'windGust'):
                new_state = round(float(loc_data[param]['@mps']) * 3.6, 1)
            elif param == 'windDirection':
                new_state = round(float(loc_data[param]['@deg']), 1)
            elif param in ('fog', 'cloudiness', 'lowClouds',
                           'mediumClouds', 'highClouds'):
                new_state = round(float(loc_data[param]['@percent']), 1)
            return new_state
    except (ValueError, IndexError, KeyError):
        return None
507786
def decode_data(self, data):
        '''
        
        '''
        try:
            byte_data = bytearray.fromhex(data)
            acc_x, acc_y, acc_z = self._get_acceleration(byte_data)
            return {
                'data_format': 3,
                'humidity': self._get_humidity(byte_data),
                'temperature': self._get_temperature(byte_data),
                'pressure': self._get_pressure(byte_data),
                'acceleration': math.sqrt(acc_x * acc_x + acc_y * acc_y + acc_z * acc_z),
                'acceleration_x': acc_x,
                'acceleration_y': acc_y,
                'acceleration_z': acc_z,
                'battery': self._get_battery(byte_data)
            }
        except Exception:
            log.exception('Value: %s not valid', data)
            return None
136750
def __parse_loc_data(loc_data, result):
    """"""
    result[DATA] = {ATTRIBUTION: ATTRIBUTION_INFO,
                    FORECAST: [],
                    PRECIPITATION_FORECAST: None}

    for key, [value, func] in SENSOR_TYPES.items():
        result[DATA][key] = None
        try:
            from buienradar.buienradar import condition_from_code
            sens_data = loc_data[value]
            if key == CONDITION:
                # update weather symbol & status text
                code = sens_data[__BRID][:1]
                result[DATA][CONDITION] = condition_from_code(code)
                result[DATA][CONDITION][IMAGE] = sens_data[__BRTEXT]
            else:
                if key == STATIONNAME:
                    name = sens_data[__BRTEXT].replace("Meetstation", "")
                    name = name.strip()
                    name += " (%s)" % loc_data[__BRSTATIONCODE]
                    result[DATA][key] = name
                else:
                    # update all other data
                    if func is not None:
                        result[DATA][key] = func(sens_data)
                    else:
                        result[DATA][key] = sens_data
        except KeyError:
            if result[MESSAGE] is None:
                result[MESSAGE] = "Missing key(s) in br data: "
            result[MESSAGE] += "%s " % value
            log.warning("Data element with key='%s' "
                        "not loaded from br data!", key)
    result[SUCCESS] = True
    return result
909936
def create_single_weather(df, rename_dc):
    """"""
    my_weather = weather.FeedinWeather()
    data_height = {}
    name = None
    # Create a pandas.DataFrame with the time series of the weather data set
    weather_df = pd.DataFrame(index=df.time_series.iloc[0].index)
    for row in df.iterrows():
        key = rename_dc[row[1].type]
        weather_df[key] = row[1].time_series
        data_height[key] = row[1].height if not np.isnan(row[1].height) else 0
        name = row[1].gid
    my_weather.data = weather_df
    my_weather.timezone = weather_df.index.tz
    my_weather.longitude = df.geom_point.iloc[0].x
    my_weather.latitude = df.geom_point.iloc[0].y
    my_weather.geometry = df.geom_point.iloc[0]
    my_weather.data_height = data_height
    my_weather.name = name
    return my_weather
119589
def load_data_old(self):
        
        units = ""
        if len(self.file_objects) == 1 and self.file_objects[0] is not None:
            data = self.file_objects[0].variables[self.variable][self.forecast_hours]
            if hasattr(self.file_objects[0].variables[self.variable], "units"):
                units = self.file_objects[0].variables[self.variable].units
        elif len(self.file_objects) > 1:
            grid_shape = [len(self.file_objects), 1, 1]
            for file_object in self.file_objects:
                if file_object is not None:
                    if self.variable in file_object.variables.keys():
                        grid_shape = file_object.variables[self.variable].shape
                    elif self.variable.ljust(6, "_") in file_object.variables.keys():
                        grid_shape = file_object.variables[self.variable.ljust(6, "_")].shape

                    else:
                        print("{0} not found".format(self.variable))
                        raise KeyError
                    break
            data = np.zeros((len(self.file_objects), grid_shape[1], grid_shape[2]))
            for f, file_object in enumerate(self.file_objects):
                if file_object is not None:
                    if self.variable in file_object.variables.keys():
                        var_name = self.variable
                    elif self.variable.ljust(6, "_") in file_object.variables.keys():
                        var_name = self.variable.ljust(6, "_")
                    else:
                        print("{0} not found".format(self.variable))
                        raise KeyError
                    data[f] = file_object.variables[var_name][0]
                    if units == "" and hasattr(file_object.variables[var_name], "units"):
                        units = file_object.variables[var_name].units
        else:
            data = None
        return data, units