Import data from an epw file.

        Hourly data will be saved in self.data and the various header data
        will be saved in the properties above.
6139
def _import_data(self, import_header_only=False):
        
        # perform checks on the file before opening it.
        assert os.path.isfile(self._file_path), 'Cannot find an epw file at {}'.format(
            self._file_path)
        assert self._file_path.lower().endswith('epw'), '{} is not an .epw file. \n' \
            'It does not possess the .epw file extension.'.format(self._file_path)

        with open(self._file_path, readmode) as epwin:
            line = epwin.readline()
            original_header_load = bool(self._is_header_loaded)

            if not self._is_header_loaded:
                # import location data
                # first line has location data - Here is an example
                # LOCATION,Denver Golden Nr,CO,USA,TMY3,724666,39.74,-105.18,-7.0,1829.0
                location_data = line.strip().split(',')
                self._location = Location()
                self._location.city = location_data[1].replace('\\', ' ') \
                    .replace('/', ' ')
                self._location.state = location_data[2]
                self._location.country = location_data[3]
                self._location.source = location_data[4]
                self._location.station_id = location_data[5]
                self._location.latitude = location_data[6]
                self._location.longitude = location_data[7]
                self._location.time_zone = location_data[8]
                self._location.elevation = location_data[9]

                # asemble a dictionary of metadata
                self._metadata = {
                    'source': self._location.source,
                    'country': self._location.country,
                    'city': self._location.city
                }

                self._header = [line] + [epwin.readline() for i in xrange(7)]

                # parse the heating, cooling and extreme design conditions.
                dday_data = self._header[1].strip().split(',')
                if len(dday_data) >= 2 and int(dday_data[1]) == 1:
                    if dday_data[4] == 'Heating':
                        for key, val in zip(DesignDay.heating_keys, dday_data[5:20]):
                            self._heating_dict[key] = val
                    if dday_data[20] == 'Cooling':
                        for key, val in zip(DesignDay.cooling_keys, dday_data[21:53]):
                            self._cooling_dict[key] = val
                    if dday_data[53] == 'Extremes':
                        for key, val in zip(DesignDay.extreme_keys, dday_data[54:70]):
                            self._extremes_dict[key] = val

                # parse typical and extreme periods into analysis periods.
                week_data = self._header[2].split(',')
                num_weeks = int(week_data[1]) if len(week_data) >= 2 else 0
                st_ind = 2
                for i in xrange(num_weeks):
                    week_dat = week_data[st_ind:st_ind + 4]
                    st_ind += 4
                    st = [int(num) for num in week_dat[2].split('/')]
                    end = [int(num) for num in week_dat[3].split('/')]
                    if len(st) == 3:
                        a_per = AnalysisPeriod(st[1], st[2], 0, end[1], end[2], 23)
                    elif len(st) == 2:
                        a_per = AnalysisPeriod(st[0], st[1], 0, end[0], end[1], 23)
                    if 'Max' in week_dat[0] and week_dat[1] == 'Extreme':
                        self._extreme_hot_weeks[week_dat[0]] = a_per
                    elif 'Min' in week_dat[0] and week_dat[1] == 'Extreme':
                        self._extreme_cold_weeks[week_dat[0]] = a_per
                    elif week_dat[1] == 'Typical':
                        self._typical_weeks[week_dat[0]] = a_per

                # parse the monthly ground temperatures in the header.
                grnd_data = self._header[3].strip().split(',')
                num_depths = int(grnd_data[1]) if len(grnd_data) >= 2 else 0
                st_ind = 2
                for i in xrange(num_depths):
                    header_meta = dict(self._metadata)  # copying the metadata dictionary
                    header_meta['depth'] = float(grnd_data[st_ind])
                    header_meta['soil conductivity'] = grnd_data[st_ind + 1]
                    header_meta['soil density'] = grnd_data[st_ind + 2]
                    header_meta['soil specific heat'] = grnd_data[st_ind + 3]
                    grnd_header = Header(temperature.GroundTemperature(), 'C',
                                         AnalysisPeriod(), header_meta)
                    grnd_vlas = [float(x) for x in grnd_data[st_ind + 4: st_ind + 16]]
                    self._monthly_ground_temps[float(grnd_data[st_ind])] = \
                        MonthlyCollection(grnd_header, grnd_vlas, list(xrange(12)))
                    st_ind += 16

                # parse leap year, daylight savings and comments.
                leap_dl_sav = self._header[4].strip().split(',')
                self._is_leap_year = True if leap_dl_sav[1] == 'Yes' else False
                self.daylight_savings_start = leap_dl_sav[2]
                self.daylight_savings_end = leap_dl_sav[3]
                comments_1 = self._header[5].strip().split(',')
                if len(comments_1) > 0:
                    self.comments_1 = ','.join(comments_1[1:])
                comments_2 = self._header[6].strip().split(',')
                if len(comments_2) > 0:
                    self.comments_2 = ','.join(comments_2[1:])

                self._is_header_loaded = True

            if import_header_only:
                return

            # read first line of data to overwrite the number of fields
            if original_header_load is True:
                for i in xrange(7):
                    epwin.readline()
            line = epwin.readline()
            self._num_of_fields = min(len(line.strip().split(',')), 35)

            # create an annual analysis period
            analysis_period = AnalysisPeriod(is_leap_year=self.is_leap_year)

            # create headers and an empty list for each field in epw file
            headers = []
            for field_number in xrange(self._num_of_fields):
                field = EPWFields.field_by_number(field_number)
                header = Header(data_type=field.name, unit=field.unit,
                                analysis_period=analysis_period,
                                metadata=dict(self._metadata))
                headers.append(header)
                self._data.append([])

            # collect hourly data
            while line:
                data = line.strip().split(',')

                for field_number in xrange(self._num_of_fields):
                    value_type = EPWFields.field_by_number(field_number).value_type
                    try:
                        value = value_type(data[field_number])
                    except ValueError as e:
                        # failed to convert the value for the specific TypeError
                        if value_type != int:
                            raise ValueError(e)
                        value = int(round(float(data[field_number])))

                    self._data[field_number].append(value)

                line = epwin.readline()

            # if the first value is at 1 AM, move last item to start position
            for field_number in xrange(self._num_of_fields):
                point_in_time = headers[field_number].data_type.point_in_time
                if point_in_time is True:
                    # move the last hour to first position
                    last_hour = self._data[field_number].pop()
                    self._data[field_number].insert(0, last_hour)

            # finally, build the data collection objects from the headers and data
            for i in xrange(self._num_of_fields):
                self._data[i] = HourlyContinuousCollection(headers[i], self._data[i])

            self._is_data_loaded = True
463957
def import_dashboards(session, data_stream, import_time=None):
    """"""
    current_tt = int(time.time())
    import_time = current_tt if import_time is None else import_time
    data = json.loads(data_stream.read(), object_hook=decode_dashboards)
    # TODO: import DRUID datasources
    for table in data['datasources']:
        type(table).import_obj(table, import_time=import_time)
    session.commit()
    for dashboard in data['dashboards']:
        Dashboard.import_obj(
            dashboard, import_time=import_time)
    session.commit()
254141
def my_import(file):
    
    # , help="The path to the CSV file to import. The first row must contain column names."
    lines = None
    with open(file) as csv_file:
        lines = csv_file.readlines()

    # Header, the first line.
    header = lines[0]
    lines.remove(header)
    header = header.rstrip()

    # Parse records from a csv row.
    counter = 0
    app = AppAggregate()
    app.open_session()
    for line in lines:
        # Create insert statements
        line = line.rstrip()
        command = f"insert into AssetClass ({header}) values ({line});"
        # insert records
        app.session.execute(command)
        try:
            app.save()
        except:
            print(f"error: ", sys.exc_info()[0])
            app.session.close()
        counter += 1
    print(f"Data imported. {counter} rows created.")
183579
def next(self):
        
        record = self.epw_data.next()
        local_time = _muck_w_date(record)
        record['datetime'] = local_time
        # does this fix a specific data set or a general issue?
        if self.DST:
            localdt = self.local_tz.localize(record['datetime'])
            record['utc_datetime'] = localdt.astimezone(pytz.UTC)
        else:
            record['utc_datetime'] = local_time - \
                datetime.timedelta(hours=self.TZ)

        return record
1203291
def add_time(data):
    
    payload = data['data']
    updated = data['updated'].date()
    if updated == date.today():
        payload['last_updated'] = data['updated'].strftime('today at %H:%M:%S')
    elif updated >= (date.today() - timedelta(days=1)):
        payload['last_updated'] = 'yesterday'
    elif updated >= (date.today() - timedelta(days=7)):
        payload['last_updated'] = updated.strftime('on %A')
    else:
        payload['last_updated'] = updated.strftime('%Y-%m-%d')
    return payload
248993
def import_data(self, data):
        
        if self.get_read_only() and self.is_locked():
            return

        if isinstance(data, BaseModel):
            data = data.export_data()

        if not isinstance(data, (dict, Mapping)):
            raise TypeError('Impossible to import data')

        self._import_data(data)
397965
def _loadData(self, data):
        
        self._data = data
        self.id = utils.cast(int, data.attrib.get('id'))
        self.serverId = utils.cast(int, data.attrib.get('serverId'))
        self.machineIdentifier = data.attrib.get('machineIdentifier')
        self.name = data.attrib.get('name')
        self.lastSeenAt = utils.toDatetime(data.attrib.get('lastSeenAt'))
        self.numLibraries = utils.cast(int, data.attrib.get('numLibraries'))
        self.allLibraries = utils.cast(bool, data.attrib.get('allLibraries'))
        self.owned = utils.cast(bool, data.attrib.get('owned'))
        self.pending = utils.cast(bool, data.attrib.get('pending'))
784931
function (file, importOptions = {}) {
        var self = this;

        // Step 1: Handle converting the file to usable data
        return this.loadFile(file).then(function (importData) {
            // Step 2: Let the importers pre-process the data
            return self.preProcess(importData);
        }).then(function (importData) {
            // Step 3: Actually do the import
            // @TODO: It would be cool to have some sort of dry run flag here
            return self.doImport(importData, importOptions);
        }).then(function (importData) {
            // Step 4: Report on the import
            return self.generateReport(importData);
        }).finally(() => self.cleanUp()); // Step 5: Cleanup any files
    }
1142502
def add_content(self, data):
        # type: (Optional[Union[Dict[str, Any], ET.Element]]) -> None
        
        if data is None:
            return

        if isinstance(data, ET.Element):
            bytes_data = ET.tostring(data, encoding="utf8")
            self.headers['Content-Length'] = str(len(bytes_data))
            self.data = bytes_data
            return

        # By default, assume JSON
        try:
            self.data = json.dumps(data)
            self.headers['Content-Length'] = str(len(self.data))
        except TypeError:
            self.data = data
1226416
function importItem(data) {
  // Legacy format converter
  data = legacy.convertPackage(data);

  const now = new Date();
  data.id = misc.uuidv4().slice(0, 8);
  data.sessionStarted = now.toString();
  return putItem(data);
}
609647
def get_hours_data(self, entries, date_headers):
        """"""
        project_totals = get_project_totals(
            entries, date_headers, total_column=False) if entries else []

        data_map = {}
        for rows, totals in project_totals:
            for user, user_id, periods in rows:
                for period in periods:
                    day = period['day']
                    if day not in data_map:
                        data_map[day] = {'billable': 0, 'nonbillable': 0}
                    data_map[day]['billable'] += period['billable']
                    data_map[day]['nonbillable'] += period['nonbillable']

        return data_map
10395
def data_import(self, json_response):
        """"""
        if 'data' not in json_response:
            raise PyVLXException('no element data found: {0}'.format(
                json.dumps(json_response)))
        data = json_response['data']

        for item in data:
            if 'category' not in item:
                raise PyVLXException('no element category: {0}'.format(
                    json.dumps(item)))
            category = item['category']
            if category == 'Window opener':
                self.load_window_opener(item)
            elif category in ['Roller shutter', 'Dual Shutter']:
                self.load_roller_shutter(item)
            elif category in ['Blind']:
                self.load_blind(item)
            else:
                self.pyvlx.logger.warning(
                    'WARNING: Could not parse product: %s', category)
491104
def write(self, data):
        """"""

        report_size = self.packet_size
        if self.ep_out:
            report_size = self.ep_out.wMaxPacketSize

        for _ in range(report_size - len(data)):
            data.append(0)

        self.read_sem.release()

        self.ep_out.write(data)
255131
def load_data(self):
        
        units = ""
        if self.file_objects[0] is None:
            raise IOError()
        var_name, z_index = self.format_var_name(self.variable, list(self.file_objects[0].variables.keys()))
        ntimes = 0
        if 'time' in self.file_objects[0].variables[var_name].dimensions:
            ntimes = len(self.file_objects[0].dimensions['time'])

        if ntimes > 1:
            if z_index is None:
                data = self.file_objects[0].variables[var_name][self.forecast_hours].astype(np.float32)
            else:
                data = self.file_objects[0].variables[var_name][self.forecast_hours, z_index].astype(np.float32)
        else:
            y_dim, x_dim = self.file_objects[0].variables[var_name].shape[-2:]
            data = np.zeros((len(self.valid_dates), y_dim, x_dim), dtype=np.float32)
            for f, file_object in enumerate(self.file_objects):
                if file_object is not None:
                    if z_index is None:
                        data[f] = file_object.variables[var_name][0]
                    else:
                        data[f] = file_object.variables[var_name][0, z_index]
        if hasattr(self.file_objects[0].variables[var_name], "units"):
            units = self.file_objects[0].variables[var_name].units
        return data, units
1311405
def load_data(self):
        
        for m, member in enumerate(self.members):
            mo = ModelOutput(self.ensemble_name, member, self.run_date, self.variable,
                             self.start_date, self.end_date, self.path, self.map_file, self.single_step)
            mo.load_data()
            if self.data is None:
                self.data = np.zeros((len(self.members), mo.data.shape[0], mo.data.shape[1], mo.data.shape[2]),
                                     dtype=np.float32)
            if mo.units == "m":
                self.data[m] = mo.data * 1000
                self.units = "mm"
            else:
                self.data[m] = mo.data
            if self.units == "":
                self.units = mo.units
            del mo.data
            del mo
1109274
def import_data(self, data):
        
        _completed_num = 0
        for trial_info in data:
            logger.info("Importing data, current processing progress %s / %s" %(_completed_num, len(data)))
            _completed_num += 1
            assert "parameter" in trial_info
            _params = trial_info["parameter"]
            assert "value" in trial_info
            _value = trial_info['value']
            if not _value:
                logger.info("Useless trial data, value is %s, skip this trial data." %_value)
                continue
            _params_tuple = convert_dict2tuple(_params)
            self.supplement_data[_params_tuple] = True
        logger.info("Successfully import data to grid search tuner.")
1016971
def _add_data(self, eopatch, data):
        
        valid_mask = data[..., -1]
        data = data[..., :-1]

        if data.ndim == 3:
            data = data.reshape(data.shape + (1,))
        if not self.feature_type.is_time_dependent():
            if data.shape[0] > 1:
                raise ValueError('Cannot save time dependent data to time independent feature')
            data = data.squeeze(axis=0)
        if self.feature_type.is_discrete():
            data = data.astype(np.int32)

        eopatch[self.feature_type][self.feature_name] = data

        mask_feature_type, mask_feature_name = next(self.valid_data_mask_feature())

        max_value = self.image_format.get_expected_max_value()
        valid_data = (valid_mask == max_value).astype(np.bool).reshape(valid_mask.shape + (1,))

        if mask_feature_name not in eopatch[mask_feature_type]:
            eopatch[mask_feature_type][mask_feature_name] = valid_data