Load table data to :class:`EuroStatsValue` objects
767698
def run(self):
        
        # -- start documentation include: eurostats-run-1
        # create a new indicator metadata object
        indicator = models.EuroStatIndicator(
            number=self.number,
            description=self.description,
            url="http://ec.europa.eu/eurostat/web/products-datasets/-/tgs" + self.number)

        # add/commit to get the object ID filled
        self.session.add(indicator)
        self.session.commit()
        # -- end documentation include: eurostats-run-1

        # -- start documentation include: eurostats-run-2
        # load data from input file task
        df = next(self.requires()).load(key_filter=self.key_filter,
                                        header_preproc=self.header_preproc)

        # Transform data: DataFrame from loading has NUTS2 key and years as columns.
        # Index by key, then stack years as second level of index. Reset the index
        # to get year and key as regular columns, with one value column left.
        values = df.set_index('key').stack()
        values.index.levels[1].name = 'year'
        values.name = 'value'
        df = values.reset_index()
        # -- end documentation include: eurostats-run-2

        # -- start documentation include: eurostats-run-3
        # get current max ID for EuroStatValue objects, for manual ID generation
        max_id = models.EuroStatValue.get_max_id(self.session)

        # append an ID column, starting with the current max ID of the object class plus one
        df['id'] = list(range(max_id + 1, max_id + 1 + len(df)))
        # -- end documentation include: eurostats-run-3

        # -- start documentation include: eurostats-run-4
        # append indicator ID (constant)
        df['indicator_id'] = indicator.id

        # append region ID column, by mapping NUTS2 region keys to DB object IDs
        regions = self.client.df_query(self.session.query(models.NUTS2Region)) \
            .set_index('key')['id']
        df['region_id'] = df['key'].map(regions)

        # drop columns that are not part of the data model
        df = df.drop(['key'], axis=1)  # type: pd.DataFrame
        # -- end documentation include: eurostats-run-4

        # -- start documentation include: eurostats-run-5
        # store, done
        df.to_sql(name=models.EuroStatValue.__tablename__,
                  con=client.get_client().engine,
                  if_exists='append',
                  index=False)

        self.done()
274939
function loadKeyValueCSV (filename, callback) {
  var out = {}

  csv()
  .from.path(filename, {
    delimiter: '\t'
  })
  .on('record', function (data, index) {
    if (data[0].match(/^[\w\d]/)) {
      out[data[0]] = data[1]
    }
  })
  .on('end', function (count) {
    callback(null, out)
  })
  .on('error', function (err) {
    callback(err)
  })
}
172715
def _load_data(self,
                   equities,
                   futures,
                   exchanges,
                   root_symbols,
                   equity_supplementary_mappings):
        
        # Set named identifier columns as indices, if provided.
        _normalize_index_columns_in_place(
            equities=equities,
            equity_supplementary_mappings=equity_supplementary_mappings,
            futures=futures,
            exchanges=exchanges,
            root_symbols=root_symbols,
        )

        futures_output = self._normalize_futures(futures)

        equity_supplementary_mappings_output = (
            self._normalize_equity_supplementary_mappings(
                equity_supplementary_mappings,
            )
        )

        exchanges_output = _generate_output_dataframe(
            data_subset=exchanges,
            defaults=_exchanges_defaults,
        )

        equities_output, equities_mappings = self._normalize_equities(
            equities,
            exchanges_output,
        )

        root_symbols_output = _generate_output_dataframe(
            data_subset=root_symbols,
            defaults=_root_symbols_defaults,
        )

        return AssetData(
            equities=equities_output,
            equities_mappings=equities_mappings,
            futures=futures_output,
            exchanges=exchanges_output,
            root_symbols=root_symbols_output,
            equity_supplementary_mappings=equity_supplementary_mappings_output,
        )
1048370
def load_table(self, table):
        
        table = self._valid_table_name(table)
        schema, table = self.parse_table_name(table)
        if not schema:
            schema = self.schema
            tables = self.tables
        else:
            tables = self.tables_in_schema(schema)
        if table in tables:
            return Table(self, schema, table)
        else:
            return None
1021789
def record_to_objects(self):
        """"""
        from ambry.orm import SourceTable

        bsfile = self.record

        failures = set()

        # Clear out all of the columns from existing tables. We don't clear out the
        # tables, since they may be referenced by sources

        for row in bsfile.dict_row_reader:
            st = self._dataset.source_table(row['table'])

            if st:
                st.columns[:] = []

        self._dataset.commit()

        for row in bsfile.dict_row_reader:
            st = self._dataset.source_table(row['table'])

            if not st:
                st = self._dataset.new_source_table(row['table'])
                # table_number += 1

            if 'datatype' not in row:
                row['datatype'] = 'unknown'

            del row['table']

            st.add_column(**row)  # Create or update

        if failures:
            raise ConfigurationError('Failed to load source schema, missing sources: {} '.format(failures))

        self._dataset.commit()
914522
def load_table_data(self):
        
        # We only want the data to be loaded once, so we track if we have...
        if not self._table_data_loaded:
            for table_name, table in self._tables.items():
                # Fetch the data function.
                func_name = "get_%s_data" % table_name
                data_func = getattr(self, func_name, None)
                if data_func is None:
                    cls_name = self.__class__.__name__
                    raise NotImplementedError(
                        "You must define a %(func_name)s method on"
                        " %(cls_name)s."
                        % {'func_name': func_name, 'cls_name': cls_name})

                # Load the data.
                table.data = data_func()
                table._meta.has_prev_data = self.has_prev_data(table)
                table._meta.has_more_data = self.has_more_data(table)
            # Mark our data as loaded so we don't run the loaders again.
            self._table_data_loaded = True
1322486
def set_countriesdata(cls, countries):
        # type: (str) -> None
        
        cls._countriesdata = dict()
        cls._countriesdata['countries'] = dict()
        cls._countriesdata['iso2iso3'] = dict()
        cls._countriesdata['m49iso3'] = dict()
        cls._countriesdata['countrynames2iso3'] = dict()
        cls._countriesdata['regioncodes2countries'] = dict()
        cls._countriesdata['regioncodes2names'] = dict()
        cls._countriesdata['regionnames2codes'] = dict()
        cls._countriesdata['aliases'] = dict()

        for country in countries:
            iso3 = country.get('#country+code+v_iso3')
            if not iso3:
                continue
            iso3 = iso3.upper()
            cls._add_countriesdata(iso3, country)
            cls._countriesdata['countries'][iso3] = country.dictionary

        def sort_list(colname):
            for idval in cls._countriesdata[colname]:
                cls._countriesdata[colname][idval] = \
                    sorted(list(cls._countriesdata[colname][idval]))

        sort_list('regioncodes2countries')
620541
def loads(inputStr):
    """"""
    inCat = json.loads(inputStr)
    assert CATALOGUE_TYPE in _values(inCat[CATALOGUE_METADATA], ISCONTENTTYPE_RELATION)
    # Manually copy mandatory fields, to check that they are they, and exclude other garbage
    desc = _values(inCat[CATALOGUE_METADATA], DESCRIPTION_RELATION)[0]  # TODO: We are ASSUMING just one description, which may not be true
    outCat = Hypercat(desc)
    for i in inCat[ITEMS]:
        href = i[HREF]
        contentType = _values(i[ITEM_METADATA], ISCONTENTTYPE_RELATION) [0]
        desc = _values(i[ITEM_METADATA], DESCRIPTION_RELATION) [0]
        if contentType == CATALOGUE_TYPE:
            r = Hypercat(desc)
        else:
            r = Resource(desc, contentType)
        outCat.addItem(r, href)

    return outCat
1267959
def load_table(self, table):
        
        region = table.database if table.database else self.default_region
        resource_name, collection_name = table.table.split('_', 1)
        # we use underscore "_" instead of dash "-" for region name but boto3 need dash
        boto_region_name = region.replace('_', '-')
        resource = self.boto3_session.resource(resource_name, region_name=boto_region_name)
        if not hasattr(resource, collection_name):
            raise QueryError(
                'Unknown collection <{0}> of resource <{1}>'.format(collection_name, resource_name))

        self.attach_region(region)
        self.refresh_table(region, table.table, resource, getattr(resource, collection_name))
866149
def load_stock_prices(self):
        
        from pricedb import SecuritySymbol

        info = StocksInfo(self.config)
        for item in self.model.stocks:
            symbol = SecuritySymbol("", "")
            symbol.parse(item.symbol)

            price: PriceModel = info.load_latest_price(symbol)
            if not price:
                # Use a dummy price of 1, effectively keeping the original amount.
                price = PriceModel()
                price.currency = self.config.get(ConfigKeys.default_currency)
                price.value = Decimal(1)
            item.price = price.value
            if isinstance(item, Stock):
                item.currency = price.currency
                # Do not set currency for Cash balance records.
        info.close_databases()
776186
def loadModel(self, model_file):
        """"""
        with open(model_file) as f:
            self.q_table = json.load(f)
123374
def load_scatter_table(self, fn):
        
        data = pickle.load(file(fn))

        if ("version" not in data) or (data["version"]!=tmatrix_aux.VERSION):
            warnings.warn("Loading data saved with another version.", Warning)

        (self.num_points, self.D_max, self._psd_D, self._S_table, 
            self._Z_table, self._angular_table, self._m_table, 
            self.geometries) = data["psd_scatter"]
        return (data["time"], data["description"])
1084327
def _load(self, data):
        
        self.raw_data = data
        self.creation_date = _convert_tstamp(data['creationDate'])
        self.economics = data['economics']
        self.id = data['id']
        self.last_modified_date = _convert_tstamp(data['lastModifiedDate'])
        self.length = data['length']
        self.link_text = data['linkText']
        self.link_url = data['linkURL']
        self.long_description = data['longDescription']
        self.name = data['name']
        self.plays_total = data['playsTotal']
        self.plays_trailing_week = data['playsTrailingWeek']
        self.published_date = _convert_tstamp(data['publishedDate'])
        self.start_date = _convert_tstamp(data.get('startDate', None))
        self.end_date = _convert_tstamp(data.get('endDate', None))
        self.reference_id = data['referenceId']
        self.short_description = data['shortDescription']
        self.tags = []
        for tag in data['tags']:
            self.tags.append(tag)
        self.thumbnail_url = data['thumbnailURL']
        self.video_still_url = data['videoStillURL']
1344416
def _convert_to_floats(self, data):
        
        for key, value in data.items():
            data[key] = float(value)

        return data
1190277
def record_to_objects(self):
        """"""
        from ambry.orm.exc import NotFoundError

        fr = self.record

        contents = fr.unpacked_contents

        if not contents:
            return

        # Zip transposes an array when in the form of a list of lists, so this transposes so
        # each row starts with the heading and the rest of the row are the values
        # for that row. The bool and filter return false when none of the values
        # are non-empty. Then zip again to transpose to original form.

        non_empty_rows = drop_empty(contents)

        s = self._dataset._database.session

        for i, row in enumerate(non_empty_rows):

            if i == 0:
                header = row
            else:
                d = dict(six.moves.zip(header, row))

                if 'widths' in d:
                    del d['widths']  # Obsolete column in old spreadsheets.

                if 'table' in d:
                    d['dest_table_name'] = d['table']
                    del d['table']

                if 'order' in d:
                    d['stage'] = d['order']
                    del d['order']

                if 'dest_table' in d:
                    d['dest_table_name'] = d['dest_table']
                    del d['dest_table']

                if 'source_table' in d:
                    d['source_table_name'] = d['source_table']
                    del d['source_table']

                d['d_vid'] = self._dataset.vid

                d['state'] = 'synced'

                try:
                    ds = self._dataset.source_file(str(d['name']))
                    ds.update(**d)
                except NotFoundError:
                    name = d['name']
                    del d['name']

                    try:
                        ds = self._dataset.new_source(name, **d)
                    except:
                        print(name, d)
                        import pprint
                        pprint.pprint(d)
                        raise
                except:  # Odd error with 'none' in keys for d
                    print('!!!', header)
                    print('!!!', row)
                    raise

                s.merge(ds)

        self._dataset._database.commit()
276493
def load(self, model, value):
        
        try:
            return self._cattrs_converter.structure(value, model)
        except (ValueError, TypeError) as e:
            raise SerializationException(str(e))
1269738
def load_data_table(table_name, meta_file, meta):
    
    for table in meta['tables']:
        if table['name'] == table_name:
            prefix = os.path.dirname(meta_file)
            relative_path = os.path.join(prefix, meta['path'], table['path'])
            return pd.read_csv(relative_path), table