Loading random time series data from a zip file in the repo
501770
def load_flights():
    """"""
    tbl_name = 'flights'
    data = get_example_data('flight_data.csv.gz', make_bytes=True)
    pdf = pd.read_csv(data, encoding='latin-1')

    # Loading airports info to join and get lat/long
    airports_bytes = get_example_data('airports.csv.gz', make_bytes=True)
    airports = pd.read_csv(airports_bytes, encoding='latin-1')
    airports = airports.set_index('IATA_CODE')

    pdf['ds'] = pdf.YEAR.map(str) + '-0' + pdf.MONTH.map(str) + '-0' + pdf.DAY.map(str)
    pdf.ds = pd.to_datetime(pdf.ds)
    del pdf['YEAR']
    del pdf['MONTH']
    del pdf['DAY']

    pdf = pdf.join(airports, on='ORIGIN_AIRPORT', rsuffix='_ORIG')
    pdf = pdf.join(airports, on='DESTINATION_AIRPORT', rsuffix='_DEST')
    pdf.to_sql(
        tbl_name,
        db.engine,
        if_exists='replace',
        chunksize=500,
        dtype={
            'ds': DateTime,
        },
        index=False)
    tbl = db.session.query(TBL).filter_by(table_name=tbl_name).first()
    if not tbl:
        tbl = TBL(table_name=tbl_name)
    tbl.description = 'Random set of flights in the US'
    tbl.database = utils.get_or_create_main_db()
    db.session.merge(tbl)
    db.session.commit()
    tbl.fetch_metadata()
    print('Done loading table!')
991032
def extract_zip(fileobj):
    """"""
    archive = zipfile.ZipFile(fileobj, mode='r')
    filenames = archive.namelist()

    for src_name, dst_name in filter_tzfiles(filenames):
        modified = archive.getinfo(src_name).date_time
        bytes = archive.read(src_name)

        yield dst_name, modified, bytes
968934
def load_data(verbose=False):
    ""

    df = pd.read_csv(STOPS_PATH, usecols=['stop_id', 'stop_name'])
    if verbose:
        print('- Loaded %d entries from "%s".' % (len(df), STOPS_PATH))
    return df
1019815
def _load_timestamps(self):
        """"""
        timestamp_file = os.path.join(self.sequence_path, 'times.txt')

        # Read and parse the timestamps
        self.timestamps = []
        with open(timestamp_file, 'r') as f:
            for line in f.readlines():
                t = dt.timedelta(seconds=float(line))
                self.timestamps.append(t)

        # Subselect the chosen range of frames, if any
        if self.frames is not None:
            self.timestamps = [self.timestamps[i] for i in self.frames]
730229
private static Data loadLeapSeconds(URL url) throws ClassNotFoundException, IOException {
        List<String> lines;
        try (BufferedReader reader = new BufferedReader(new InputStreamReader(url.openStream(), StandardCharsets.UTF_8))) {
            lines = reader.lines().collect(Collectors.toList());
        }
        List<Long> dates = new ArrayList<>();
        List<Integer> offsets = new ArrayList<>();
        for (String line : lines) {
            line = line.trim();
            if (line.isEmpty() || line.startsWith("#")) {
                continue;
            }
            Matcher matcher = LEAP_FILE_FORMAT.matcher(line);
            if (matcher.matches() == false) {
                throw new StreamCorruptedException("Invalid leap second file");
            }
            dates.add(LocalDate.parse(matcher.group(1)).getLong(JulianFields.MODIFIED_JULIAN_DAY));
            offsets.add(Integer.valueOf(matcher.group(2)));
        }
        long[] datesData = new long[dates.size()];
        int[] offsetsData = new int[dates.size()];
        long[] taiData = new long[dates.size()];
        for (int i = 0; i < datesData.length; i++) {
            datesData[i] = dates.get(i);
            offsetsData[i] = offsets.get(i);
            taiData[i] = tai(datesData[i], offsetsData[i]);
        }
        return new Data(datesData, offsetsData, taiData);
    }
335646
def make_rawr_zip_payload(rawr_tile, date_time=None):
    """"""
    if date_time is None:
        date_time = gmtime()[0:6]

    buf = StringIO()
    with zipfile.ZipFile(buf, mode='w') as z:
        for fmt_data in rawr_tile.all_formatted_data:
            zip_info = zipfile.ZipInfo(fmt_data.name, date_time)
            z.writestr(zip_info, fmt_data.data, zipfile.ZIP_DEFLATED)
    return buf.getvalue()
1227329
def load_data():
    ''''''
    boston = load_boston()
    X_train, X_test, y_train, y_test = train_test_split(boston.data, boston.target, random_state=99, test_size=0.25)
    #normalize data
    ss_X = StandardScaler()
    ss_y = StandardScaler()

    X_train = ss_X.fit_transform(X_train)
    X_test = ss_X.transform(X_test)
    y_train = ss_y.fit_transform(y_train[:, None])[:,0]
    y_test = ss_y.transform(y_test[:, None])[:,0]

    return X_train, X_test, y_train, y_test
96168
def _fetch_losc_data_file(url, *args, **kwargs):
    
    cls = kwargs.pop('cls', TimeSeries)
    cache = kwargs.pop('cache', None)
    verbose = kwargs.pop('verbose', False)

    # match file format
    if url.endswith('.gz'):
        ext = os.path.splitext(url[:-3])[-1]
    else:
        ext = os.path.splitext(url)[-1]
    if ext == '.hdf5':
        kwargs.setdefault('format', 'hdf5.losc')
    elif ext == '.txt':
        kwargs.setdefault('format', 'ascii.losc')
    elif ext == '.gwf':
        kwargs.setdefault('format', 'gwf')

    with _download_file(url, cache, verbose=verbose) as rem:
        # get channel for GWF if not given
        if ext == ".gwf" and (not args or args[0] is None):
            args = (_gwf_channel(rem, cls, kwargs.get("verbose")),)
        if verbose:
            print('Reading data...', end=' ')
        try:
            series = cls.read(rem, *args, **kwargs)
        except Exception as exc:
            if verbose:
                print('')
            exc.args = ("Failed to read LOSC data from %r: %s"
                        % (url, str(exc)),)
            raise
        else:
            # parse bits from unit in GWF
            if ext == '.gwf' and isinstance(series, StateVector):
                try:
                    bits = {}
                    for bit in str(series.unit).split():
                        a, b = bit.split(':', 1)
                        bits[int(a)] = b
                    series.bits = bits
                    series.override_unit('')
                except (TypeError, ValueError):  # don't care, bad LOSC
                    pass

            if verbose:
                print('[Done]')
            return series
196610
def load_stats(self, fdump):
        
        if isinstance(fdump, type('')):
            fdump = open(fdump, 'rb')
        self.index = pickle.load(fdump)
        self.snapshots = pickle.load(fdump)
        self.sorted = []
1046185
def fetch_losc_data(detector, start, end, cls=TimeSeries, **kwargs):
    
    # format arguments
    start = to_gps(start)
    end = to_gps(end)
    span = Segment(start, end)
    kwargs.update({
        'start': start,
        'end': end,
    })

    # find URLs (requires gwopensci)
    url_kw = {key: kwargs.pop(key) for key in GWOSC_LOCATE_KWARGS if
              key in kwargs}
    if 'sample_rate' in url_kw:  # format as Hertz
        url_kw['sample_rate'] = Quantity(url_kw['sample_rate'], 'Hz').value
    cache = get_urls(detector, int(start), int(ceil(end)), **url_kw)
    # if event dataset, pick shortest file that covers the request
    # -- this is a bit hacky, and presumes that only an event dataset
    # -- would be produced with overlapping files.
    # -- This should probably be improved to use dataset information
    if len(cache) and _overlapping(cache):
        cache.sort(key=lambda x: abs(file_segment(x)))
        for url in cache:
            a, b = file_segment(url)
            if a <= start and b >= end:
                cache = [url]
                break
    if kwargs.get('verbose', False):  # get_urls() guarantees len(cache) >= 1
        host = urlparse(cache[0]).netloc
        print("Fetched {0} URLs from {1} for [{2} .. {3}))".format(
            len(cache), host, int(start), int(ceil(end))))

    is_gwf = cache[0].endswith('.gwf')
    if is_gwf and len(cache):
        args = (kwargs.pop('channel', None),)
    else:
        args = ()

    # read data
    out = None
    kwargs['cls'] = cls
    for url in cache:
        keep = file_segment(url) & span
        new = _fetch_losc_data_file(url, *args, **kwargs).crop(
            *keep, copy=False)
        if is_gwf and (not args or args[0] is None):
            args = (new.name,)
        if out is None:
            out = new.copy()
        else:
            out.append(new, resize=True)
    return out
1350819
def from_url(cls, path):
        
        if os.path.isfile(path):
            with open(path) as fd:
                data = fd.read()
        else:
            try:
                response = urllib.urlopen(path)
                if response.code >= 300:
                    raise RuntimeError('Unable to load repo from %s' % path)

                data = response.read()
                response.close()
            except IOError:
                raise RuntimeError(
                    'Unable to load repo from %s (IO error)' % path
                )

        return cls(json.loads(data), path)
637989
def download_extract(url):
    """"""
    logger.info("Downloading %s", url)
    request = urllib2.Request(url)
    request.add_header('User-Agent',
                       'caelum/0.1 +https://github.com/nrcharles/caelum')
    opener = urllib2.build_opener()
    with tempfile.TemporaryFile(suffix='.zip', dir=env.WEATHER_DATA_PATH) \
            as local_file:
        logger.debug('Saving to temporary file %s', local_file.name)
        local_file.write(opener.open(request).read())
        compressed_file = zipfile.ZipFile(local_file, 'r')
        logger.debug('Extracting %s', compressed_file)
        compressed_file.extractall(env.WEATHER_DATA_PATH)
        local_file.close()
538203
def load_data(self):
        
        data = []
        loaded_dates = []
        loaded_indices = []
        for t, timestamp in enumerate(self.all_dates):
            date_str = timestamp.date().strftime("%Y%m%d")
            full_path = self.path_start + date_str + "/"
            if self.variable in os.listdir(full_path):
                full_path += self.variable + "/"
                data_files = sorted(os.listdir(full_path))
                file_dates = pd.to_datetime([d.split("_")[-1][0:13] for d in data_files])
                if timestamp in file_dates:
                    data_file = data_files[np.where(timestamp==file_dates)[0][0]]
                    print(full_path + data_file)
                    if data_file[-2:] == "gz":
                        subprocess.call(["gunzip", full_path + data_file])
                        file_obj = Nio.open_file(full_path + data_file[:-3])
                    else:
                        file_obj = Nio.open_file(full_path + data_file)
                    var_name = sorted(file_obj.variables.keys())[0]
                    data.append(file_obj.variables[var_name][:])
                    if self.lon is None:
                        self.lon = file_obj.variables["lon_0"][:]
                        # Translates longitude values from 0:360 to -180:180
                        if np.count_nonzero(self.lon > 180) > 0:
                            self.lon -= 360
                        self.lat = file_obj.variables["lat_0"][:]
                    file_obj.close()
                    if data_file[-2:] == "gz":
                        subprocess.call(["gzip", full_path + data_file[:-3]])
                    else:
                        subprocess.call(["gzip", full_path + data_file])
                    loaded_dates.append(timestamp)
                    loaded_indices.append(t)
        if len(loaded_dates) > 0:
            self.loaded_dates = pd.DatetimeIndex(loaded_dates)
            self.data = np.ones((self.all_dates.shape[0], data[0].shape[0], data[0].shape[1])) * -9999
            self.data[loaded_indices] = np.array(data)
746901
def _load_at(self, time, channels=None):
        """"""
        if channels is None:
            channels = slice(None, None, None)
        time = int(time)
        time_o = time
        ns = self.n_samples_trace
        if not (0 <= time_o < ns):
            raise ValueError("Invalid time {0:d}/{1:d}.".format(time_o, ns))
        slice_extract = _slice(time_o,
                               self.n_samples_before_after,
                               self._filter_margin)
        extract = self._traces[slice_extract][:, channels].astype(np.float32)

        # Pad the extracted chunk if needed.
        if slice_extract.start <= 0:
            extract = _pad(extract, self._n_samples_extract, 'left')
        elif slice_extract.stop >= ns - 1:
            extract = _pad(extract, self._n_samples_extract, 'right')

        assert extract.shape[0] == self._n_samples_extract
        return extract
766974
def _load_from_tar(self, dtype_out_time, dtype_out_vert=False):
        """"""
        path = os.path.join(self.dir_tar_out, 'data.tar')
        utils.io.dmget([path])
        with tarfile.open(path, 'r') as data_tar:
            ds = xr.open_dataset(
                data_tar.extractfile(self.file_name[dtype_out_time])
            )
            return ds[self.name]
1374551
def load_repo(client, path=None, index='git'):
    
    path = dirname(dirname(abspath(__file__))) if path is None else path
    repo_name = basename(path)
    repo = git.Repo(path)

    create_git_index(client, index)

    # we let the streaming bulk continuously process the commits as they come
    # in - since the `parse_commits` function is a generator this will avoid
    # loading all the commits into memory
    for ok, result in streaming_bulk(
            client,
            parse_commits(repo.refs.master.commit, repo_name),
            index=index,
            doc_type='doc',
            chunk_size=50 # keep the batch sizes small for appearances only
        ):
        action, result = result.popitem()
        doc_id = '/%s/doc/%s' % (index, result['_id'])
        # process the information from ES whether the document has been
        # successfully indexed
        if not ok:
            print('Failed to %s document %s: %r' % (action, doc_id, result))
        else:
            print(doc_id)
68578
def _memcache_key(self, timestamped=False):
        """"""
        request = tuple(map(str, self.package_requests))
        repo_ids = []
        for path in self.package_paths:
            repo = package_repository_manager.get_repository(path)
            repo_ids.append(repo.uid)

        t = ["resolve",
             request,
             tuple(repo_ids),
             self.package_filter_hash,
             self.package_orderers_hash,
             self.building,
             config.prune_failed_graph]

        if timestamped and self.timestamp:
            t.append(self.timestamp)

        return str(tuple(t))