Read data from a source into a `gwpy.timeseries` object.

    This method is just the internal worker for `TimeSeries.read`, and
    `TimeSeriesDict.read`, and isn't meant to be called directly.
506267
def read(cls, source, *args, **kwargs):
    
    # if reading a cache, read it now and sieve
    if io_cache.is_cache(source):
        from .cache import preformat_cache
        source = preformat_cache(source, *args[1:],
                                 start=kwargs.get('start'),
                                 end=kwargs.get('end'))

    # get join arguments
    pad = kwargs.pop('pad', None)
    gap = kwargs.pop('gap', 'raise' if pad is None else 'pad')
    joiner = _join_factory(cls, gap, pad)

    # read
    return io_read_multi(joiner, cls, source, *args, **kwargs)
5054
def _parse_data(self, raw_data, var_filter, time_extents):
        

        retval = defaultdict(list)
        p = parser()

        begin_time, end_time = time_extents

        for line in raw_data.splitlines():
            if len(line) == 0:
                continue

            fields = line.split("|")[0:-1]
            if var_filter is None or fields[2] in var_filter:
                dt = p.parse(fields[3]).replace(tzinfo=pytz.utc)
                if (begin_time is None or dt >= begin_time) and (
                    end_time is None or dt <= end_time
                ):
                    try:
                        value = (
                            float(fields[4]) if fields[4] != "NaN" else npNan
                        )
                    except ValueError:
                        value = npNan
                    retval[fields[0]].append((fields[2], dt, value))

        return dict(retval)
1294403
def read_dataset(args, dataset):
    
    path = os.path.join(vars(args)[dataset])
    logger.info('reading data from {}'.format(path))
    examples = [line.strip().split('\t') for line in open(path)]
    if args.max_num_examples > 0:
        examples = examples[:args.max_num_examples]
    # NOTE: assume data has been tokenized
    dataset = gluon.data.SimpleDataset([(e[0], e[1], LABEL_TO_IDX[e[2]]) for e in examples])
    dataset = dataset.transform(lambda s1, s2, label: (
        ['NULL'] + s1.lower().split(),
        ['NULL'] + s2.lower().split(), label),
                                lazy=False)
    logger.info('read {} examples'.format(len(dataset)))
    return dataset
117191
def ReadAllClientGraphSeries(
      self,
      client_label,
      report_type,
      time_range = None,
  ):
    """"""
    series_with_timestamps = {}
    for series_key, series in iteritems(self.client_graph_series):
      series_label, series_type, timestamp = series_key
      if series_label == client_label and series_type == report_type:
        if time_range is not None and not time_range.Includes(timestamp):
          continue
        series_with_timestamps[timestamp.Copy()] = series.Copy()
    return series_with_timestamps
803599
def read(self):
        """"""
        line = self.trace_file.readline()
        if line == '':
            if self.loop:
                self._reopen_file()
            else:
                self.trace_file.close()
                self.trace_file = None
                raise DataSourceError()

        message = JsonFormatter.deserialize(line)
        timestamp = message.get('timestamp', None)
        if self.realtime and timestamp is not None:
            self._store_timestamp(timestamp)
            self._wait(self.starting_time, self.first_timestamp, timestamp)
        return line + "\x00"
1030626
def internal_reader(self, input_stream):
        
        for input_line in input_stream:
            yield list(map(self.deserialize, input_line.split("\t")))
850332
def asdict(self):
        

        timestamp_str = None
        if self.reading_time is not None:
            timestamp_str = self.reading_time.isoformat()

        return {
            'stream': self.stream,
            'device_timestamp': self.raw_time,
            'streamer_local_id': self.reading_id,
            'timestamp': timestamp_str,
            'value': self.value
        }
1220050
def load(self, dtype_out_time, dtype_out_vert=False, region=False,
             plot_units=False, mask_unphysical=False):
        """"""
        msg = ("Loading data from disk for object={0}, dtype_out_time={1}, "
               "dtype_out_vert={2}, and region="
               "{3}".format(self, dtype_out_time, dtype_out_vert, region))
        logging.info(msg + ' ({})'.format(ctime()))
        # Grab from the object if its there.
        try:
            data = self.data_out[dtype_out_time]
        except (AttributeError, KeyError):
            # Otherwise get from disk.  Try scratch first, then archive.
            try:
                data = self._load_from_disk(dtype_out_time, dtype_out_vert,
                                            region=region)
            except IOError:
                data = self._load_from_tar(dtype_out_time, dtype_out_vert)
        # Copy the array to self.data_out for ease of future access.
        self._update_data_out(data, dtype_out_time)
        # Apply desired plotting/cleanup methods.
        if mask_unphysical:
            data = self.var.mask_unphysical(data)
        if plot_units:
            data = self.var.to_plot_units(data, dtype_vert=dtype_out_vert)
        return data
537226
def to_python(self, data):
        
        if data is None:
            return data
        if self.datatype:
            return self.datatype(data)
        else:
            return data
663657
def from_series(cls, series, offset=0):
        
        return cls(data=series.data, index=series.index, data_name=series.data_name, index_name=series.index_name,
                   sort=series.sort, offset=offset)
569633
def load_stream(self, stream):
        
        batches = super(ArrowStreamPandasSerializer, self).load_stream(stream)
        import pyarrow as pa
        for batch in batches:
            yield [self.arrow_to_pandas(c) for c in pa.Table.from_batches([batch]).itercolumns()]
991338
def _read(cls, filepath_or_buffer, **kwargs):
        
        # The intention of the inspection code is to reduce the amount of
        # communication we have to do between processes and nodes. We take a quick
        # pass over the arguments and remove those that are default values so we
        # don't have to serialize and send them to the workers. Because the
        # arguments list is so long, this does end up saving time based on the
        # number of nodes in the cluster.
        try:
            args, _, _, defaults, _, _, _ = inspect.getfullargspec(cls.read_csv)
            defaults = dict(zip(args[2:], defaults))
            filtered_kwargs = {
                kw: kwargs[kw]
                for kw in kwargs
                if kw in defaults
                and not isinstance(kwargs[kw], type(defaults[kw]))
                or kwargs[kw] != defaults[kw]
            }
        # This happens on Python2, we will just default to serializing the entire dictionary
        except AttributeError:
            filtered_kwargs = kwargs

        if isinstance(filepath_or_buffer, str):
            if not file_exists(filepath_or_buffer):
                ErrorMessage.default_to_pandas("File path could not be resolved")
                return cls._read_csv_from_pandas(filepath_or_buffer, filtered_kwargs)
        elif not isinstance(filepath_or_buffer, py.path.local):
            read_from_pandas = True
            # Pandas read_csv supports pathlib.Path
            try:
                import pathlib

                if isinstance(filepath_or_buffer, pathlib.Path):
                    read_from_pandas = False
            except ImportError:  # pragma: no cover
                pass
            if read_from_pandas:
                ErrorMessage.default_to_pandas("Reading from buffer.")
                return cls._read_csv_from_pandas(filepath_or_buffer, kwargs)
        if (
            _infer_compression(filepath_or_buffer, kwargs.get("compression"))
            is not None
        ):
            ErrorMessage.default_to_pandas("Compression detected.")
            return cls._read_csv_from_pandas(filepath_or_buffer, filtered_kwargs)

        chunksize = kwargs.get("chunksize")
        if chunksize is not None:
            ErrorMessage.default_to_pandas("Reading chunks from a file.")
            return cls._read_csv_from_pandas(filepath_or_buffer, filtered_kwargs)

        skiprows = kwargs.get("skiprows")
        if skiprows is not None and not isinstance(skiprows, int):
            ErrorMessage.default_to_pandas("skiprows parameter not optimized yet.")
            return cls._read_csv_from_pandas(filepath_or_buffer, kwargs)
        # TODO: replace this by reading lines from file.
        if kwargs.get("nrows") is not None:
            ErrorMessage.default_to_pandas("`read_csv` with `nrows`")
            return cls._read_csv_from_pandas(filepath_or_buffer, filtered_kwargs)
        else:
            return cls._read_csv_from_file_pandas_on_ray(
                filepath_or_buffer, filtered_kwargs
            )
302662
def read(self, ncfile, timegrid_data) -> None:
        
        array = query_array(ncfile, self.name)
        subdev2index = self.query_subdevice2index(ncfile)
        for subdevice, sequence in self.sequences.items():
            idx = subdev2index.get_index(subdevice)
            values = array[self.get_slices(idx, sequence.shape)]
            sequence.series = sequence.adjust_series(
                timegrid_data, values)
416496
def read(cls, data):
        
        if isinstance(data, OrderedDict):
            return cls(data)
        elif isinstance(data, basestring)\
             and data.startswith(("http://", "https://", "ftp://", "ftps://")):
            return cls(request(data))
        elif isinstance(data, basestring):
            try:
                json_dict = json.loads(data, object_pairs_hook=OrderedDict)
                return cls(json_dict)
            except ValueError:
                raise
        else:
            try:
                json_dict = json.load(data, object_pairs_hook=OrderedDict)
                return cls(json_dict)
            except ValueError:
                raise
57202
def _postprocess_data(self, data):
        
        new_data = {}
        for k, values in data.items():
            values = decode_bytes(values) # Bytes need decoding to strings

            # Certain datetime types need to be converted
            if len(values) and isinstance(values[0], cftime_types):
                if any(v.calendar not in _STANDARD_CALENDARS for v in values):
                    self.param.warning(
                        'Converting cftime.datetime from a non-standard '
                        'calendar (%s) to a standard calendar for plotting. '
                        'This may lead to subtle errors in formatting '
                        'dates, for accurate tick formatting switch to '
                        'the matplotlib backend.' % values[0].calendar)
                values = cftime_to_timestamp(values, 'ms')
            new_data[k] = values
        return new_data
146190
def reindex_timeseries(ts_string, new_timestamps):
    
    #If a single timestamp is passed in, turn it into a list
    #Reindexing can't work if it's not a list
    if not isinstance(new_timestamps, list):
        new_timestamps = [new_timestamps]

    #Convert the incoming timestamps to datetimes
    #if they are not datetimes.
    new_timestamps_converted = []
    for t in new_timestamps:
        new_timestamps_converted.append(get_datetime(t))

    new_timestamps = new_timestamps_converted

    seasonal_year = config.get('DEFAULT','seasonal_year', '1678')
    seasonal_key = config.get('DEFAULT', 'seasonal_key', '9999')

    ts = ts_string.replace(seasonal_key, seasonal_year)

    timeseries = pd.read_json(ts)

    idx = timeseries.index

    ts_timestamps = new_timestamps

    #'Fix' the incoming timestamp in case it's a seasonal value
    if type(idx) == pd.DatetimeIndex:
        if set(idx.year) == set([int(seasonal_year)]):
            if isinstance(new_timestamps,  list):
                seasonal_timestamp = []
                for t in ts_timestamps:
                    t_1900 = t.replace(year=int(seasonal_year))
                    seasonal_timestamp.append(t_1900)
                ts_timestamps = seasonal_timestamp

    #Reindex the timeseries to reflect the requested timestamps
    reindexed_ts = timeseries.reindex(ts_timestamps, method='ffill')

    i = reindexed_ts.index

    reindexed_ts.index = pd.Index(new_timestamps, names=i.names)

    #If there are no values at all, just return None
    if len(reindexed_ts.dropna()) == 0:
        return None

    #Replace all numpy NAN values with None
    pandas_ts = reindexed_ts.where(reindexed_ts.notnull(), None)

    return pandas_ts
1282521
def get_data(self):
        
        if self.handle is None:
            raise Exception("Cannot get data before construct Dataset")
        if self.data is not None and self.used_indices is not None and self.need_slice:
            if isinstance(self.data, np.ndarray) or scipy.sparse.issparse(self.data):
                self.data = self.data[self.used_indices, :]
            elif isinstance(self.data, DataFrame):
                self.data = self.data.iloc[self.used_indices].copy()
            elif isinstance(self.data, DataTable):
                self.data = self.data[self.used_indices, :]
            else:
                warnings.warn("Cannot subset {} type of raw data.\n"
                              "Returning original raw data".format(type(self.data).__name__))
            self.need_slice = False
        return self.data