Indexes all datasets of the library.

        Args:
            tick_f (callable, optional): callable of one argument. Gets string with index state.
9534
def index_library_datasets(self, tick_f=None):
        

        dataset_n = 0
        partition_n = 0

        def tick(d, p):
            if tick_f:
                tick_f('datasets: {} partitions: {}'.format(d, p))

        for dataset in self.library.datasets:

            if self.backend.dataset_index.index_one(dataset):
                # dataset added to index
                dataset_n += 1
                tick(dataset_n, partition_n)
                for partition in dataset.partitions:
                    self.backend.partition_index.index_one(partition)
                    partition_n += 1
                    tick(dataset_n, partition_n)
            else:
                # dataset already indexed
                pass
320756
def _index_fs(self):
        

        indexed_objects = self._return_deque()

        directory = self.job_args.get('directory')
        if directory:
            indexed_objects = self._return_deque(
                deque=indexed_objects,
                item=self._drectory_local_files(
                    directory=directory
                )
            )

        object_names = self.job_args.get('object')
        if object_names:
            indexed_objects = self._return_deque(
                deque=indexed_objects,
                item=self._named_local_files(
                    object_names=object_names
                )
            )

        return indexed_objects
1342428
def get_indexes_from_base(self):
        
        if self.is_indexed:
            return np.copy(self.order[i])
        if self.data.base is None:
            i = 0
        else:
            i = self.get_raw_index(0)
        return np.arange(i, i + len(self), dtype=np.uint32)
517671
def queryables(self):
        

        # compute the values_axes queryables
        return dict(
            [(a.cname, a) for a in self.index_axes] +
            [(self.storage_obj_type._AXIS_NAMES[axis], None)
             for axis, values in self.non_index_axes] +
            [(v.cname, v) for v in self.values_axes
             if v.name in set(self.data_columns)]
        )
299184
def get_dataframes(symbols=("sne", "goog", "tsla"), source='yahoo', refresh=False):
    
    symbols = util.make_symbols(list(symbols))
    if refresh:
        symbols_to_refresh = symbols
    else:
        symbols_to_refresh = [sym for sym in symbols if not Equity.objects.filter(symbol=sym).exists()]
    source = source.lower().strip()
    if source in ('yahoo', 'google'):
        source += '_finance'
    if source[:3] == 'fed':
        source = 'federal_reserve_economic_data'
    ccpanda = ccp.ConcurrentPandas()
    # set the data source
    getattr(ccpanda, "set_source_" + source)()
    if symbols_to_refresh:
        # tell concurrent pandas which keys/symbols to retrieve
        ccpanda.insert_keys(symbols_to_refresh)
        # start concurrentpandas threads
        ccpanda.consume_keys_asynchronous_threads()
        # FIXME: is there a better/faster iterator to use like `ccpanda.output_map` attribute?
        pseudodict = ccpanda.return_map()
    else:
        pseudodict = {}
    table = {}
    for sym in symbols:
        e, created = None, False
        if not sym in symbols_to_refresh:
            e, created = Equity.objects.get_or_create(symbol=sym)
        if created or not e or not e.time_series or sym in symbols_to_refresh:
            e, created = Equity.objects.get_or_create(
                symbol=sym,
                name=sym,  # FIXME: use data source to find equity name!
                time_series=pseudodict[sym].to_json(),
                )
        table[sym] = pd.io.json.read_json(path_or_buf=e.time_series, orient='columns', typ='frame', convert_dates=True)
    return table
282061
def on_drag_data_get(self, widget, context, data, info, time):
        
        library_state = self._get_selected_library_state()
        import rafcon.gui.helpers.state_machine as gui_helper_state_machine
        gui_helper_state_machine.add_state_by_drag_and_drop(library_state, data)
351906
def get_pk_index(self, collection):
        
        cls = self.collections[collection]

        if not cls.get_pk_name() in self.indexes[collection]:
            self.create_index(cls.get_pk_name(), collection)
        return self.indexes[collection][cls.get_pk_name()]
449060
def datagetter(cls):
        
        with open('myfile', 'rt') as f:
            rows = [r for r in csv.reader(f)]
        dothing = lambda _: [i for i, v in enumerate(_)]
        rows = [dothing(_) for _ in rows]
        raise NotImplementedError('You need to implement this yourlself!')
        return rows
1066733
def _set_TS_index(self, data):
        
        
        # Set index
        data.index = pd.to_datetime(data.index, error= "ignore")

        # Format types to numeric
        for col in data.columns:
            data[col] = pd.to_numeric(data[col], errors="coerce")

        return data
1038750
def get_all_resources(datasets):
        # type: (List['Dataset']) -> List[hdx.data.resource.Resource]
        
        resources = []
        for dataset in datasets:
            for resource in dataset.get_resources():
                resources.append(resource)
        return resources
475631
def get_dataframe(self, sort_key="wall_time", **kwargs):
        
        import pandas as pd
        frame = pd.DataFrame(columns=AbinitTimerSection.FIELDS)

        for osect in self.order_sections(sort_key):
            frame = frame.append(osect.to_dict(), ignore_index=True)

        # Monkey patch
        frame.info =  self.info
        frame.cpu_time =  self.cpu_time
        frame.wall_time =  self.wall_time
        frame.mpi_nprocs =  self.mpi_nprocs
        frame.omp_nthreads =  self.omp_nthreads
        frame.mpi_rank =  self.mpi_rank
        frame.fname =  self.fname

        return frame
1214495
def _get_index(self, dim, index):
    """"""
    if isinstance(index, six.string_types):
      try:
        return self._index_names[dim][index]
      except KeyError:
        raise KeyError("Name '%s' is invalid for axis %s." % (index, dim))
      except TypeError:
        raise TypeError(
            "Trying to access an unnamed axis %s by name: '%s'" % (dim, index))
    else:
      return index
344887
def get_dataframe():
        
        names = []
        [names.extend(line.split()) for line in CONTROL_VARIABLE_LINES]

        defaults = []
        [defaults.extend(line.split()) for line in CONTROL_DEFAULT_LINES]

        types, required,cast_defaults,formats = [],[],[],[]
        for name,default in zip(names,defaults):
            if '[' in name or ']' in name:
                required.append(False)
            else:
                required.append(True)
            v,t,f = ControlData._parse_value(default)
            types.append(t)
            formats.append(f)
            cast_defaults.append(v)
        return pandas.DataFrame({"name":names,"type":types,
                                     "value":cast_defaults,"required":required,
                                    "format":formats})
1099326
def _check_timeindex(self):
        
        try:
            self.timeseries.generation_fluctuating
            self.timeseries.generation_dispatchable
            self.timeseries.load
            self.timeseries.generation_reactive_power
            self.timeseries.load_reactive_power
        except:
            message = 'Time index of feed-in and load time series does ' \
                      'not match.'
            logging.error(message)
            raise KeyError(message)
402495
def get_timeseries(self, *args, **kwargs):
        
        import predix.data.timeseries
        ts = predix.data.timeseries.TimeSeries(*args, **kwargs)
        return ts
85378
def get_dataframes():
    """"""

    for k, v in caller_locals().items():
        if k.startswith('_'):
            continue

        if isinstance(v, pd.core.frame.DataFrame):
            yield k, v
1350307
def _index_file(self):
        
        with self.filename.open() as fd:
            data = fd.read()

        ident = "Contour in frame"
        self._data = data.split(ident)[1:]
        self._initialized = True