Build tabula-py option from template file

    Args:
        file_like_obj: File like object of Tabula app template

    Returns:
        `obj`:dict: tabula-py options
772443
def load_template(path_or_buffer):
    '''
    '''

    from itertools import groupby
    from operator import itemgetter

    path_or_buffer = _stringify_path(path_or_buffer)

    if is_file_like(path_or_buffer):
        templates = json.load(path_or_buffer)
    else:
        with open(path_or_buffer, 'r') as f:
            templates = json.load(f)

    options = []

    grouper = itemgetter('page', 'extraction_method')

    for key, grp in groupby(sorted(templates, key=grouper), grouper):
        tmp_options = [_convert_template_option(e) for e in grp]

        if len(tmp_options) == 1:
            options.append(tmp_options[0])
            continue

        option = tmp_options[0]
        areas = [e.get('area') for e in tmp_options]
        option['area'] = areas
        option['multiple_tables'] = True
        options.append(option)

    return options
988775
def _parse_file(self, file_obj):
        
        byte_data = file_obj.read(self.size)
        self._parse_byte_data(byte_data)
218353
def get_template(template_file='', **kwargs):
    
    template = get_template_object(template_file)

    LOG.info('Rendering template %s', template.filename)
    for key, value in kwargs.items():
        LOG.debug('%s => %s', key, value)

    rendered_json = template.render(**kwargs)
    LOG.debug('Rendered JSON:\n%s', rendered_json)

    return rendered_json
243155
def ParseFileObject(self, parser_mediator, file_object):
    
    olecf_file = pyolecf.file()
    olecf_file.set_ascii_codepage(parser_mediator.codepage)

    try:
      olecf_file.open_file_object(file_object)
    except IOError as exception:
      parser_mediator.ProduceExtractionWarning(
          'unable to open file with error: {0!s}'.format(exception))
      return

    root_item = olecf_file.root_item
    if not root_item:
      return

    # Get a list of all items in the root item from the OLECF file.
    item_names = [item.name for item in root_item.sub_items]

    # Compare the list of available plugin objects.
    # We will try to use every plugin against the file (except
    # the default plugin) and run it. Only if none of the plugins
    # works will we use the default plugin.

    item_names = frozenset(item_names)

    try:
      for plugin in self._plugins:
        if parser_mediator.abort:
          break

        if not plugin.REQUIRED_ITEMS.issubset(item_names):
          continue

        try:
          plugin.UpdateChainAndProcess(parser_mediator, root_item=root_item)

        except Exception as exception:  # pylint: disable=broad-except
          parser_mediator.ProduceExtractionWarning((
              'plugin: {0:s} unable to parse OLECF file with error: '
              '{1!s}').format(plugin.NAME, exception))

      if self._default_plugin and not parser_mediator.abort:
        try:
          self._default_plugin.UpdateChainAndProcess(
              parser_mediator, root_item=root_item)

        except Exception as exception:  # pylint: disable=broad-except
          parser_mediator.ProduceExtractionWarning((
              'plugin: {0:s} unable to parse OLECF file with error: '
              '{1!s}').format(self._default_plugin.NAME, exception))

    finally:
      olecf_file.close()
234365
function(file, asObject) {
		var self = this,
			template = {
				'background' : 'url(\'{url}\') 0 0 no-repeat',
				'background-size' : 'contain'
			},
			style = '',
			cssObj = {},
			i = 0;
		if (file.icon) {
			style = 'style="';
			$.each(template, function(k, v) {
				if (i++ === 0) {
					v = v.replace('{url}', self.escape(file.icon));
				}
				if (asObject) {
					cssObj[k] = v;
				} else {
					style += k+':'+v+';';
				}
			});
			style += '"';
		}
		return asObject? cssObj : style;
	}
422316
def excel_to_dict(excel_filepath, encapsulate_filepath=False, **kwargs):
    
    result = {}
    try:
        callbacks = {'to_dictlist': excel_todictlist}  # Default callback
        callbacks.update(kwargs.get('alt_callbacks', {}))

        # Retrieve excel data as dict of sheets lists
        excel_data = callbacks.get('to_dictlist')(excel_filepath, **kwargs)
        for sheet in excel_data.keys():
            try:
                kwargs['rows'] = excel_data.get(sheet, [])
                result[sheet] = csv_to_dict(excel_filepath, **kwargs)
            except Exception as ex:
                logger.error('Fail to parse sheet {} - {}'.format(sheet, ex))
                result[sheet] = []
                continue

        if encapsulate_filepath:
            result = {excel_filepath: result}

    except Exception as ex:
        msg = 'Fail transform excel to dict - {}'.format(ex)
        logger.error(msg, excel_filepath=excel_filepath)

    return result
763143
def to_df(obj, cfg={}, raise_on_error=True, silent=False, verbose=False):
    
    
    if type(obj) == str:
        name = obj
    else:
        name = obj.name
    name = name[name.rfind('/') + 1:]
    
    if not raise_on_error:
        try:
            return to_df(obj=obj, cfg=cfg, raise_on_error=True)
        except (pd.parser.CParserError, AttributeError, ValueError, TypeError, IOError) as e:
            if not silent:
                print('WARNING in {}: {} {}'.format(name, e.__class__, e))
            return None
        except:
            if not silent:
                print('WARNING in {}: {}'.format(name, sys.exc_info()[0]))
            return None

    params = {}
    if 'default' in cfg:
        params = deepcopy(cfg['default'])
    if name in cfg:
        for e in cfg[name]:
            params[e] = deepcopy(cfg[name][e])
    if 'custom_date_parser' in params:
        params['date_parser'] = DtParser(params['custom_date_parser']).parse
        del params['custom_date_parser']

    if verbose:
        print(name, params)
    
    if name.lower().startswith('s3:'):
        if not find_spec('boto'):
            raise ImportError('reading from aws-s3 requires the boto package to be installed')
    
    if '.csv' in name.lower():
        # name can be .csv.gz or .csv.bz2
        return pd.read_csv(obj, **params)
        
    elif '.tsv' in name.lower() or '.txt' in name.lower():
        # name can be .tsv.gz or .txt.bz2
        return pd.read_table(obj, **params)
    
    elif name.lower().endswith(('.htm', '.html')):
        if not find_spec('lxml'):
            params['flavor'] = 'bs4'
            if not find_spec('bs4') and not find_spec('html5lib'):
                raise ImportError('reading html requires the lxml or bs4 + html5lib packages to be installed')

        if 'nrows' in params:
            del params['nrows']
        
        if type(obj) == zipfile.ZipExtFile:
            obj = obj.read()
        data = pd.read_html(obj, **params)
        data = {str(i): data[i] for i in range(len(data))}
        return data
    
    elif name.lower().endswith('.xml'):
        if 'nrows' in params:
            del params['nrows']
        
        from icy.utils import xml_to_json
        
        with open(obj) as f:
            json = xml_to_json(str(f.read()))
        
        return pd.read_json(json, **params)
        
    elif name.lower().endswith('.json'):
        if 'nrows' in params:
            del params['nrows']
        
        return pd.read_json(obj, **params)
    
    elif name.lower().endswith(('.xls', '.xlsx')):
        if not find_spec('xlrd'):
            raise ImportError('reading excel files requires the xlrd package to be installed')
        
        if 'nrows' in params:
            del params['nrows']
        
        data = {}
        xls = pd.ExcelFile(obj)
        for key in xls.sheet_names:
            data[key] = xls.parse(key, **params)
        return data
    
    elif name.lower().endswith(('.h5', '.hdf5')):
        if not find_spec('tables'):
            raise ImportError('reading hdf5 files requires the pytables package to be installed')
        
        if 'nrows' in params:
            del params['nrows']
            # params['chunksize'] = params.pop('nrows') # returns iterator
        
        with pd.HDFStore(obj) as store:
            data = {}
            for key in store.keys():
                data[key[1:]] = store[key]
        return data
    
    elif name.lower().endswith(('.sqlite', '.sql', '.db')):
        import sqlite3
        if type(obj) != str:
            raise IOError('sqlite-database must be decompressed before import')
        
        if 'nrows' in params:
            del params['nrows']
            # params['chunksize'] = params.pop('nrows') # returns iterator
        
        with sqlite3.connect(obj) as con:
            data = {}
            cursor = con.cursor()
            cursor.execute('SELECT name FROM sqlite_master WHERE type="table"')
            tables = [t[0] for t in cursor.fetchall()]
            for t in tables:
                sql = 'SELECT * FROM ' + t
                data[t] = pd.read_sql_query(sql, con, **params)
        return data
    
    else:
        try:
            data = {name: odo(obj, pd.DataFrame)}
            if type(data[name]) == pd.DataFrame:
                return data
        except NotImplementedError:
            pass
        raise NotImplementedError('Error creating DataFrame from object', obj)
834999
def set_fortpy_templates(obj, fortpy_templates=None):
    
    #If they didn't specify a custom templates directory, use the default
    #one that shipped with the package.
    from os import path
    if fortpy_templates is not None:
        obj.fortpy_templates = path.abspath(path.expanduser(fortpy_templates))
    else:
        from fortpy.utility import get_fortpy_templates_dir
        obj.fortpy_templates = get_fortpy_templates_dir()
551434
def from_file(cls, file):
        """"""
        opf_xml = etree.parse(file)
        # Check if ``file`` is file-like.
        if hasattr(file, 'read'):
            name = os.path.basename(file.name)
            root = os.path.abspath(os.path.dirname(file.name))
        else:  # ...a filepath
            name = os.path.basename(file)
            root = os.path.abspath(os.path.dirname(file))
        parser = OPFParser(opf_xml)

        # Roll through the item entries
        manifest = opf_xml.xpath('/opf:package/opf:manifest/opf:item',
                                 namespaces=EPUB_OPF_NAMESPACES)
        pkg_items = []
        for item in manifest:
            absolute_filepath = os.path.join(root, item.get('href'))
            properties = item.get('properties', '').split()
            is_navigation = 'nav' in properties
            media_type = item.get('media-type')
            pkg_items.append(Item.from_file(absolute_filepath,
                                            media_type=media_type,
                                            is_navigation=is_navigation,
                                            properties=properties))
        # Ignore spine ordering, because it is not important
        #   for our use cases.
        return cls(name, pkg_items, parser.metadata)
730217
def fromtab(args):
    
    p = OptionParser(fromtab.__doc__)
    p.set_sep(sep=None)
    p.add_option("--noheader", default=False, action="store_true",
                 help="Ignore first line")
    p.add_option("--replace",
                 help="Replace spaces in name to char [default: %default]")
    opts, args = p.parse_args(args)

    if len(args) != 2:
        sys.exit(not p.print_help())

    tabfile, fastafile = args
    sep = opts.sep
    replace = opts.replace
    fp = must_open(tabfile)
    fw = must_open(fastafile, "w")
    nseq = 0
    if opts.noheader:
        next(fp)
    for row in fp:
        row = row.strip()
        if not row or row[0] == '#':
            continue

        name, seq = row.rsplit(sep, 1)
        if replace:
            name = name.replace(" ", replace)
        print(">{0}\n{1}".format(name, seq), file=fw)
        nseq += 1
    fw.close()

    logging.debug("A total of {0} sequences written to `{1}`.".\
                   format(nseq, fastafile))
278868
def convert_ipynb_to_gallery(file_name):
    
    python_file = ""

    nb_dict = json.load(open(file_name))
    cells = nb_dict['cells']

    for i, cell in enumerate(cells):
        if i == 0:
            assert cell['cell_type'] == 'markdown', \
                'First cell has to be markdown'

            md_source = ''.join(cell['source'])
            rst_source = pdoc.convert_text(md_source, 'rst', 'md')
            python_file = '"""\n' + rst_source + '\n"""'
        else:
            if cell['cell_type'] == 'markdown':
                md_source = ''.join(cell['source'])
                rst_source = pdoc.convert_text(md_source, 'rst', 'md')
                commented_source = '\n'.join([
                    '# ' + x for x in rst_source.split('\n')
                ])
                python_file = python_file + '\n\n\n' + '#' * 70 + '\n' + \
                    commented_source
            elif cell['cell_type'] == 'code':
                source = ''.join(cell['source'])
                python_file = python_file + '\n' * 2 + source

    open(file_name.replace('.ipynb', '.py'), 'w').write(python_file)
244093
def _parse_multifile(self, desired_type: Type[T], obj: PersistedObject,
                         parsing_plan_for_children: Dict[str, ParsingPlan], logger: Logger,
                         options: Dict[str, Dict[str, Any]]) -> T:
        
        pass
547969
def statstable(args):
    
    p = OptionParser(statstable.__doc__)
    opts, args = p.parse_args(args)

    if len(args) < 1:
        sys.exit(not p.print_help())

    gff_files = args
    for metric in metrics:
        logging.debug("Parsing files in `{0}`..".format(metric))

        table = {}
        for x in gff_files:
            pf = op.basename(x).split(".")[0]
            numberfile = op.join(metric, pf + ".txt")
            ar = [int(x.strip()) for x in open(numberfile)]
            sum = SummaryStats(ar).todict().items()
            keys, vals = zip(*sum)
            keys = [(pf, x) for x in keys]
            table.update(dict(zip(keys, vals)))

        print(tabulate(table), file=sys.stderr)
482530
def iter_(obj):
    
    if six.PY2:
        file_types = file,  # noqa
    if six.PY3:
        file_types = io.IOBase,
        dict_items = {}.items().__class__
        dict_values = {}.values().__class__
        dict_keys = {}.keys().__class__
        dict_view = (dict_items, dict_values, dict_keys)

    if isinstance(obj, dict):
        return ordered_sequence_iterator(list(obj.keys()))
    if isinstance(obj, file_types):
        return file_iterator(obj)
    if six.PY2:
        if isinstance(obj, (list, tuple)):
            return ordered_sequence_iterator(obj)
        if isinstance(obj, xrange):  # noqa
            return range_iterator(obj)
        if NUMPY_AVAILABLE and isinstance(obj, numpy.ndarray):
            return ordered_sequence_iterator(obj)
    if six.PY3 and isinstance(obj, dict_view):
        return ordered_sequence_iterator(list(obj))
    return iter(obj)
67525
def obj_to_file(obj, filename, filetype='auto', ndarray_to_list=False, squeeze=True):
    '''

    '''
    # import json
    # with open(filename, mode='w') as f:
    #    json.dump(annotation,f)
    if ndarray_to_list:
        obj = ndarray_to_list_in_structure(obj, squeeze=squeeze)

    # write to yaml
    d = os.path.dirname(os.path.abspath(filename))
    if not os.path.exists(d):
        os.makedirs(d)

    if filetype == 'auto':
        _, ext = os.path.splitext(filename)
        filetype = ext[1:]

    if filetype in ('yaml', 'yml'):
        # import yaml
        from ruamel.yaml import YAML
        yaml = YAML(typ="unsafe")
        with open(filename, 'wt', encoding="utf-8") as f:
            yaml.dump(obj, f)
        # if sys.version_info.major == 2:
        #     with open(filename, 'wb') as f:
        #         yaml.dump(obj, f, encoding="utf-8")
        # else:
        #     with open(filename, "w", encoding="utf-8") as f:
        #         yaml.dump(obj, f)
    elif filetype in ('pickle', 'pkl'):
        f = open(filename, 'wb')
        logger.info("filename " + filename)
        # if sys.version_info[0] < 3: import cPickle as pickle
        # else: import _pickle as pickle
        import pickle
        pickle.dump(obj, f, -1)
        f.close
    elif filetype in ('streamingpicklezip', 'spklz'):
        # this is not working :-(
        import gzip
        import sPickle as pickle
        f = gzip.open(filename, 'wb', compresslevel=1)
        # f = open(filename, 'wb')
        pickle.s_dump(obj, f)
        f.close
    elif filetype in ('picklezip', 'pklz'):
        import gzip
        if sys.version_info[0] < 3: import cPickle as pickle
        else: import _pickle as pickle
        f = gzip.open(filename, 'wb', compresslevel=1)
        # f = open(filename, 'wb')
        pickle.dump(obj, f)
        f.close
    elif filetype in('mat'):

        import scipy.io as sio
        sio.savemat(filename, obj)
    else:
        logger.error('Unknown filetype ' + filetype)
1358042
def ParseFileObject(self, parser_mediator, file_object):
    
    esedb_file = pyesedb.file()

    try:
      esedb_file.open_file_object(file_object)
    except IOError as exception:
      parser_mediator.ProduceExtractionWarning(
          'unable to open file with error: {0!s}'.format(exception))
      return

    # Compare the list of available plugin objects.
    cache = ESEDBCache()
    try:
      table_names = frozenset(self._GetTableNames(esedb_file))

      for plugin in self._plugins:
        if parser_mediator.abort:
          break

        if not plugin.required_tables.issubset(table_names):
          continue

        try:
          plugin.UpdateChainAndProcess(
              parser_mediator, cache=cache, database=esedb_file)

        except Exception as exception:  # pylint: disable=broad-except
          parser_mediator.ProduceExtractionWarning((
              'plugin: {0:s} unable to parse ESE database with error: '
              '{1!s}').format(plugin.NAME, exception))

    finally:
      # TODO: explicitly clean up cache.

      esedb_file.close()
1269506
def load_data(self, table_name, obj, **kwargs):
        
        # kwargs is a catch all for any options required by other backends.
        self.dictionary[table_name] = pd.DataFrame(obj)