Loads data from a directory.
    Returns tuple (config_dict, wordlists).
    Raises Exception on failure (e.g. if data is corrupted).
344518
def _load_data(path):
    
    path = os.path.abspath(path)
    if not os.path.isdir(path):
        raise InitializationError('Directory not found: {0}'.format(path))
    wordlists = {}
    for file_name in os.listdir(path):
        if os.path.splitext(file_name)[1] != '.txt':
            continue
        file_path = os.path.join(path, file_name)
        name = os.path.splitext(os.path.split(file_path)[1])[0]
        try:
            with codecs.open(file_path, encoding='utf-8') as file:
                wordlists[name] = _load_wordlist(name, file)
        except OSError as ex:
            raise InitializationError('Failed to read {}: {}'.format(file_path, ex))
    config = _load_config(os.path.join(path, 'config.json'))
    return (config, wordlists)
379466
def config_dict(name: str) -> Dict[str, Any]:
    
    try:
        content = resource_string(PACKAGE, DATADIR.format(name)).decode()
    except DistributionNotFound as error:
        LOGGER.warning("Cannot load %s from packages: %s", name, error)
        content = DATA_FALLBACK.joinpath(name).read_text()

    return cast(Dict[str, Any], json.loads(content))
1356368
def load_dict(self, source, namespace='', make_namespaces=False):
        ''' 
        '''
        stack = [(namespace, source)]
        while stack:
            prefix, source = stack.pop()
            if not isinstance(source, dict):
                raise TypeError('Source is not a dict (r)' % type(key))
            for key, value in source.items():
                if not isinstance(key, str):
                    raise TypeError('Key is not a string (%r)' % type(key))
                full_key = prefix + '.' + key if prefix else key
                if isinstance(value, dict):
                    stack.append((full_key, value))
                    if make_namespaces:
                        self[full_key] = self.Namespace(self, full_key)
                else:
                    self[full_key] = value
        return self
643035
def _load_config(self, path):
        '''
        
        '''
        config = {}
        log.debug('Reading configuration from %s', path)
        if not os.path.isdir(path):
            msg = (
                'Unable to read from {path}: '
                'the directory does not exist!'
            ).format(path=path)
            log.error(msg)
            raise IOError(msg)
        # The directory tree should look like the following:
        # .
        # ├── __init__.py
        # ├── eos
        # │   └── init.yml
        # ├── iosxr
        # │   └── __init__.py
        # ├── junos
        # │   └── init.yml
        # │   └── bgp_read_message.py
        # │   └── BGP_PREFIX_THRESH_EXCEEDED.py
        # └── nxos
        #     └── init.yml
        os_subdirs = [sdpath[0] for sdpath in os.walk(path)][1:]
        if not os_subdirs:
            log.error('%s does not contain any OS subdirectories', path)
        for os_dir in os_subdirs:
            os_name = os.path.split(os_dir)[1]  # the network OS name
            if os_name.startswith('__'):
                log.debug('Ignoring %s', os_name)
                continue
            if not self._whitelist_blacklist(os_name):
                log.debug('Not building config for %s (whitelist-blacklist logic)', os_name)
                # Ignore devices that are not in the whitelist (if defined),
                #   or those operating systems that are on the blacklist.
                # This way we can prevent starting unwanted sub-processes.
                continue
            log.debug('Building config for %s:', os_name)
            log.debug('='*40)
            if os_name not in config:
                config[os_name] = {}
            files = os.listdir(os_dir)
            # Read all files under the OS dir
            for file_ in files:
                log.debug('Inspecting %s', file_)
                file_name, file_extension = os.path.splitext(file_)
                file_extension = file_extension.replace('.', '')
                filepath = os.path.join(os_dir, file_)
                comment = ''
                if file_extension in ('yml', 'yaml'):
                    try:
                        log.debug('Loading %s as YAML', file_)
                        with open(filepath, 'r') as fstream:
                            cfg = yaml.load(fstream)
                            # Reposition at the top and read the comments.
                            if file_name not in CONFIG.OS_INIT_FILENAMES:
                                # If the file name is not a profile init.
                                fstream.seek(0)
                                comment = self._extract_yaml_docstring(fstream)
                                if 'messages' in cfg:
                                    for message in cfg['messages']:
                                        message['__doc__'] = comment
                            napalm_logs.utils.dictupdate(config[os_name], cfg)
                    except yaml.YAMLError as yamlexc:
                        log.error('Invalid YAML file: %s', filepath, exc_info=True)
                        if file_name in CONFIG.OS_INIT_FILENAMES:
                            # Raise exception and break only when the init file is borked
                            #   otherwise, it will try loading best efforts.
                            raise IOError(yamlexc)
                elif file_extension == 'py':
                    log.debug('Lazy loading Python module %s', file_)
                    mod_fp, mod_file, mod_data = imp.find_module(file_name, [os_dir])
                    mod = imp.load_module(file_name, mod_fp, mod_file, mod_data)
                    if file_name in CONFIG.OS_INIT_FILENAMES:
                        # Init file defined as Python module
                        log.debug('%s seems to be a Python profiler', filepath)
                        # Init files require to define the `extract` function.
                        # Sample init file:
                        # def extract(message):
                        #     return {'tag': 'A_TAG', 'host': 'hostname'}
                        if hasattr(mod, CONFIG.INIT_RUN_FUN) and\
                           hasattr(getattr(mod, CONFIG.INIT_RUN_FUN), '__call__'):
                            # if extract is defined and is callable
                            if 'prefixes' not in config[os_name]:
                                config[os_name]['prefixes'] = []
                            config[os_name]['prefixes'].append({
                                'values': {'tag': ''},
                                'line': '',
                                '__python_fun__': getattr(mod, CONFIG.INIT_RUN_FUN),
                                '__python_mod__': filepath  # Will be used for debugging
                            })
                            log.info('Adding the prefix function defined under %s to %s',
                                     filepath, os_name)
                        elif file_name != '__init__':
                            # If __init__.py does not have the extractor function, no problem.
                            log.warning('%s does not have the "%s" function defined. Ignoring.',
                                        filepath, CONFIG.INIT_RUN_FUN)
                    else:
                        # Other python files require the `emit` function.
                        if hasattr(mod, '__tag__'):
                            mod_tag = getattr(mod, '__tag__')
                        else:
                            log.info('%s does not have __tag__, defaulting the tag to %s', filepath, file_name)
                            mod_tag = file_name
                        if hasattr(mod, '__error__'):
                            mod_err = getattr(mod, '__error__')
                        else:
                            log.info('%s does not have __error__, defaulting the error to %s', filepath, file_name)
                            mod_err = file_name
                        if hasattr(mod, '__match_on__'):
                            err_match = getattr(mod, '__match_on__')
                        else:
                            err_match = 'tag'
                        model = CONFIG.OPEN_CONFIG_NO_MODEL
                        if hasattr(mod, '__yang_model__'):
                            model = getattr(mod, '__yang_model__')
                        log.debug('Mathing on %s', err_match)
                        if hasattr(mod, CONFIG.CONFIG_RUN_FUN) and\
                           hasattr(getattr(mod, CONFIG.CONFIG_RUN_FUN), '__call__'):
                            log.debug('Adding %s with tag:%s, error:%s, matching on:%s',
                                      file_, mod_tag, mod_err, err_match)
                            # the structure below must correspond to the VALID_CONFIG structure enforcement
                            if 'messages' not in config[os_name]:
                                config[os_name]['messages'] = []
                            config[os_name]['messages'].append({
                                'tag': mod_tag,
                                'error': mod_err,
                                'match_on': err_match,
                                '__doc__': mod.__doc__,
                                '__python_fun__': getattr(mod, CONFIG.CONFIG_RUN_FUN),
                                '__python_mod__': filepath,  # Will be used for debugging
                                'line': '',
                                'model': model,
                                'values': {},
                                'mapping': {'variables': {}, 'static': {}}
                            })
                        else:
                            log.warning('%s does not have the "%s" function defined. Ignoring.',
                                        filepath, CONFIG.CONFIG_RUN_FUN)
                else:
                    log.info('Ignoring %s (extension not allowed)', filepath)
            log.debug('-'*40)
        if not config:
            msg = 'Could not find proper configuration files under {path}'.format(path=path)
            log.error(msg)
            raise IOError(msg)
        log.debug('Complete config:')
        log.debug(config)
        log.debug('ConfigParserg size in bytes: %d', sys.getsizeof(config))
        return config
801847
def _load_wordlist(name, stream):
    
    items = []
    max_length = None
    multiword = False
    multiword_start = None
    number_of_words = None
    for i, line in enumerate(stream, start=1):
        line = line.strip()
        if not line or line.startswith('#'):
            continue
        # Is it an option line, e.g. 'max_length = 10'?
        if '=' in line:
            if items:
                raise ConfigurationError('Invalid assignment at list {!r} line {}: {!r} '
                                         '(options must be defined before words)'
                                         .format(name, i, line))
            try:
                option, option_value = _parse_option(line)
            except ValueError as ex:
                raise ConfigurationError('Invalid assignment at list {!r} line {}: {!r} '
                                         '({})'
                                         .format(name, i, line, ex))
            if option == _CONF.FIELD.MAX_LENGTH:
                max_length = option_value
            elif option == _CONF.FIELD.NUMBER_OF_WORDS:
                number_of_words = option_value
            continue  # pragma: no cover
        # Parse words
        if not multiword and _WORD_REGEX.match(line):
            if max_length is not None and len(line) > max_length:
                raise ConfigurationError('Word is too long at list {!r} line {}: {!r}'
                                         .format(name, i, line))
            items.append(line)
        elif _PHRASE_REGEX.match(line):
            if not multiword:
                multiword = True
                multiword_start = len(items)
            phrase = tuple(line.split(' '))
            if number_of_words is not None and len(phrase) != number_of_words:
                raise ConfigurationError('Phrase has {} word(s) (while number_of_words={}) '
                                         'at list {!r} line {}: {!r}'
                                         .format(len(phrase), number_of_words, name, i, line))
            if max_length is not None and sum(len(x) for x in phrase) > max_length:
                raise ConfigurationError('Phrase is too long at list {!r} line {}: {!r}'
                                         .format(name, i, line))
            items.append(phrase)
        else:
            raise ConfigurationError('Invalid syntax at list {!r} line {}: {!r}'
                                     .format(name, i, line))
    if multiword:
        # If in phrase mode, convert everything to tuples
        for i in range(0, multiword_start):
            items[i] = (items[i], )
        result = {
            _CONF.FIELD.TYPE: _CONF.TYPE.PHRASES,
            _CONF.FIELD.PHRASES: items
        }
        if number_of_words is not None:
            result[_CONF.FIELD.NUMBER_OF_WORDS] = number_of_words
    else:
        result = {
            _CONF.FIELD.TYPE: _CONF.TYPE.WORDS,
            _CONF.FIELD.WORDS: items
        }
    if max_length is not None:
        result[_CONF.FIELD.MAX_LENGTH] = max_length
    return result
836357
def load(self, train=True, test=True, shuffle=True) -> tuple:
        
        return self.__load(self.__load_files, train, test, shuffle=shuffle)
903888
def load_dictionary(filename):
    """"""
    filename = osp.abspath(filename)
    old_cwd = getcwd()
    tmp_folder = tempfile.mkdtemp()
    os.chdir(tmp_folder)
    data = None
    error_message = None
    try:
        with tarfile.open(filename, "r") as tar:
            tar.extractall()
        pickle_filename = glob.glob('*.pickle')[0]
        # 'New' format (Spyder >=2.2 for Python 2 and Python 3)
        with open(pickle_filename, 'rb') as fdesc:
            data = pickle.loads(fdesc.read())
        saved_arrays = {}
        if load_array is not None:
            # Loading numpy arrays saved with np.save
            try:
                saved_arrays = data.pop('__saved_arrays__')
                for (name, index), fname in list(saved_arrays.items()):
                    arr = np.load( osp.join(tmp_folder, fname) )
                    if index is None:
                        data[name] = arr
                    elif isinstance(data[name], dict):
                        data[name][index] = arr
                    else:
                        data[name].insert(index, arr)
            except KeyError:
                pass
    # Except AttributeError from e.g. trying to load function no longer present
    except (AttributeError, EOFError, ValueError) as error:
        error_message = to_text_string(error)
    # To ensure working dir gets changed back and temp dir wiped no matter what
    finally:
        os.chdir(old_cwd)
        try:
            shutil.rmtree(tmp_folder)
        except OSError as error:
            error_message = to_text_string(error)
    return data, error_message
683326
def load_from_dict(self, data: dict, overwrite: bool=True):
        
        if data is None or data == {}:
            return False
        # Loop over items
        if isinstance(data, list) or isinstance(data, tuple):
            # Pick a random item from the tuple.
            if len(data[0]) != 2:
                raise exc.LoaderException("Cannot load data with length {}".format(len(data[0])))
            items = data
        elif isinstance(data, dict) or isinstance(data, self.__class__):
            items = data.items()
        else:
            raise exc.LoaderException("Cannot load data of type {}".format(type(data)))
        for key, item in items:
            assert isinstance(key, str)
            if hasattr(self, key) and not overwrite:
                # Refuse to overwrite existing data.
                continue
            # Check name to verify it's safe.
            if self.safe_load:
                if key.startswith("__") or key in ['dump', 'items', 'keys', 'values',
                                                   'iter_list', 'load_from_dict', 'iter_list_dump',
                                                   'parsed', 'safe_load']:
                    # It's evil!
                    key = "unsafe_" + key
            if '.' in key:
                # Doubly evil!
                key = key.replace('.', '_')
            if isinstance(item, dict):
                # Create a new ConfigKey object with the dict.
                ncfg = ConfigKey()
                # Parse the data.
                ncfg.load_from_dict(item)
                # Set our new ConfigKey as an attribute of ourselves.
                setattr(self, key, ncfg)
            elif isinstance(item, list):
                # Iterate over the list, creating ConfigKey items as appropriate.
                nlst = self.iter_list(item)
                # Set our new list as an attribute of ourselves.
                setattr(self, key, nlst)
            else:
                # Set the item as an attribute of ourselves.
                setattr(self, key, item)
        # Flip the parsed flag,
        self.parsed = True
739618
def read_folder(directory):
    
    res = []
    for filename in os.listdir(directory):
        with io.open(os.path.join(directory, filename), encoding="utf-8") as f:
            content = f.read()
            res.append(content)
    return res
1264413
def _load_data(self, reset=False):
        ''' 
        '''

        log = logging.getLogger("%s.%s" % (self.log_name,
                                           inspect.stack()[0][3]))
        log.setLevel(self.log_level)
        for attr, obj in self.datafile_obj.items():
            if reset or obj['latest_mod'] > obj['last_json_mod']:
                conn = obj['conn']
                sparql = "DROP ALL;"
                if os.path.isdir(obj['cache_path']):
                    shutil.rmtree(obj['cache_path'], ignore_errors=True)
                os.makedirs(obj['cache_path'])
                drop_extensions = conn.update_query(sparql)
                rdf_resource_templates = []
                rdf_data = []
                for path, files in obj['files'].items():
                    for file in files:
                        file_path = os.path.join(path, file)
                        # data = open(file_path).read()
                        # log.info(" uploading file: %s | namespace: %s",
                        #          file,
                        #          conn.namespace)
                        # data_type = file.split('.')[-1]
                        result = conn.load_data(file_path,
                                                #datatype=data_type,
                                                graph=str(getattr(NSM.kdr,
                                                                  file)),
                                                is_file=True)
                        if result.status_code > 399:
                            raise ValueError("Cannot load '{}' into {}".format(
                                file_name, conn))
341623
def load(self , filename=None, data=None):
        

        self.__load(filename=filename, data=data)
98606
def read_dictionary_file(dictionary_path):
    """"""
    try:
        return _user_dictionary_cache[dictionary_path]
    except KeyError:
        if dictionary_path and os.path.exists(dictionary_path):
            with open(dictionary_path, "rt") as dict_f:
                words = set(re.findall(r"(\w[\w']*\w|\w)",
                                       " ".join(dict_f.read().splitlines())))
                return words

        return set()
585176
def _load_names(self) -> List[str]:
        
        names = []
        for path in self._get_files():
            for name in self._get_names(path):
                names.append(self._normalize_name(name))
        return names
475648
def _loadConfiguration(self):
        
        configPath = os.path.join(self.path, "config")
        if not os.path.isdir(configPath):
            return

        config = Config(configPath)

        Config.mergeDictionaries(config.getData(), self.application.config)
179371
def load_items():
    

    filename = os.path.join(os.path.dirname(__file__), "data", "items.json")

    with open(filename) as f:
        items = json.loads(f.read())["result"]["items"]
        for item in items:
            ITEMS_CACHE[item["id"]] = item
18254
def load_dale_chall(self):
        """"""
        self.dale_chall_dict = os.path.join(self.base_dir, 'dicts', 'dale_chall.txt')
        with open(self.dale_chall_dict) as words_file:
            data = words_file.read()
        self.dale_chall_words = set(data.splitlines())
1283402
def loadFileList(self):
        """"""
        try:
            data = open(self.filelist_file, 'rb')
        except IOError:
            '''print "No SRTM cached file list. Creating new one!"'''
            if self.offline == 0:
                self.createFileList()
            return
        try:
            self.filelist = pickle.load(data)
            data.close()
            if len(self.filelist) < self.min_filelist_len:
                self.filelist = {}
                if self.offline == 0:
                    self.createFileList()
        except:
            '''print "Unknown error loading cached SRTM file list. Creating new one!"'''
            if self.offline == 0:
                self.createFileList()