Create collection from locally discovered data, e.g. /etc/passwd.
760267
def from_passwd(uid_min=None, uid_max=None):
        """"""
        import pwd
        users = Users(oktypes=User)
        passwd_list = pwd.getpwall()
        if not uid_min:
            uid_min = UID_MIN
        if not uid_max:
            uid_max = UID_MAX
        sudoers_entries = read_sudoers()
        for pwd_entry in passwd_list:
            if uid_min <= pwd_entry.pw_uid <= uid_max:
                user = User(name=text_type(pwd_entry.pw_name),
                            passwd=text_type(pwd_entry.pw_passwd),
                            uid=pwd_entry.pw_uid,
                            gid=pwd_entry.pw_gid,
                            gecos=text_type(pwd_entry.pw_gecos),
                            home_dir=text_type(pwd_entry.pw_dir),
                            shell=text_type(pwd_entry.pw_shell),
                            public_keys=read_authorized_keys(username=pwd_entry.pw_name),
                            sudoers_entry=get_sudoers_entry(username=pwd_entry.pw_name,
                                                            sudoers_entries=sudoers_entries))
                users.append(user)
        return users
761378
def create_payload(self):
        """"""
        payload = super(HostCollection, self).create_payload()
        if 'system_ids' in payload:
            payload['system_uuids'] = payload.pop('system_ids')
        return payload
163506
def create_result(self, local_path, container_path, permissions, meta, val, dividers):
        """"""
        if permissions is NotSpecified:
            permissions = 'rw'
        return Mount(local_path, container_path, permissions)
29247
def download_data():
    """"""

    n_datasets = 2
    for d in range(1, n_datasets + 1):
        if not os.path.isdir('enron%d' % d):

            URL = url[d-1]
            print("Downloading %d/%d: %s" % (d, n_datasets, URL))
            folderzip = 'enron%d.zip' % d

            with urlopen(URL) as remotedata:
                with open(folderzip, 'wb') as z:
                    z.write(remotedata.read())

            with ZipFile(folderzip) as z:
                z.extractall()
            os.remove(folderzip)
81118
def resource_collection_create(self, data, **kwargs):
        ""
        api_path = "/api/v2/resource_collections.json"
        return self.call(api_path, method="POST", data=data, **kwargs)
730547
def create(self):
        """"""
        # get json content from POST request
        content = request.environ['wsgi.input'].read(int(request.environ['CONTENT_LENGTH']))
        #content = content.decode('utf8')   mapfile interface don't like unicode strings... bad...

        # load mapfile
        mapfile = Mapfile()
        dict = simplejson.loads(content)
        mapfile.from_dict(dict)

        # create mapfile
        mapname = mapfile.get_name()
        map_pathname = h.gen_mapname()
        mapfile.to_file(os.path.join(config['mapfiles_dir'], map_pathname))

        # create map in db
        map = self._new_map_from_user(c.user, mapname, map_pathname)

        response.status = 201

        href = h.url_for(controller="mapfiles", action="show", id=map.id)
        wmsproxyurl = h.url_for(controller='mapfiles', action='wms_proxy', id=map.id)
        wmsurl = "%s?%s" %(config['mapserver_url'], urlencode({'map':os.path.join(config['mapfiles_dir'], map.filepath)}))
        return {'name': map.name, 'id': map.id, 'href': href, 'wmsurl': wmsurl, 'wmsproxyurl': wmsproxyurl}
981164
def extractall(filename, directory, backend='auto', auto_create_dir=False):
    '''
    
    '''
    Archive(filename, backend).extractall(directory,
                                          auto_create_dir=auto_create_dir)
956405
def dataset(ctx, client, revision, datadir, format):
    """"""
    ctx.meta['renku.datasets.datadir'] = datadir

    if ctx.invoked_subcommand is not None:
        return

    if revision is None:
        datasets = client.datasets.values()
    else:
        datasets = client.datasets_from_commit(client.repo.commit(revision))

    DATASETS_FORMATS[format](client, datasets)
1099684
def _file_local_list(self, dest):
        '''
        
        '''
        if os.path.isdir(dest):
            destdir = dest
        else:
            destdir = os.path.dirname(dest)

        filelist = set()

        for root, dirs, files in salt.utils.path.os_walk(destdir, followlinks=True):
            for name in files:
                path = os.path.join(root, name)
                filelist.add(path)

        return filelist
312320
def from_config(cls, config):
        """"""
        auth_key = None
        password_file = config["authentication.passwordFile"]
        if password_file is not None:
            try:
                auth_key = open(config["authentication.passwordFile"]).read()
                auth_key = re.sub(r"\s", "", auth_key)
            except IOError:
                LOG.error("Could not load password file!")
                sys.exit(1)
        password = config["authentication.password"]
        if password is not None:
            auth_key = password
        connector = Connector(
            mongo_address=config["mainAddress"],
            doc_managers=config["docManagers"],
            oplog_checkpoint=os.path.abspath(config["oplogFile"]),
            collection_dump=config["onlyDump"] or not config["noDump"],
            only_dump=config["onlyDump"],
            batch_size=config["batchSize"],
            continue_on_error=config["continueOnError"],
            auth_username=config["authentication.adminUsername"],
            auth_key=auth_key,
            fields=config["fields"],
            exclude_fields=config["exclude_fields"],
            ns_set=config["namespaces.include"],
            ex_ns_set=config["namespaces.exclude"],
            dest_mapping=config["namespaces.mapping"],
            namespace_options=config["namespaces.namespace_options"],
            gridfs_set=config["namespaces.gridfs"],
            ssl_certfile=config["ssl.sslCertfile"],
            ssl_keyfile=config["ssl.sslKeyfile"],
            ssl_ca_certs=config["ssl.sslCACerts"],
            ssl_cert_reqs=config["ssl.sslCertificatePolicy"],
            tz_aware=config["timezoneAware"],
        )
        return connector
473997
def from_folder(cls, path:PathOrStr, extensions:Collection[str]=None, recurse:bool=True,
                    include:Optional[Collection[str]]=None, processor:PreProcessors=None, **kwargs)->'ItemList':
        """"""
        path = Path(path)
        return cls(get_files(path, extensions, recurse=recurse, include=include), path=path, processor=processor, **kwargs)
304237
def _secret_generator(baseline):
    """"""
    for filename, secrets in baseline['results'].items():
        for secret in secrets:
            yield filename, secret
604647
def collect(self):
        """"""
        local_df = self._schema_rdd.toPandas()
        correct_idx_df = _update_index_on_df(local_df, self._index_names)
        return correct_idx_df
105537
private void createEmptyDataCollection()
    {
        Class<?> collectionClass = getRelation().getProperty().getType();
        if (collectionClass.isAssignableFrom(Set.class))
        {
            dataCollection = new HashSet();
        }
        else if (collectionClass.isAssignableFrom(List.class))
        {
            dataCollection = new ArrayList();
        }
    }
303666
def local_open(url):
    """"""
    scheme, server, path, param, query, frag = urllib.parse.urlparse(url)
    filename = urllib.request.url2pathname(path)
    if os.path.isfile(filename):
        return urllib.request.urlopen(url)
    elif path.endswith('/') and os.path.isdir(filename):
        files = []
        for f in os.listdir(filename):
            filepath = os.path.join(filename, f)
            if f == 'index.html':
                with open(filepath, 'r') as fp:
                    body = fp.read()
                break
            elif os.path.isdir(filepath):
                f += '/'
            files.append('<a href="{name}">{name}</a>'.format(name=f))
        else:
            tmpl = (
                "<html><head><title>{url}</title>"
                "</head><body>{files}</body></html>")
            body = tmpl.format(url=url, files='\n'.join(files))
        status, message = 200, "OK"
    else:
        status, message, body = 404, "Path not found", "Not found"

    headers = {'content-type': 'text/html'}
    body_stream = six.StringIO(body)
    return urllib.error.HTTPError(url, status, message, headers, body_stream)
919118
def create_rack(self):
        """"""
        return Rack(
            self.networkapi_url,
            self.user,
            self.password,
            self.user_ldap)
1320440
def open_dataset(self, service):
        """"""

        if not self.dataset:
            path = os.path.join(SERVICE_DATA_ROOT, service.data_path)
            self.dataset = netCDF4.Dataset(path, 'r')
        return self.dataset