Retrieves a file from a URL.

    Returns a tuple (mimetype, filename, data)
519535
def _fetch_remote_file(service_url, params=None, use_http_post=False):
    
    if not params:
        params = {}

    request_url, response = _fetch_remote(service_url, params, use_http_post)
    dummy, params = cgi.parse_header(
            response.headers.get('Content-Disposition', ''))
    fn = params['filename']

    return (response.headers.get('content-type'),
            fn, response.read(), response.geturl())
166111
def get(self, url):
        
        self._load(url)
        self._flush()

        return self.metadata[url]
908542
def upload_file_from_url(self, url, file_name, dir_name=None):
        
        real_file_name = str(int(time.time()*1000))
        urllib.request.urlretrieve(url, real_file_name)
        data = self.upload_file(real_file_name, file_name, dir_name)
        os.remove(real_file_name)
        return data
35782
def _get_data(url):
    """"""
    if urllib_parse.urlparse(url).scheme in ('http', 'https'):
        resp = urllib_request.urlopen(url)
        encoding = resp.headers.get('content-encoding', 'plain')
        data = resp.read()
        if encoding == 'plain':
            data = data.decode('utf-8')
        elif encoding == 'gzip':
            data = BytesIO(data)
            data = gzip.GzipFile(fileobj=data).read().decode('utf-8')
        else:
            raise RuntimeError('unknown encoding')
    else:
        with codecs.open(url, mode='r', encoding='utf-8') as fid:
            data = fid.read()

    return data
562612
def read_file(self, url, location=None):
        
        response = requests_retry_session().get(url, timeout=10.0)
        response.raise_for_status()
        text = response.text
        if 'tab-width' in self.options:
            text = text.expandtabs(self.options['tab-width'])

        return text.splitlines(True)
541763
def get_mime(self, path, isdir):
        ''''''
        if isdir:
            file_type = FOLDER
        else:
            file_type = mimetypes.guess_type(path)[0]
            if not file_type:
                file_type = UNKNOWN
        return file_type
1248349
def get_file(file_id: str, download):
    
    data = current_app.config["data"]  # type: DataStorage
    dao = data.get_files_dao()
    file_fp, filename, upload_date = dao.get(file_id)

    if download:
        mime = mimetypes.guess_type(filename)[0]
        if mime is None:
            # unknown type
            mime = "binary/octet-stream"

        basename = os.path.basename(filename)
        return send_file(file_fp, mimetype=mime, attachment_filename=basename, as_attachment=True)

    else:
        rawdata = file_fp.read()
        try:
            text = rawdata.decode('utf-8')
        except UnicodeDecodeError:
            # not decodable as utf-8
            text = _get_binary_info(rawdata)
        html = render_template("api/file_view.html", content=text)
        file_fp.close()
        return Response(html)
1152409
def fetch(self, url):
        '''
        
        '''
        opener = self._urllib.build_opener()
        opener.addheaders = self._requestHeaders.items()
        response = opener.open(url)
        headers = response.info()
        raw = response.read()
        raw = raw.decode('utf8')

        if not 'Content-Type' in headers:
            raise OEmbedError('Missing mime-type in response')

        if headers['Content-Type'].find('application/xml') != -1 or \
           headers['Content-Type'].find('text/xml') != -1:
            response = OEmbedResponse.newFromXML(raw)
        elif headers['Content-Type'].find('application/json') != -1 or \
             headers['Content-Type'].find('text/javascript') != -1 or \
             headers['Content-Type'].find('text/json') != -1:
            response = OEmbedResponse.newFromJSON(raw)
        else:
            raise OEmbedError('Invalid mime-type in response - %s' % headers['Content-Type'])

        return response
650940
def download(self, url, file):
        
        if isinstance(file, str) or isinstance(file, unicode):
            file = open(file, 'wb')
        webclient = self.dupe()
        webclient.request('GET', url)
        rsp = webclient.getresponse()
        self._currdl = rsp
        self._dlfile = file
        for chunk in iter(lambda: rsp.read(16384), ''):
            file.write(chunk)
        self._currdl = None
        file.close()
1205451
def _get_url(self, url):
        
        url = str(url)
        if url != 'default' and not '://' in url:
            url = ':///'.join(('file', url))
        return url
210584
def download_data(url:str, fname:PathOrStr=None, data:bool=True, ext:str='.tgz') -> Path:
    ""
    fname = Path(ifnone(fname, _url2tgz(url, data, ext=ext)))
    os.makedirs(fname.parent, exist_ok=True)
    if not fname.exists():
        print(f'Downloading {url}')
        download_url(f'{url}{ext}', fname)
    return fname
1272292
def get_file(f, decoder, read):
    """"""
    with f as f:
        if decoder is None:
            return list(f)
        else:
            d = f.read() if read else f
            out = decoder(d)
            if isinstance(out, (tuple, list)):
                return out
            else:
                return [out]
877335
def get_file_link(self, file_key):
		'''
		'''
		#does not work
		self._raise_unimplemented_error()

		uri = '/'.join([self.api_uri,
						self.files_suffix,
						file_key,
						self.file_link_suffix,
						])
		return self._req('get', uri)
268405
def get(url):
    
    url = text_type(url)
    if url.find("://") == -1:
        Log.error("{{url}} must have a prototcol (eg http://) declared", url=url)

    base = URL("")
    if url.startswith("file://") and url[7] != "/":
        if os.sep=="\\":
            base = URL("file:///" + os.getcwd().replace(os.sep, "/").rstrip("/") + "/.")
        else:
            base = URL("file://" + os.getcwd().rstrip("/") + "/.")
    elif url[url.find("://") + 3] != "/":
        Log.error("{{url}} must be absolute", url=url)

    phase1 = _replace_ref(wrap({"$ref": url}), base)  # BLANK URL ONLY WORKS IF url IS ABSOLUTE
    try:
        phase2 = _replace_locals(phase1, [phase1])
        return wrap(phase2)
    except Exception as e:
        Log.error("problem replacing locals in\n{{phase1}}", phase1=phase1, cause=e)
486834
def get_file(self, file_route, file_name=''):

        ''' 
        '''

        title = '%s.get_file' % self.__class__.__name__

    # construct file url
        file_url = '%s%s' % (self.file_endpoint, file_route)

    # send request for file data
        data_buffer = self._get_data(file_url, file_name, method_title=title)

        return data_buffer
29851
def fetch(url, timeout=5, userAgent=None, only_mime_types=None):
    
    headers = {}
    if userAgent:
        headers['User-agent'] = str(userAgent)
    else:
        headers['User-agent'] = ua.random

    #request = Request(url=url, headers=headers)
    #response = urlopen(request, timeout=timeout)
    response = requests.get(url, headers=headers, timeout=timeout)

    # Return nothing of the content isn't one of the target mime-types.
    if only_mime_types:
        assert isinstance(only_mime_types, (tuple, list))
        # Check for mimetype by looking at pattern in the URL.
        # Not super accurate, but very fast.
        ct, mt_encoding = mimetypes.guess_type(url)
        # Then check for mimetype by actually requesting the resource and
        # looking at the response.
        # More accurate, but slower since we actually have to send a request.
        if not ct:
            response_info = response.info()
            ct = (response_info.getheader('Content-Type') or '').split(';')[0]
        #TODO:if still undefined, use magic.Magic(mime=True).from_file(url)?
        if ct not in only_mime_types:
            return
    try:
        #return response.read()
        return response.text
    except httplib.IncompleteRead as e:
        # This should rarely happen, and is often the fault of the server
        # sending a malformed response.
        #TODO:just abandon all content and return '' instead?
        return e.partial
524702
def urllib_retrieve(url, data=None):
    
    range_limit()
    print "uGET", url
    f = urlopen(url, data)
    html = f.read().decode('utf-8', 'ignore')
    return f.getcode(), html