Return whether the URL can be fetched.
165702
def can_fetch(self, url_info: URLInfo, user_agent: str):
        ''''''
        key = self.url_info_key(url_info)

        parser = self._parsers[key]
        return parser.is_allowed(user_agent, url_info.url)
141630
def _get(self, **kwargs):
        """"""
        path = self._construct_path_to_item()
        return self._http.get(path)
710150
async def bounded_fetch(session, url):
    
    async with sem, session.get(url) as response:
        return await response.json()
597585
def __valid_url(cls, url):
        """"""
        bits = urlparse(url)
        return ((bits.scheme == "http" or bits.scheme == "https") and
                _PATTERN_URL_PART.match(bits.netloc) and
                _PATTERN_URL_PART.match(bits.path))
147023
def geturl(self):
        """"""
        return urlparse.urlunparse((
            self.scheme,
            self.netloc,
            self.path,
            self.params,
            self.query_str,
            self.fragment,
        ))
968526
def __get_url(url):
    """"""
    log.info("Retrieving  weather data (%s)...", url)
    result = {SUCCESS: False, MESSAGE: None}
    try:
        r = requests.get(url)
        result[STATUS_CODE] = r.status_code
        result[HEADERS] = r.headers
        result[CONTENT] = r.text
        if (200 == r.status_code):
            result[SUCCESS] = True
        else:
            result[MESSAGE] = "Got http statuscode: %d." % (r.status_code)
        return result
    except requests.RequestException as ose:
        result[MESSAGE] = 'Error getting url data. %s' % ose
        log.error(result[MESSAGE])

    return result
1190379
def fetch(destination):
    """"""
    with io.open(destination, mode="w", encoding="utf-8") as dest:
        for url in TLE_URLS:
            response = urlopen(url)
            dest.write(response.read().decode("utf-8"))
1286605
def fetch(self, url, encoding=None, force_refetch=False, nocache=False, quiet=True):
        ''' '''
        try:
            if not force_refetch and self.cache is not None and url in self.cache:
                # try to look for content in cache
                logging.debug('Retrieving content from cache for {}'.format(url))
                return self.cache.retrieve_blob(url, encoding)
            encoded_url = WebHelper.encode_url(url)
            req = Request(encoded_url, headers={'User-Agent': 'Mozilla/5.0'})
            # support gzip
            req.add_header('Accept-encoding', 'gzip, deflate')
            # Open URL
            getLogger().info("Fetching: {url} |".format(url=url))
            response = urlopen(req)
            content = response.read()
            # unzip if required
            if 'Content-Encoding' in response.info() and response.info().get('Content-Encoding') == 'gzip':
                # unzip
                with gzip.open(BytesIO(content)) as gzfile:
                    content = gzfile.read()
            # update cache if required
            if self.cache is not None and not nocache:
                if url not in self.cache:
                    self.cache.insert_blob(url, content)
            return content.decode(encoding) if content and encoding else content
        except URLError as e:
            if hasattr(e, 'reason'):
                getLogger().exception('We failed to reach {}. Reason: {}'.format(url, e.reason))
            elif hasattr(e, 'code'):
                getLogger().exception('The server couldn\'t fulfill the request. Error code: {}'.format(e.code))
            else:
                # Other exception ...
                getLogger().exception("Fetching error")
            if not quiet:
                raise
        return None
1141136
def is_parsable(url):
        

        try:
            parsed = urlparse(url)
            URLHelper.__cache[url] = parsed
            return True
        except:
            return False
1147101
def http_url_exists(url):
    """"""
    class HEADRequest(Request):
        def get_method(self):
            return "HEAD"
    try:
        response = urlopen(HEADRequest(url))
        return True
    except URLError:
        return False
149226
def get(self, url, data=None, params=None):
        
        return self.request("get", url, data, params)
568049
def _is_valid_api_url(self, url):
        """"""
        # Check response is a JSON with ok: 1
        data = {}
        try:
            r = requests.get(url, proxies=self.proxy_servers)
            content = to_text_string(r.content, encoding='utf-8')
            data = json.loads(content)
        except Exception as error:
            logger.error(str(error))

        return data.get('ok', 0) == 1
29851
def fetch(url, timeout=5, userAgent=None, only_mime_types=None):
    
    headers = {}
    if userAgent:
        headers['User-agent'] = str(userAgent)
    else:
        headers['User-agent'] = ua.random

    #request = Request(url=url, headers=headers)
    #response = urlopen(request, timeout=timeout)
    response = requests.get(url, headers=headers, timeout=timeout)

    # Return nothing of the content isn't one of the target mime-types.
    if only_mime_types:
        assert isinstance(only_mime_types, (tuple, list))
        # Check for mimetype by looking at pattern in the URL.
        # Not super accurate, but very fast.
        ct, mt_encoding = mimetypes.guess_type(url)
        # Then check for mimetype by actually requesting the resource and
        # looking at the response.
        # More accurate, but slower since we actually have to send a request.
        if not ct:
            response_info = response.info()
            ct = (response_info.getheader('Content-Type') or '').split(';')[0]
        #TODO:if still undefined, use magic.Magic(mime=True).from_file(url)?
        if ct not in only_mime_types:
            return
    try:
        #return response.read()
        return response.text
    except httplib.IncompleteRead as e:
        # This should rarely happen, and is often the fault of the server
        # sending a malformed response.
        #TODO:just abandon all content and return '' instead?
        return e.partial
221777
def _fetch(self):
        """"""
        if 'uri' in self._meta_data:
            error = "There was an attempt to assign a new uri to this "\
                    "resource, the _meta_data['uri'] is %s and it should"\
                    " not be changed." % (self._meta_data['uri'])
            raise URICreationCollision(error)
        # Make convenience variable with short names for this method.
        _create_uri = self._meta_data['container']._meta_data['uri']
        session = self._meta_data['bigip']._meta_data['icr_session']
        # Invoke the REST operation on the device.
        response = session.post(_create_uri, json={})
        # Make new instance of self
        return self._produce_instance(response)
230540
def get(self, url, **kwargs):
        
        """

        kwargs.setdefault('allow_redirects', True)
        return self.request('GET', url, **kwargs)
871330
def _get(self, url):
        """"""
        self.logger.info(f'Sending request: {RequestHandler.BASE_URL+url}')
        req = requests.get(RequestHandler.BASE_URL+url, headers=self.headers)
        self.logger.info(f'Request returned with status code {req.status_code}')

        if req.status_code == requests.codes.ok:
            return req

        if req.status_code == requests.codes.bad:
            raise APIErrorException('Invalid request. Check parameters.', req.status_code)

        if req.status_code == requests.codes.forbidden:
            raise APIErrorException('This resource is restricted', req.status_code)

        if req.status_code == requests.codes.not_found:
            raise APIErrorException('This resource does not exist. Check parameters', req.status_code)

        if req.status_code == requests.codes.too_many_requests:
            raise APIErrorException('You have exceeded your allowed requests per minute/day', req.status_code)
426247
def get(self, uri, query=None, **kwargs):
        """"""
        return self.fetch('get', uri, query, **kwargs)