A minimal urlopen replacement hack that supports timeouts for http.
       Note that this supports GET only.
357337
def urlopen(url, timeout=20, redirects=None):
    """"""
    scheme, host, path, params, query, frag = urlparse(url)

    if not scheme in ('http', 'https'):
        return urllib.urlopen(url)
    if params: path = '%s;%s' % (path, params)
    if query:  path = '%s?%s' % (path, query)
    if frag:   path = '%s#%s' % (path, frag)

    if scheme == 'https':
        # If ssl is not compiled into Python, you will not get an exception
        # until a conn.endheaders() call.   We need to know sooner, so use
        # getattr.
        try:
            import M2Crypto
        except ImportError:
            if not hasattr(socket, 'ssl'):
                raise RuntimeError, 'no built-in SSL Support'

            conn = TimeoutHTTPS(host, None, timeout)
        else:
            ctx = M2Crypto.SSL.Context()
            ctx.set_session_timeout(timeout)
            conn = M2Crypto.httpslib.HTTPSConnection(host, ssl_context=ctx)
            conn.set_debuglevel(1)

    else:
        conn = TimeoutHTTP(host, None, timeout)

    conn.putrequest('GET', path)
    conn.putheader('Connection', 'close')
    conn.endheaders()
    response = None
    while 1:
        response = conn.getresponse()
        if response.status != 100:
            break
        conn._HTTPConnection__state = httplib._CS_REQ_SENT
        conn._HTTPConnection__response = None

    status = response.status

    # If we get an HTTP redirect, we will follow it automatically.
    if status >= 300 and status < 400:
        location = response.msg.getheader('location')
        if location is not None:
            response.close()
            if redirects is not None and redirects.has_key(location):
                raise RecursionError(
                    'Circular HTTP redirection detected.'
                    )
            if redirects is None:
                redirects = {}
            redirects[location] = 1
            return urlopen(location, timeout, redirects)
        raise HTTPResponse(response)

    if not (status >= 200 and status < 300):
        raise HTTPResponse(response)

    body = StringIO(response.read())
    response.close()
    return body
723656
def urlopen(self, method, url, **kw):
        ""
        kw['assert_same_host'] = False
        return self.proxy_pool.urlopen(method, url, **kw)
638576
def api(self, name, url, **kwargs):
        """"""
        if name not in self._apis:
            raise ValueError('API name must be one of {0}, not {1!r}.'.format(
                tuple(self._apis), name))
        fields = kwargs.get('fields')
        timeout = kwargs.get('timeout')
        text = kwargs.get('text')
        html = kwargs.get('html')
        if text and html:
            raise ValueError(u'Both `text` and `html` arguments provided!')
        params = {'url': url, 'token': self._token}
        if timeout:
            params['timeout'] = timeout
        if fields:
            if not isinstance(fields, str):
                fields = ','.join(sorted(fields))
            params['fields'] = fields
        url = self.endpoint(name)
        if text or html:
            content_type = html and 'text/html' or 'text/plain'
            return self._post(url, text or html, content_type, params=params)
        return self._get(url, params=params)
727161
public static HttpResponse getResponse(String urls, HttpRequest request,
                                           HttpMethod method, int connectTimeoutMillis, int readTimeoutMillis) throws IOException {
        OutputStream out = null;
        InputStream content = null;
        HttpResponse response = null;
        HttpURLConnection httpConn = request
            .getHttpConnection(urls, method.name());
        httpConn.setConnectTimeout(connectTimeoutMillis);
        httpConn.setReadTimeout(readTimeoutMillis);

        try {
            httpConn.connect();
            if (null != request.getPayload() && request.getPayload().length > 0) {
                out = httpConn.getOutputStream();
                out.write(request.getPayload());
            }
            content = httpConn.getInputStream();
            response = new HttpResponse();
            parseHttpConn(response, httpConn, content);
            return response;
        } catch (SocketTimeoutException e) {
            throw e;
        } catch (IOException e) {
            content = httpConn.getErrorStream();
            response = new HttpResponse();
            parseHttpConn(response, httpConn, content);
            return response;
        } finally {
            if (content != null) {
                content.close();
            }
            httpConn.disconnect();
        }
    }
1300553
public static String get(String urlString, int timeout) {
		return HttpRequest.get(urlString).timeout(timeout).execute().body();
	}
1166465
public static HttpRequestBase get(String path,
	                                  Map<String, String> headers,
	                                  Integer timeOutInSeconds) {


		HttpGet get = new HttpGet(path);
		addHeaders(get, headers);
		get.setConfig(getConfig(timeOutInSeconds));
		return get;
	}
49780
def create_http(_url = nil)


      if !request_endpoint.nil?
       _url = request_endpoint
      end


      if _url.nil? || _url[0] =~ /^\//
        our_uri = URI.parse(site)
      else
        our_uri = URI.parse(_url)
      end


      if proxy.nil?
        http_object = Net::HTTP.new(our_uri.host, our_uri.port)
      else
        proxy_uri = proxy.is_a?(URI) ? proxy : URI.parse(proxy)
        http_object = Net::HTTP.new(our_uri.host, our_uri.port, proxy_uri.host, proxy_uri.port, proxy_uri.user, proxy_uri.password)
      end

      http_object.use_ssl = (our_uri.scheme == 'https')

      if @options[:ca_file] || CA_FILE
        http_object.ca_file = @options[:ca_file] || CA_FILE
        http_object.verify_mode = OpenSSL::SSL::VERIFY_PEER
        http_object.verify_depth = 5
      else
        http_object.verify_mode = OpenSSL::SSL::VERIFY_NONE
      end

      http_object.read_timeout = http_object.open_timeout = @options[:timeout] || 30
      http_object.open_timeout = @options[:open_timeout] if @options[:open_timeout]
      http_object.ssl_version = @options[:ssl_version] if @options[:ssl_version]
      http_object.set_debug_output(debug_output) if debug_output

      http_object
    end
1260557
def _make_request(self, conn, method, url, timeout=_Default,
                      **httplib_request_kw):
        
        self.num_requests += 1

        if timeout is _Default:
            timeout = self.timeout

        conn.request(method, url, **httplib_request_kw)
        conn.sock.settimeout(timeout)
        httplib_response = conn.getresponse()

        log.debug("\"%s %s %s\" %s %s" %
                  (method, url,
                   conn._http_vsn_str, # pylint: disable-msg=W0212
                   httplib_response.status, httplib_response.length))

        return httplib_response
150952
def fetch(self, url):
        
        try:
            r = requests.get(url, timeout=self.timeout)
        except requests.exceptions.Timeout:
            if not self.safe:
                raise
            else:
                return None
        
        # Raise 404/500 error if any
        if r and not self.safe:
            r.raise_for_status()
        
        return r.text
182097
function breakableUrl(url) {
        // This is for displaying in UI, so always want it escaped
        var escUrl = _.escape(url);

        // Inject zero-width space character (U+200B) near path separators (/) to allow line breaking there
        return escUrl.replace(
            new RegExp(regexEscape("/"), "g"),
            "/" + "&#8203;"
        );
    }
188676
function truncateUrl(url, limit) {
  limit = limit || 30
  url = url.replace(/http[s]?:\/\/(www\.)?/i, '')
  if (url.length <= limit) return url
  var segment = Math.floor(limit / 2) - 1
  return url.slice(0, segment) + '..' + url.slice(url.length - segment) 
}
1022444
def urlopen(self, method, url, body=None, headers=None, **kwargs):
        """"""
        # pylint: disable=arguments-differ
        # We use kwargs to collect additional args that we don't need to
        # introspect here. However, we do explicitly collect the two
        # positional arguments.

        # Use a kwarg for this instead of an attribute to maintain
        # thread-safety.
        _credential_refresh_attempt = kwargs.pop(
            '_credential_refresh_attempt', 0)

        if headers is None:
            headers = self.headers

        # Make a copy of the headers. They will be modified by the credentials
        # and we want to pass the original headers if we recurse.
        request_headers = headers.copy()

        self.credentials.before_request(
            self._request, method, url, request_headers)

        response = self.http.urlopen(
            method, url, body=body, headers=request_headers, **kwargs)

        # If the response indicated that the credentials needed to be
        # refreshed, then refresh the credentials and re-attempt the
        # request.
        # A stored token may expire between the time it is retrieved and
        # the time the request is made, so we may need to try twice.
        # The reason urllib3's retries aren't used is because they
        # don't allow you to modify the request headers. :/
        if (response.status in self._refresh_status_codes
                and _credential_refresh_attempt < self._max_refresh_attempts):

            _LOGGER.info(
                'Refreshing credentials due to a %s response. Attempt %s/%s.',
                response.status, _credential_refresh_attempt + 1,
                self._max_refresh_attempts)

            self.credentials.refresh(self._request)

            # Recurse. Pass in the original headers, not our modified set.
            return self.urlopen(
                method, url, body=body, headers=headers,
                _credential_refresh_attempt=_credential_refresh_attempt + 1,
                **kwargs)

        return response
479285
def urlopen(link):
    
    try:
        return urllib2.urlopen(link)
    except urllib2.URLError:
        pass
    except ValueError:
        return ""
    except KeyboardInterrupt:
        print("")
        raise SystemExit()
1245445
def requester(
        url,
        main_url=None,
        delay=0,
        cook=None,
        headers=None,
        timeout=10,
        host=None,
        proxies=[None],
        user_agents=[None],
        failed=None,
        processed=None
    ):
    """"""
    cook = cook or set()
    headers = headers or set()
    user_agents = user_agents or ['Photon']
    failed = failed or set()
    processed = processed or set()
    # Mark the URL as crawled
    processed.add(url)
    # Pause/sleep the program for specified time
    time.sleep(delay)

    def make_request(url):
        """Default request"""
        final_headers = headers or {
            'Host': host,
            # Selecting a random user-agent
            'User-Agent': random.choice(user_agents),
            'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',
            'Accept-Language': 'en-US,en;q=0.5',
            'Accept-Encoding': 'gzip',
            'DNT': '1',
            'Connection': 'close',
        }
        try:
            response = SESSION.get(
                url,
                cookies=cook,
                headers=final_headers,
                verify=False,
                timeout=timeout,
                stream=True,
                proxies=random.choice(proxies)
            )
        except TooManyRedirects:
            return 'dummy'

        if 'text/html' in response.headers['content-type'] or \
           'text/plain' in response.headers['content-type']:
            if response.status_code != '404':
                return response.text
            else:
                response.close()
                failed.add(url)
                return 'dummy'
        else:
            response.close()
            return 'dummy'

    return make_request(url)
387443
def _do_http(opts, profile='default'):
    '''
    
    '''

    ret = {}

    url = __salt__['config.get']('modjk:{0}:url'.format(profile), '')
    user = __salt__['config.get']('modjk:{0}:user'.format(profile), '')
    passwd = __salt__['config.get']('modjk:{0}:pass'.format(profile), '')
    realm = __salt__['config.get']('modjk:{0}:realm'.format(profile), '')
    timeout = __salt__['config.get']('modjk:{0}:timeout'.format(profile), '')

    if not url:
        raise Exception('missing url in profile {0}'.format(profile))

    if user and passwd:
        auth = _auth(url=url, realm=realm, user=user, passwd=passwd)
        _install_opener(auth)

    url += '?{0}'.format(_urlencode(opts))

    for line in _urlopen(url, timeout=timeout).read().splitlines():
        splt = line.split('=', 1)
        if splt[0] in ret:
            ret[splt[0]] += ',{0}'.format(splt[1])
        else:
            ret[splt[0]] = splt[1]

    return ret
60184
def hijack_require_http_methods(fn):
    
    required_methods = ['POST']
    if hijack_settings.HIJACK_ALLOW_GET_REQUESTS:
        required_methods.append('GET')
    return require_http_methods(required_methods)(fn)
562178
def recoverURL(self, url):
        
        # Configuring user agents...
        self.setUserAgent()

        # Configuring proxies
        if "https://" in url:
            self.setProxy(protocol = "https")
        else:
            self.setProxy(protocol = "http")

        # Giving special treatment for .onion platforms
        if ".onion" in url:
            try:
                # TODO: configuring manually the tor bundle
                pass
            except:
                # TODO: capturing the error and eventually trying the tor2web approach
                #url = url.replace(".onion", ".tor2web.org")
                pass
            url = url.replace(".onion", ".onion.cab")

        # Opening the resource
        try:
            recurso = self.br.open(url)
        except:
            # Something happened. Maybe the request was forbidden?
            return None

        html = recurso.read()

        return html