Parse endpoint URL if single URL is provided. Extend proxy options by allowing and parsing keyFile, certFile and caFile.
352423
function readCertificateDataFromFile ({ keyFile, certFile, caFile }) {
  let key, cert, ca;

  if (keyFile) {
    key = fs.readFileSync(keyFile);
  }

  if (certFile) {
    cert = fs.readFileSync(certFile);
  }

  if (caFile) {
    ca = fs.readFileSync(caFile);
  }

  return { key, cert, ca };
}
580818
def endpoint(url, methods=['GET']):
  
  def decorator(function, methods=methods):
    # Always allow OPTIONS since CORS requests will need it.
    methods = set(methods)
    methods.add('OPTIONS')

    @wraps(function)
    def wrapper(environment, start_response):
      try:
        start_time = time.time()

        if function.func_name not in (_serving_mode_endpoints
                                      [settings.serving_mode]):
          start_response('403 Forbidden',
                         [('Content-Type', 'application/json')])
          return marshal.dumps({
            ERRORS_FIELD: ['kronosd is configured to block access to this '
                           'endpoint.'],
            SUCCESS_FIELD: False,
            TOOK_FIELD: '%fms' % (1000 * (time.time() - start_time))
          })
        req_method = environment['REQUEST_METHOD']

        # If the request method is not allowed, return 405.
        if req_method not in methods:
          start_response('405 Method Not Allowed',
                         [('Allow', ', '.join(methods)),
                          ('Content-Type', 'application/json')])
          return marshal.dumps({
            ERRORS_FIELD: ['%s method not allowed' % req_method],
            SUCCESS_FIELD: False,
            TOOK_FIELD: '%fms' % (1000 * (time.time() - start_time))
          })

        headers = []
        remote_origin = environment.get('HTTP_ORIGIN')

        if req_method == 'OPTIONS':
          # This is a CORS preflight request so check that the remote domain is
          # allowed and respond with appropriate CORS headers.
          # http://www.html5rocks.com/static/images/cors_server_flowchart.png
          if is_remote_allowed(remote_origin):
            headers.extend([
              ('Access-Control-Allow-Origin', remote_origin),
              ('Access-Control-Allow-Credentials', 'true'),
              ('Access-Control-Allow-Headers', ', '.join(
                ('Accept', 'Content-Type', 'Origin', 'X-Requested-With'))),
              ('Access-Control-Allow-Methods', ', '.join(methods))
            ])
          # We just tell the client that CORS is ok. Client will follow up
          # with another request to get the answer.
          start_response('200 OK', headers)
          return ''

        # All POST bodies must be json, so decode it here.
        if req_method == 'POST':
          try:
            environment['json'] = marshal.loads(environment['wsgi.input']
                                                .read())
          except ValueError:
            start_response('400 Bad Request',
                           [('Content-Type', 'application/json')])
            return marshal.dumps({
              ERRORS_FIELD: ['Request body must be valid JSON.'],
              SUCCESS_FIELD: False,
              TOOK_FIELD: '%fms' % (1000 * (time.time() - start_time))
            })

        # All responses are JSON.
        headers.append(('Content-Type', 'application/json'))

        if remote_origin:
          headers.append(('Access-Control-Allow-Origin', remote_origin))

        response = function(environment, start_response, headers)
        if not isinstance(response, types.GeneratorType):
          response[TOOK_FIELD] = '%fms' % (1000 * (time.time() - start_time))
          response = marshal.dumps(response)
        return response
      except Exception, e:
        log.exception('endpoint: uncaught exception!')
        start_response('400 Bad Request',
                       [('Content-Type', 'application/json')])
        return marshal.dumps({
          ERRORS_FIELD: [repr(e)],
          SUCCESS_FIELD: False,
          TOOK_FIELD: '%fms' % (1000 * (time.time() - start_time))
        })

    if settings.profile:
      wrapper = profile(wrapper)

    # Map the URL to serve to this function. Only map certain
    # endpoints if serving_mode is restrictive.
    global ENDPOINTS
    ENDPOINTS[url] = wrapper

    return wrapper

  return decorator
180745
def parse_url(url):
    
    if ":" not in url:
        raise ValueError("url is invalid")

    scheme, url = url.split(":", 1)

    parsed = urlparse(url, scheme="ws")
    if parsed.hostname:
        hostname = parsed.hostname
    else:
        raise ValueError("hostname is invalid")
    port = 0
    if parsed.port:
        port = parsed.port

    is_secure = False
    if scheme == "ws":
        if not port:
            port = 80
    elif scheme == "wss":
        is_secure = True
        if not port:
            port = 443
    else:
        raise ValueError("scheme %s is invalid" % scheme)

    if parsed.path:
        resource = parsed.path
    else:
        resource = "/"

    if parsed.query:
        resource += "?" + parsed.query

    return hostname, port, resource, is_secure
872898
def open_file(self, url):
        """"""
        if not isinstance(url, str):
            raise URLError('file error: proxy support for file protocol currently not implemented')
        if url[:2] == '//' and url[2:3] != '/' and url[2:12].lower() != 'localhost/':
            raise ValueError("file:// scheme is supported only on localhost")
        else:
            return self.open_local_file(url)
304710
async def convert_endpoint(url_string, ts, is_just_checking):
    '''
    
    '''
    response = singletons.server.response

    # Prep ForeignResource and ensure does not validate security settings
    singletons.settings
    foreign_res = ForeignResource(url_string)

    target_ts = TypeString(ts)
    target_resource = TypedResource(url_string, target_ts)

    # Send back cache if it exists
    if target_resource.cache_exists():
        if is_just_checking:
            return _just_checking_response(True, target_resource)
        return await response.file(target_resource.cache_path, headers={
            'Content-Type': target_ts.mimetype,
        })

    # Check if already downloaded. If not, queue up download.
    if not foreign_res.cache_exists():
        singletons.workers.enqueue_download(foreign_res)

    # Queue up a single function that will in turn queue up conversion
    # process
    singletons.workers.enqueue_sync(
        enqueue_conversion_path,
        url_string,
        str(target_ts),
        singletons.workers.enqueue_convert
    )

    if is_just_checking:
        return _just_checking_response(False, target_resource)

    # Respond with placeholder
    return singletons.placeholders.stream_response(target_ts, response)
983628
def parse_url(url)
      uri = URI(url)
      if URI.split(url)[3].nil?
        uri.port = uri.scheme == 'https' ? DEFAULT_HTTPS_PORT : DEFAULT_HTTP_PORT
      end

      raise ArgumentError, "URL #{url} has invalid scheme" unless uri.scheme =~ /^https?$/
      parsed_url = { ssl: uri.scheme == 'https', host: uri.host, port: uri.port }
      if uri.user || uri.password
        parsed_url[:username] = uri.user
        parsed_url[:password] = uri.password
      end

      parsed_url
    rescue URI::InvalidURIError => exc
      raise exc.class, "Invalid URL: #{url} (#{exc.message})"
    end
941778
def create_net_http(uri, options)
      http_class = Net::HTTP

      if options[:proxy]
        proxy = URI(options[:proxy])
        http_class = Net::HTTP::Proxy(proxy.hostname, proxy.port, proxy.user, proxy.password)
      end

      http = http_class.new(uri.host, uri.port)

      # Handle SSL parameters (taken from the open-uri implementation).
      if uri.is_a?(URI::HTTPS)
        http.use_ssl = true
        http.verify_mode = options[:ssl_verify_mode] || OpenSSL::SSL::VERIFY_PEER
        store = OpenSSL::X509::Store.new
        if options[:ssl_ca_cert]
          Array(options[:ssl_ca_cert]).each do |cert|
            File.directory?(cert) ? store.add_path(cert) : store.add_file(cert)
          end
        else
          store.set_default_paths
        end
        http.cert_store = store
      end

      http.read_timeout = options[:read_timeout] if options.key?(:read_timeout)
      http.open_timeout = options[:open_timeout] if options.key?(:open_timeout)

      headers = options.select { |key, value| key.is_a?(String) }
      headers.merge!(options[:headers]) if options[:headers]
      headers["Accept-Encoding"] = "" # Net::HTTP's inflater causes FiberErrors

      get = Net::HTTP::Get.new(uri.request_uri, headers)
      get.basic_auth(uri.user, uri.password) if uri.user || uri.password

      [http, get]
    end
393486
def root_endpoint(request, format=None):
    
    endpoints = []
    # loop over url modules
    for urlmodule in urlpatterns:
        # is it a urlconf module?
        if hasattr(urlmodule, 'urlconf_module'):
            is_urlconf_module = True
        else:
            is_urlconf_module = False
        # if url is really a urlmodule
        if is_urlconf_module:
            # loop over urls of that module
            for url in urlmodule.urlconf_module.urlpatterns:
                # TODO: configurable skip url in settings
                # skip api-docs url
                if url.name in ['django.swagger.resources.view']:
                    continue
                # try adding url to list of urls to show
                try:
                    endpoints.append({
                        'name': url.name.replace('api_', ''),
                        'url': reverse(url.name, request=request, format=format)
                    })
                # urls of object details will fail silently (eg: /nodes/<slug>/)
                except NoReverseMatch:
                    pass
    return Response(endpoints)
1080455
def just_find_proxy(pacfile, url, host=None):
  
  if not os.path.isfile(pacfile):
    raise IOError('Pac file does not exist: {}'.format(pacfile))
  init()
  parse_pac(pacfile)
  proxy = find_proxy(url,host)
  cleanup()
  return proxy
53529
def parser_endpoint_to_web_url
      # @type [String]
      prop_name = nil
      if Constants::RAPID_ENDPOINT_PRODUCTION.casecmp(@rapid_endpoint).zero?
        prop_name = Constants::GLOBAL_RAPID_PRODUCTION_REST_URL_PARAM
      elsif Constants::RAPID_ENDPOINT_SANDBOX.casecmp(@rapid_endpoint).zero?
        prop_name = Constants::GLOBAL_RAPID_SANDBOX_REST_URL_PARAM
      end
      if prop_name.nil?
        set_web_url(@rapid_endpoint)
      else
        property_array = YAML.load_file(File.join(File.dirname(__FILE__), 'resources', 'rapid-api.yml'))
        property_array.each do |h|
          if prop_name.casecmp(h.keys.first).zero?
            set_web_url(h[h.keys.first])
          end
        end
        if @web_url.nil?
          fail Exception, "The endpoint #{prop_name} is invalid."
        end
      end
      # verify_endpoint_url(@web_url) # this is unreliable
    end
193935
function (url) {
        url = _.trimStart(url);
        var pathVariables,
            p = {
                raw: url
            };

        // extract the protocol
        p.protocol = _.get(url.match(regexes.extractProtocol), GET_1);
        _.isString(p.protocol) && (url = url.substr(p.protocol.length + 3)); // remove that damn protocol from url

        // extract the host
        p.host = _.get(url.match(regexes.extractHost), GET_1);

        // if host exists there are a lot you can extract from it
        if (_.isString(p.host)) {
            url = url.substr(p.host.length); // remove host from url

            if ((p.auth = _.get(p.host.match(regexes.extractHostAuth), GET_1))) {
                p.host = p.host.substr(p.auth.length + 1); // remove auth from host
                p.auth = p.auth.split(AUTH_SEPARATOR);
                p.auth = {
                    user: p.auth[0],
                    password: p.auth[1]
                };
            }

            // extract the port from the host
            p.port = _.get(p.host.match(regexes.extractPort), GET_1);
            p.port && (p.host = p.host.substring(0, p.host.length - p.port.length - 1)); // remove port from url

            p.host = _.trim(p.host, DOMAIN_SEPARATOR).split(regexes.splitDomain); // split host by subdomains
        }

        // extract the path
        p.path = _.get(url.match(regexes.extractPath), GET_0);
        if (_.isString(p.path)) {
            url = url.substr(p.path.length);
            p.path && (p.path = p.path.replace(regexes.trimPath, MATCH_1)); // remove leading slash for valid path
            // if path is blank string, we set it to undefined, if '/' then single blank string array
            p.path = p.path ? (p.path === PATH_SEPARATOR ? [E] : p.path.split(PATH_SEPARATOR)) : undefined;
        }

        // extract the query string
        p.query = _.get(url.match(regexes.extractQuery), GET_1);
        _.isString(p.query) && ((url = url.substr(p.query.length + 1)), (p.query = QueryParam.parse(p.query)));

        // extract the hash
        p.hash = _.get(url.match(regexes.extractSearch), GET_1);

        // extract path variables
        pathVariables = _.transform(p.path, function (res, segment) {
            // check if the segment has path variable prefix followed by the variable name.
            if (_.startsWith(segment, PATH_VARIABLE_IDENTIFIER) && segment !== PATH_VARIABLE_IDENTIFIER) {
                res.push({ key: segment.slice(1) }); // remove path variable prefix.
            }
        }, []);
        p.variable = pathVariables.length ? pathVariables : undefined;

        return p;
    }
1029761
def filter(self, endpoint, params):
        
        params = self.parse_params(params)
        params = urlencode(params)
        path = '{0}?{1}'.format(endpoint, params)
        return self.get(path)
1319494
def configure(self, endpoint=None, **kwargs):
        """"""
        if endpoint:
            kwargs['endpoint'] = endpoint
        keywords = self._keywords.copy()
        keywords.update(kwargs)
        if 'endpoint' in kwargs:
            # Then we need to correctly format the endpoint.
            endpoint = kwargs['endpoint']
            keywords['endpoint'] = self._configure_endpoint(endpoint)
        self.api_key = keywords['api_key'] or self._global_api_key()
        self.endpoint = keywords['endpoint']
        self.format = keywords['format'] or 'json'
        self.jurisdiction = keywords['jurisdiction']
        self.proxy = keywords['proxy']
        self.discovery_url = keywords['discovery'] or None

        # Use a custom requests session and set the correct SSL version if
        # specified.
        self.session = requests.Session()
        if 'ssl_version' in keywords:
            self.session.mount('https://', SSLAdapter(keywords['ssl_version']))
687827
def grab_keyfile(cert_url):
    
    key_cache = caches[getattr(settings, 'BOUNCY_KEY_CACHE', 'default')]

    pemfile = key_cache.get(cert_url)
    if not pemfile:
        response = urlopen(cert_url)
        pemfile = response.read()
        # Extract the first certificate in the file and confirm it's a valid
        # PEM certificate
        certificates = pem.parse(smart_bytes(pemfile))

        # A proper certificate file will contain 1 certificate
        if len(certificates) != 1:
            logger.error('Invalid Certificate File: URL %s', cert_url)
            raise ValueError('Invalid Certificate File')

        key_cache.set(cert_url, pemfile)
    return pemfile
747690
def _allows_url (self, url_data, roboturl):
        """"""
        with cache_lock:
            if roboturl in self.cache:
                self.hits += 1
                rp = self.cache[roboturl]
                return rp.can_fetch(self.useragent, url_data.url)
            self.misses += 1
        kwargs = dict(auth=url_data.auth, session=url_data.session)
        if hasattr(url_data, "proxy") and hasattr(url_data, "proxy_type"):
            kwargs["proxies"] = {url_data.proxytype: url_data.proxy}
        rp = robotparser2.RobotFileParser(**kwargs)
        rp.set_url(roboturl)
        rp.read()
        with cache_lock:
            self.cache[roboturl] = rp
        self.add_sitemap_urls(rp, url_data, roboturl)
        return rp.can_fetch(self.useragent, url_data.url)
305521
def parse_url(url):
    
    split_url = url.split('/', 3)
    if len(split_url) == 3:
        # http://foo.com
        path = '/'
    elif len(split_url) == 4:
        path = '/' + split_url[3]
    else:
        raise ValueError('Invalid URL: %s' % url)

    try:
        parse_result = urlparse(url)
    except Exception:
        raise ValueError('Invalid URL: %s' % url)

    protocol = parse_result.scheme
    protocol = protocol.lower()
    if protocol not in ('http', 'https'):
        raise ValueError('Invalid URL protocol "%s"' % protocol)

    split_netloc = parse_result.netloc.split(':')
    domain = split_netloc[0]
    domain = domain.lower()

    if len(split_netloc) == 2:
        try:
            port = int(split_netloc[1])
        except:
            raise ValueError('Invalid port: "%s"' % split_netloc[1])
    elif protocol == 'https':
        port = 443
    elif protocol == 'http':
        port = 80
    else:
        raise ValueError('Invalid scheme: "%s"' % protocol)

    return protocol, domain, port, path
32423
def _build_url(self, endpoint):
        
        if not issubclass(type(self), ChatApiBase) and not self.subdomain:
            raise ZenpyException("subdomain is required when accessing the Zendesk API!")

        if self.subdomain:
            endpoint.netloc = '{}.{}'.format(self.subdomain, self.domain)
        else:
            endpoint.netloc = self.domain

        endpoint.prefix_path(self.api_prefix)
        return endpoint.build()