Is a given URL crawlable?
@param  string $url
@return bool
352008
public function isCrawlable()
    {
        if (empty($this->getPath()) === true && 
                ($this->parentLink && empty($this->parentLink->getPath()))) {
            return false;
        }

        $stop_links = array(
            '@^javascript\:.*$@i',           
            '@^mailto\:.*@i',
            '@^tel\:.*@i',
            '@^skype\:.*@i',
            '@^fax\:.*@i',            
        );

        foreach ($stop_links as $ptrn) {
            if (preg_match($ptrn, $this->getOriginalUrl()) === 1) {                
                return false;
            }
        }
        
        return true;
    }
1218630
public function matchDirectly($url)
    {
        if (!$this->isAcceptedRequestMethod()) {
            return false;
        }

        if (rtrim($this->url, "/") === rtrim($url, "/")) {
            return true;
        }
        return false;
    }
1042878
public function crawl( $url ) {

		$this->parse($url);

		while( $url = array_shift($this->sitemaps) ) {

			$this->parse($url);
		}

		return $this->urls;
	}
1320996
private static function httpGet($url)
    {
        $oCurl = curl_init();
        curl_setopt($oCurl, CURLOPT_URL, $url);
        curl_setopt($oCurl, CURLOPT_TIMEOUT, 10);
        curl_setopt($oCurl, CURLOPT_RETURNTRANSFER, 1);
        $sContent = curl_exec($oCurl);
        $aStatus = curl_getinfo($oCurl);
        curl_close($oCurl);
        if (intval($aStatus["http_code"]) == 200) {
            return $sContent;
        } else {
            return false;
        }
    }
311896
public function hasAccess($url)
    {
        if ($this->isGuestAction($url)) {
            return true;
        }

        $path = $this->getPathFromUrl($url);

        $groupId = $this->Auth->user('group_id');

        if ($groupId === null) {
            return false;
        }

        if (!is_array($groupId)) {
            $groupId = [$groupId];
        }

        if (in_array(1, $groupId)) {
            return true;
        }

        $user = Wasabi::user();
        if (empty($user->permissions)) {
            Wasabi::user()->permissions = $this->_getGroupPermissions()->findAllForGroup($groupId);
        }

        if (array_key_exists($path, Wasabi::user()->permissions)) {
            return true;
        }

        return false;
    }
43291
public function isAdminRoute($url)
    {
        if ($this->matchesPreviewRoute($url)) {
            return false;
        }

        preg_match(sprintf(self::$ADMIN_MATCH_REGEX, $this->adminKey), $url, $matches);

        // Check if path is part of admin area
        if (count($matches) === 0) {
            return false;
        }

        return true;
    }
1349105
private function resourceExists($url) {
        try {
            $response = $this->head((string) $url)->send();

            return $response->getStatusCode() === 200;
        } catch (GuzzleException $e) {
            if ($e->getResponse()->getStatusCode() === 404) {
                return false;
            }

            throw $e;
        }
    }
1158938
public static function is_external_url($url)
    {
        if (\parse_url($url, PHP_URL_HOST) === Request::get_host()) {
            return false;
        }

        if (\strpos($url, "/") === 0) {
            return false;
        }

        return true;
    }
740764
static function isShortUrl($url){
        // Overall URL length - May be a max of 30 characters
        if (strlen($url) > 30) return false;

        $parts = parse_url($url);

        if(isset($parts["path"])){
            $path = $parts["path"];
            $pathParts = explode("/", $path);

            // Number of '/' after protocol (http://) - Max 2
            if (count($pathParts) > 2) return false;

            // URL length after last '/' - May be a max of 10 characters
            $lastPath = array_pop($pathParts);
            if (strlen($lastPath) > 10) return false;

            // Max length of host
            if (strlen($parts["host"]) > 10) return false;
        } else {
            return false;
        }

        // Get headers and see if Location is set.
        $headers = get_headers($url, 1);
        if(!is_array($headers) || !isset($headers['Location'])){
            return false;
        }

        return true;
    }
173844
public function contains($url)
	{
		$url = trim($url, '/');

		foreach ($this->entries as $entry) {
			if ($entry['loc'] == $url) {
				return TRUE;
			}
		}

		return FALSE;
	}
1085159
public function verify($url)
    {
        $url = parse_url($url);
        if ($url === false) {
            throw new \Exception('Bad url to match (not an url ?)');
        }

        $path = isset($url['path']) ? $url['path'] : '';

        if ((bool) preg_match($this->pattern, $path, $matches)) {
            array_shift($matches);

            foreach ($matches as $position => $value) {
                $this->parameterCollection->getByPosition($position)
                    ->setValue($value);
            }

            return true;
        }

        return false;
    }
557053
public static function isUrlArchive($url)
    {
        if (0 === strpos($url, 'http')) {
            foreach (self::$extensions as $extension) {
                if (substr($url, -\strlen($extension)) === $extension) {
                    return true;
                }
            }
        }

        return false;
    }
596448
public static function isInternal($url)
    {
        $relative = static::makeRelative($url);

        if (substr($relative, 0, 1) !== '/') {
            return false;
        }

        $path = static::getInternalPath($relative);

        return !URLFacade::isAvailable($path);
    }
1288812
public function isMatching($url)
    {
        if ($this->getRegex() === null) {
            return true;
        }

        if (preg_match($this->getRegex(), $url)) {
            return true;
        }

        return false;
    }
231645
public static function isLocalUrl($url)
    {
        if (empty($url)) {
            return true;
        }

        // handle host name mangling
        $requestUri = isset($_SERVER['SCRIPT_URI']) ? $_SERVER['SCRIPT_URI'] : '';
        $parseRequest = @parse_url($requestUri);
        $hosts = array(self::getHost(), self::getCurrentHost());
        if (!empty($parseRequest['host'])) {
            $hosts[] = $parseRequest['host'];
        }

        // drop port numbers from hostnames and IP addresses
        $hosts = array_map(array('self', 'getHostSanitized'), $hosts);

        $disableHostCheck = Config::getInstance()->General['enable_trusted_host_check'] == 0;
        // compare scheme and host
        $parsedUrl = @parse_url($url);
        $host = IPUtils::sanitizeIp(@$parsedUrl['host']);
        return !empty($host)
        && ($disableHostCheck || in_array($host, $hosts))
        && !empty($parsedUrl['scheme'])
        && in_array($parsedUrl['scheme'], array('http', 'https'));
    }
981034
protected function urlParts(string $url)
    {
        if (false === filter_var($url, FILTER_VALIDATE_URL, FILTER_FLAG_PATH_REQUIRED)) {
            return false;
        }
        return parse_url($url) ?: false;
    }
863669
public function validate($url)
    {
        $url = UrlImmutable::createFromUrl($url);

        $query = $url->getQuery();

        if ($this->isMissingAQueryParameter($query)) {
            return false;
        }

        $expiration = $query[$this->expiresParameter];

        if (!$this->isFuture($expiration)) {
            return false;
        }

        if (!$this->hasValidSignature($url)) {
            return false;
        }

        return true;
    }