Extracts urls from the file.
517754
def check(self, url_data):
        """"""
        content = url_data.get_content()
        self._check_by_re(url_data, content)
        self._check_inline_links(url_data, content)
1031137
def file_to_url(self, file_rel_path):
        """"""
        _abs_path = os.path.abspath(file_rel_path)
        return urlparse.urlparse(_abs_path, scheme='file').geturl()
158121
def _get_url_hashes(path):
  """"""
  urls = _read_text_file(path)
  def url_hash(u):
    h = hashlib.sha1()
    try:
      u = u.encode('utf-8')
    except UnicodeDecodeError:
      logging.error('Cannot hash url: %s', u)
    h.update(u)
    return h.hexdigest()
  return {url_hash(u): True for u in urls}
443872
def url_or_file_open(uri):
    """"""
    if (not re.match(r'''\w+:''', uri)):
        uri = 'file:' + uri
    return(urlopen(uri))
333094
def getDownloadUrls(self):
		""""""
		data = self.searchIndex(False)
		fileUrls = []
		for datum in data:
			fileUrl = self.formatDownloadUrl(datum[0])
			fileUrls.append(fileUrl)
		return fileUrls
1058500
private List<String> parse( URL url )
    {
        InputStream is = null;
        BufferedReader reader = null;
        List<String> names = new ArrayList<String>();
        try
        {
            is = url.openStream();
            reader = new BufferedReader( new InputStreamReader( is, "UTF-8" ) );
            String line = null;
            while( ( line = reader.readLine() ) != null )
            {
                parseLine( names, line );
            }
        }
        catch ( IOException exc )
        {
            throw new NoServiceProviderException( exc );
        }
        finally
        {
            closeSilently( reader );
        }
        return names;
    }
109982
def links(self):
        """"""
        for match in self._href_re.finditer(self.content):
            url = match.group(1) or match.group(2) or match.group(3)
            url = self.clean_link(urlparse.urljoin(self.base_url, url))
            yield Link(url, self)
1191722
public static List<String> extractPathParams(final String url) {

    final List<String> pathParams = new ArrayList<>();
    int index = url.indexOf('{');

    while (index >= 0) {

      final int endIndex = url.indexOf('}', index);
      final String pathParam = url.substring(index + 1, endIndex);
      pathParams.add(pathParam);
      index = url.indexOf('{', endIndex);
    }

    return pathParams;
  }
912330
def _extract_image_urls(self):
        """"""

        resultsPage = self._chromeDriver.page_source

        resultsPageSoup = BeautifulSoup(resultsPage, 'html.parser')
        images = resultsPageSoup.find_all('div', class_='rg_meta')
        images = [json.loads(image.contents[0]) for image in images]

        [self._imageURLs.append(image['ou']) for image in images]

        self._imageURLsExtractedCount += len(images)
823492
def get_url_file_name(url):
        
        url_root_ext = os.path.splitext(url)

        if len(url_root_ext[1]) <= MAX_FILE_EXTENSION_LENGTH:
            return os.path.split(url_root_ext[0])[1]
        else:
            return os.path.split(url)[1]
1069015
def from_url(cls, url, filename=None, chunk_size=CHUNK_SIZE):
        
        pipe = _WebPipe(url, chunk_size=chunk_size)
        if filename is None:
            filename = pipe.name

        return cls(pipe, filename, chunk_size)
256648
def read
      files = Dir.glob(File.join(path, "**/*"))
      
      regex = Regexp.new("(?<=#{path}).*")

      files.map do |file|
        unless ((File.directory? file) || (File.extname(file).upcase != ".URL"))
          url = File.read(file).scan(/(?<=\nURL=)(.*)$/).flatten.first.chomp
          name = url_name(File.basename(file, ".*"))
          description = ""
          tag         = extract_tags(File.dirname(file).scan(regex))
          [url, name, description, tag]
        end
      end.compact
    end
532001
private URL[] pathToURLs(String path) {
        Set<URL> urls = new LinkedHashSet<URL>();
        for (String s: path.split(File.pathSeparator)) {
            if (s.isEmpty()) continue;
            try {
                urls.add(new File(s).getAbsoluteFile().toURI().toURL());
            } catch (MalformedURLException e) {
                message.error("doclet.MalformedURL", s);
            }
        }
        return urls.toArray(new URL[urls.size()]);
    }
1201183
def read_links(self, file, encoding=None):
        '''
        '''
        return [item[0] for item in self.iter_text(file, encoding) if item[1]]
496370
def extracturls(mesg):
    
    lines = NLRE.split(mesg)

    # The number of lines of context above to provide.
    # above_context = 1
    # The number of lines of context below to provide.
    # below_context = 1

    # Plan here is to first transform lines into the form
    # [line_fragments] where each fragment is a chunk as
    # seen by parse_text_urls.  Note that this means that
    # lines with more than one entry or one entry that's
    # a URL are the only lines containing URLs.

    linechunks = [parse_text_urls(l) for l in lines]

    return extract_with_context(linechunks,
                                lambda chunk: len(chunk) > 1 or
                                (len(chunk) == 1 and chunk[0].url is not None),
                                1, 1)
172969
def get_pages(url):
    
    while True:
        yield url
        doc = html.parse(url).find("body")
        links = [a for a in doc.findall(".//a") if a.text and a.text.startswith("next ")]
        if not links:
            break
        url = urljoin(url, links[0].get('href'))
846286
def text_extract(path, password=None):
    """"""
    pdf = Info(path, password).pdf

    return [pdf.getPage(i).extractText() for i in range(pdf.getNumPages())]