Build and return url. Also update max_page.
        URL structure for user torrent lists differs from other result lists
        as the page number is part of the query string and not the URL path
517724
def build(self, update=True):
        
        query_str = "?page={}".format(self.page)
        if self.order:
            query_str += "".join(("&field=", self.order[0], "&sorder=",self.order[1]))

        ret = "".join((self.base, self.user, "/uploads/", query_str))

        if update:
            self.max_page = self._get_max_page(ret)
        return ret
35116
def _BuildUrl(self, url, path_elements=None, extra_params=None):
        
        # Break url into constituent parts
        (scheme, netloc, path, params, query, fragment) = urlparse(url)

        # Add any additional path elements to the path
        if path_elements:
            # Filter out the path elements that have a value of None
            p = [i for i in path_elements if i]
            if not path.endswith('/'):
                path += '/'
            path += '/'.join(p)

        # Add any additional query parameters to the query string
        if extra_params and len(extra_params) > 0:
            extra_query = self._EncodeParameters(extra_params)
            # Add it to the existing query
            if query:
                query += '&' + extra_query
            else:
                query = extra_query

        # Return the rebuilt URL
        return urlunparse((scheme, netloc, path, params, query, fragment))
189719
def get_url_and_headers_for_range(self, file_download):
        
        headers = self.get_range_headers()
        if file_download.http_headers:
            headers.update(file_download.http_headers)
        separator = ""
        if not file_download.url.startswith("/"):
            separator = "/"
        url = '{}{}{}'.format(file_download.host, separator, file_download.url)
        return url, headers
42590
def list(self, page=1, per_page=50):
        
        data = {"page": page,
                "per_page": per_page}
        return self.get(self.base_url, data=data)
181015
def crawl_links(self, seed_url=None):
        
        if seed_url is not None:
            self.seed_url = seed_url

        if self.seed_url is None:
            sys.stderr.write('Crawling requires a seed URL.\n')
            return []

        prev_part_num = utils.get_num_part_files()
        crawled_links = set()
        uncrawled_links = OrderedSet()

        uncrawled_links.add(self.seed_url)
        try:
            while uncrawled_links:
                # Check limit on number of links and pages to crawl
                if self.limit_reached(len(crawled_links)):
                    break
                url = uncrawled_links.pop(last=False)

                # Remove protocol, fragments, etc. to get unique URLs
                unique_url = utils.remove_protocol(utils.clean_url(url))
                if unique_url not in crawled_links:
                    raw_resp = utils.get_raw_resp(url)
                    if raw_resp is None:
                        if not self.args['quiet']:
                            sys.stderr.write('Failed to parse {0}.\n'.format(url))
                        continue

                    resp = lh.fromstring(raw_resp)
                    if self.page_crawled(resp):
                        continue

                    crawled_links.add(unique_url)
                    new_links = self.get_new_links(url, resp)
                    uncrawled_links.update(new_links)
                    if not self.args['quiet']:
                        print('Crawled {0} (#{1}).'.format(url, len(crawled_links)))

                    # Write page response to PART.html file
                    utils.write_part_file(self.args, url, raw_resp, resp, len(crawled_links))
        except (KeyboardInterrupt, EOFError):
            pass

        curr_part_num = utils.get_num_part_files()
        return utils.get_part_filenames(curr_part_num, prev_part_num)
463264
def next_page_url(self):
        
        if 'meta' in self._payload and 'next_page_url' in self._payload['meta']:
            return self._payload['meta']['next_page_url']
        elif 'next_page_uri' in self._payload and self._payload['next_page_uri']:
            return self._version.domain.absolute_url(self._payload['next_page_uri'])

        return None
909437
def _new_url_record(cls, request: Request) -> URLRecord:
        ''''''
        url_record = URLRecord()

        url_record.url = request.url_info.url
        url_record.status = Status.in_progress
        url_record.try_count = 0
        url_record.level = 0

        return url_record
209647
def _build_urlmapping(urls, strict_slashes=False, **kwargs):
    

    rules = _build_rules(urls)
    return Map(rules=list(rules), strict_slashes=strict_slashes, **kwargs)
1059059
def page_url(self, page):
        '''
        
        '''
        if page is not None and page != 1:
            return self.url.qs_set(**{self.page_param: page})
        elif page is not None:
            return self.url.qs_delete('page')
227737
def items(self):
        
        if self._multipage:
            while True:
                # Pool for more torrents
                items = super(Paginated, self).items()
                # Stop if no more torrents
                first = next(items, None)
                if first is None:
                    raise StopIteration()
                # Yield them if not
                else:
                    yield first
                    for item in items:
                        yield item
                # Go to the next page
                self.next()
        else:
            for item in super(Paginated, self).items():
                yield item
785303
def _get_max_page(self, url):
        
        html = requests.get(url).text
        pq = PyQuery(html)
        try:
            tds = int(pq("h2").text().split()[-1])
            if tds % 25:
                return tds / 25 + 1
            return tds / 25
        except ValueError:
            raise ValueError("No results found!")
52600
def _build_versioned_url(self, url):
        
        return '{}/v{}{}'.format(self.host, str(self._version), url)
1020234
def build_url (self):
        
        # norm base url - can raise UnicodeError from url.idna_encode()
        base_url, is_idn = url_norm(self.base_url, self.encoding)
        # make url absolute
        if self.base_ref:
            # use base reference as parent url
            if ":" not in self.base_ref:
                # some websites have a relative base reference
                self.base_ref = urljoin(self.parent_url, self.base_ref)
            self.url = urljoin(self.base_ref, base_url)
        elif self.parent_url:
            # strip the parent url query and anchor
            urlparts = list(urlparse.urlsplit(self.parent_url))
            urlparts[4] = ""
            parent_url = urlutil.urlunsplit(urlparts)
            self.url = urljoin(parent_url, base_url)
        else:
            self.url = base_url
        # urljoin can unnorm the url path, so norm it again
        urlparts = list(urlparse.urlsplit(self.url))
        if urlparts[2]:
            urlparts[2] = urlutil.collapse_segments(urlparts[2])
        self.url = urlutil.urlunsplit(urlparts)
        # split into (modifiable) list
        self.urlparts = strformat.url_unicode_split(self.url)
        self.build_url_parts()
        # and unsplit again
        self.url = urlutil.urlunsplit(self.urlparts)
1069289
def _build_path(self):
        '''
        

        '''
        if not self.path:
            self.path = '/'

        if self.uri_parameters:
            self.path = self.path + ';' + requote_uri(self.uri_parameters)

        if self.query:
            self.path = (self.path + '?' + self.query)

        if self.params:
            try:
                if self.query:
                    self.path = self.path + self._dict_to_query(
                        self.params, base_query=True)
                else:
                    self.path = self.path + self._dict_to_query(self.params)
            except AttributeError:
                self.path = self.path + '?' + self.params

        self.path = requote_uri(self.path)

        self.req_url = urlunparse(
            (self.scheme, self.host, (self.path or ''), '', '', ''))
346832
def all_resource_urls(query):
    '''  '''
    urls = []
    next = True
    while next:
        response = requests.get(query)
        json_data = json.loads(response.content)
        for resource in json_data['results']:
            urls.append(resource['url'])
        if bool(json_data['next']):
            query = json_data['next']
        else:
            next = False
    return urls
1250997
def get_page(url, stash=false, options={})
      return EMPTY_STRING if url.nil? || url.empty? #url is nil if the <a> lacks an `href` attribute.
      global_options = {
        :cache => stash,
        :verbose => @verbose
      }
      if @readable_filenames
        global_options[:readable_filenames] = true
      end
      if @stash_folder
        global_options[:readable_filenames] = true
        global_options[:cache_location] = @stash_folder
      end
      resp_and_cache = Downloader.new(url, global_options.merge(options)).get
      if resp_and_cache[:from_resource]
        puts "sleeping #{@sleep_time_between_requests} secs" if @verbose
        sleep @sleep_time_between_requests
      end
      resp_and_cache[:resp]
    end
354081
def get(url, last_modified=None):
    
    try:
        with closing(urllib2.urlopen(url)) as page:
            if last_modified is not None:
                last_mod = dateutil.parser.parse(dict(page.info())['last-modified'])
                if last_mod <= last_modified:
                    return ""
            return page.read()
    except urllib2.URLError:
        return ""