Given a list of tags from either a search page or the
		   KAT home page parse the details and return a list of
		   Torrents
356760
def _parse_details(self, tag_list):
		""""""

		result = list()
		for i, item in enumerate(tag_list):
			title = item.find("a", class_="cellMainLink")
			title_text = title.text
			link = title.get("href")
			tds = item.find_all("td", class_="center") # Better name here.
			size = tds[0].text
			files = tds[1].text
			age = tds[2].text
			seed = tds[3].text
			leech = tds[4].text
			magnet = item.find("a", class_="imagnet icon16")
			download = item.find("a", class_="idownload icon16")
			isVerified = item.find("a", class_="iverify icon16") != None

			# Home page doesn't have magnet or download links
			if magnet:
				magnet = magnet.get("href")
			if download:
				download = download.get("href")

			# Get category changes depending on if we're parsing
			# the home page or a search page.
			if self.current_url == self.base_url:
				category = self._get_torrent_category(item, result=i)
			else:
				category = self._get_torrent_category(item)

			result.append(Torrent(title_text, category, link, size, seed,
								leech, magnet, download,files, age, isVerified))

		return result
1136679
def buildlist(self, category=tpb.CATEGORIES.VIDEO.TV_SHOWS, limit=1000):
        

        try:
            s = self.source.search(self.title.lower(), limit)
        except Exception as e:
            logging.error("Can not send search request to the t411 server")
            logging.error(e.message)
            sys.exit(1)
        

        try:
            for t in s.items():
                pass
        except:
            logging.error("t411 server returned an invalid result")
            sys.exit(1)

        torrentlist = []
        for torrent in s['torrents']:
            if isinstance(torrent, dict):    
                #logging.debug("Compare regex to: %s" % t.title.lower())
                if (re.search(self.regexp, torrent['name'].lower()) and (int(torrent['seeders']) >= self.seeders_min)):
                    # logging.debug("Matched")
                    torrentlist.append((torrent['name'], torrent['seeders'], torrent['id']))

        logging.debug("Found %d matching items " % (len(torrentlist)))

        # Return the list
        return torrentlist
730590
def tag_stuff(self):
        """"""

        # For each packet in the pcap process the contents
        for item in self.input_stream:

            # Make sure it has a tags field (which is a set)
            if 'tags' not in item:
                item['tags'] = set()

            # For each tag_methods run it on the item
            for tag_method in self.tag_methods:
                item['tags'].add(tag_method(item))

            # Not interested in None tags
            if None in item['tags']:
                item['tags'].remove(None)
            # All done
            yield item
819402
def parse_magnet(magnet_uri):
    """"""
    data = defaultdict(list)
    if not magnet_uri.startswith('magnet:'):
        return data
    else:
        magnet_uri = magnet_uri.strip('magnet:?')
        for segment in magnet_uri.split('&'):
            key, value = segment.split('=')
            if key == 'dn':
                data['name'] = requests.utils.unquote(value).replace('+', '.')
            elif key == 'xt':
                data['infoHash'] = value.strip('urn:btih:')
            elif key == 'tr':
                data['trackers'].append(requests.utils.unquote(value))
            else:
                data[key] = value
    return data
675384
def get_tags(self, use_cached=True):
        """"""
        device_json = self.get_device_json(use_cached)
        potential_tags = device_json.get("dpTags")
        if potential_tags:
            return list(filter(None, potential_tags.split(",")))
        else:
            return []
196103
def analyze_url_list(self, response):
        
        response = json.loads(response.text)
        for _, block in response['blocks'].items():
            for attribute in block:
                parsed = urlparse(block[attribute])
                # find urls in the JSON response
                if parsed.scheme and parsed.netloc:
                    yield self.make_requests_from_url(block[attribute])
337216
def retrieve_list(self, parsed_args):
        """"""
        neutron_client = self.get_client()
        _extra_values = parse_args_to_dict(self.values_specs)
        _merge_args(self, parsed_args, _extra_values,
                    self.values_specs)
        search_opts = self.args2search_opts(parsed_args)
        search_opts.update(_extra_values)
        if self.pagination_support:
            page_size = parsed_args.page_size
            if page_size:
                search_opts.update({'limit': page_size})
        if self.sorting_support:
            keys = parsed_args.sort_key
            if keys:
                search_opts.update({'sort_key': keys})
            dirs = parsed_args.sort_dir
            len_diff = len(keys) - len(dirs)
            if len_diff > 0:
                dirs += ['asc'] * len_diff
            elif len_diff < 0:
                dirs = dirs[:len(keys)]
            if dirs:
                search_opts.update({'sort_dir': dirs})
        data = self.call_server(neutron_client, search_opts, parsed_args)
        collection = _get_resource_plural(self.resource, neutron_client)
        return data.get(collection, [])
420778
def get_list(self, size=100, startIndex=0, searchText="", sortProperty="", sortOrder='ASC', status='Active,Pending'):
        

        url = urljoin(BASEURL, "sites", "list")

        params = {
            'api_key': self.token,
            'size': size,
            'startIndex': startIndex,
            'sortOrder': sortOrder,
            'status': status
        }

        if searchText:
            params['searchText'] = searchText

        if sortProperty:
            params['sortProperty'] = sortProperty

        r = requests.get(url, params)
        r.raise_for_status()
        return r.json()
747917
def parse(url):
    
    result = urlparse.urlparse(nstr(url))

    path = result.scheme + '://' + result.netloc
    if result.path:
        path += result.path

    query = {}

    # extract the python information from the query
    if result.query:
        url_query = urlparse.parse_qs(result.query)
        for key, value in url_query.items():
            if type(value) == list and len(value) == 1:
                value = value[0]

            query[key] = value

    return path, query, result.fragment
868711
def get_parser():
    
    desc = Colors.LIGHTBLUE + textwrap.dedent(
        '''\
        Welcome to
                      _                           _                            _
           __ _ _   _| |_ ___      _ __  _   _   | |_ ___  _ __ _ __ ___ _ __ | |_
          / _` | | | | __/ _ \    | '_ \| | | |  | __/ _ \| '__| '__/ _ \ '_ \| __|
         | (_| | |_| | || (_) |   | |_) | |_| |  | || (_) | |  | | |  __/ | | | |_
          \__,_|\__,_|\__\___/____| .__/ \__, |___\__\___/|_|  |_|  \___|_| |_|\__|
                            |_____|_|    |___/_____|

        ------------------------------------
          auto_py_torrent is an automated tool for download files by obtaining
        torrents or magnets that are in different provided pages that the
        user can choose.

          Its goal is to make it easier for users to find the files they want
        and download them instantly.

          An auto_py_torrent command is provided in which the user can
        currently choose between two modes, best_rated and list mode, then it
        selects one of the torrent tracking pages for multimedia content and
        finally enter the text of what you want to download.
        ------------------------------------
        ''') + Colors.ENDC
    usage_info = Colors.LGREEN + textwrap.dedent(
        '''\

        Use "%(prog)s --help" for more information.
        Examples:
            use "%(prog)s MODE SELECTED_PAGE STRING_TO_SEARCH # generic.
            use "%(prog)s 0 0 "The simpsons" # best rated.
            use "%(prog)s 1 0 "The simpsons" # list rated.

        Mode options:
            0: best_rated. # Download the most rated file.
            1: list. # Get a list, and select one of them.

        Page list options:
            0: torrent project.
            1: the pirate bay.
            2: 1337x.
            3: eztv.
            4: limetorrents.
            5: isohunt.
        ''') + Colors.ENDC
    epi = Colors.LIGHTPURPLE + textwrap.dedent(
        '''\
        -> Thanks for using auto_py_torrent!
        ''') + Colors.ENDC

    # Parent and only parser.
    parser = argparse.ArgumentParser(
        add_help=True,
        formatter_class=argparse.RawTextHelpFormatter,
        usage=usage_info,
        description=desc,
        epilog=epi)
    parser.add_argument('mode', action='store',
                        choices=range(len(MODES)),
                        type=int,
                        help='Select mode of file download.\n'
                             '    e.g: 0(rated) or 1(list).')
    parser.add_argument('torr_page', action='store',
                        choices=range(len(TORRENTS)),
                        type=int,
                        help='Select tracking page to download from.\n'
                             '    e.g: 0 to .. ' + str(len(TORRENTS)-1) + '.')
    parser.add_argument('str_search', action='store',
                        type=str,
                        help='Input torrent string to search.\n'
                             '    e.g: "String search"')
    return(parser)
553431
def build_list_result(results, xml):
    
    xml_count = xml.xpath('//div[@class="paginator"]/span[@class="count"]/text()')
    xml_next = xml.xpath('//div[@class="paginator"]/span[@class="next"]/a/@href')
    count = int(re.search(r'\d+', xml_count[0]).group()) if xml_count else len(results)
    next_start = int(re.search(r'start=(\d+)', xml_next[0]).groups()[0]) if xml_next else None
    return {'results': results, 'count': count, 'next_start': next_start}
85915
private static List<ShowInfo> processShowInfo(String searchUrl, String tagName) throws TVRageException {
        List<ShowInfo> showList = new ArrayList<>();
        ShowInfo showInfo;

        Document doc = getDocFromUrl(searchUrl);
        NodeList nlShowInfo = doc.getElementsByTagName(tagName);

        if (nlShowInfo == null || nlShowInfo.getLength() == 0) {
            return showList;
        }

        for (int loop = 0; loop < nlShowInfo.getLength(); loop++) {
            Node nShowInfo = nlShowInfo.item(loop);
            if (nShowInfo.getNodeType() == Node.ELEMENT_NODE) {
                Element eShowInfo = (Element) nShowInfo;
                showInfo = parseNextShowInfo(eShowInfo);
                showList.add(showInfo);
            }
        }

        return showList;
    }
172969
def get_pages(url):
    
    while True:
        yield url
        doc = html.parse(url).find("body")
        links = [a for a in doc.findall(".//a") if a.text and a.text.startswith("next ")]
        if not links:
            break
        url = urljoin(url, links[0].get('href'))
1244012
def parse_list(self, entries):
        """"""
        result_entries = SearchableList()
        for entry in entries:
            result_entries.append(self.instance.parse(self.requester, entry))
        return result_entries
974529
def gen_tasks(self, urls):
        
        tasks = []
        if isinstance(urls, list):
            for url in urls:
                dat = {
                    'url': url,
                    'name': url.split('/')[-1]
                }
                if not self.overwrite and helper.is_file_ok(dat['name']):
                    self.result['skip'] += 1
                else:
                    tasks.append(dat)
        elif isinstance(urls, dict):
            # {k: v} => k is name, v is url value
            for k, v in urls.items():
                dat = {
                    'url': v,
                    'name': k,
                }
                if not self.overwrite and helper.is_file_ok(dat['name']):
                    self.result['skip'] += 1
                else:
                    tasks.append(dat)
        else:
            raise CrawlerParamsError('urls should be list/dict')
        return tasks
1295928
def fetchUrls(cls, url, data, urlSearch):
        """"""
        searchUrls = []
        if cls.css:
            searchFun = data.cssselect
        else:
            searchFun = data.xpath
        searches = makeSequence(urlSearch)
        for search in searches:
            for match in searchFun(search):
                try:
                    for attrib in html_link_attrs:
                        if attrib in match.attrib:
                            searchUrls.append(match.get(attrib))
                except AttributeError:
                    searchUrls.append(str(match))
            if not cls.multipleImagesPerStrip and searchUrls:
                # do not search other links if one pattern matched
                break
        if not searchUrls:
            raise ValueError("XPath %s not found at URL %s." % (searches, url))
        return searchUrls
434688
def parse_text_urls(mesg):
    """"""

    rval = []

    loc = 0

    for match in URLRE.finditer(mesg):
        if loc < match.start():
            rval.append(Chunk(mesg[loc:match.start()], None))
        # Turn email addresses into mailto: links
        email = match.group("email")
        if email and "mailto" not in email:
            mailto = "mailto:{}".format(email)
        else:
            mailto = match.group(1)
        rval.append(Chunk(None, mailto))
        loc = match.end()

    if loc < len(mesg):
        rval.append(Chunk(mesg[loc:], None))

    return rval