code:
# coding: utf-8
from __future__ import unicode_literals

import functools
import itertools
import operator
import re

from .common import InfoExtractor
from ..compat import (
    compat_HTTPError,
    compat_str,
    compat_urllib_request,
)
from .openload import PhantomJSwrapper
from ..utils import (
    determine_ext,
    ExtractorError,
    int_or_none,
    merge_dicts,
    NO_DEFAULT,
    orderedSet,
    remove_quotes,
    str_to_int,
    url_or_none,
)

class PornHubBaseIE(InfoExtractor):
    def _download_webpage_handle(self, *args, **kwargs):
        def dl(*args, **kwargs):
            return super(PornHubBaseIE, self)._download_webpage_handle(*args, **kwargs)

        webpage, urlh = dl(*args, **kwargs)

        if any(re.search(p, webpage) for p in (
                r']+\bonload=["\']go\(\)',
                r'document\.cookie\s*=\s*["\']RNKEY=',
                r'document\.location\.reload\(true\)')):
            url_or_request = args[0]
            url = (url_or_request.get_full_url()
                   if isinstance(url_or_request, compat_urllib_request.Request)
                   else url_or_request)
            phantom = PhantomJSwrapper(self, required_version='2.0')
            phantom.get(url, html=webpage)
            webpage, urlh = dl(*args, **kwargs)

        return webpage, urlh

class PornHubIE(PornHubBaseIE):
    IE_DESC = 'PornHub and Thumbzilla'
    _VALID_URL = r'''(?x)
                    https?://
                        (?:
                            (?:[^/]+\.)?(?P<host>pornhub(?:premium)?\.(?:com|net))/(?:(?:view_video\.php|video/show)\?viewkey=|embed/)|
                            (?:www\.)?thumbzilla\.com/video/
                        )
                        (?P<id>[\da-z]+)
                    '''
    _TESTS = [{
        'url': 'http://www.pornhub.com/view_video.php?viewkey=648719015',
        'md5': 'a6391306d050e4547f62b3f485dd9ba9',
        'info_dict': {
            'id': '648719015',
            'ext': 'mp4',
            'title': 'Seductive Indian beauty strips down and fingers her pink pussy',
            'uploader': 'Babes',
            'upload_date': '20130628',
            'timestamp': 1372447216,
            'duration': 361,
            'view_count': int,
            'like_count': int,
            'dislike_count': int,
            'comment_count': int,
            'age_limit': 18,
            'tags': list,
            'categories': list,
        },
    }, {
        # non-ASCII title
        'url': 'http://www.pornhub.com/view_video.php?viewkey=1331683002',
        'info_dict': {
            'id': '1331683002',
            'ext': 'mp4',
            'title': '重庆婷婷女王足交',
            'upload_date': '20150213',
            'timestamp': 1423804862,
            'duration': 1753,
            'view_count': int,
            'like_count': int,
            'dislike_count': int,
            'comment_count': int,
            'age_limit': 18,
            'tags': list,
            'categories': list,
        },
        'params': {
            'skip_download': True,
        },
    }, {
        # subtitles
        'url': 'https://www.pornhub.com/view_video.php?viewkey=ph5af5fef7c2aa7',
        'info_dict': {
            'id': 'ph5af5fef7c2aa7',
            'ext': 'mp4',
            'title': 'BFFS - Cute Teen Girls Share Cock On the Floor',
            'uploader': 'BFFs',
            'duration': 622,
            'view_count': int,
            'like_count': int,
            'dislike_count': int,
            'comment_count': int,
            'age_limit': 18,
            'tags': list,
            'categories': list,
            'subtitles': {
                'en': [{
                    "ext": 'srt'
                }]
            },
        },
        'params': {
            'skip_download': True,
        },
        'skip': 'This video has been disabled',
    }, {
        'url': 'http://www.pornhub.com/view_video.php?viewkey=ph557bbb6676d2d',
        'only_matching': True,
    }, {
        # removed at the request of cam4.com
        'url': 'http://fr.pornhub.com/view_video.php?viewkey=ph55ca2f9760862',
        'only_matching': True,
    }, {
        # removed at the request of the copyright owner
        'url': 'http://www.pornhub.com/view_video.php?viewkey=788152859',
        'only_matching': True,
    }, {
        # removed by uploader
        'url': 'http://www.pornhub.com/view_video.php?viewkey=ph572716d15a111',
        'only_matching': True,
    }, {
        # private video
        'url': 'http://www.pornhub.com/view_video.php?viewkey=ph56fd731fce6b7',
        'only_matching': True,
    }, {
        'url': 'https://www.thumbzilla.com/video/ph56c6114abd99a/horny-girlfriend-sex',
        'only_matching': True,
    }, {
        'url': 'http://www.pornhub.com/video/show?viewkey=648719015',
        'only_matching': True,
    }, {
        'url': 'https://www.pornhub.net/view_video.php?viewkey=203640933',
        'only_matching': True,
    }, {
        'url': 'https://www.pornhubpremium.com/view_video.php?viewkey=ph5e4acdae54a82',
        'only_matching': True,
    }]

    @staticmethod
    def _extract_urls(webpage):
        return re.findall(
            r']+?src=["\'](?P<url>(?:https?:)?//(?:www\.)?pornhub\.(?:com|net)/embed/[\da-z]+)',
            webpage)

    def _extract_count(self, pattern, webpage, name):
        return str_to_int(self._search_regex(
            pattern, webpage, '%s count' % name, fatal=False))

    def _real_extract(self, url):
        mobj = re.match(self._VALID_URL, url)
        host = mobj.group('host') or 'pornhub.com'
        video_id = mobj.group('id')

        if 'premium' in host:
            if not self._downloader.params.get('cookiefile'):
                raise ExtractorError(
                    'PornHub Premium requires authentication.'
                    ' You may want to use --cookies.',
                    expected=True)

        self._set_cookie(host, 'age_verified', '1')

        def dl_webpage(platform):
            self._set_cookie(host, 'platform', platform)
            return self._download_webpage(
                'https://www.%s/view_video.php?viewkey=%s' % (host, video_id),
                video_id, 'Downloading %s webpage' % platform)

        webpage = dl_webpage('pc')

        error_msg = self._html_search_regex(
            r'(?s)
]+class=(["\'])(?:(?!\1).)*\b(?:removed|userMessageSection)\b(?:(?!\1).)*\1[^>]*>(?P<error>.+?)
',
            webpage, 'error message', default=None, group='error')
        if error_msg:
            error_msg = re.sub(r'\s+', ' ', error_msg)
            raise ExtractorError(
                'PornHub said: %s' % error_msg,
                expected=True, video_id=video_id)

        # video_title from flashvars contains whitespace instead of non-ASCII (see
        # http://www.pornhub.com/view_video.php?viewkey=1331683002), not relying
        # on that anymore.
        title = self._html_search_meta(
            'twitter:title', webpage, default=None) or self._html_search_regex(
            (r'(?s)
]+class=["\']title["\'][^>]*>(?P<title>.+?)
',
             r'
]+data-video-title=(["\'])(?P<title>(?:(?!\1).)+)\1',
             r'shareTitle["\']\s*[=:]\s*(["\'])(?P<title>(?:(?!\1).)+)\1'),
            webpage, 'title', group='title')

        video_urls = []
        video_urls_set = set()
        subtitles = {}

        flashvars = self._parse_json(
            self._search_regex(
                r'var\s+flashvars_\d+\s*=\s*({.+?});', webpage, 'flashvars', default='{}'),
            video_id)
        if flashvars:
            subtitle_url = url_or_none(flashvars.get('closedCaptionsFile'))
            if subtitle_url:
                subtitles.setdefault('en', []).append({
                    'url': subtitle_url,
                    'ext': 'srt',
                })
            thumbnail = flashvars.get('image_url')
            duration = int_or_none(flashvars.get('video_duration'))
            media_definitions = flashvars.get('mediaDefinitions')
            if isinstance(media_definitions, list):
                for definition in media_definitions:
                    if not isinstance(definition, dict):
                        continue
                    video_url = definition.get('videoUrl')
                    if not video_url or not isinstance(video_url, compat_str):
                        continue
                    if video_url in video_urls_set:
                        continue
                    video_urls_set.add(video_url)
                    video_urls.append(
                        (video_url, int_or_none(definition.get('quality'))))
        else:
            thumbnail, duration = [None] * 2

        def extract_js_vars(webpage, pattern, default=NO_DEFAULT):
            assignments = self._search_regex(
                pattern, webpage, 'encoded url', default=default)
            if not assignments:
                return {}

            assignments = assignments.split(';')

            js_vars = {}

            def parse_js_value(inp):
                inp = re.sub(r'/\*(?:(?!\*/).)*?\*/', '', inp)
                if '+' in inp:
                    inps = inp.split('+')
                    return functools.reduce(
                        operator.concat, map(parse_js_value, inps))
                inp = inp.strip()
                if inp in js_vars:
                    return js_vars[inp]
                return remove_quotes(inp)

            for assn in assignments:
                assn = assn.strip()
                if not assn:
                    continue
                assn = re.sub(r'var\s+', '', assn)
                vname, value = assn.split('=', 1)
                js_vars[vname] = parse_js_value(value)
            return js_vars

        def add_video_url(video_url):
            v_url = url_or_none(video_url)
            if not v_url:
                return
            if v_url in video_urls_set:
                return
            video_urls.append((v_url, None))
            video_urls_set.add(v_url)

        if not video_urls:
            FORMAT_PREFIXES = ('media', 'quality')
            js_vars = extract_js_vars(
                webpage, r'(var\s+(?:%s)_.+)' % '|'.join(FORMAT_PREFIXES),
                default=None)
            if js_vars:
                for key, format_url in js_vars.items():
                    if any(key.startswith(p) for p in FORMAT_PREFIXES):
                        add_video_url(format_url)
            if not video_urls and re.search(
                    r'<[^>]+\bid=["\']lockedPlayer', webpage):
                raise ExtractorError(
                    'Video %s is locked' % video_id, expected=True)

        if not video_urls:
            js_vars = extract_js_vars(
                dl_webpage('tv'), r'(var.+?mediastring.+?)</script>')
            add_video_url(js_vars['mediastring'])

        for mobj in re.finditer(
                r']+\bclass=["\']downloadBtn\b[^>]+\bhref=(["\'])(?P<url>(?:(?!\1).)+)\1',
                webpage):
            video_url = mobj.group('url')
            if video_url not in video_urls_set:
                video_urls.append((video_url, None))
                video_urls_set.add(video_url)

        upload_date = None
        formats = []
        for video_url, height in video_urls:
            if not upload_date:
                upload_date = self._search_regex(
                    r'/(\d{6}/\d{2})/', video_url, 'upload data', default=None)
                if upload_date:
                    upload_date = upload_date.replace('/', '')
            ext = determine_ext(video_url)
            if ext == 'mpd':
                formats.extend(self._extract_mpd_formats(
                    video_url, video_id, mpd_id='dash', fatal=False))
                continue
            elif ext == 'm3u8':
                formats.extend(self._extract_m3u8_formats(
                    video_url, video_id, 'mp4', entry_protocol='m3u8_native',
                    m3u8_id='hls', fatal=False))
                continue
            tbr = None
            mobj = re.search(r'(?P<height>\d+)[pP]?_(?P<tbr>\d+)[kK]', video_url)
            if mobj:
                if not height:
                    height = int(mobj.group('height'))
                tbr = int(mobj.group('tbr'))
            formats.append({
                'url': video_url,
                'format_id': '%dp' % height if height else None,
                'height': height,
                'tbr': tbr,
            })
        self._sort_formats(formats)

        video_uploader = self._html_search_regex(
            r'(?s)From: .+?<(?:a\b[^>]+\bhref=["\']/(?:(?:user|channel)s|model|pornstar)/|span\b[^>]+\bclass=["\']username)[^>]+>(.+?)<',
            webpage, 'uploader', default=None)

        view_count = self._extract_count(
            r'([\d,\.]+) [Vv]iews', webpage, 'view')
        like_count = self._extract_count(
            r'([\d,\.]+)', webpage, 'like')
        dislike_count = self._extract_count(
            r'([\d,\.]+)', webpage, 'dislike')
        comment_count = self._extract_count(
            r'All Comments\s*\(([\d,.]+)\)', webpage, 'comment')

        def extract_list(meta_key):
            div = self._search_regex(
                r'(?s)
]+\bclass=["\'].*?\b%sWrapper[^>]*>(.+?)
'
                % meta_key, webpage, meta_key, default=None)
            if div:
                return re.findall(r']+\bhref=[^>]+>([^<]+)', div)

        info = self._search_json_ld(webpage, video_id, default={})
        # description provided in JSON-LD is irrelevant
        info['description'] = None

        return merge_dicts({
            'id': video_id,
            'uploader': video_uploader,
            'upload_date': upload_date,
            'title': title,
            'thumbnail': thumbnail,
            'duration': duration,
            'view_count': view_count,
            'like_count': like_count,
            'dislike_count': dislike_count,
            'comment_count': comment_count,
            'formats': formats,
            'age_limit': 18,
            'tags': extract_list('tags'),
            'categories': extract_list('categories'),
            'subtitles': subtitles,
        }, info)

class PornHubPlaylistBaseIE(PornHubBaseIE):
    def _extract_entries(self, webpage, host):
        # Only process container div with main playlist content skipping
        # drop-down menu that uses similar pattern for videos (see
        # https://github.com/ytdl-org/youtube-dl/issues/11594).
        container = self._search_regex(
            r'(?s)(
]+class=["\']container.+)', webpage,
            'container', default=webpage)

        return [
            self.url_result(
                'http://www.%s/%s' % (host, video_url),
                PornHubIE.ie_key(), video_title=title)
            for video_url, title in orderedSet(re.findall(
                r'href="/?(view_video\.php\?.*\bviewkey=[\da-z]+[^"]*)"[^>]*\s+title="([^"]+)"',
                container))
        ]

    def _real_extract(self, url):
        mobj = re.match(self._VALID_URL, url)
        host = mobj.group('host')
        playlist_id = mobj.group('id')

        webpage = self._download_webpage(url, playlist_id)

        entries = self._extract_entries(webpage, host)

        playlist = self._parse_json(
            self._search_regex(
                r'(?:playlistObject|PLAYLIST_VIEW)\s*=\s*({.+?});', webpage,
                'playlist', default='{}'),
            playlist_id, fatal=False)
        title = playlist.get('title') or self._search_regex(
            r'>Videos\s+in\s+(.+?)\s+[Pp]laylist<', webpage, 'title', fatal=False)

        return self.playlist_result(
            entries, playlist_id, title, playlist.get('description'))

class PornHubUserIE(PornHubPlaylistBaseIE):
    _VALID_URL = r'(?P<url>https?://(?:[^/]+\.)?(?P<host>pornhub(?:premium)?\.(?:com|net))/(?:(?:user|channel)s|model|pornstar)/(?P<id>[^/?#&]+))(?:[?#&]|/(?!videos)|$)'
    _TESTS = [{
        'url': 'https://www.pornhub.com/model/zoe_ph',
        'playlist_mincount': 118,
    }, {
        'url': 'https://www.pornhub.com/pornstar/liz-vicious',
        'info_dict': {
            'id': 'liz-vicious',
        },
        'playlist_mincount': 118,
    }, {
        'url': 'https://www.pornhub.com/users/russianveet69',
        'only_matching': True,
    }, {
        'url': 'https://www.pornhub.com/channels/povd',
        'only_matching': True,
    }, {
        'url': 'https://www.pornhub.com/model/zoe_ph?abc=1',
        'only_matching': True,
    }]

    def _real_extract(self, url):
        mobj = re.match(self._VALID_URL, url)
        user_id = mobj.group('id')
        return self.url_result(
            '%s/videos' % mobj.group('url'), ie=PornHubPagedVideoListIE.ie_key(),
            video_id=user_id)

class PornHubPagedPlaylistBaseIE(PornHubPlaylistBaseIE):
    @staticmethod
    def _has_more(webpage):
        return re.search(
            r'''(?x)

]+\bclass=["\']page_next|
                <link[^>]+\brel=["\']next|
                <button[^>]+\bid=["\']moreDataBtn
            ''', webpage) is not None

    def _real_extract(self, url):
        mobj = re.match(self._VALID_URL, url)
        host = mobj.group('host')
        item_id = mobj.group('id')

        page = int_or_none(self._search_regex(
            r'\bpage=(\d+)', url, 'page', default=None))

        entries = []
        for page_num in (page, ) if page is not None else itertools.count(1):
            try:
                webpage = self._download_webpage(
                    url, item_id, 'Downloading page %d' % page_num,
                    query={'page': page_num})
            except ExtractorError as e:
                if isinstance(e.cause, compat_HTTPError) and e.cause.code == 404:
                    break
                raise
            page_entries = self._extract_entries(webpage, host)
            if not page_entries:
                break
            entries.extend(page_entries)
            if not self._has_more(webpage):
                break

        return self.playlist_result(orderedSet(entries), item_id)

class PornHubPagedVideoListIE(PornHubPagedPlaylistBaseIE):
    _VALID_URL = r'https?://(?:[^/]+\.)?(?P<host>pornhub(?:premium)?\.(?:com|net))/(?P<id>(?:[^/]+/)*[^/?#&]+)'
    _TESTS = [{
        'url': 'https://www.pornhub.com/model/zoe_ph/videos',
        'only_matching': True,
    }, {
        'url': 'http://www.pornhub.com/users/rushandlia/videos',
        'only_matching': True,
    }, {
        'url': 'https://www.pornhub.com/pornstar/jenny-blighe/videos',
        'info_dict': {
            'id': 'pornstar/jenny-blighe/videos',
        },
        'playlist_mincount': 149,
    }, {
        'url': 'https://www.pornhub.com/pornstar/jenny-blighe/videos?page=3',
        'info_dict': {
            'id': 'pornstar/jenny-blighe/videos',
        },
        'playlist_mincount': 40,
    }, {
        # default sorting as Top Rated Videos
        'url': 'https://www.pornhub.com/channels/povd/videos',
        'info_dict': {
            'id': 'channels/povd/videos',
        },
        'playlist_mincount': 293,
    }, {
        # Top Rated Videos
        'url': 'https://www.pornhub.com/channels/povd/videos?o=ra',
        'only_matching': True,
    }, {
        # Most Recent Videos
        'url': 'https://www.pornhub.com/channels/povd/videos?o=da',
        'only_matching': True,
    }, {
        # Most Viewed Videos
        'url': 'https://www.pornhub.com/channels/povd/videos?o=vi',
        'only_matching': True,
    }, {
        'url': 'http://www.pornhub.com/users/zoe_ph/videos/public',
        'only_matching': True,
    }, {
        # Most Viewed Videos
        'url': 'https://www.pornhub.com/pornstar/liz-vicious/videos?o=mv',
        'only_matching': True,
    }, {
        # Top Rated Videos
        'url': 'https://www.pornhub.com/pornstar/liz-vicious/videos?o=tr',
        'only_matching': True,
    }, {
        # Longest Videos
        'url': 'https://www.pornhub.com/pornstar/liz-vicious/videos?o=lg',
        'only_matching': True,
    }, {
        # Newest Videos
        'url': 'https://www.pornhub.com/pornstar/liz-vicious/videos?o=cm',
        'only_matching': True,
    }, {
        'url': 'https://www.pornhub.com/pornstar/liz-vicious/videos/paid',
        'only_matching': True,
    }, {
        'url': 'https://www.pornhub.com/pornstar/liz-vicious/videos/fanonly',
        'only_matching': True,
    }, {
        'url': 'https://www.pornhub.com/video',
        'only_matching': True,
    }, {
        'url': 'https://www.pornhub.com/video?page=3',
        'only_matching': True,
    }, {
        'url': 'https://www.pornhub.com/video/search?search=123',
        'only_matching': True,
    }, {
        'url': 'https://www.pornhub.com/categories/teen',
        'only_matching': True,
    }, {
        'url': 'https://www.pornhub.com/categories/teen?page=3',
        'only_matching': True,
    }, {
        'url': 'https://www.pornhub.com/hd',
        'only_matching': True,
    }, {
        'url': 'https://www.pornhub.com/hd?page=3',
        'only_matching': True,
    }, {
        'url': 'https://www.pornhub.com/described-video',
        'only_matching': True,
    }, {
        'url': 'https://www.pornhub.com/described-video?page=2',
        'only_matching': True,
    }, {
        'url': 'https://www.pornhub.com/video/incategories/60fps-1/hd-porn',
        'only_matching': True,
    }, {
        'url': 'https://www.pornhub.com/playlist/44121572',
        'info_dict': {
            'id': 'playlist/44121572',
        },
        'playlist_mincount': 132,
    }, {
        'url': 'https://www.pornhub.com/playlist/4667351',
        'only_matching': True,
    }, {
        'url': 'https://de.pornhub.com/playlist/4667351',
        'only_matching': True,
    }]

    @classmethod
    def suitable(cls, url):
        return (False
                if PornHubIE.suitable(url) or PornHubUserIE.suitable(url) or PornHubUserVideosUploadIE.suitable(url)
                else super(PornHubPagedVideoListIE, cls).suitable(url))

class PornHubUserVideosUploadIE(PornHubPagedPlaylistBaseIE):
    _VALID_URL = r'(?P<url>https?://(?:[^/]+\.)?(?P<host>pornhub(?:premium)?\.(?:com|net))/(?:(?:user|channel)s|model|pornstar)/(?P<id>[^/]+)/videos/upload)'
    _TESTS = [{
        'url': 'https://www.pornhub.com/pornstar/jenny-blighe/videos/upload',
        'info_dict': {
            'id': 'jenny-blighe',
        },
        'playlist_mincount': 129,
    }, {
        'url': 'https://www.pornhub.com/model/zoe_ph/videos/upload',
        'only_matching': True,
    }]


code:
from __future__ import unicode_literals

import json

from .common import InfoExtractor
from ..utils import int_or_none

class PornotubeIE(InfoExtractor):
    _VALID_URL = r'https?://(?:\w+\.)?pornotube\.com/(?:[^?#]*?)/video/(?P<id>[0-9]+)'
    _TEST = {
        'url': 'http://www.pornotube.com/orientation/straight/video/4964/title/weird-hot-and-wet-science',
        'md5': '60fc5a4f0d93a97968fc7999d98260c9',
        'info_dict': {
            'id': '4964',
            'ext': 'mp4',
            'upload_date': '20141203',
            'title': 'Weird Hot and Wet Science',
            'description': 'md5:a8304bef7ef06cb4ab476ca6029b01b0',
            'categories': ['Adult Humor', 'Blondes'],
            'uploader': 'Alpha Blue Archives',
            'thumbnail': r're:^https?://.*\.jpg$',
            'timestamp': 1417582800,
            'age_limit': 18,
        }
    }

    def _real_extract(self, url):
        video_id = self._match_id(url)

        token = self._download_json(
            'https://api.aebn.net/auth/v2/origins/authenticate',
            video_id, note='Downloading token',
            data=json.dumps({'credentials': 'Clip Application'}).encode('utf-8'),
            headers={
                'Content-Type': 'application/json',
                'Origin': 'http://www.pornotube.com',
            })['tokenKey']

        video_url = self._download_json(
            'https://api.aebn.net/delivery/v1/clips/%s/MP4' % video_id,
            video_id, note='Downloading delivery information',
            headers={'Authorization': token})['mediaUrl']

        FIELDS = (
            'title', 'description', 'startSecond', 'endSecond', 'publishDate',
            'studios{name}', 'categories{name}', 'movieId', 'primaryImageNumber'
        )

        info = self._download_json(
            'https://api.aebn.net/content/v2/clips/%s?fields=%s'
            % (video_id, ','.join(FIELDS)), video_id,
            note='Downloading metadata',
            headers={'Authorization': token})

        if isinstance(info, list):
            info = info[0]

        title = info['title']

        timestamp = int_or_none(info.get('publishDate'), scale=1000)
        uploader = info.get('studios', [{}])[0].get('name')
        movie_id = info.get('movieId')
        primary_image_number = info.get('primaryImageNumber')
        thumbnail = None
        if movie_id and primary_image_number:
            thumbnail = 'http://pic.aebn.net/dis/t/%s/%s_%08d.jpg' % (
                movie_id, movie_id, primary_image_number)
        start = int_or_none(info.get('startSecond'))
        end = int_or_none(info.get('endSecond'))
        duration = end - start if start and end else None
        categories = [c['name'] for c in info.get('categories', []) if c.get('name')]

        return {
            'id': video_id,
            'url': video_url,
            'title': title,
            'description': info.get('description'),
            'duration': duration,
            'timestamp': timestamp,
            'uploader': uploader,
            'thumbnail': thumbnail,
            'categories': categories,
            'age_limit': 18,
        }

repo_name: rrosajp/youtube


code:
# coding: utf-8
from __future__ import unicode_literals

import re

from .common import InfoExtractor
from ..utils import (
    int_or_none,
    float_or_none,
    unified_strdate,
)

class PornoVoisinesIE(InfoExtractor):
    _VALID_URL = r'https?://(?:www\.)?pornovoisines\.com/videos/show/(?P<id>\d+)/(?P<display_id>[^/.]+)'

    _TEST = {
        'url': 'http://www.pornovoisines.com/videos/show/919/recherche-appartement.html',
        'md5': '6f8aca6a058592ab49fe701c8ba8317b',
        'info_dict': {
            'id': '919',
            'display_id': 'recherche-appartement',
            'ext': 'mp4',
            'title': 'Recherche appartement',
            'description': 'md5:fe10cb92ae2dd3ed94bb4080d11ff493',
            'thumbnail': r're:^https?://.*\.jpg$',
            'upload_date': '20140925',
            'duration': 120,
            'view_count': int,
            'average_rating': float,
            'categories': ['Débutante', 'Débutantes', 'Scénario', 'Sodomie'],
            'age_limit': 18,
            'subtitles': {
                'fr': [{
                    'ext': 'vtt',
                }]
            },
        }
    }

    def _real_extract(self, url):
        mobj = re.match(self._VALID_URL, url)
        video_id = mobj.group('id')
        display_id = mobj.group('display_id')

        settings_url = self._download_json(
            'http://www.pornovoisines.com/api/video/%s/getsettingsurl/' % video_id,
            video_id, note='Getting settings URL')['video_settings_url']
        settings = self._download_json(settings_url, video_id)['data']

        formats = []
        for kind, data in settings['variants'].items():
            if kind == 'HLS':
                formats.extend(self._extract_m3u8_formats(
                    data, video_id, ext='mp4', entry_protocol='m3u8_native', m3u8_id='hls'))
            elif kind == 'MP4':
                for item in data:
                    formats.append({
                        'url': item['url'],
                        'height': item.get('height'),
                        'bitrate': item.get('bitrate'),
                    })
        self._sort_formats(formats)

        webpage = self._download_webpage(url, video_id)

        title = self._og_search_title(webpage)
        description = self._og_search_description(webpage)

        # The webpage has a bug - there's no space between "thumb" and src=
        thumbnail = self._html_search_regex(
            r'<img[^>]+class=([\'"])thumb\1[^>]*src=([\'"])(?P<url>[^"]+)\2',
            webpage, 'thumbnail', fatal=False, group='url')

        upload_date = unified_strdate(self._search_regex(
            r'Le\s*<b>([\d/]+)', webpage, 'upload date', fatal=False))
        duration = settings.get('main', {}).get('duration')
        view_count = int_or_none(self._search_regex(
            r'(\d+) vues', webpage, 'view count', fatal=False))
        average_rating = self._search_regex(
            r'Note\s*:\s*(\d+(?:,\d+)?)', webpage, 'average rating', fatal=False)
        if average_rating:
            average_rating = float_or_none(average_rating.replace(',', '.'))

        categories = self._html_search_regex(
            r'(?s)Catégories\s*:\s*<b>(.+?)</b>', webpage, 'categories', fatal=False)
        if categories:
            categories = [category.strip() for category in categories.split(',')]

        subtitles = {'fr': [{
            'url': subtitle,
        } for subtitle in settings.get('main', {}).get('vtt_tracks', {}).values()]}

        return {
            'id': video_id,
            'display_id': display_id,
            'formats': formats,
            'title': title,
            'description': description,
            'thumbnail': thumbnail,
            'upload_date': upload_date,
            'duration': duration,
            'view_count': view_count,
            'average_rating': average_rating,
            'categories': categories,
            'age_limit': 18,
            'subtitles': subtitles,
        }


code:
from __future__ import unicode_literals

import re

from .common import InfoExtractor
from ..utils import (
    str_to_int,
)

class PornoXOIE(InfoExtractor):
    _VALID_URL = r'https?://(?:www\.)?pornoxo\.com/videos/(?P<id>\d+)/(?P<display_id>[^/]+)\.html'
    _TEST = {
        'url': 'http://www.pornoxo.com/videos/7564/striptease-from-sexy-secretary.html',
        'md5': '582f28ecbaa9e6e24cb90f50f524ce87',
        'info_dict': {
            'id': '7564',
            'ext': 'flv',
            'title': 'Striptease From Sexy Secretary!',
            'display_id': 'striptease-from-sexy-secretary',
            'description': 'md5:0ee35252b685b3883f4a1d38332f9980',
            'categories': list,  # NSFW
            'thumbnail': r're:https?://.*\.jpg$',
            'age_limit': 18,
        }
    }

    def _real_extract(self, url):
        mobj = re.match(self._VALID_URL, url)
        video_id, display_id = mobj.groups()

        webpage = self._download_webpage(url, video_id)
        video_data = self._extract_jwplayer_data(webpage, video_id, require_title=False)

        title = self._html_search_regex(
            r'<title>([^<]+)\s*-\s*PornoXO', webpage, 'title')

        view_count = str_to_int(self._html_search_regex(
            r'[vV]iews:\s*([0-9,]+)', webpage, 'view count', fatal=False))

        categories_str = self._html_search_regex(
            r'<meta name="description" content=".*featuring\s*([^"]+)"',
            webpage, 'categories', fatal=False)
        categories = (
            None if categories_str is None
            else categories_str.split(','))

        video_data.update({
            'id': video_id,
            'title': title,
            'display_id': display_id,
            'description': self._html_search_meta('description', webpage),
            'categories': categories,
            'view_count': view_count,
            'age_limit': 18,
        })

        return video_data


code:
# coding: utf-8
from __future__ import unicode_literals

import re

from .common import InfoExtractor
from ..utils import remove_start

class PressTVIE(InfoExtractor):
    _VALID_URL = r'https?://(?:www\.)?presstv\.ir/[^/]+/(?P<y>\d+)/(?P<m>\d+)/(?P<d>\d+)/(?P<id>\d+)/(?P<display_id>[^/]+)?'

    _TEST = {
        'url': 'http://www.presstv.ir/Detail/2016/04/09/459911/Australian-sewerage-treatment-facility-/',
        'md5': '5d7e3195a447cb13e9267e931d8dd5a5',
        'info_dict': {
            'id': '459911',
            'display_id': 'Australian-sewerage-treatment-facility-',
            'ext': 'mp4',
            'title': 'Organic mattresses used to clean waste water',
            'upload_date': '20160409',
            'thumbnail': r're:^https?://.*\.jpg',
            'description': 'md5:20002e654bbafb6908395a5c0cfcd125'
        }
    }

    def _real_extract(self, url):
        mobj = re.match(self._VALID_URL, url)
        video_id = mobj.group('id')
        display_id = mobj.group('display_id') or video_id

        webpage = self._download_webpage(url, display_id)

        # extract video URL from webpage
        video_url = self._hidden_inputs(webpage)['inpPlayback']

        # build list of available formats
        # specified in http://www.presstv.ir/Scripts/playback.js
        base_url = 'http://192.99.219.222:82/presstv'
        _formats = [
            (180, '_low200.mp4'),
            (360, '_low400.mp4'),
            (720, '_low800.mp4'),
            (1080, '.mp4')
        ]

        formats = [{
            'url': base_url + video_url[:-4] + extension,
            'format_id': '%dp' % height,
            'height': height,
        } for height, extension in _formats]

        # extract video metadata
        title = remove_start(
            self._html_search_meta('title', webpage, fatal=True), 'PressTV-')

        thumbnail = self._og_search_thumbnail(webpage)
        description = self._og_search_description(webpage)

        upload_date = '%04d%02d%02d' % (
            int(mobj.group('y')),
            int(mobj.group('m')),
            int(mobj.group('d')),
        )

        return {
            'id': video_id,
            'display_id': display_id,
            'title': title,
            'formats': formats,
            'thumbnail': thumbnail,
            'upload_date': upload_date,
            'description': description
        }


code:
# coding: utf-8
from __future__ import unicode_literals

import re

from hashlib import sha1
from .common import InfoExtractor
from ..compat import compat_str
from ..utils import (
    ExtractorError,
    determine_ext,
    float_or_none,
    int_or_none,
    merge_dicts,
    unified_strdate,
)

class ProSiebenSat1BaseIE(InfoExtractor):
    _GEO_BYPASS = False
    _ACCESS_ID = None
    _SUPPORTED_PROTOCOLS = 'dash:clear,hls:clear,progressive:clear'
    _V4_BASE_URL = 'https://vas-v4.p7s1video.net/4.0/get'

    def _extract_video_info(self, url, clip_id):
        client_location = url

        video = self._download_json(
            'http://vas.sim-technik.de/vas/live/v2/videos',
            clip_id, 'Downloading videos JSON', query={
                'access_token': self._TOKEN,
                'client_location': client_location,
                'client_name': self._CLIENT_NAME,
                'ids': clip_id,
            })[0]

        if video.get('is_protected') is True:
            raise ExtractorError('This video is DRM protected.', expected=True)

        formats = []
        if self._ACCESS_ID:
            raw_ct = self._ENCRYPTION_KEY + clip_id + self._IV + self._ACCESS_ID
            protocols = self._download_json(
                self._V4_BASE_URL + 'protocols', clip_id,
                'Downloading protocols JSON',
                headers=self.geo_verification_headers(), query={
                    'access_id': self._ACCESS_ID,
                    'client_token': sha1((raw_ct).encode()).hexdigest(),
                    'video_id': clip_id,
                }, fatal=False, expected_status=(403,)) or {}
            error = protocols.get('error') or {}
            if error.get('title') == 'Geo check failed':
                self.raise_geo_restricted(countries=['AT', 'CH', 'DE'])
            server_token = protocols.get('server_token')
            if server_token:
                urls = (self._download_json(
                    self._V4_BASE_URL + 'urls', clip_id, 'Downloading urls JSON', query={
                        'access_id': self._ACCESS_ID,
                        'client_token': sha1((raw_ct + server_token + self._SUPPORTED_PROTOCOLS).encode()).hexdigest(),
                        'protocols': self._SUPPORTED_PROTOCOLS,
                        'server_token': server_token,
                        'video_id': clip_id,
                    }, fatal=False) or {}).get('urls') or {}
                for protocol, variant in urls.items():
                    source_url = variant.get('clear', {}).get('url')
                    if not source_url:
                        continue
                    if protocol == 'dash':
                        formats.extend(self._extract_mpd_formats(
                            source_url, clip_id, mpd_id=protocol, fatal=False))
                    elif protocol == 'hls':
                        formats.extend(self._extract_m3u8_formats(
                            source_url, clip_id, 'mp4', 'm3u8_native',
                            m3u8_id=protocol, fatal=False))
                    else:
                        formats.append({
                            'url': source_url,
                            'format_id': protocol,
                        })
        if not formats:
            source_ids = [compat_str(source['id']) for source in video['sources']]

            client_id = self._SALT[:2] + sha1(''.join([clip_id, self._SALT, self._TOKEN, client_location, self._SALT, self._CLIENT_NAME]).encode('utf-8')).hexdigest()

            sources = self._download_json(
                'http://vas.sim-technik.de/vas/live/v2/videos/%s/sources' % clip_id,
                clip_id, 'Downloading sources JSON', query={
                    'access_token': self._TOKEN,
                    'client_id': client_id,
                    'client_location': client_location,
                    'client_name': self._CLIENT_NAME,
                })
            server_id = sources['server_id']

            def fix_bitrate(bitrate):
                bitrate = int_or_none(bitrate)
                if not bitrate:
                    return None
                return (bitrate // 1000) if bitrate % 1000 == 0 else bitrate

            for source_id in source_ids:
                client_id = self._SALT[:2] + sha1(''.join([self._SALT, clip_id, self._TOKEN, server_id, client_location, source_id, self._SALT, self._CLIENT_NAME]).encode('utf-8')).hexdigest()
                urls = self._download_json(
                    'http://vas.sim-technik.de/vas/live/v2/videos/%s/sources/url' % clip_id,
                    clip_id, 'Downloading urls JSON', fatal=False, query={
                        'access_token': self._TOKEN,
                        'client_id': client_id,
                        'client_location': client_location,
                        'client_name': self._CLIENT_NAME,
                        'server_id': server_id,
                        'source_ids': source_id,
                    })
                if not urls:
                    continue
                if urls.get('status_code') != 0:
                    raise ExtractorError('This video is unavailable', expected=True)
                urls_sources = urls['sources']
                if isinstance(urls_sources, dict):
                    urls_sources = urls_sources.values()
                for source in urls_sources:
                    source_url = source.get('url')
                    if not source_url:
                        continue
                    protocol = source.get('protocol')
                    mimetype = source.get('mimetype')
                    if mimetype == 'application/f4m+xml' or 'f4mgenerator' in source_url or determine_ext(source_url) == 'f4m':
                        formats.extend(self._extract_f4m_formats(
                            source_url, clip_id, f4m_id='hds', fatal=False))
                    elif mimetype == 'application/x-mpegURL':
                        formats.extend(self._extract_m3u8_formats(
                            source_url, clip_id, 'mp4', 'm3u8_native',
                            m3u8_id='hls', fatal=False))
                    elif mimetype == 'application/dash+xml':
                        formats.extend(self._extract_mpd_formats(
                            source_url, clip_id, mpd_id='dash', fatal=False))
                    else:
                        tbr = fix_bitrate(source['bitrate'])
                        if protocol in ('rtmp', 'rtmpe'):
                            mobj = re.search(r'^(?P<url>rtmpe?://[^/]+)/(?P<path>.+)$', source_url)
                            if not mobj:
                                continue
                            path = mobj.group('path')
                            mp4colon_index = path.rfind('mp4:')
                            app = path[:mp4colon_index]
                            play_path = path[mp4colon_index:]
                            formats.append({
                                'url': '%s/%s' % (mobj.group('url'), app),
                                'app': app,
                                'play_path': play_path,
                                'player_url': 'http://livepassdl.conviva.com/hf/ver/2.79.0.17083/LivePassModuleMain.swf',
                                'page_url': 'http://www.prosieben.de',
                                'tbr': tbr,
                                'ext': 'flv',
                                'format_id': 'rtmp%s' % ('-%d' % tbr if tbr else ''),
                            })
                        else:
                            formats.append({
                                'url': source_url,
                                'tbr': tbr,
                                'format_id': 'http%s' % ('-%d' % tbr if tbr else ''),
                            })
        self._sort_formats(formats)

        return {
            'duration': float_or_none(video.get('duration')),
            'formats': formats,
        }

class ProSiebenSat1IE(ProSiebenSat1BaseIE):
    IE_NAME = 'prosiebensat1'
    IE_DESC = 'ProSiebenSat.1 Digital'
    _VALID_URL = r'''(?x)
                    https?://
                        (?:www\.)?
                        (?:
                            (?:beta\.)?
                            (?:
                                prosieben(?:maxx)?|sixx|sat1(?:gold)?|kabeleins(?:doku)?|the-voice-of-germany|advopedia
                            )\.(?:de|at|ch)|
                            ran\.de|fem\.com|advopedia\.de|galileo\.tv/video
                        )
                        /(?P<id>.+)
                    '''

    _TESTS = [
        {
            # Tests changes introduced in https://github.com/ytdl-org/youtube-dl/pull/6242
            # in response to fixing https://github.com/ytdl-org/youtube-dl/issues/6215:
            # - malformed f4m manifest support
            # - proper handling of URLs starting with `https?://` in 2.0 manifests
            # - recursive child f4m manifests extraction
            'url': 'http://www.prosieben.de/tv/circus-halligalli/videos/218-staffel-2-episode-18-jahresrueckblick-ganze-folge',
            'info_dict': {
                'id': '2104602',
                'ext': 'mp4',
                'title': 'CIRCUS HALLIGALLI - Episode 18 - Staffel 2',
                'description': 'md5:8733c81b702ea472e069bc48bb658fc1',
                'upload_date': '20131231',
                'duration': 5845.04,
                'series': 'CIRCUS HALLIGALLI',
                'season_number': 2,
                'episode': 'Episode 18 - Staffel 2',
                'episode_number': 18,
            },
        },
        {
            'url': 'http://www.prosieben.de/videokatalog/Gesellschaft/Leben/Trends/video-Lady-Umstyling-f%C3%BCr-Audrina-Rebekka-Audrina-Fergen-billig-aussehen-Battal-Modica-700544.html',
            'info_dict': {
                'id': '2570327',
                'ext': 'mp4',
                'title': 'Lady-Umstyling für Audrina',
                'description': 'md5:4c16d0c17a3461a0d43ea4084e96319d',
                'upload_date': '20131014',
                'duration': 606.76,
            },
            'params': {
                # rtmp download
                'skip_download': True,
            },
            'skip': 'Seems to be broken',
        },
        {
            'url': 'http://www.prosiebenmaxx.de/tv/experience/video/144-countdown-fuer-die-autowerkstatt-ganze-folge',
            'info_dict': {
                'id': '2429369',
                'ext': 'mp4',
                'title': 'Countdown für die Autowerkstatt',
                'description': 'md5:809fc051a457b5d8666013bc40698817',
                'upload_date': '20140223',
                'duration': 2595.04,
            },
            'params': {
                # rtmp download
                'skip_download': True,
            },
            'skip': 'This video is unavailable',
        },
        {
            'url': 'http://www.sixx.de/stars-style/video/sexy-laufen-in-ugg-boots-clip',
            'info_dict': {
                'id': '2904997',
                'ext': 'mp4',
                'title': 'Sexy laufen in Ugg Boots',
                'description': 'md5:edf42b8bd5bc4e5da4db4222c5acb7d6',
                'upload_date': '20140122',
                'duration': 245.32,
            },
            'params': {
                # rtmp download
                'skip_download': True,
            },
            'skip': 'This video is unavailable',
        },
        {
            'url': 'http://www.sat1.de/film/der-ruecktritt/video/im-interview-kai-wiesinger-clip',
            'info_dict': {
                'id': '2906572',
                'ext': 'mp4',
                'title': 'Im Interview: Kai Wiesinger',
                'description': 'md5:e4e5370652ec63b95023e914190b4eb9',
                'upload_date': '20140203',
                'duration': 522.56,
            },
            'params': {
                # rtmp download
                'skip_download': True,
            },
            'skip': 'This video is unavailable',
        },
        {
            'url': 'http://www.kabeleins.de/tv/rosins-restaurants/videos/jagd-auf-fertigkost-im-elsthal-teil-2-ganze-folge',
            'info_dict': {
                'id': '2992323',
                'ext': 'mp4',
                'title': 'Jagd auf Fertigkost im Elsthal - Teil 2',
                'description': 'md5:2669cde3febe9bce13904f701e774eb6',
                'upload_date': '20141014',
                'duration': 2410.44,
            },
            'params': {
                # rtmp download
                'skip_download': True,
            },
            'skip': 'This video is unavailable',
        },
        {
            'url': 'http://www.ran.de/fussball/bundesliga/video/schalke-toennies-moechte-raul-zurueck-ganze-folge',
            'info_dict': {
                'id': '3004256',
                'ext': 'mp4',
                'title': 'Schalke: Tönnies möchte Raul zurück',
                'description': 'md5:4b5b271d9bcde223b54390754c8ece3f',
                'upload_date': '20140226',
                'duration': 228.96,
            },
            'params': {
                # rtmp download
                'skip_download': True,
            },
            'skip': 'This video is unavailable',
        },
        {
            'url': 'http://www.the-voice-of-germany.de/video/31-andreas-kuemmert-rocket-man-clip',
            'info_dict': {
                'id': '2572814',
                'ext': 'mp4',
                'title': 'The Voice of Germany - Andreas Kümmert: Rocket Man',
                'description': 'md5:6ddb02b0781c6adf778afea606652e38',
                'timestamp': 1382041620,
                'upload_date': '20131017',
                'duration': 469.88,
            },
            'params': {
                'skip_download': True,
            },
        },
        {
            'url': 'http://www.fem.com/videos/beauty-lifestyle/kurztrips-zum-valentinstag',
            'info_dict': {
                'id': '2156342',
                'ext': 'mp4',
                'title': 'Kurztrips zum Valentinstag',
                'description': 'Romantischer Kurztrip zum Valentinstag? Nina Heinemann verrät, was sich hier wirklich lohnt.',
                'duration': 307.24,
            },
            'params': {
                'skip_download': True,
            },
        },
        {
            'url': 'http://www.prosieben.de/tv/joko-gegen-klaas/videos/playlists/episode-8-ganze-folge-playlist',
            'info_dict': {
                'id': '439664',
                'title': 'Episode 8 - Ganze Folge - Playlist',
                'description': 'md5:63b8963e71f481782aeea877658dec84',
            },
            'playlist_count': 2,
            'skip': 'This video is unavailable',
        },
        {
            # title in

            'url': 'http://www.prosieben.de/stars/oscar-award/videos/jetzt-erst-enthuellt-das-geheimnis-von-emma-stones-oscar-robe-clip',
            'info_dict': {
                'id': '4895826',
                'ext': 'mp4',
                'title': 'Jetzt erst enthüllt: Das Geheimnis von Emma Stones Oscar-Robe',
                'description': 'md5:e5ace2bc43fadf7b63adc6187e9450b9',
                'upload_date': '20170302',
            },
            'params': {
                'skip_download': True,
            },
            'skip': 'geo restricted to Germany',
        },
        {
            # geo restricted to Germany
            'url': 'http://www.kabeleinsdoku.de/tv/mayday-alarm-im-cockpit/video/102-notlandung-im-hudson-river-ganze-folge',
            'only_matching': True,
        },
        {
            # geo restricted to Germany
            'url': 'http://www.sat1gold.de/tv/edel-starck/video/11-staffel-1-episode-1-partner-wider-willen-ganze-folge',
            'only_matching': True,
        },
        {
            # geo restricted to Germany
            'url': 'https://www.galileo.tv/video/diese-emojis-werden-oft-missverstanden',
            'only_matching': True,
        },
        {
            'url': 'http://www.sat1gold.de/tv/edel-starck/playlist/die-gesamte-1-staffel',
            'only_matching': True,
        },
        {
            'url': 'http://www.advopedia.de/videos/lenssen-klaert-auf/lenssen-klaert-auf-folge-8-staffel-3-feiertage-und-freie-tage',
            'only_matching': True,
        },
    ]

    _TOKEN = 'prosieben'
    _SALT = '01!8d8F_)r9]4s[qeuXfP%'
    _CLIENT_NAME = 'kolibri-2.0.19-splec4'

    _ACCESS_ID = 'x_prosiebenmaxx-de'
    _ENCRYPTION_KEY = 'Eeyeey9oquahthainoofashoyoikosag'
    _IV = 'Aeluchoc6aevechuipiexeeboowedaok'

    _CLIPID_REGEXES = [
        r'"clip_id"\s*:\s+"(\d+)"',
        r'clipid: "(\d+)"',
        r'clip[iI]d=(\d+)',
        r'clip[iI][dD]\s*=\s*["\'](\d+)',
        r"'itemImageUrl'\s*:\s*'/dynamic/thumbnails/full/\d+/(\d+)",
        r'proMamsId"\s*:\s*"(\d+)',
        r'proMamsId"\s*:\s*"(\d+)',
    ]
    _TITLE_REGEXES = [
        r'
\s*(.+?)
',
        r'\s*
(.+?)
',
        r'\s*
(.+?)
',
        r'
\s*(.+?)
',
        r'\s*
([^<]+)
\s*',
        r'
\s*(.+?)
',
        r'
]+id="veeseoTitle"[^>]*>(.+?)
',
        r'
]+class="subtitle"[^>]*>([^<]+)
',
    ]
    _DESCRIPTION_REGEXES = [
        r'
\s*(.+?)
',
        r'
\s*
Beschreibung: (.+?)
',
        r'

\s*
\s*\s*(.+?)\s*',
        r'
\s*(.+?)\s*
',
        r'
\s*(.+?)
',
        r'
]+id="veeseoDescription"[^>]*>(.+?)
',
    ]
    _UPLOAD_DATE_REGEXES = [
        r'\s*(\d{2}\.\d{2}\.\d{4} \d{2}:\d{2}) \|\s*<span itemprop="duration"',
        r'\s*(\d{2}\.\d{2}\.\d{4}) \d{2}:\d{2} Uhr',
        r'(\d{2}\.\d{2}\.\d{4})',
        r'(\d{2}\.\d{2}\.\d{4}) \| \d{2}:\d{2} Min
',
    ]
    _PAGE_TYPE_REGEXES = [
        r'<meta name="page_type" content="([^"]+)">',
        r"'itemType'\s*:\s*'([^']*)'",
    ]
    _PLAYLIST_ID_REGEXES = [
        r'content[iI]d=(\d+)',
        r"'itemId'\s*:\s*'([^']*)'",
    ]
    _PLAYLIST_CLIP_REGEXES = [
        r'(?s)data-qvt=.+?<a href="([^"]+)"',
    ]

    def _extract_clip(self, url, webpage):
        clip_id = self._html_search_regex(
            self._CLIPID_REGEXES, webpage, 'clip id')
        title = self._html_search_regex(
            self._TITLE_REGEXES, webpage, 'title',
            default=None) or self._og_search_title(webpage)
        info = self._extract_video_info(url, clip_id)
        description = self._html_search_regex(
            self._DESCRIPTION_REGEXES, webpage, 'description', default=None)
        if description is None:
            description = self._og_search_description(webpage)
        thumbnail = self._og_search_thumbnail(webpage)
        upload_date = unified_strdate(
            self._html_search_meta('og:published_time', webpage,
                                   'upload date', default=None)
            or self._html_search_regex(self._UPLOAD_DATE_REGEXES,
                                       webpage, 'upload date', default=None))

        json_ld = self._search_json_ld(webpage, clip_id, default={})

        return merge_dicts(info, {
            'id': clip_id,
            'title': title,
            'description': description,
            'thumbnail': thumbnail,
            'upload_date': upload_date,
        }, json_ld)

    def _extract_playlist(self, url, webpage):
        playlist_id = self._html_search_regex(
            self._PLAYLIST_ID_REGEXES, webpage, 'playlist id')
        playlist = self._parse_json(
            self._search_regex(
                r'var\s+contentResources\s*=\s*(\[.+?\]);\s*</script',
                webpage, 'playlist'),
            playlist_id)
        entries = []
        for item in playlist:
            clip_id = item.get('id') or item.get('upc')
            if not clip_id:
                continue
            info = self._extract_video_info(url, clip_id)
            info.update({
                'id': clip_id,
                'title': item.get('title') or item.get('teaser', {}).get('headline'),
                'description': item.get('teaser', {}).get('description'),
                'thumbnail': item.get('poster'),
                'duration': float_or_none(item.get('duration')),
                'series': item.get('tvShowTitle'),
                'uploader': item.get('broadcastPublisher'),
            })
            entries.append(info)
        return self.playlist_result(entries, playlist_id)

    def _real_extract(self, url):
        video_id = self._match_id(url)
        webpage = self._download_webpage(url, video_id)
        page_type = self._search_regex(
            self._PAGE_TYPE_REGEXES, webpage,
            'page type', default='clip').lower()
        if page_type == 'clip':
            return self._extract_clip(url, webpage)
        elif page_type == 'playlist':
            return self._extract_playlist(url, webpage)
        else:
            raise ExtractorError(
                'Unsupported page type %s' % page_type, expected=True)


code:
# coding: utf-8
from __future__ import unicode_literals

from .common import InfoExtractor
from ..compat import (
    compat_HTTPError,
    compat_str,
)
from ..utils import (
    ExtractorError,
    int_or_none,
    float_or_none,
    parse_resolution,
    str_or_none,
    try_get,
    unified_timestamp,
    url_or_none,
    urljoin,
)

class PuhuTVIE(InfoExtractor):
    _VALID_URL = r'https?://(?:www\.)?puhutv\.com/(?P<id>[^/?#&]+)-izle'
    IE_NAME = 'puhutv'
    _TESTS = [{
        # film
        'url': 'https://puhutv.com/sut-kardesler-izle',
        'md5': 'a347470371d56e1585d1b2c8dab01c96',
        'info_dict': {
            'id': '5085',
            'display_id': 'sut-kardesler',
            'ext': 'mp4',
            'title': 'Süt Kardeşler',
            'description': 'md5:ca09da25b7e57cbb5a9280d6e48d17aa',
            'thumbnail': r're:^https?://.*\.jpg$',
            'duration': 4832.44,
            'creator': 'Arzu Film',
            'timestamp': 1561062602,
            'upload_date': '20190620',
            'release_year': 1976,
            'view_count': int,
            'tags': list,
        },
    }, {
        # episode, geo restricted, bypassable with --geo-verification-proxy
        'url': 'https://puhutv.com/jet-sosyete-1-bolum-izle',
        'only_matching': True,
    }, {
        # 4k, with subtitles
        'url': 'https://puhutv.com/dip-1-bolum-izle',
        'only_matching': True,
    }]
    _SUBTITLE_LANGS = {
        'English': 'en',
        'Deutsch': 'de',
        'عربى': 'ar'
    }

    def _real_extract(self, url):
        display_id = self._match_id(url)

        info = self._download_json(
            urljoin(url, '/api/slug/%s-izle' % display_id),
            display_id)['data']

        video_id = compat_str(info['id'])
        show = info.get('title') or {}
        title = info.get('name') or show['name']
        if info.get('display_name'):
            title = '%s %s' % (title, info['display_name'])

        try:
            videos = self._download_json(
                'https://puhutv.com/api/assets/%s/videos' % video_id,
                display_id, 'Downloading video JSON',
                headers=self.geo_verification_headers())
        except ExtractorError as e:
            if isinstance(e.cause, compat_HTTPError) and e.cause.code == 403:
                self.raise_geo_restricted()
            raise

        urls = []
        formats = []

        for video in videos['data']['videos']:
            media_url = url_or_none(video.get('url'))
            if not media_url or media_url in urls:
                continue
            urls.append(media_url)

            playlist = video.get('is_playlist')
            if (video.get('stream_type') == 'hls' and playlist is True) or 'playlist.m3u8' in media_url:
                formats.extend(self._extract_m3u8_formats(
                    media_url, video_id, 'mp4', entry_protocol='m3u8_native',
                    m3u8_id='hls', fatal=False))
                continue

            quality = int_or_none(video.get('quality'))
            f = {
                'url': media_url,
                'ext': 'mp4',
                'height': quality
            }
            video_format = video.get('video_format')
            is_hls = (video_format == 'hls' or '/hls/' in media_url or '/chunklist.m3u8' in media_url) and playlist is False
            if is_hls:
                format_id = 'hls'
                f['protocol'] = 'm3u8_native'
            elif video_format == 'mp4':
                format_id = 'http'
            else:
                continue
            if quality:
                format_id += '-%sp' % quality
            f['format_id'] = format_id
            formats.append(f)
        self._sort_formats(formats)

        creator = try_get(
            show, lambda x: x['producer']['name'], compat_str)

        content = info.get('content') or {}

        images = try_get(
            content, lambda x: x['images']['wide'], dict) or {}
        thumbnails = []
        for image_id, image_url in images.items():
            if not isinstance(image_url, compat_str):
                continue
            if not image_url.startswith(('http', '//')):
                image_url = 'https://%s' % image_url
            t = parse_resolution(image_id)
            t.update({
                'id': image_id,
                'url': image_url
            })
            thumbnails.append(t)

        tags = []
        for genre in show.get('genres') or []:
            if not isinstance(genre, dict):
                continue
            genre_name = genre.get('name')
            if genre_name and isinstance(genre_name, compat_str):
                tags.append(genre_name)

        subtitles = {}
        for subtitle in content.get('subtitles') or []:
            if not isinstance(subtitle, dict):
                continue
            lang = subtitle.get('language')
            sub_url = url_or_none(subtitle.get('url') or subtitle.get('file'))
            if not lang or not isinstance(lang, compat_str) or not sub_url:
                continue
            subtitles[self._SUBTITLE_LANGS.get(lang, lang)] = [{
                'url': sub_url
            }]

        return {
            'id': video_id,
            'display_id': display_id,
            'title': title,
            'description': info.get('description') or show.get('description'),
            'season_id': str_or_none(info.get('season_id')),
            'season_number': int_or_none(info.get('season_number')),
            'episode_number': int_or_none(info.get('episode_number')),
            'release_year': int_or_none(show.get('released_at')),
            'timestamp': unified_timestamp(info.get('created_at')),
            'creator': creator,
            'view_count': int_or_none(content.get('watch_count')),
            'duration': float_or_none(content.get('duration_in_ms'), 1000),
            'tags': tags,
            'subtitles': subtitles,
            'thumbnails': thumbnails,
            'formats': formats
        }

class PuhuTVSerieIE(InfoExtractor):
    _VALID_URL = r'https?://(?:www\.)?puhutv\.com/(?P<id>[^/?#&]+)-detay'
    IE_NAME = 'puhutv:serie'
    _TESTS = [{
        'url': 'https://puhutv.com/deniz-yildizi-detay',
        'info_dict': {
            'title': 'Deniz Yıldızı',
            'id': 'deniz-yildizi',
        },
        'playlist_mincount': 205,
    }, {
        # a film detail page which is using same url with serie page
        'url': 'https://puhutv.com/kaybedenler-kulubu-detay',
        'only_matching': True,
    }]

    def _extract_entries(self, seasons):
        for season in seasons:
            season_id = season.get('id')
            if not season_id:
                continue
            page = 1
            has_more = True
            while has_more is True:
                season = self._download_json(
                    'https://galadriel.puhutv.com/seasons/%s' % season_id,
                    season_id, 'Downloading page %s' % page, query={
                        'page': page,
                        'per': 40,
                    })
                episodes = season.get('episodes')
                if isinstance(episodes, list):
                    for ep in episodes:
                        slug_path = str_or_none(ep.get('slugPath'))
                        if not slug_path:
                            continue
                        video_id = str_or_none(int_or_none(ep.get('id')))
                        yield self.url_result(
                            'https://puhutv.com/%s' % slug_path,
                            ie=PuhuTVIE.ie_key(), video_id=video_id,
                            video_title=ep.get('name') or ep.get('eventLabel'))
                page += 1
                has_more = season.get('hasMore')

    def _real_extract(self, url):
        playlist_id = self._match_id(url)

        info = self._download_json(
            urljoin(url, '/api/slug/%s-detay' % playlist_id),
            playlist_id)['data']

        seasons = info.get('seasons')
        if seasons:
            return self.playlist_result(
                self._extract_entries(seasons), playlist_id, info.get('name'))

        # For films, these are using same url with series
        video_id = info.get('slug') or info['assets'][0]['slug']
        return self.url_result(
            'https://puhutv.com/%s-izle' % video_id,
            PuhuTVIE.ie_key(), video_id)


code:
# coding: utf-8
from __future__ import unicode_literals

from .prosiebensat1 import ProSiebenSat1BaseIE
from ..utils import (
    unified_strdate,
    parse_duration,
    compat_str,
)

class Puls4IE(ProSiebenSat1BaseIE):
    _VALID_URL = r'https?://(?:www\.)?puls4\.com/(?P<id>[^?#&]+)'
    _TESTS = [{
        'url': 'http://www.puls4.com/2-minuten-2-millionen/staffel-3/videos/2min2miotalk/Tobias-Homberger-von-myclubs-im-2min2miotalk-118118',
        'md5': 'fd3c6b0903ac72c9d004f04bc6bb3e03',
        'info_dict': {
            'id': '118118',
            'ext': 'flv',
            'title': 'Tobias Homberger von myclubs im #2min2miotalk',
            'description': 'md5:f9def7c5e8745d6026d8885487d91955',
            'upload_date': '20160830',
            'uploader': 'PULS_4',
        },
    }, {
        'url': 'http://www.puls4.com/pro-und-contra/wer-wird-prasident/Ganze-Folgen/Wer-wird-Praesident.-Norbert-Hofer',
        'only_matching': True,
    }, {
        'url': 'http://www.puls4.com/pro-und-contra/wer-wird-prasident/Ganze-Folgen/Wer-wird-Praesident-Analyse-des-Interviews-mit-Norbert-Hofer-416598',
        'only_matching': True,
    }]
    _TOKEN = 'puls4'
    _SALT = '01!kaNgaiNgah1Ie4AeSha'
    _CLIENT_NAME = ''

    def _real_extract(self, url):
        path = self._match_id(url)
        content_path = self._download_json(
            'http://www.puls4.com/api/json-fe/page/' + path, path)['content'][0]['url']
        media = self._download_json(
            'http://www.puls4.com' + content_path,
            content_path)['mediaCurrent']
        player_content = media['playerContent']
        info = self._extract_video_info(url, player_content['id'])
        info.update({
            'id': compat_str(media['objectId']),
            'title': player_content['title'],
            'description': media.get('description'),
            'thumbnail': media.get('previewLink'),
            'upload_date': unified_strdate(media.get('date')),
            'duration': parse_duration(player_content.get('duration')),
            'episode': player_content.get('episodePartName'),
            'show': media.get('channel'),
            'season_id': player_content.get('seasonId'),
            'uploader': player_content.get('sourceCompany'),
        })
        return info


code:
from __future__ import unicode_literals

import re

from .common import InfoExtractor
from ..compat import compat_str
from ..utils import int_or_none

class PyvideoIE(InfoExtractor):
    _VALID_URL = r'https?://(?:www\.)?pyvideo\.org/(?P<category>[^/]+)/(?P<id>[^/?#&.]+)'

    _TESTS = [{
        'url': 'http://pyvideo.org/pycon-us-2013/become-a-logging-expert-in-30-minutes.html',
        'info_dict': {
            'id': 'become-a-logging-expert-in-30-minutes',
        },
        'playlist_count': 2,
    }, {
        'url': 'http://pyvideo.org/pygotham-2012/gloriajw-spotifywitherikbernhardsson182m4v.html',
        'md5': '5fe1c7e0a8aa5570330784c847ff6d12',
        'info_dict': {
            'id': '2542',
            'ext': 'm4v',
            'title': 'Gloriajw-SpotifyWithErikBernhardsson182.m4v',
        },
    }]

    def _real_extract(self, url):
        mobj = re.match(self._VALID_URL, url)
        category = mobj.group('category')
        video_id = mobj.group('id')

        entries = []

        data = self._download_json(
            'https://raw.githubusercontent.com/pyvideo/data/master/%s/videos/%s.json'
            % (category, video_id), video_id, fatal=False)

        if data:
            for video in data['videos']:
                video_url = video.get('url')
                if video_url:
                    if video.get('type') == 'youtube':
                        entries.append(self.url_result(video_url, 'Youtube'))
                    else:
                        entries.append({
                            'id': compat_str(data.get('id') or video_id),
                            'url': video_url,
                            'title': data['title'],
                            'description': data.get('description') or data.get('summary'),
                            'thumbnail': data.get('thumbnail_url'),
                            'duration': int_or_none(data.get('duration')),
                        })
        else:
            webpage = self._download_webpage(url, video_id)
            title = self._og_search_title(webpage)
            media_urls = self._search_regex(
                r'(?s)Media URL:(.+?)</li>', webpage, 'media urls')
            for m in re.finditer(
                    r'<a[^>]+href=(["\'])(?P<url>http.+?)\1', media_urls):
                media_url = m.group('url')
                if re.match(r'https?://www\.youtube\.com/watch\?v=.*', media_url):
                    entries.append(self.url_result(media_url, 'Youtube'))
                else:
                    entries.append({
                        'id': video_id,
                        'url': media_url,
                        'title': title,
                    })

        return self.playlist_result(entries, video_id)


code:
# coding: utf-8
from __future__ import unicode_literals

import random
import re
import time

from .common import InfoExtractor
from ..utils import (
    clean_html,
    ExtractorError,
    strip_jsonp,
    unescapeHTML,
)

class QQMusicIE(InfoExtractor):
    IE_NAME = 'qqmusic'
    IE_DESC = 'QQ音乐'
    _VALID_URL = r'https?://y\.qq\.com/n/yqq/song/(?P<id>[0-9A-Za-z]+)\.html'
    _TESTS = [{
        'url': 'https://y.qq.com/n/yqq/song/004295Et37taLD.html',
        'md5': '5f1e6cea39e182857da7ffc5ef5e6bb8',
        'info_dict': {
            'id': '004295Et37taLD',
            'ext': 'mp3',
            'title': '可惜没如果',
            'release_date': '20141227',
            'creator': '林俊杰',
            'description': 'md5:d85afb3051952ecc50a1ee8a286d1eac',
            'thumbnail': r're:^https?://.*\.jpg$',
        }
    }, {
        'note': 'There is no mp3-320 version of this song.',
        'url': 'https://y.qq.com/n/yqq/song/004MsGEo3DdNxV.html',
        'md5': 'fa3926f0c585cda0af8fa4f796482e3e',
        'info_dict': {
            'id': '004MsGEo3DdNxV',
            'ext': 'mp3',
            'title': '如果',
            'release_date': '20050626',
            'creator': '李季美',
            'description': 'md5:46857d5ed62bc4ba84607a805dccf437',
            'thumbnail': r're:^https?://.*\.jpg$',
        }
    }, {
        'note': 'lyrics not in .lrc format',
        'url': 'https://y.qq.com/n/yqq/song/001JyApY11tIp6.html',
        'info_dict': {
            'id': '001JyApY11tIp6',
            'ext': 'mp3',
            'title': 'Shadows Over Transylvania',
            'release_date': '19970225',
            'creator': 'Dark Funeral',
            'description': 'md5:c9b20210587cbcd6836a1c597bab4525',
            'thumbnail': r're:^https?://.*\.jpg$',
        },
        'params': {
            'skip_download': True,
        },
    }]

    _FORMATS = {
        'mp3-320': {'prefix': 'M800', 'ext': 'mp3', 'preference': 40, 'abr': 320},
        'mp3-128': {'prefix': 'M500', 'ext': 'mp3', 'preference': 30, 'abr': 128},
        'm4a': {'prefix': 'C200', 'ext': 'm4a', 'preference': 10}
    }

    # Reference: m_r_GetRUin() in top_player.js
    # http://imgcache.gtimg.cn/music/portal_v3/y/top_player.js
    @staticmethod
    def m_r_get_ruin():
        curMs = int(time.time() * 1000) % 1000
        return int(round(random.random() * 2147483647) * curMs % 1E10)

    def _real_extract(self, url):
        mid = self._match_id(url)

        detail_info_page = self._download_webpage(
            'http://s.plcloud.music.qq.com/fcgi-bin/fcg_yqq_song_detail_info.fcg?songmid=%s&play=0' % mid,
            mid, note='Download song detail info',
            errnote='Unable to get song detail info', encoding='gbk')

        song_name = self._html_search_regex(
            r"songname:\s*'([^']+)'", detail_info_page, 'song name')

        publish_time = self._html_search_regex(
            r'发行时间：(\d{4}-\d{2}-\d{2})', detail_info_page,
            'publish time', default=None)
        if publish_time:
            publish_time = publish_time.replace('-', '')

        singer = self._html_search_regex(
            r"singer:\s*'([^']+)", detail_info_page, 'singer', default=None)

        lrc_content = self._html_search_regex(
            r'<div class="content" id="lrc_content"[^<>]*>([^<>]+)
',
            detail_info_page, 'LRC lyrics', default=None)
        if lrc_content:
            lrc_content = lrc_content.replace('\\n', '\n')

        thumbnail_url = None
        albummid = self._search_regex(
            [r'albummid:\'([0-9a-zA-Z]+)\'', r'"albummid":"([0-9a-zA-Z]+)"'],
            detail_info_page, 'album mid', default=None)
        if albummid:
            thumbnail_url = 'http://i.gtimg.cn/music/photo/mid_album_500/%s/%s/%s.jpg' \
                            % (albummid[-2:-1], albummid[-1], albummid)

        guid = self.m_r_get_ruin()

        vkey = self._download_json(
            'http://base.music.qq.com/fcgi-bin/fcg_musicexpress.fcg?json=3&guid=%s' % guid,
            mid, note='Retrieve vkey', errnote='Unable to get vkey',
            transform_source=strip_jsonp)['key']

        formats = []
        for format_id, details in self._FORMATS.items():
            formats.append({
                'url': 'http://cc.stream.qqmusic.qq.com/%s%s.%s?vkey=%s&guid=%s&fromtag=0'
                       % (details['prefix'], mid, details['ext'], vkey, guid),
                'format': format_id,
                'format_id': format_id,
                'preference': details['preference'],
                'abr': details.get('abr'),
            })
        self._check_formats(formats, mid)
        self._sort_formats(formats)

        actual_lrc_lyrics = ''.join(
            line + '\n' for line in re.findall(
                r'(?m)^(\[[0-9]{2}:[0-9]{2}(?:\.[0-9]{2,})?\][^\n]*|\[[^\]]*\])', lrc_content))

        info_dict = {
            'id': mid,
            'formats': formats,
            'title': song_name,
            'release_date': publish_time,
            'creator': singer,
            'description': lrc_content,
            'thumbnail': thumbnail_url
        }
        if actual_lrc_lyrics:
            info_dict['subtitles'] = {
                'origin': [{
                    'ext': 'lrc',
                    'data': actual_lrc_lyrics,
                }]
            }
        return info_dict

class QQPlaylistBaseIE(InfoExtractor):
    @staticmethod
    def qq_static_url(category, mid):
        return 'http://y.qq.com/y/static/%s/%s/%s/%s.html' % (category, mid[-2], mid[-1], mid)

    def get_singer_all_songs(self, singmid, num):
        return self._download_webpage(
            r'https://c.y.qq.com/v8/fcg-bin/fcg_v8_singer_track_cp.fcg', singmid,
            query={
                'format': 'json',
                'inCharset': 'utf8',
                'outCharset': 'utf-8',
                'platform': 'yqq',
                'needNewCode': 0,
                'singermid': singmid,
                'order': 'listen',
                'begin': 0,
                'num': num,
                'songstatus': 1,
            })

    def get_entries_from_page(self, singmid):
        entries = []

        default_num = 1
        json_text = self.get_singer_all_songs(singmid, default_num)
        json_obj_all_songs = self._parse_json(json_text, singmid)

        if json_obj_all_songs['code'] == 0:
            total = json_obj_all_songs['data']['total']
            json_text = self.get_singer_all_songs(singmid, total)
            json_obj_all_songs = self._parse_json(json_text, singmid)

        for item in json_obj_all_songs['data']['list']:
            if item['musicData'].get('songmid') is not None:
                songmid = item['musicData']['songmid']
                entries.append(self.url_result(
                    r'https://y.qq.com/n/yqq/song/%s.html' % songmid, 'QQMusic', songmid))

        return entries

class QQMusicSingerIE(QQPlaylistBaseIE):
    IE_NAME = 'qqmusic:singer'
    IE_DESC = 'QQ音乐 - 歌手'
    _VALID_URL = r'https?://y\.qq\.com/n/yqq/singer/(?P<id>[0-9A-Za-z]+)\.html'
    _TEST = {
        'url': 'https://y.qq.com/n/yqq/singer/001BLpXF2DyJe2.html',
        'info_dict': {
            'id': '001BLpXF2DyJe2',
            'title': '林俊杰',
            'description': 'md5:870ec08f7d8547c29c93010899103751',
        },
        'playlist_mincount': 12,
    }

    def _real_extract(self, url):
        mid = self._match_id(url)

        entries = self.get_entries_from_page(mid)
        singer_page = self._download_webpage(url, mid, 'Download singer page')
        singer_name = self._html_search_regex(
            r"singername\s*:\s*'(.*?)'", singer_page, 'singer name', default=None)
        singer_desc = None

        if mid:
            singer_desc_page = self._download_xml(
                'http://s.plcloud.music.qq.com/fcgi-bin/fcg_get_singer_desc.fcg', mid,
                'Donwload singer description XML',
                query={'utf8': 1, 'outCharset': 'utf-8', 'format': 'xml', 'singermid': mid},
                headers={'Referer': 'https://y.qq.com/n/yqq/singer/'})

            singer_desc = singer_desc_page.find('./data/info/desc').text

        return self.playlist_result(entries, mid, singer_name, singer_desc)

class QQMusicAlbumIE(QQPlaylistBaseIE):
    IE_NAME = 'qqmusic:album'
    IE_DESC = 'QQ音乐 - 专辑'
    _VALID_URL = r'https?://y\.qq\.com/n/yqq/album/(?P<id>[0-9A-Za-z]+)\.html'

    _TESTS = [{
        'url': 'https://y.qq.com/n/yqq/album/000gXCTb2AhRR1.html',
        'info_dict': {
            'id': '000gXCTb2AhRR1',
            'title': '我们都是这样长大的',
            'description': 'md5:179c5dce203a5931970d306aa9607ea6',
        },
        'playlist_count': 4,
    }, {
        'url': 'https://y.qq.com/n/yqq/album/002Y5a3b3AlCu3.html',
        'info_dict': {
            'id': '002Y5a3b3AlCu3',
            'title': '그리고...',
            'description': 'md5:a48823755615508a95080e81b51ba729',
        },
        'playlist_count': 8,
    }]

    def _real_extract(self, url):
        mid = self._match_id(url)

        album = self._download_json(
            'http://i.y.qq.com/v8/fcg-bin/fcg_v8_album_info_cp.fcg?albummid=%s&format=json' % mid,
            mid, 'Download album page')['data']

        entries = [
            self.url_result(
                'https://y.qq.com/n/yqq/song/' + song['songmid'] + '.html', 'QQMusic', song['songmid']
            ) for song in album['list']
        ]
        album_name = album.get('name')
        album_detail = album.get('desc')
        if album_detail is not None:
            album_detail = album_detail.strip()

        return self.playlist_result(entries, mid, album_name, album_detail)

class QQMusicToplistIE(QQPlaylistBaseIE):
    IE_NAME = 'qqmusic:toplist'
    IE_DESC = 'QQ音乐 - 排行榜'
    _VALID_URL = r'https?://y\.qq\.com/n/yqq/toplist/(?P<id>[0-9]+)\.html'

    _TESTS = [{
        'url': 'https://y.qq.com/n/yqq/toplist/123.html',
        'info_dict': {
            'id': '123',
            'title': '美国iTunes榜',
            'description': 'md5:89db2335fdbb10678dee2d43fe9aba08',
        },
        'playlist_count': 100,
    }, {
        'url': 'https://y.qq.com/n/yqq/toplist/3.html',
        'info_dict': {
            'id': '3',
            'title': '巅峰榜·欧美',
            'description': 'md5:5a600d42c01696b26b71f8c4d43407da',
        },
        'playlist_count': 100,
    }, {
        'url': 'https://y.qq.com/n/yqq/toplist/106.html',
        'info_dict': {
            'id': '106',
            'title': '韩国Mnet榜',
            'description': 'md5:cb84b325215e1d21708c615cac82a6e7',
        },
        'playlist_count': 50,
    }]

    def _real_extract(self, url):
        list_id = self._match_id(url)

        toplist_json = self._download_json(
            'http://i.y.qq.com/v8/fcg-bin/fcg_v8_toplist_cp.fcg', list_id,
            note='Download toplist page',
            query={'type': 'toplist', 'topid': list_id, 'format': 'json'})

        entries = [self.url_result(
            'https://y.qq.com/n/yqq/song/' + song['data']['songmid'] + '.html', 'QQMusic',
            song['data']['songmid'])
            for song in toplist_json['songlist']]

        topinfo = toplist_json.get('topinfo', {})
        list_name = topinfo.get('ListName')
        list_description = topinfo.get('info')
        return self.playlist_result(entries, list_id, list_name, list_description)

class QQMusicPlaylistIE(QQPlaylistBaseIE):
    IE_NAME = 'qqmusic:playlist'
    IE_DESC = 'QQ音乐 - 歌单'
    _VALID_URL = r'https?://y\.qq\.com/n/yqq/playlist/(?P<id>[0-9]+)\.html'

    _TESTS = [{
        'url': 'http://y.qq.com/n/yqq/playlist/3462654915.html',
        'info_dict': {
            'id': '3462654915',
            'title': '韩国5月新歌精选下旬',
            'description': 'md5:d2c9d758a96b9888cf4fe82f603121d4',
        },
        'playlist_count': 40,
        'skip': 'playlist gone',
    }, {
        'url': 'https://y.qq.com/n/yqq/playlist/1374105607.html',
        'info_dict': {
            'id': '1374105607',
            'title': '易入人心的华语民谣',
            'description': '民谣的歌曲易于传唱、、歌词朗朗伤口、旋律简单温馨。属于那种才入耳孔。却上心头的感觉。没有太多的复杂情绪。简单而直接地表达乐者的情绪，就是这样的简单才易入人心。',
        },
        'playlist_count': 20,
    }]

    def _real_extract(self, url):
        list_id = self._match_id(url)

        list_json = self._download_json(
            'http://i.y.qq.com/qzone-music/fcg-bin/fcg_ucc_getcdinfo_byids_cp.fcg',
            list_id, 'Download list page',
            query={'type': 1, 'json': 1, 'utf8': 1, 'onlysong': 0, 'disstid': list_id},
            transform_source=strip_jsonp)
        if not len(list_json.get('cdlist', [])):
            if list_json.get('code'):
                raise ExtractorError(
                    'QQ Music said: error %d in fetching playlist info' % list_json['code'],
                    expected=True)
            raise ExtractorError('Unable to get playlist info')

        cdlist = list_json['cdlist'][0]
        entries = [self.url_result(
            'https://y.qq.com/n/yqq/song/' + song['songmid'] + '.html', 'QQMusic', song['songmid'])
            for song in cdlist['songlist']]

        list_name = cdlist.get('dissname')
        list_description = clean_html(unescapeHTML(cdlist.get('desc')))
        return self.playlist_result(entries, list_id, list_name, list_description)


code:
# coding: utf-8
from __future__ import unicode_literals

from .common import InfoExtractor
from ..utils import int_or_none

class R7IE(InfoExtractor):
    _VALID_URL = r'''(?x)
                        https?://
                        (?:
                            (?:[a-zA-Z]+)\.r7\.com(?:/[^/]+)+/idmedia/|
                            noticias\.r7\.com(?:/[^/]+)+/[^/]+-|
                            player\.r7\.com/video/i/
                        )
                        (?P<id>[\da-f]{24})
                    '''
    _TESTS = [{
        'url': 'http://videos.r7.com/policiais-humilham-suspeito-a-beira-da-morte-morre-com-dignidade-/idmedia/54e7050b0cf2ff57e0279389.html',
        'md5': '403c4e393617e8e8ddc748978ee8efde',
        'info_dict': {
            'id': '54e7050b0cf2ff57e0279389',
            'ext': 'mp4',
            'title': 'Policiais humilham suspeito à beira da morte: "Morre com dignidade"',
            'description': 'md5:01812008664be76a6479aa58ec865b72',
            'thumbnail': r're:^https?://.*\.jpg$',
            'duration': 98,
            'like_count': int,
            'view_count': int,
        },
    }, {
        'url': 'http://esportes.r7.com/videos/cigano-manda-recado-aos-fas/idmedia/4e176727b51a048ee6646a1b.html',
        'only_matching': True,
    }, {
        'url': 'http://noticias.r7.com/record-news/video/representante-do-instituto-sou-da-paz-fala-sobre-fim-do-estatuto-do-desarmamento-5480fc580cf2285b117f438d/',
        'only_matching': True,
    }, {
        'url': 'http://player.r7.com/video/i/54e7050b0cf2ff57e0279389?play=true&video=http://vsh.r7.com/54e7050b0cf2ff57e0279389/ER7_RE_BG_MORTE_JOVENS_570kbps_2015-02-2009f17818-cc82-4c8f-86dc-89a66934e633-ATOS_copy.mp4&linkCallback=http://videos.r7.com/policiais-humilham-suspeito-a-beira-da-morte-morre-com-dignidade-/idmedia/54e7050b0cf2ff57e0279389.html&thumbnail=http://vtb.r7.com/ER7_RE_BG_MORTE_JOVENS_570kbps_2015-02-2009f17818-cc82-4c8f-86dc-89a66934e633-thumb.jpg&idCategory=192&share=true&layout=full&full=true',
        'only_matching': True,
    }]

    def _real_extract(self, url):
        video_id = self._match_id(url)

        video = self._download_json(
            'http://player-api.r7.com/video/i/%s' % video_id, video_id)

        title = video['title']

        formats = []
        media_url_hls = video.get('media_url_hls')
        if media_url_hls:
            formats.extend(self._extract_m3u8_formats(
                media_url_hls, video_id, 'mp4', entry_protocol='m3u8_native',
                m3u8_id='hls', fatal=False))
        media_url = video.get('media_url')
        if media_url:
            f = {
                'url': media_url,
                'format_id': 'http',
            }
            # m3u8 format always matches the http format, let's copy metadata from
            # one to another
            m3u8_formats = list(filter(
                lambda f: f.get('vcodec') != 'none', formats))
            if len(m3u8_formats) == 1:
                f_copy = m3u8_formats[0].copy()
                f_copy.update(f)
                f_copy['protocol'] = 'http'
                f = f_copy
            formats.append(f)
        self._sort_formats(formats)

        description = video.get('description')
        thumbnail = video.get('thumb')
        duration = int_or_none(video.get('media_duration'))
        like_count = int_or_none(video.get('likes'))
        view_count = int_or_none(video.get('views'))

        return {
            'id': video_id,
            'title': title,
            'description': description,
            'thumbnail': thumbnail,
            'duration': duration,
            'like_count': like_count,
            'view_count': view_count,
            'formats': formats,
        }

class R7ArticleIE(InfoExtractor):
    _VALID_URL = r'https?://(?:[a-zA-Z]+)\.r7\.com/(?:[^/]+/)+[^/?#&]+-(?P<id>\d+)'
    _TEST = {
        'url': 'http://tv.r7.com/record-play/balanco-geral/videos/policiais-humilham-suspeito-a-beira-da-morte-morre-com-dignidade-16102015',
        'only_matching': True,
    }

    @classmethod
    def suitable(cls, url):
        return False if R7IE.suitable(url) else super(R7ArticleIE, cls).suitable(url)

    def _real_extract(self, url):
        display_id = self._match_id(url)

        webpage = self._download_webpage(url, display_id)

        video_id = self._search_regex(
            r'<div[^>]+(?:id=["\']player-|class=["\']embed["\'][^>]+id=["\'])([\da-f]{24})',
            webpage, 'video id')

        return self.url_result('http://player.r7.com/video/i/%s' % video_id, R7IE.ie_key())


code:
# coding: utf-8

from __future__ import unicode_literals

import re

from .common import InfoExtractor
from ..utils import parse_duration

class RadioBremenIE(InfoExtractor):
    _VALID_URL = r'http?://(?:www\.)?radiobremen\.de/mediathek/(?:index\.html)?\?id=(?P<id>[0-9]+)'
    IE_NAME = 'radiobremen'

    _TEST = {
        'url': 'http://www.radiobremen.de/mediathek/?id=141876',
        'info_dict': {
            'id': '141876',
            'ext': 'mp4',
            'duration': 178,
            'width': 512,
            'title': 'Druck auf Patrick Öztürk',
            'thumbnail': r're:https?://.*\.jpg$',
            'description': 'Gegen den SPD-Bürgerschaftsabgeordneten Patrick Öztürk wird wegen Beihilfe zum gewerbsmäßigen Betrug ermittelt. Am Donnerstagabend sollte er dem Vorstand des SPD-Unterbezirks Bremerhaven dazu Rede und Antwort stehen.',
        },
    }

    def _real_extract(self, url):
        video_id = self._match_id(url)

        meta_url = 'http://www.radiobremen.de/apps/php/mediathek/metadaten.php?id=%s' % video_id
        meta_doc = self._download_webpage(
            meta_url, video_id, 'Downloading metadata')
        title = self._html_search_regex(
            r'
(?P<title>.+)
', meta_doc, 'title')
        description = self._html_search_regex(
            r'
(?P<description>.*)
', meta_doc, 'description', fatal=False)
        duration = parse_duration(self._html_search_regex(
            r'Länge:	\s+(?P<duration>[0-9]+:[0-9]+)	',
            meta_doc, 'duration', fatal=False))

        page_doc = self._download_webpage(
            url, video_id, 'Downloading video information')
        mobj = re.search(
            r"ardformatplayerclassic\(\'playerbereich\',\'(?P<width>[0-9]+)\',\'.*\',\'(?P<video_id>[0-9]+)\',\'(?P<secret>[0-9]+)\',\'(?P<thumbnail>.+)\',\'\'\)",
            page_doc)
        video_url = (
            "http://dl-ondemand.radiobremen.de/mediabase/%s/%s_%s_%s.mp4" %
            (video_id, video_id, mobj.group("secret"), mobj.group('width')))

        formats = [{
            'url': video_url,
            'ext': 'mp4',
            'width': int(mobj.group('width')),
        }]
        return {
            'id': video_id,
            'title': title,
            'description': description,
            'duration': duration,
            'formats': formats,
            'thumbnail': mobj.group('thumbnail'),
        }


code:
# coding: utf-8
from __future__ import unicode_literals

import re

from .common import InfoExtractor
from ..compat import compat_HTTPError
from ..utils import (
    determine_ext,
    ExtractorError,
    int_or_none,
    unified_strdate,
)

class RadioCanadaIE(InfoExtractor):
    IE_NAME = 'radiocanada'
    _VALID_URL = r'(?:radiocanada:|https?://ici\.radio-canada\.ca/widgets/mediaconsole/)(?P<app_code>[^:/]+)[:/](?P<id>[0-9]+)'
    _TESTS = [
        {
            'url': 'http://ici.radio-canada.ca/widgets/mediaconsole/medianet/7184272',
            'info_dict': {
                'id': '7184272',
                'ext': 'mp4',
                'title': 'Le parcours du tireur capté sur vidéo',
                'description': 'Images des caméras de surveillance fournies par la GRC montrant le parcours du tireur d\'Ottawa',
                'upload_date': '20141023',
            },
            'params': {
                # m3u8 download
                'skip_download': True,
            }
        },
        {
            # empty Title
            'url': 'http://ici.radio-canada.ca/widgets/mediaconsole/medianet/7754998/',
            'info_dict': {
                'id': '7754998',
                'ext': 'mp4',
                'title': 'letelejournal22h',
                'description': 'INTEGRALE WEB 22H-TJ',
                'upload_date': '20170720',
            },
            'params': {
                # m3u8 download
                'skip_download': True,
            },
        },
        {
            # with protectionType but not actually DRM protected
            'url': 'radiocanada:toutv:140872',
            'info_dict': {
                'id': '140872',
                'title': 'Épisode 1',
                'series': 'District 31',
            },
            'only_matching': True,
        }
    ]
    _GEO_COUNTRIES = ['CA']
    _access_token = None
    _claims = None

    def _call_api(self, path, video_id=None, app_code=None, query=None):
        if not query:
            query = {}
        query.update({
            'client_key': '773aea60-0e80-41bb-9c7f-e6d7c3ad17fb',
            'output': 'json',
        })
        if video_id:
            query.update({
                'appCode': app_code,
                'idMedia': video_id,
            })
        if self._access_token:
            query['access_token'] = self._access_token
        try:
            return self._download_json(
                'https://services.radio-canada.ca/media/' + path, video_id, query=query)
        except ExtractorError as e:
            if isinstance(e.cause, compat_HTTPError) and e.cause.code in (401, 422):
                data = self._parse_json(e.cause.read().decode(), None)
                error = data.get('error_description') or data['errorMessage']['text']
                raise ExtractorError(error, expected=True)
            raise

    def _extract_info(self, app_code, video_id):
        metas = self._call_api('meta/v1/index.ashx', video_id, app_code)['Metas']

        def get_meta(name):
            for meta in metas:
                if meta.get('name') == name:
                    text = meta.get('text')
                    if text:
                        return text

        # protectionType does not necessarily mean the video is DRM protected (see
        # https://github.com/ytdl-org/youtube-dl/pull/18609).
        if get_meta('protectionType'):
            self.report_warning('This video is probably DRM protected.')

        query = {
            'connectionType': 'hd',
            'deviceType': 'ipad',
            'multibitrate': 'true',
        }
        if self._claims:
            query['claims'] = self._claims
        v_data = self._call_api('validation/v2/', video_id, app_code, query)
        v_url = v_data.get('url')
        if not v_url:
            error = v_data['message']
            if error == "Le contenu sélectionné n'est pas disponible dans votre pays":
                raise self.raise_geo_restricted(error, self._GEO_COUNTRIES)
            if error == 'Le contenu sélectionné est disponible seulement en premium':
                self.raise_login_required(error)
            raise ExtractorError(
                '%s said: %s' % (self.IE_NAME, error), expected=True)
        formats = self._extract_m3u8_formats(v_url, video_id, 'mp4')
        self._sort_formats(formats)

        subtitles = {}
        closed_caption_url = get_meta('closedCaption') or get_meta('closedCaptionHTML5')
        if closed_caption_url:
            subtitles['fr'] = [{
                'url': closed_caption_url,
                'ext': determine_ext(closed_caption_url, 'vtt'),
            }]

        return {
            'id': video_id,
            'title': get_meta('Title') or get_meta('AV-nomEmission'),
            'description': get_meta('Description') or get_meta('ShortDescription'),
            'thumbnail': get_meta('imageHR') or get_meta('imageMR') or get_meta('imageBR'),
            'duration': int_or_none(get_meta('length')),
            'series': get_meta('Emission'),
            'season_number': int_or_none('SrcSaison'),
            'episode_number': int_or_none('SrcEpisode'),
            'upload_date': unified_strdate(get_meta('Date')),
            'subtitles': subtitles,
            'formats': formats,
        }

    def _real_extract(self, url):
        return self._extract_info(*re.match(self._VALID_URL, url).groups())

class RadioCanadaAudioVideoIE(InfoExtractor):
    IE_NAME = 'radiocanada:audiovideo'
    _VALID_URL = r'https?://ici\.radio-canada\.ca/([^/]+/)*media-(?P<id>[0-9]+)'
    _TESTS = [{
        'url': 'http://ici.radio-canada.ca/audio-video/media-7527184/barack-obama-au-vietnam',
        'info_dict': {
            'id': '7527184',
            'ext': 'mp4',
            'title': 'Barack Obama au Vietnam',
            'description': 'Les États-Unis lèvent l\'embargo sur la vente d\'armes qui datait de la guerre du Vietnam',
            'upload_date': '20160523',
        },
        'params': {
            # m3u8 download
            'skip_download': True,
        },
    }, {
        'url': 'https://ici.radio-canada.ca/info/videos/media-7527184/barack-obama-au-vietnam',
        'only_matching': True,
    }]

    def _real_extract(self, url):
        return self.url_result('radiocanada:medianet:%s' % self._match_id(url))


code:
from __future__ import unicode_literals

from .common import InfoExtractor

class RadioDeIE(InfoExtractor):
    IE_NAME = 'radio.de'
    _VALID_URL = r'https?://(?P<id>.+?)\.(?:radio\.(?:de|at|fr|pt|es|pl|it)|rad\.io)'
    _TEST = {
        'url': 'http://ndr2.radio.de/',
        'info_dict': {
            'id': 'ndr2',
            'ext': 'mp3',
            'title': 're:^NDR 2 [0-9]{4}-[0-9]{2}-[0-9]{2} [0-9]{2}:[0-9]{2}$',
            'description': 'md5:591c49c702db1a33751625ebfb67f273',
            'thumbnail': r're:^https?://.*\.png',
            'is_live': True,
        },
        'params': {
            'skip_download': True,
        }
    }

    def _real_extract(self, url):
        radio_id = self._match_id(url)
        webpage = self._download_webpage(url, radio_id)
        jscode = self._search_regex(
            r"'components/station/stationService':\s*\{\s*'?station'?:\s*(\{.*?\s*\}),\n",
            webpage, 'broadcast')

        broadcast = self._parse_json(jscode, radio_id)
        title = self._live_title(broadcast['name'])
        description = broadcast.get('description') or broadcast.get('shortDescription')
        thumbnail = broadcast.get('picture4Url') or broadcast.get('picture4TransUrl') or broadcast.get('logo100x100')

        formats = [{
            'url': stream['streamUrl'],
            'ext': stream['streamContentFormat'].lower(),
            'acodec': stream['streamContentFormat'],
            'abr': stream['bitRate'],
            'asr': stream['sampleRate']
        } for stream in broadcast['streamUrls']]
        self._sort_formats(formats)

        return {
            'id': radio_id,
            'title': title,
            'description': description,
            'thumbnail': thumbnail,
            'is_live': True,
            'formats': formats,
        }


code:
# coding: utf-8
from __future__ import unicode_literals

import re

from .common import InfoExtractor

class RadioFranceIE(InfoExtractor):
    _VALID_URL = r'^https?://maison\.radiofrance\.fr/radiovisions/(?P<id>[^?#]+)'
    IE_NAME = 'radiofrance'

    _TEST = {
        'url': 'http://maison.radiofrance.fr/radiovisions/one-one',
        'md5': 'bdbb28ace95ed0e04faab32ba3160daf',
        'info_dict': {
            'id': 'one-one',
            'ext': 'ogg',
            'title': 'One to one',
            'description': "Plutôt que d'imaginer la radio de demain comme technologie ou comme création de contenu, je veux montrer que quelles que soient ses évolutions, j'ai l'intime conviction que la radio continuera d'être un grand média de proximité pour les auditeurs.",
            'uploader': 'Thomas Hercouët',
        },
    }

    def _real_extract(self, url):
        m = re.match(self._VALID_URL, url)
        video_id = m.group('id')

        webpage = self._download_webpage(url, video_id)
        title = self._html_search_regex(r'
(.*?)
', webpage, 'title')
        description = self._html_search_regex(
            r'

(.*?)
',
            webpage, 'description', fatal=False)
        uploader = self._html_search_regex(
            r'
  © (.*?)
',
            webpage, 'uploader', fatal=False)

        formats_str = self._html_search_regex(
            r'class="jp-jplayer[^"]*" data-source="([^"]+)">',
            webpage, 'audio URLs')
        formats = [
            {
                'format_id': fm[0],
                'url': fm[1],
                'vcodec': 'none',
                'preference': i,
            }
            for i, fm in
            enumerate(re.findall(r"([a-z0-9]+)\s*:\s*'([^']+)'", formats_str))
        ]
        self._sort_formats(formats)

        return {
            'id': video_id,
            'title': title,
            'formats': formats,
            'description': description,
            'uploader': uploader,
        }