code:
from __future__ import unicode_literals

from .common import InfoExtractor
from ..utils import (
    int_or_none,
    determine_ext,
    mimetype2ext,
)

class TweakersIE(InfoExtractor):
    _VALID_URL = r'https?://tweakers\.net/video/(?P<id>\d+)'
    _TEST = {
        'url': 'https://tweakers.net/video/9926/new-nintendo-3ds-xl-op-alle-fronten-beter.html',
        'md5': 'fe73e417c093a788e0160c4025f88b15',
        'info_dict': {
            'id': '9926',
            'ext': 'mp4',
            'title': 'New Nintendo 3DS XL - Op alle fronten beter',
            'description': 'md5:3789b21fed9c0219e9bcaacd43fab280',
            'thumbnail': r're:^https?://.*\.jpe?g$',
            'duration': 386,
            'uploader_id': 's7JeEm',
        }
    }

    def _real_extract(self, url):
        video_id = self._match_id(url)
        video_data = self._download_json(
            'https://tweakers.net/video/s1playlist/%s/1920/1080/playlist.json' % video_id,
            video_id)['items'][0]

        title = video_data['title']

        formats = []
        for location in video_data.get('locations', {}).get('progressive', []):
            format_id = location.get('label')
            width = int_or_none(location.get('width'))
            height = int_or_none(location.get('height'))
            for source in location.get('sources', []):
                source_url = source.get('src')
                if not source_url:
                    continue
                ext = mimetype2ext(source.get('type')) or determine_ext(source_url)
                formats.append({
                    'format_id': format_id,
                    'url': source_url,
                    'width': width,
                    'height': height,
                    'ext': ext,
                })
        self._sort_formats(formats)

        return {
            'id': video_id,
            'title': title,
            'description': video_data.get('description'),
            'thumbnail': video_data.get('poster'),
            'duration': int_or_none(video_data.get('duration')),
            'uploader_id': video_data.get('account'),
            'formats': formats,
        }


code:
# coding: utf-8
from __future__ import unicode_literals

import re

from .common import InfoExtractor
from ..utils import (
    parse_iso8601,
    int_or_none,
    xpath_attr,
    xpath_element,
)

class TwentyFourVideoIE(InfoExtractor):
    IE_NAME = '24video'
    _VALID_URL = r'''(?x)
                    https?://
                        (?P<host>
                            (?:(?:www|porno?)\.)?24video\.
                            (?:net|me|xxx|sexy?|tube|adult|site|vip)
                        )/
                        (?:
                            video/(?:(?:view|xml)/)?|
                            player/new24_play\.swf\?id=
                        )
                        (?P<id>\d+)
                    '''

    _TESTS = [{
        'url': 'http://www.24video.net/video/view/1044982',
        'md5': 'e09fc0901d9eaeedac872f154931deeb',
        'info_dict': {
            'id': '1044982',
            'ext': 'mp4',
            'title': 'Эротика каменного века',
            'description': 'Как смотрели порно в каменном веке.',
            'thumbnail': r're:^https?://.*\.jpg$',
            'uploader': 'SUPERTELO',
            'duration': 31,
            'timestamp': 1275937857,
            'upload_date': '20100607',
            'age_limit': 18,
            'like_count': int,
            'dislike_count': int,
        },
    }, {
        'url': 'http://www.24video.net/player/new24_play.swf?id=1044982',
        'only_matching': True,
    }, {
        'url': 'http://www.24video.me/video/view/1044982',
        'only_matching': True,
    }, {
        'url': 'http://www.24video.tube/video/view/2363750',
        'only_matching': True,
    }, {
        'url': 'https://www.24video.site/video/view/2640421',
        'only_matching': True,
    }, {
        'url': 'https://porno.24video.net/video/2640421-vsya-takaya-gibkaya-i-v-masle',
        'only_matching': True,
    }, {
        'url': 'https://www.24video.vip/video/view/1044982',
        'only_matching': True,
    }, {
        'url': 'https://porn.24video.net/video/2640421-vsya-takay',
        'only_matching': True,
    }]

    def _real_extract(self, url):
        mobj = re.match(self._VALID_URL, url)
        video_id = mobj.group('id')
        host = mobj.group('host')

        webpage = self._download_webpage(
            'http://%s/video/view/%s' % (host, video_id), video_id)

        title = self._og_search_title(webpage)
        description = self._html_search_regex(
            r'<(p|span)[^>]+itemprop="description"[^>]*>(?P<description>[^<]+)</\1>',
            webpage, 'description', fatal=False, group='description')
        thumbnail = self._og_search_thumbnail(webpage)
        duration = int_or_none(self._og_search_property(
            'duration', webpage, 'duration', fatal=False))
        timestamp = parse_iso8601(self._search_regex(
            r'<time[^>]+\bdatetime="([^"]+)"[^>]+itemprop="uploadDate"',
            webpage, 'upload date', fatal=False))

        uploader = self._html_search_regex(
            r'class="video-uploaded"[^>]*>\s*]*>([^<]+)',
            webpage, 'uploader', fatal=False)

        view_count = int_or_none(self._html_search_regex(
            r'(\d+) просмотр',
            webpage, 'view count', fatal=False))
        comment_count = int_or_none(self._html_search_regex(
            r']+href="#tab-comments"[^>]*>(\d+) комментари',
            webpage, 'comment count', default=None))

        # Sets some cookies
        self._download_xml(
            r'http://%s/video/xml/%s?mode=init' % (host, video_id),
            video_id, 'Downloading init XML')

        video_xml = self._download_xml(
            'http://%s/video/xml/%s?mode=play' % (host, video_id),
            video_id, 'Downloading video XML')

        video = xpath_element(video_xml, './/video', 'video', fatal=True)

        formats = [{
            'url': xpath_attr(video, '', 'url', 'video URL', fatal=True),
        }]

        like_count = int_or_none(video.get('ratingPlus'))
        dislike_count = int_or_none(video.get('ratingMinus'))
        age_limit = 18 if video.get('adult') == 'true' else 0

        return {
            'id': video_id,
            'title': title,
            'description': description,
            'thumbnail': thumbnail,
            'uploader': uploader,
            'duration': duration,
            'timestamp': timestamp,
            'view_count': view_count,
            'comment_count': comment_count,
            'like_count': like_count,
            'dislike_count': dislike_count,
            'age_limit': age_limit,
            'formats': formats,
        }

repo_name: rrosajp/youtube


code:
# coding: utf-8
from __future__ import unicode_literals

import re

from .common import InfoExtractor
from ..utils import (
    int_or_none,
    try_get,
)

class TwentyMinutenIE(InfoExtractor):
    IE_NAME = '20min'
    _VALID_URL = r'''(?x)
                    https?://
                        (?:www\.)?20min\.ch/
                        (?:
                            videotv/*\?.*?\bvid=|
                            videoplayer/videoplayer\.html\?.*?\bvideoId@
                        )
                        (?P<id>\d+)
                    '''
    _TESTS = [{
        'url': 'http://www.20min.ch/videotv/?vid=469148&cid=2',
        'md5': 'e7264320db31eed8c38364150c12496e',
        'info_dict': {
            'id': '469148',
            'ext': 'mp4',
            'title': '85 000 Franken für 15 perfekte Minuten',
            'thumbnail': r're:https?://.*\.jpg$',
        },
    }, {
        'url': 'http://www.20min.ch/videoplayer/videoplayer.html?params=client@twentyDE|videoId@523629',
        'info_dict': {
            'id': '523629',
            'ext': 'mp4',
            'title': 'So kommen Sie bei Eis und Schnee sicher an',
            'description': 'md5:117c212f64b25e3d95747e5276863f7d',
            'thumbnail': r're:https?://.*\.jpg$',
        },
        'params': {
            'skip_download': True,
        },
    }, {
        'url': 'http://www.20min.ch/videotv/?cid=44&vid=468738',
        'only_matching': True,
    }]

    @staticmethod
    def _extract_urls(webpage):
        return [m.group('url') for m in re.finditer(
            r'<iframe[^>]+src=(["\'])(?P<url>(?:(?:https?:)?//)?(?:www\.)?20min\.ch/videoplayer/videoplayer.html\?.*?\bvideoId@\d+.*?)\1',
            webpage)]

    def _real_extract(self, url):
        video_id = self._match_id(url)

        video = self._download_json(
            'http://api.20min.ch/video/%s/show' % video_id,
            video_id)['content']

        title = video['title']

        formats = [{
            'format_id': format_id,
            'url': 'http://podcast.20min-tv.ch/podcast/20min/%s%s.mp4' % (video_id, p),
            'quality': quality,
        } for quality, (format_id, p) in enumerate([('sd', ''), ('hd', 'h')])]
        self._sort_formats(formats)

        description = video.get('lead')
        thumbnail = video.get('thumbnail')

        def extract_count(kind):
            return try_get(
                video,
                lambda x: int_or_none(x['communityobject']['thumbs_%s' % kind]))

        like_count = extract_count('up')
        dislike_count = extract_count('down')

        return {
            'id': video_id,
            'title': title,
            'description': description,
            'thumbnail': thumbnail,
            'like_count': like_count,
            'dislike_count': dislike_count,
            'formats': formats,
        }


code:
from __future__ import unicode_literals

import re

from .common import InfoExtractor
from ..utils import int_or_none

class TwentyThreeVideoIE(InfoExtractor):
    IE_NAME = '23video'
    _VALID_URL = r'https?://(?P<domain>[^.]+\.(?:twentythree\.net|23video\.com|filmweb\.no))/v\.ihtml/player\.html\?(?P<query>.*?\bphoto(?:_|%5f)id=(?P<id>\d+).*)'
    _TESTS = [{
        'url': 'https://video.twentythree.net/v.ihtml/player.html?showDescriptions=0&source=site&photo%5fid=20448876&autoPlay=1',
        'md5': '75fcf216303eb1dae9920d651f85ced4',
        'info_dict': {
            'id': '20448876',
            'ext': 'mp4',
            'title': 'Video Marketing Minute: Personalized Video',
            'timestamp': 1513855354,
            'upload_date': '20171221',
            'uploader_id': '12258964',
            'uploader': 'Rasmus Bysted',
        }
    }, {
        'url': 'https://bonnier-publications-danmark.23video.com/v.ihtml/player.html?token=f0dc46476e06e13afd5a1f84a29e31e8&source=embed&photo%5fid=36137620',
        'only_matching': True,
    }]

    def _real_extract(self, url):
        domain, query, photo_id = re.match(self._VALID_URL, url).groups()
        base_url = 'https://%s' % domain
        photo_data = self._download_json(
            base_url + '/api/photo/list?' + query, photo_id, query={
                'format': 'json',
            }, transform_source=lambda s: self._search_regex(r'(?s)({.+})', s, 'photo data'))['photo']
        title = photo_data['title']

        formats = []

        audio_path = photo_data.get('audio_download')
        if audio_path:
            formats.append({
                'format_id': 'audio',
                'url': base_url + audio_path,
                'filesize': int_or_none(photo_data.get('audio_size')),
                'vcodec': 'none',
            })

        def add_common_info_to_list(l, template, id_field, id_value):
            f_base = template % id_value
            f_path = photo_data.get(f_base + 'download')
            if not f_path:
                return
            l.append({
                id_field: id_value,
                'url': base_url + f_path,
                'width': int_or_none(photo_data.get(f_base + 'width')),
                'height': int_or_none(photo_data.get(f_base + 'height')),
                'filesize': int_or_none(photo_data.get(f_base + 'size')),
            })

        for f in ('mobile_high', 'medium', 'hd', '1080p', '4k'):
            add_common_info_to_list(formats, 'video_%s_', 'format_id', f)

        thumbnails = []
        for t in ('quad16', 'quad50', 'quad75', 'quad100', 'small', 'portrait', 'standard', 'medium', 'large', 'original'):
            add_common_info_to_list(thumbnails, '%s_', 'id', t)

        return {
            'id': photo_id,
            'title': title,
            'timestamp': int_or_none(photo_data.get('creation_date_epoch')),
            'duration': int_or_none(photo_data.get('video_length')),
            'view_count': int_or_none(photo_data.get('view_count')),
            'comment_count': int_or_none(photo_data.get('number_of_comments')),
            'uploader_id': photo_data.get('user_id'),
            'uploader': photo_data.get('display_name'),
            'thumbnails': thumbnails,
            'formats': formats,
        }


code:
# coding: utf-8
from __future__ import unicode_literals

from .common import InfoExtractor
from ..utils import urlencode_postdata

import re

class TwitCastingIE(InfoExtractor):
    _VALID_URL = r'https?://(?:[^/]+\.)?twitcasting\.tv/(?P<uploader_id>[^/]+)/movie/(?P<id>\d+)'
    _TESTS = [{
        'url': 'https://twitcasting.tv/ivetesangalo/movie/2357609',
        'md5': '745243cad58c4681dc752490f7540d7f',
        'info_dict': {
            'id': '2357609',
            'ext': 'mp4',
            'title': 'Live #2357609',
            'uploader_id': 'ivetesangalo',
            'description': "Moi! I'm live on TwitCasting from my iPhone.",
            'thumbnail': r're:^https?://.*\.jpg$',
        },
        'params': {
            'skip_download': True,
        },
    }, {
        'url': 'https://twitcasting.tv/mttbernardini/movie/3689740',
        'info_dict': {
            'id': '3689740',
            'ext': 'mp4',
            'title': 'Live playing something #3689740',
            'uploader_id': 'mttbernardini',
            'description': "I'm live on TwitCasting from my iPad. password: abc (Santa Marinella/Lazio, Italia)",
            'thumbnail': r're:^https?://.*\.jpg$',
        },
        'params': {
            'skip_download': True,
            'videopassword': 'abc',
        },
    }]

    def _real_extract(self, url):
        mobj = re.match(self._VALID_URL, url)
        video_id = mobj.group('id')
        uploader_id = mobj.group('uploader_id')

        video_password = self._downloader.params.get('videopassword')
        request_data = None
        if video_password:
            request_data = urlencode_postdata({
                'password': video_password,
            })
        webpage = self._download_webpage(url, video_id, data=request_data)

        title = self._html_search_regex(
            r'(?s)<[^>]+id=["\']movietitle[^>]+>(.+?)</',
            webpage, 'title', default=None) or self._html_search_meta(
            'twitter:title', webpage, fatal=True)

        m3u8_url = self._search_regex(
            (r'data-movie-url=(["\'])(?P<url>(?:(?!\1).)+)\1',
             r'(["\'])(?P<url>http.+?\.m3u8.*?)\1'),
            webpage, 'm3u8 url', group='url')

        formats = self._extract_m3u8_formats(
            m3u8_url, video_id, ext='mp4', entry_protocol='m3u8_native',
            m3u8_id='hls')

        thumbnail = self._og_search_thumbnail(webpage)
        description = self._og_search_description(
            webpage, default=None) or self._html_search_meta(
            'twitter:description', webpage)

        return {
            'id': video_id,
            'title': title,
            'description': description,
            'thumbnail': thumbnail,
            'uploader_id': uploader_id,
            'formats': formats,
        }


code:
# coding: utf-8
from __future__ import unicode_literals

import collections
import itertools
import json
import random
import re

from .common import InfoExtractor
from ..compat import (
    compat_kwargs,
    compat_parse_qs,
    compat_str,
    compat_urlparse,
    compat_urllib_parse_urlencode,
    compat_urllib_parse_urlparse,
)
from ..utils import (
    clean_html,
    ExtractorError,
    float_or_none,
    int_or_none,
    parse_duration,
    parse_iso8601,
    qualities,
    try_get,
    unified_timestamp,
    update_url_query,
    url_or_none,
    urljoin,
)

class TwitchBaseIE(InfoExtractor):
    _VALID_URL_BASE = r'https?://(?:(?:www|go|m)\.)?twitch\.tv'

    _API_BASE = 'https://api.twitch.tv'
    _USHER_BASE = 'https://usher.ttvnw.net'
    _LOGIN_FORM_URL = 'https://www.twitch.tv/login'
    _LOGIN_POST_URL = 'https://passport.twitch.tv/login'
    _CLIENT_ID = 'kimne78kx3ncx6brgo4mv6wki5h1ko'
    _NETRC_MACHINE = 'twitch'

    def _handle_error(self, response):
        if not isinstance(response, dict):
            return
        error = response.get('error')
        if error:
            raise ExtractorError(
                '%s returned error: %s - %s' % (self.IE_NAME, error, response.get('message')),
                expected=True)

    def _call_api(self, path, item_id, *args, **kwargs):
        headers = kwargs.get('headers', {}).copy()
        headers.update({
            'Accept': 'application/vnd.twitchtv.v5+json; charset=UTF-8',
            'Client-ID': self._CLIENT_ID,
        })
        kwargs.update({
            'headers': headers,
            'expected_status': (400, 410),
        })
        response = self._download_json(
            '%s/%s' % (self._API_BASE, path), item_id,
            *args, **compat_kwargs(kwargs))
        self._handle_error(response)
        return response

    def _real_initialize(self):
        self._login()

    def _login(self):
        username, password = self._get_login_info()
        if username is None:
            return

        def fail(message):
            raise ExtractorError(
                'Unable to login. Twitch said: %s' % message, expected=True)

        def login_step(page, urlh, note, data):
            form = self._hidden_inputs(page)
            form.update(data)

            page_url = urlh.geturl()
            post_url = self._search_regex(
                r'<form[^>]+action=(["\'])(?P<url>.+?)\1', page,
                'post url', default=self._LOGIN_POST_URL, group='url')
            post_url = urljoin(page_url, post_url)

            headers = {
                'Referer': page_url,
                'Origin': page_url,
                'Content-Type': 'text/plain;charset=UTF-8',
            }

            response = self._download_json(
                post_url, None, note, data=json.dumps(form).encode(),
                headers=headers, expected_status=400)
            error = response.get('error_description') or response.get('error_code')
            if error:
                fail(error)

            if 'Authenticated successfully' in response.get('message', ''):
                return None, None

            redirect_url = urljoin(
                post_url,
                response.get('redirect') or response['redirect_path'])
            return self._download_webpage_handle(
                redirect_url, None, 'Downloading login redirect page',
                headers=headers)

        login_page, handle = self._download_webpage_handle(
            self._LOGIN_FORM_URL, None, 'Downloading login page')

        # Some TOR nodes and public proxies are blocked completely
        if 'blacklist_message' in login_page:
            fail(clean_html(login_page))

        redirect_page, handle = login_step(
            login_page, handle, 'Logging in', {
                'username': username,
                'password': password,
                'client_id': self._CLIENT_ID,
            })

        # Successful login
        if not redirect_page:
            return

        if re.search(r'(?i)<form[^>]+id="two-factor-submit"', redirect_page) is not None:
            # TODO: Add mechanism to request an SMS or phone call
            tfa_token = self._get_tfa_info('two-factor authentication token')
            login_step(redirect_page, handle, 'Submitting TFA token', {
                'authy_token': tfa_token,
                'remember_2fa': 'true',
            })

    def _prefer_source(self, formats):
        try:
            source = next(f for f in formats if f['format_id'] == 'Source')
            source['quality'] = 10
        except StopIteration:
            for f in formats:
                if '/chunked/' in f['url']:
                    f.update({
                        'quality': 10,
                        'format_note': 'Source',
                    })
        self._sort_formats(formats)

    def _download_access_token(self, channel_name):
        return self._call_api(
            'api/channels/%s/access_token' % channel_name, channel_name,
            'Downloading access token JSON')

    def _extract_channel_id(self, token, channel_name):
        return compat_str(self._parse_json(token, channel_name)['channel_id'])

class TwitchVodIE(TwitchBaseIE):
    IE_NAME = 'twitch:vod'
    _VALID_URL = r'''(?x)
                    https?://
                        (?:
                            (?:(?:www|go|m)\.)?twitch\.tv/(?:[^/]+/v(?:ideo)?|videos)/|
                            player\.twitch\.tv/\?.*?\bvideo=v?
                        )
                        (?P<id>\d+)
                    '''
    _ITEM_TYPE = 'vod'
    _ITEM_SHORTCUT = 'v'

    _TESTS = [{
        'url': 'http://www.twitch.tv/riotgames/v/6528877?t=5m10s',
        'info_dict': {
            'id': 'v6528877',
            'ext': 'mp4',
            'title': 'LCK Summer Split - Week 6 Day 1',
            'thumbnail': r're:^https?://.*\.jpg$',
            'duration': 17208,
            'timestamp': 1435131709,
            'upload_date': '20150624',
            'uploader': 'Riot Games',
            'uploader_id': 'riotgames',
            'view_count': int,
            'start_time': 310,
        },
        'params': {
            # m3u8 download
            'skip_download': True,
        },
    }, {
        # Untitled broadcast (title is None)
        'url': 'http://www.twitch.tv/belkao_o/v/11230755',
        'info_dict': {
            'id': 'v11230755',
            'ext': 'mp4',
            'title': 'Untitled Broadcast',
            'thumbnail': r're:^https?://.*\.jpg$',
            'duration': 1638,
            'timestamp': 1439746708,
            'upload_date': '20150816',
            'uploader': 'BelkAO_o',
            'uploader_id': 'belkao_o',
            'view_count': int,
        },
        'params': {
            # m3u8 download
            'skip_download': True,
        },
        'skip': 'HTTP Error 404: Not Found',
    }, {
        'url': 'http://player.twitch.tv/?t=5m10s&video=v6528877',
        'only_matching': True,
    }, {
        'url': 'https://www.twitch.tv/videos/6528877',
        'only_matching': True,
    }, {
        'url': 'https://m.twitch.tv/beagsandjam/v/247478721',
        'only_matching': True,
    }, {
        'url': 'https://www.twitch.tv/northernlion/video/291940395',
        'only_matching': True,
    }, {
        'url': 'https://player.twitch.tv/?video=480452374',
        'only_matching': True,
    }]

    def _download_info(self, item_id):
        return self._extract_info(
            self._call_api(
                'kraken/videos/%s' % item_id, item_id,
                'Downloading video info JSON'))

    @staticmethod
    def _extract_info(info):
        status = info.get('status')
        if status == 'recording':
            is_live = True
        elif status == 'recorded':
            is_live = False
        else:
            is_live = None
        _QUALITIES = ('small', 'medium', 'large')
        quality_key = qualities(_QUALITIES)
        thumbnails = []
        preview = info.get('preview')
        if isinstance(preview, dict):
            for thumbnail_id, thumbnail_url in preview.items():
                thumbnail_url = url_or_none(thumbnail_url)
                if not thumbnail_url:
                    continue
                if thumbnail_id not in _QUALITIES:
                    continue
                thumbnails.append({
                    'url': thumbnail_url,
                    'preference': quality_key(thumbnail_id),
                })
        return {
            'id': info['_id'],
            'title': info.get('title') or 'Untitled Broadcast',
            'description': info.get('description'),
            'duration': int_or_none(info.get('length')),
            'thumbnails': thumbnails,
            'uploader': info.get('channel', {}).get('display_name'),
            'uploader_id': info.get('channel', {}).get('name'),
            'timestamp': parse_iso8601(info.get('recorded_at')),
            'view_count': int_or_none(info.get('views')),
            'is_live': is_live,
        }

    def _real_extract(self, url):
        vod_id = self._match_id(url)

        info = self._download_info(vod_id)
        access_token = self._call_api(
            'api/vods/%s/access_token' % vod_id, vod_id,
            'Downloading %s access token' % self._ITEM_TYPE)

        formats = self._extract_m3u8_formats(
            '%s/vod/%s.m3u8?%s' % (
                self._USHER_BASE, vod_id,
                compat_urllib_parse_urlencode({
                    'allow_source': 'true',
                    'allow_audio_only': 'true',
                    'allow_spectre': 'true',
                    'player': 'twitchweb',
                    'playlist_include_framerate': 'true',
                    'nauth': access_token['token'],
                    'nauthsig': access_token['sig'],
                })),
            vod_id, 'mp4', entry_protocol='m3u8_native')

        self._prefer_source(formats)
        info['formats'] = formats

        parsed_url = compat_urllib_parse_urlparse(url)
        query = compat_parse_qs(parsed_url.query)
        if 't' in query:
            info['start_time'] = parse_duration(query['t'][0])

        if info.get('timestamp') is not None:
            info['subtitles'] = {
                'rechat': [{
                    'url': update_url_query(
                        'https://api.twitch.tv/v5/videos/%s/comments' % vod_id, {
                            'client_id': self._CLIENT_ID,
                        }),
                    'ext': 'json',
                }],
            }

        return info

def _make_video_result(node):
    assert isinstance(node, dict)
    video_id = node.get('id')
    if not video_id:
        return
    return {
        '_type': 'url_transparent',
        'ie_key': TwitchVodIE.ie_key(),
        'id': video_id,
        'url': 'https://www.twitch.tv/videos/%s' % video_id,
        'title': node.get('title'),
        'thumbnail': node.get('previewThumbnailURL'),
        'duration': float_or_none(node.get('lengthSeconds')),
        'view_count': int_or_none(node.get('viewCount')),
    }

class TwitchGraphQLBaseIE(TwitchBaseIE):
    _PAGE_LIMIT = 100

    _OPERATION_HASHES = {
        'CollectionSideBar': '27111f1b382effad0b6def325caef1909c733fe6a4fbabf54f8d491ef2cf2f14',
        'FilterableVideoTower_Videos': 'a937f1d22e269e39a03b509f65a7490f9fc247d7f83d6ac1421523e3b68042cb',
        'ClipsCards__User': 'b73ad2bfaecfd30a9e6c28fada15bd97032c83ec77a0440766a56fe0bd632777',
        'ChannelCollectionsContent': '07e3691a1bad77a36aba590c351180439a40baefc1c275356f40fc7082419a84',
        'StreamMetadata': '1c719a40e481453e5c48d9bb585d971b8b372f8ebb105b17076722264dfa5b3e',
        'ComscoreStreamingQuery': 'e1edae8122517d013405f237ffcc124515dc6ded82480a88daef69c83b53ac01',
        'VideoPreviewOverlay': '3006e77e51b128d838fa4e835723ca4dc9a05c5efd4466c1085215c6e437e65c',
    }

    def _download_gql(self, video_id, ops, note, fatal=True):
        for op in ops:
            op['extensions'] = {
                'persistedQuery': {
                    'version': 1,
                    'sha256Hash': self._OPERATION_HASHES[op['operationName']],
                }
            }
        return self._download_json(
            'https://gql.twitch.tv/gql', video_id, note,
            data=json.dumps(ops).encode(),
            headers={
                'Content-Type': 'text/plain;charset=UTF-8',
                'Client-ID': self._CLIENT_ID,
            }, fatal=fatal)

class TwitchCollectionIE(TwitchGraphQLBaseIE):
    _VALID_URL = r'https?://(?:(?:www|go|m)\.)?twitch\.tv/collections/(?P<id>[^/]+)'

    _TESTS = [{
        'url': 'https://www.twitch.tv/collections/wlDCoH0zEBZZbQ',
        'info_dict': {
            'id': 'wlDCoH0zEBZZbQ',
            'title': 'Overthrow Nook, capitalism for children',
        },
        'playlist_mincount': 13,
    }]

    _OPERATION_NAME = 'CollectionSideBar'

    def _real_extract(self, url):
        collection_id = self._match_id(url)
        collection = self._download_gql(
            collection_id, [{
                'operationName': self._OPERATION_NAME,
                'variables': {'collectionID': collection_id},
            }],
            'Downloading collection GraphQL')[0]['data']['collection']
        title = collection.get('title')
        entries = []
        for edge in collection['items']['edges']:
            if not isinstance(edge, dict):
                continue
            node = edge.get('node')
            if not isinstance(node, dict):
                continue
            video = _make_video_result(node)
            if video:
                entries.append(video)
        return self.playlist_result(
            entries, playlist_id=collection_id, playlist_title=title)

class TwitchPlaylistBaseIE(TwitchGraphQLBaseIE):
    def _entries(self, channel_name, *args):
        cursor = None
        variables_common = self._make_variables(channel_name, *args)
        entries_key = '%ss' % self._ENTRY_KIND
        for page_num in itertools.count(1):
            variables = variables_common.copy()
            variables['limit'] = self._PAGE_LIMIT
            if cursor:
                variables['cursor'] = cursor
            page = self._download_gql(
                channel_name, [{
                    'operationName': self._OPERATION_NAME,
                    'variables': variables,
                }],
                'Downloading %ss GraphQL page %s' % (self._NODE_KIND, page_num),
                fatal=False)
            if not page:
                break
            edges = try_get(
                page, lambda x: x[0]['data']['user'][entries_key]['edges'], list)
            if not edges:
                break
            for edge in edges:
                if not isinstance(edge, dict):
                    continue
                if edge.get('__typename') != self._EDGE_KIND:
                    continue
                node = edge.get('node')
                if not isinstance(node, dict):
                    continue
                if node.get('__typename') != self._NODE_KIND:
                    continue
                entry = self._extract_entry(node)
                if entry:
                    cursor = edge.get('cursor')
                    yield entry
            if not cursor or not isinstance(cursor, compat_str):
                break

    # Deprecated kraken v5 API
    def _entries_kraken(self, channel_name, broadcast_type, sort):
        access_token = self._download_access_token(channel_name)
        channel_id = self._extract_channel_id(access_token['token'], channel_name)
        offset = 0
        counter_override = None
        for counter in itertools.count(1):
            response = self._call_api(
                'kraken/channels/%s/videos/' % channel_id,
                channel_id,
                'Downloading video JSON page %s' % (counter_override or counter),
                query={
                    'offset': offset,
                    'limit': self._PAGE_LIMIT,
                    'broadcast_type': broadcast_type,
                    'sort': sort,
                })
            videos = response.get('videos')
            if not isinstance(videos, list):
                break
            for video in videos:
                if not isinstance(video, dict):
                    continue
                video_url = url_or_none(video.get('url'))
                if not video_url:
                    continue
                yield {
                    '_type': 'url_transparent',
                    'ie_key': TwitchVodIE.ie_key(),
                    'id': video.get('_id'),
                    'url': video_url,
                    'title': video.get('title'),
                    'description': video.get('description'),
                    'timestamp': unified_timestamp(video.get('published_at')),
                    'duration': float_or_none(video.get('length')),
                    'view_count': int_or_none(video.get('views')),
                    'language': video.get('language'),
                }
            offset += self._PAGE_LIMIT
            total = int_or_none(response.get('_total'))
            if total and offset >= total:
                break

class TwitchVideosIE(TwitchPlaylistBaseIE):
    _VALID_URL = r'https?://(?:(?:www|go|m)\.)?twitch\.tv/(?P<id>[^/]+)/(?:videos|profile)'

    _TESTS = [{
        # All Videos sorted by Date
        'url': 'https://www.twitch.tv/spamfish/videos?filter=all',
        'info_dict': {
            'id': 'spamfish',
            'title': 'spamfish - All Videos sorted by Date',
        },
        'playlist_mincount': 924,
    }, {
        # All Videos sorted by Popular
        'url': 'https://www.twitch.tv/spamfish/videos?filter=all&sort=views',
        'info_dict': {
            'id': 'spamfish',
            'title': 'spamfish - All Videos sorted by Popular',
        },
        'playlist_mincount': 931,
    }, {
        # Past Broadcasts sorted by Date
        'url': 'https://www.twitch.tv/spamfish/videos?filter=archives',
        'info_dict': {
            'id': 'spamfish',
            'title': 'spamfish - Past Broadcasts sorted by Date',
        },
        'playlist_mincount': 27,
    }, {
        # Highlights sorted by Date
        'url': 'https://www.twitch.tv/spamfish/videos?filter=highlights',
        'info_dict': {
            'id': 'spamfish',
            'title': 'spamfish - Highlights sorted by Date',
        },
        'playlist_mincount': 901,
    }, {
        # Uploads sorted by Date
        'url': 'https://www.twitch.tv/esl_csgo/videos?filter=uploads&sort=time',
        'info_dict': {
            'id': 'esl_csgo',
            'title': 'esl_csgo - Uploads sorted by Date',
        },
        'playlist_mincount': 5,
    }, {
        # Past Premieres sorted by Date
        'url': 'https://www.twitch.tv/spamfish/videos?filter=past_premieres',
        'info_dict': {
            'id': 'spamfish',
            'title': 'spamfish - Past Premieres sorted by Date',
        },
        'playlist_mincount': 1,
    }, {
        'url': 'https://www.twitch.tv/spamfish/videos/all',
        'only_matching': True,
    }, {
        'url': 'https://m.twitch.tv/spamfish/videos/all',
        'only_matching': True,
    }, {
        'url': 'https://www.twitch.tv/spamfish/videos',
        'only_matching': True,
    }]

    Broadcast = collections.namedtuple('Broadcast', ['type', 'label'])

    _DEFAULT_BROADCAST = Broadcast(None, 'All Videos')
    _BROADCASTS = {
        'archives': Broadcast('ARCHIVE', 'Past Broadcasts'),
        'highlights': Broadcast('HIGHLIGHT', 'Highlights'),
        'uploads': Broadcast('UPLOAD', 'Uploads'),
        'past_premieres': Broadcast('PAST_PREMIERE', 'Past Premieres'),
        'all': _DEFAULT_BROADCAST,
    }

    _DEFAULT_SORTED_BY = 'Date'
    _SORTED_BY = {
        'time': _DEFAULT_SORTED_BY,
        'views': 'Popular',
    }

    _OPERATION_NAME = 'FilterableVideoTower_Videos'
    _ENTRY_KIND = 'video'
    _EDGE_KIND = 'VideoEdge'
    _NODE_KIND = 'Video'

    @classmethod
    def suitable(cls, url):
        return (False
                if any(ie.suitable(url) for ie in (
                    TwitchVideosClipsIE,
                    TwitchVideosCollectionsIE))
                else super(TwitchVideosIE, cls).suitable(url))

    @staticmethod
    def _make_variables(channel_name, broadcast_type, sort):
        return {
            'channelOwnerLogin': channel_name,
            'broadcastType': broadcast_type,
            'videoSort': sort.upper(),
        }

    @staticmethod
    def _extract_entry(node):
        return _make_video_result(node)

    def _real_extract(self, url):
        channel_name = self._match_id(url)
        qs = compat_urlparse.parse_qs(compat_urlparse.urlparse(url).query)
        filter = qs.get('filter', ['all'])[0]
        sort = qs.get('sort', ['time'])[0]
        broadcast = self._BROADCASTS.get(filter, self._DEFAULT_BROADCAST)
        return self.playlist_result(
            self._entries(channel_name, broadcast.type, sort),
            playlist_id=channel_name,
            playlist_title='%s - %s sorted by %s'
            % (channel_name, broadcast.label,
               self._SORTED_BY.get(sort, self._DEFAULT_SORTED_BY)))

class TwitchVideosClipsIE(TwitchPlaylistBaseIE):
    _VALID_URL = r'https?://(?:(?:www|go|m)\.)?twitch\.tv/(?P<id>[^/]+)/(?:clips|videos/*?\?.*?\bfilter=clips)'

    _TESTS = [{
        # Clips
        'url': 'https://www.twitch.tv/vanillatv/clips?filter=clips&range=all',
        'info_dict': {
            'id': 'vanillatv',
            'title': 'vanillatv - Clips Top All',
        },
        'playlist_mincount': 1,
    }, {
        'url': 'https://www.twitch.tv/dota2ruhub/videos?filter=clips&range=7d',
        'only_matching': True,
    }]

    Clip = collections.namedtuple('Clip', ['filter', 'label'])

    _DEFAULT_CLIP = Clip('LAST_WEEK', 'Top 7D')
    _RANGE = {
        '24hr': Clip('LAST_DAY', 'Top 24H'),
        '7d': _DEFAULT_CLIP,
        '30d': Clip('LAST_MONTH', 'Top 30D'),
        'all': Clip('ALL_TIME', 'Top All'),
    }

    # NB: values other than 20 result in skipped videos
    _PAGE_LIMIT = 20

    _OPERATION_NAME = 'ClipsCards__User'
    _ENTRY_KIND = 'clip'
    _EDGE_KIND = 'ClipEdge'
    _NODE_KIND = 'Clip'

    @staticmethod
    def _make_variables(channel_name, filter):
        return {
            'login': channel_name,
            'criteria': {
                'filter': filter,
            },
        }

    @staticmethod
    def _extract_entry(node):
        assert isinstance(node, dict)
        clip_url = url_or_none(node.get('url'))
        if not clip_url:
            return
        return {
            '_type': 'url_transparent',
            'ie_key': TwitchClipsIE.ie_key(),
            'id': node.get('id'),
            'url': clip_url,
            'title': node.get('title'),
            'thumbnail': node.get('thumbnailURL'),
            'duration': float_or_none(node.get('durationSeconds')),
            'timestamp': unified_timestamp(node.get('createdAt')),
            'view_count': int_or_none(node.get('viewCount')),
            'language': node.get('language'),
        }

    def _real_extract(self, url):
        channel_name = self._match_id(url)
        qs = compat_urlparse.parse_qs(compat_urlparse.urlparse(url).query)
        range = qs.get('range', ['7d'])[0]
        clip = self._RANGE.get(range, self._DEFAULT_CLIP)
        return self.playlist_result(
            self._entries(channel_name, clip.filter),
            playlist_id=channel_name,
            playlist_title='%s - Clips %s' % (channel_name, clip.label))

class TwitchVideosCollectionsIE(TwitchPlaylistBaseIE):
    _VALID_URL = r'https?://(?:(?:www|go|m)\.)?twitch\.tv/(?P<id>[^/]+)/videos/*?\?.*?\bfilter=collections'

    _TESTS = [{
        # Collections
        'url': 'https://www.twitch.tv/spamfish/videos?filter=collections',
        'info_dict': {
            'id': 'spamfish',
            'title': 'spamfish - Collections',
        },
        'playlist_mincount': 3,
    }]

    _OPERATION_NAME = 'ChannelCollectionsContent'
    _ENTRY_KIND = 'collection'
    _EDGE_KIND = 'CollectionsItemEdge'
    _NODE_KIND = 'Collection'

    @staticmethod
    def _make_variables(channel_name):
        return {
            'ownerLogin': channel_name,
        }

    @staticmethod
    def _extract_entry(node):
        assert isinstance(node, dict)
        collection_id = node.get('id')
        if not collection_id:
            return
        return {
            '_type': 'url_transparent',
            'ie_key': TwitchCollectionIE.ie_key(),
            'id': collection_id,
            'url': 'https://www.twitch.tv/collections/%s' % collection_id,
            'title': node.get('title'),
            'thumbnail': node.get('thumbnailURL'),
            'duration': float_or_none(node.get('lengthSeconds')),
            'timestamp': unified_timestamp(node.get('updatedAt')),
            'view_count': int_or_none(node.get('viewCount')),
        }

    def _real_extract(self, url):
        channel_name = self._match_id(url)
        return self.playlist_result(
            self._entries(channel_name), playlist_id=channel_name,
            playlist_title='%s - Collections' % channel_name)

class TwitchStreamIE(TwitchGraphQLBaseIE):
    IE_NAME = 'twitch:stream'
    _VALID_URL = r'''(?x)
                    https?://
                        (?:
                            (?:(?:www|go|m)\.)?twitch\.tv/|
                            player\.twitch\.tv/\?.*?\bchannel=
                        )
                        (?P<id>[^/#?]+)
                    '''

    _TESTS = [{
        'url': 'http://www.twitch.tv/shroomztv',
        'info_dict': {
            'id': '12772022048',
            'display_id': 'shroomztv',
            'ext': 'mp4',
            'title': 're:^ShroomzTV [0-9]{4}-[0-9]{2}-[0-9]{2} [0-9]{2}:[0-9]{2}$',
            'description': 'H1Z1 - lonewolfing with ShroomzTV | A3 Battle Royale later - @ShroomzTV',
            'is_live': True,
            'timestamp': 1421928037,
            'upload_date': '20150122',
            'uploader': 'ShroomzTV',
            'uploader_id': 'shroomztv',
            'view_count': int,
        },
        'params': {
            # m3u8 download
            'skip_download': True,
        },
    }, {
        'url': 'http://www.twitch.tv/miracle_doto#profile-0',
        'only_matching': True,
    }, {
        'url': 'https://player.twitch.tv/?channel=lotsofs',
        'only_matching': True,
    }, {
        'url': 'https://go.twitch.tv/food',
        'only_matching': True,
    }, {
        'url': 'https://m.twitch.tv/food',
        'only_matching': True,
    }]

    @classmethod
    def suitable(cls, url):
        return (False
                if any(ie.suitable(url) for ie in (
                    TwitchVodIE,
                    TwitchCollectionIE,
                    TwitchVideosIE,
                    TwitchVideosClipsIE,
                    TwitchVideosCollectionsIE,
                    TwitchClipsIE))
                else super(TwitchStreamIE, cls).suitable(url))

    def _real_extract(self, url):
        channel_name = self._match_id(url).lower()

        gql = self._download_gql(
            channel_name, [{
                'operationName': 'StreamMetadata',
                'variables': {'channelLogin': channel_name},
            }, {
                'operationName': 'ComscoreStreamingQuery',
                'variables': {
                    'channel': channel_name,
                    'clipSlug': '',
                    'isClip': False,
                    'isLive': True,
                    'isVodOrCollection': False,
                    'vodID': '',
                },
            }, {
                'operationName': 'VideoPreviewOverlay',
                'variables': {'login': channel_name},
            }],
            'Downloading stream GraphQL')

        user = gql[0]['data']['user']

        if not user:
            raise ExtractorError(
                '%s does not exist' % channel_name, expected=True)

        stream = user['stream']

        if not stream:
            raise ExtractorError('%s is offline' % channel_name, expected=True)

        access_token = self._download_access_token(channel_name)
        token = access_token['token']

        stream_id = stream.get('id') or channel_name
        query = {
            'allow_source': 'true',
            'allow_audio_only': 'true',
            'allow_spectre': 'true',
            'p': random.randint(1000000, 10000000),
            'player': 'twitchweb',
            'playlist_include_framerate': 'true',
            'segment_preference': '4',
            'sig': access_token['sig'].encode('utf-8'),
            'token': token.encode('utf-8'),
        }
        formats = self._extract_m3u8_formats(
            '%s/api/channel/hls/%s.m3u8' % (self._USHER_BASE, channel_name),
            stream_id, 'mp4', query=query)
        self._prefer_source(formats)

        view_count = stream.get('viewers')
        timestamp = unified_timestamp(stream.get('createdAt'))

        sq_user = try_get(gql, lambda x: x[1]['data']['user'], dict) or {}
        uploader = sq_user.get('displayName')
        description = try_get(
            sq_user, lambda x: x['broadcastSettings']['title'], compat_str)

        thumbnail = url_or_none(try_get(
            gql, lambda x: x[2]['data']['user']['stream']['previewImageURL'],
            compat_str))

        title = uploader or channel_name
        stream_type = stream.get('type')
        if stream_type in ['rerun', 'live']:
            title += ' (%s)' % stream_type

        return {
            'id': stream_id,
            'display_id': channel_name,
            'title': self._live_title(title),
            'description': description,
            'thumbnail': thumbnail,
            'uploader': uploader,
            'uploader_id': channel_name,
            'timestamp': timestamp,
            'view_count': view_count,
            'formats': formats,
            'is_live': stream_type == 'live',
        }

class TwitchClipsIE(TwitchBaseIE):
    IE_NAME = 'twitch:clips'
    _VALID_URL = r'''(?x)
                    https?://
                        (?:
                            clips\.twitch\.tv/(?:embed\?.*?\bclip=|(?:[^/]+/)*)|
                            (?:(?:www|go|m)\.)?twitch\.tv/[^/]+/clip/
                        )
                        (?P<id>[^/?#&]+)
                    '''

    _TESTS = [{
        'url': 'https://clips.twitch.tv/FaintLightGullWholeWheat',
        'md5': '761769e1eafce0ffebfb4089cb3847cd',
        'info_dict': {
            'id': '42850523',
            'ext': 'mp4',
            'title': 'EA Play 2016 Live from the Novo Theatre',
            'thumbnail': r're:^https?://.*\.jpg',
            'timestamp': 1465767393,
            'upload_date': '20160612',
            'creator': 'EA',
            'uploader': 'stereotype_',
            'uploader_id': '43566419',
        },
    }, {
        # multiple formats
        'url': 'https://clips.twitch.tv/rflegendary/UninterestedBeeDAESuppy',
        'only_matching': True,
    }, {
        'url': 'https://www.twitch.tv/sergeynixon/clip/StormyThankfulSproutFutureMan',
        'only_matching': True,
    }, {
        'url': 'https://clips.twitch.tv/embed?clip=InquisitiveBreakableYogurtJebaited',
        'only_matching': True,
    }, {
        'url': 'https://m.twitch.tv/rossbroadcast/clip/ConfidentBraveHumanChefFrank',
        'only_matching': True,
    }, {
        'url': 'https://go.twitch.tv/rossbroadcast/clip/ConfidentBraveHumanChefFrank',
        'only_matching': True,
    }]

    def _real_extract(self, url):
        video_id = self._match_id(url)

        clip = self._download_json(
            'https://gql.twitch.tv/gql', video_id, data=json.dumps({
                'query': '''{
  clip(slug: "%s") {
    broadcaster {
      displayName
    }
    createdAt
    curator {
      displayName
      id
    }
    durationSeconds
    id
    tiny: thumbnailURL(width: 86, height: 45)
    small: thumbnailURL(width: 260, height: 147)
    medium: thumbnailURL(width: 480, height: 272)
    title
    videoQualities {
      frameRate
      quality
      sourceURL
    }
    viewCount
  }
}''' % video_id,
            }).encode(), headers={
                'Client-ID': self._CLIENT_ID,
            })['data']['clip']

        if not clip:
            raise ExtractorError(
                'This clip is no longer available', expected=True)

        formats = []
        for option in clip.get('videoQualities', []):
            if not isinstance(option, dict):
                continue
            source = url_or_none(option.get('sourceURL'))
            if not source:
                continue
            formats.append({
                'url': source,
                'format_id': option.get('quality'),
                'height': int_or_none(option.get('quality')),
                'fps': int_or_none(option.get('frameRate')),
            })
        self._sort_formats(formats)

        thumbnails = []
        for thumbnail_id in ('tiny', 'small', 'medium'):
            thumbnail_url = clip.get(thumbnail_id)
            if not thumbnail_url:
                continue
            thumb = {
                'id': thumbnail_id,
                'url': thumbnail_url,
            }
            mobj = re.search(r'-(\d+)x(\d+)\.', thumbnail_url)
            if mobj:
                thumb.update({
                    'height': int(mobj.group(2)),
                    'width': int(mobj.group(1)),
                })
            thumbnails.append(thumb)

        return {
            'id': clip.get('id') or video_id,
            'title': clip.get('title') or video_id,
            'formats': formats,
            'duration': int_or_none(clip.get('durationSeconds')),
            'views': int_or_none(clip.get('viewCount')),
            'timestamp': unified_timestamp(clip.get('createdAt')),
            'thumbnails': thumbnails,
            'creator': try_get(clip, lambda x: x['broadcaster']['displayName'], compat_str),
            'uploader': try_get(clip, lambda x: x['curator']['displayName'], compat_str),
            'uploader_id': try_get(clip, lambda x: x['curator']['id'], compat_str),
        }


code:
# coding: utf-8
from __future__ import unicode_literals

import re

from .common import InfoExtractor
from ..compat import (
    compat_HTTPError,
    compat_parse_qs,
    compat_urllib_parse_unquote,
    compat_urllib_parse_urlparse,
)
from ..utils import (
    dict_get,
    ExtractorError,
    float_or_none,
    int_or_none,
    try_get,
    strip_or_none,
    unified_timestamp,
    update_url_query,
    xpath_text,
)

from .periscope import (
    PeriscopeBaseIE,
    PeriscopeIE,
)

class TwitterBaseIE(InfoExtractor):
    _API_BASE = 'https://api.twitter.com/1.1/'
    _BASE_REGEX = r'https?://(?:(?:www|m(?:obile)?)\.)?twitter\.com/'
    _GUEST_TOKEN = None

    def _extract_variant_formats(self, variant, video_id):
        variant_url = variant.get('url')
        if not variant_url:
            return []
        elif '.m3u8' in variant_url:
            return self._extract_m3u8_formats(
                variant_url, video_id, 'mp4', 'm3u8_native',
                m3u8_id='hls', fatal=False)
        else:
            tbr = int_or_none(dict_get(variant, ('bitrate', 'bit_rate')), 1000) or None
            f = {
                'url': variant_url,
                'format_id': 'http' + ('-%d' % tbr if tbr else ''),
                'tbr': tbr,
            }
            self._search_dimensions_in_video_url(f, variant_url)
            return [f]

    def _extract_formats_from_vmap_url(self, vmap_url, video_id):
        vmap_data = self._download_xml(vmap_url, video_id)
        formats = []
        urls = []
        for video_variant in vmap_data.findall('.//{http://twitter.com/schema/videoVMapV2.xsd}videoVariant'):
            video_variant.attrib['url'] = compat_urllib_parse_unquote(
                video_variant.attrib['url'])
            urls.append(video_variant.attrib['url'])
            formats.extend(self._extract_variant_formats(
                video_variant.attrib, video_id))
        video_url = strip_or_none(xpath_text(vmap_data, './/MediaFile'))
        if video_url not in urls:
            formats.extend(self._extract_variant_formats({'url': video_url}, video_id))
        return formats

    @staticmethod
    def _search_dimensions_in_video_url(a_format, video_url):
        m = re.search(r'/(?P<width>\d+)x(?P<height>\d+)/', video_url)
        if m:
            a_format.update({
                'width': int(m.group('width')),
                'height': int(m.group('height')),
            })

    def _call_api(self, path, video_id, query={}):
        headers = {
            'Authorization': 'Bearer AAAAAAAAAAAAAAAAAAAAAPYXBAAAAAAACLXUNDekMxqa8h%2F40K4moUkGsoc%3DTYfbDKbT3jJPCEVnMYqilB28NHfOPqkca3qaAxGfsyKCs0wRbw',
        }
        if not self._GUEST_TOKEN:
            self._GUEST_TOKEN = self._download_json(
                self._API_BASE + 'guest/activate.json', video_id,
                'Downloading guest token', data=b'',
                headers=headers)['guest_token']
        headers['x-guest-token'] = self._GUEST_TOKEN
        try:
            return self._download_json(
                self._API_BASE + path, video_id, headers=headers, query=query)
        except ExtractorError as e:
            if isinstance(e.cause, compat_HTTPError) and e.cause.code == 403:
                raise ExtractorError(self._parse_json(
                    e.cause.read().decode(),
                    video_id)['errors'][0]['message'], expected=True)
            raise

class TwitterCardIE(InfoExtractor):
    IE_NAME = 'twitter:card'
    _VALID_URL = TwitterBaseIE._BASE_REGEX + r'i/(?:cards/tfw/v1|videos(?:/tweet)?)/(?P<id>\d+)'
    _TESTS = [
        {
            'url': 'https://twitter.com/i/cards/tfw/v1/560070183650213889',
            # MD5 checksums are different in different places
            'info_dict': {
                'id': '560070183650213889',
                'ext': 'mp4',
                'title': "Twitter - You can now shoot, edit and share video on Twitter. Capture life's most moving moments from your perspective.",
                'description': 'md5:18d3e24bb4f6e5007487dd546e53bd96',
                'uploader': 'Twitter',
                'uploader_id': 'Twitter',
                'thumbnail': r're:^https?://.*\.jpg',
                'duration': 30.033,
                'timestamp': 1422366112,
                'upload_date': '20150127',
            },
        },
        {
            'url': 'https://twitter.com/i/cards/tfw/v1/623160978427936768',
            'md5': '7137eca597f72b9abbe61e5ae0161399',
            'info_dict': {
                'id': '623160978427936768',
                'ext': 'mp4',
                'title': "NASA - Fly over Pluto's icy Norgay Mountains and Sputnik Plain in this @NASANewHorizons #PlutoFlyby video.",
                'description': "Fly over Pluto's icy Norgay Mountains and Sputnik Plain in this @NASANewHorizons #PlutoFlyby video. https://t.co/BJYgOjSeGA",
                'uploader': 'NASA',
                'uploader_id': 'NASA',
                'timestamp': 1437408129,
                'upload_date': '20150720',
            },
        },
        {
            'url': 'https://twitter.com/i/cards/tfw/v1/654001591733886977',
            'md5': 'b6d9683dd3f48e340ded81c0e917ad46',
            'info_dict': {
                'id': 'dq4Oj5quskI',
                'ext': 'mp4',
                'title': 'Ubuntu 11.10 Overview',
                'description': 'md5:a831e97fa384863d6e26ce48d1c43376',
                'upload_date': '20111013',
                'uploader': 'OMG! UBUNTU!',
                'uploader_id': 'omgubuntu',
            },
            'add_ie': ['Youtube'],
        },
        {
            'url': 'https://twitter.com/i/cards/tfw/v1/665289828897005568',
            'md5': '6dabeaca9e68cbb71c99c322a4b42a11',
            'info_dict': {
                'id': 'iBb2x00UVlv',
                'ext': 'mp4',
                'upload_date': '20151113',
                'uploader_id': '1189339351084113920',
                'uploader': 'ArsenalTerje',
                'title': 'Vine by ArsenalTerje',
                'timestamp': 1447451307,
            },
            'add_ie': ['Vine'],
        }, {
            'url': 'https://twitter.com/i/videos/tweet/705235433198714880',
            'md5': '884812a2adc8aaf6fe52b15ccbfa3b88',
            'info_dict': {
                'id': '705235433198714880',
                'ext': 'mp4',
                'title': "Brent Yarina - Khalil Iverson's missed highlight dunk. And made highlight dunk. In one highlight.",
                'description': "Khalil Iverson's missed highlight dunk. And made highlight dunk. In one highlight. https://t.co/OrxcJ28Bns",
                'uploader': 'Brent Yarina',
                'uploader_id': 'BTNBrentYarina',
                'timestamp': 1456976204,
                'upload_date': '20160303',
            },
            'skip': 'This content is no longer available.',
        }, {
            'url': 'https://twitter.com/i/videos/752274308186120192',
            'only_matching': True,
        },
    ]

    def _real_extract(self, url):
        status_id = self._match_id(url)
        return self.url_result(
            'https://twitter.com/statuses/' + status_id,
            TwitterIE.ie_key(), status_id)

class TwitterIE(TwitterBaseIE):
    IE_NAME = 'twitter'
    _VALID_URL = TwitterBaseIE._BASE_REGEX + r'(?:(?:i/web|[^/]+)/status|statuses)/(?P<id>\d+)'

    _TESTS = [{
        'url': 'https://twitter.com/freethenipple/status/643211948184596480',
        'info_dict': {
            'id': '643211948184596480',
            'ext': 'mp4',
            'title': 'FREE THE NIPPLE - FTN supporters on Hollywood Blvd today!',
            'thumbnail': r're:^https?://.*\.jpg',
            'description': 'FTN supporters on Hollywood Blvd today! http://t.co/c7jHH749xJ',
            'uploader': 'FREE THE NIPPLE',
            'uploader_id': 'freethenipple',
            'duration': 12.922,
            'timestamp': 1442188653,
            'upload_date': '20150913',
            'age_limit': 18,
        },
    }, {
        'url': 'https://twitter.com/giphz/status/657991469417025536/photo/1',
        'md5': 'f36dcd5fb92bf7057f155e7d927eeb42',
        'info_dict': {
            'id': '657991469417025536',
            'ext': 'mp4',
            'title': 'Gifs - tu vai cai tu vai cai tu nao eh capaz disso tu vai cai',
            'description': 'Gifs on Twitter: "tu vai cai tu vai cai tu nao eh capaz disso tu vai cai https://t.co/tM46VHFlO5"',
            'thumbnail': r're:^https?://.*\.png',
            'uploader': 'Gifs',
            'uploader_id': 'giphz',
        },
        'expected_warnings': ['height', 'width'],
        'skip': 'Account suspended',
    }, {
        'url': 'https://twitter.com/starwars/status/665052190608723968',
        'info_dict': {
            'id': '665052190608723968',
            'ext': 'mp4',
            'title': 'Star Wars - A new beginning is coming December 18. Watch the official 60 second #TV spot for #StarWars: #TheForceAwakens.',
            'description': 'A new beginning is coming December 18. Watch the official 60 second #TV spot for #StarWars: #TheForceAwakens. https://t.co/OkSqT2fjWJ',
            'uploader_id': 'starwars',
            'uploader': 'Star Wars',
            'timestamp': 1447395772,
            'upload_date': '20151113',
        },
    }, {
        'url': 'https://twitter.com/BTNBrentYarina/status/705235433198714880',
        'info_dict': {
            'id': '705235433198714880',
            'ext': 'mp4',
            'title': "Brent Yarina - Khalil Iverson's missed highlight dunk. And made highlight dunk. In one highlight.",
            'description': "Khalil Iverson's missed highlight dunk. And made highlight dunk. In one highlight. https://t.co/OrxcJ28Bns",
            'uploader_id': 'BTNBrentYarina',
            'uploader': 'Brent Yarina',
            'timestamp': 1456976204,
            'upload_date': '20160303',
        },
        'params': {
            # The same video as https://twitter.com/i/videos/tweet/705235433198714880
            # Test case of TwitterCardIE
            'skip_download': True,
        },
    }, {
        'url': 'https://twitter.com/jaydingeer/status/700207533655363584',
        'info_dict': {
            'id': '700207533655363584',
            'ext': 'mp4',
            'title': 'simon vetugo - BEAT PROD: @suhmeduh #Damndaniel',
            'description': 'BEAT PROD: @suhmeduh  https://t.co/HBrQ4AfpvZ #Damndaniel https://t.co/byBooq2ejZ',
            'thumbnail': r're:^https?://.*\.jpg',
            'uploader': 'simon vetugo',
            'uploader_id': 'simonvertugo',
            'duration': 30.0,
            'timestamp': 1455777459,
            'upload_date': '20160218',
        },
    }, {
        'url': 'https://twitter.com/Filmdrunk/status/713801302971588609',
        'md5': '89a15ed345d13b86e9a5a5e051fa308a',
        'info_dict': {
            'id': 'MIOxnrUteUd',
            'ext': 'mp4',
            'title': 'Dr.Pepperの飲み方 #japanese #バカ #ドクペ #電動ガン',
            'uploader': 'TAKUMA',
            'uploader_id': '1004126642786242560',
            'timestamp': 1402826626,
            'upload_date': '20140615',
        },
        'add_ie': ['Vine'],
    }, {
        'url': 'https://twitter.com/captainamerica/status/719944021058060289',
        'info_dict': {
            'id': '719944021058060289',
            'ext': 'mp4',
            'title': 'Captain America - @King0fNerd Are you sure you made the right choice? Find out in theaters.',
            'description': '@King0fNerd Are you sure you made the right choice? Find out in theaters. https://t.co/GpgYi9xMJI',
            'uploader_id': 'CaptainAmerica',
            'uploader': 'Captain America',
            'duration': 3.17,
            'timestamp': 1460483005,
            'upload_date': '20160412',
        },
    }, {
        'url': 'https://twitter.com/OPP_HSD/status/779210622571536384',
        'info_dict': {
            'id': '1zqKVVlkqLaKB',
            'ext': 'mp4',
            'title': 'Sgt Kerry Schmidt - Ontario Provincial Police - Road rage, mischief, assault, rollover and fire in one occurrence',
            'upload_date': '20160923',
            'uploader_id': '1PmKqpJdOJQoY',
            'uploader': 'Sgt Kerry Schmidt - Ontario Provincial Police',
            'timestamp': 1474613214,
        },
        'add_ie': ['Periscope'],
    }, {
        # has mp4 formats via mobile API
        'url': 'https://twitter.com/news_al3alm/status/852138619213144067',
        'info_dict': {
            'id': '852138619213144067',
            'ext': 'mp4',
            'title': 'عالم الأخبار - كلمة تاريخية بجلسة الجناسي التاريخية.. النائب خالد مؤنس العتيبي للمعارضين : اتقوا الله .. الظلم ظلمات يوم القيامة',
            'description': 'كلمة تاريخية بجلسة الجناسي التاريخية.. النائب خالد مؤنس العتيبي للمعارضين : اتقوا الله .. الظلم ظلمات يوم القيامة   https://t.co/xg6OhpyKfN',
            'uploader': 'عالم الأخبار',
            'uploader_id': 'news_al3alm',
            'duration': 277.4,
            'timestamp': 1492000653,
            'upload_date': '20170412',
        },
    }, {
        'url': 'https://twitter.com/i/web/status/910031516746514432',
        'info_dict': {
            'id': '910031516746514432',
            'ext': 'mp4',
            'title': 'Préfet de Guadeloupe - [Direct] #Maria Le centre se trouve actuellement au sud de Basse-Terre. Restez confinés. Réfugiez-vous dans la pièce la + sûre.',
            'thumbnail': r're:^https?://.*\.jpg',
            'description': '[Direct] #Maria Le centre se trouve actuellement au sud de Basse-Terre. Restez confinés. Réfugiez-vous dans la pièce la + sûre. https://t.co/mwx01Rs4lo',
            'uploader': 'Préfet de Guadeloupe',
            'uploader_id': 'Prefet971',
            'duration': 47.48,
            'timestamp': 1505803395,
            'upload_date': '20170919',
        },
        'params': {
            'skip_download': True,  # requires ffmpeg
        },
    }, {
        # card via api.twitter.com/1.1/videos/tweet/config
        'url': 'https://twitter.com/LisPower1/status/1001551623938805763',
        'info_dict': {
            'id': '1001551623938805763',
            'ext': 'mp4',
            'title': 're:.*?Shep is on a roll today.*?',
            'thumbnail': r're:^https?://.*\.jpg',
            'description': 'md5:37b9f2ff31720cef23b2bd42ee8a0f09',
            'uploader': 'Lis Power',
            'uploader_id': 'LisPower1',
            'duration': 111.278,
            'timestamp': 1527623489,
            'upload_date': '20180529',
        },
        'params': {
            'skip_download': True,  # requires ffmpeg
        },
    }, {
        'url': 'https://twitter.com/foobar/status/1087791357756956680',
        'info_dict': {
            'id': '1087791357756956680',
            'ext': 'mp4',
            'title': 'Twitter - A new is coming.  Some of you got an opt-in to try it now. Check out the emoji button, quick keyboard shortcuts, upgraded trends, advanced search, and more. Let us know your thoughts!',
            'thumbnail': r're:^https?://.*\.jpg',
            'description': 'md5:6dfd341a3310fb97d80d2bf7145df976',
            'uploader': 'Twitter',
            'uploader_id': 'Twitter',
            'duration': 61.567,
            'timestamp': 1548184644,
            'upload_date': '20190122',
        },
    }, {
        # not available in Periscope
        'url': 'https://twitter.com/ViviEducation/status/1136534865145286656',
        'info_dict': {
            'id': '1vOGwqejwoWxB',
            'ext': 'mp4',
            'title': 'Vivi - Vivi founder @lior_rauchy announcing our new student feedback tool live at @EduTECH_AU #EduTECH2019',
            'uploader': 'Vivi',
            'uploader_id': '1eVjYOLGkGrQL',
        },
        'add_ie': ['TwitterBroadcast'],
    }, {
        # Twitch Clip Embed
        'url': 'https://twitter.com/GunB1g/status/1163218564784017422',
        'only_matching': True,
    }, {
        # promo_video_website card
        'url': 'https://twitter.com/GunB1g/status/1163218564784017422',
        'only_matching': True,
    }]

    def _real_extract(self, url):
        twid = self._match_id(url)
        status = self._call_api(
            'statuses/show/%s.json' % twid, twid, {
                'cards_platform': 'Web-12',
                'include_cards': 1,
                'include_reply_count': 1,
                'include_user_entities': 0,
                'tweet_mode': 'extended',
            })

        title = description = status['full_text'].replace('\n', ' ')
        # strip  'https -_t.co_BJYgOjSeGA' junk from filenames
        title = re.sub(r'\s+(https?://[^ ]+)', '', title)
        user = status.get('user') or {}
        uploader = user.get('name')
        if uploader:
            title = '%s - %s' % (uploader, title)
        uploader_id = user.get('screen_name')

        tags = []
        for hashtag in (try_get(status, lambda x: x['entities']['hashtags'], list) or []):
            hashtag_text = hashtag.get('text')
            if not hashtag_text:
                continue
            tags.append(hashtag_text)

        info = {
            'id': twid,
            'title': title,
            'description': description,
            'uploader': uploader,
            'timestamp': unified_timestamp(status.get('created_at')),
            'uploader_id': uploader_id,
            'uploader_url': 'https://twitter.com/' + uploader_id if uploader_id else None,
            'like_count': int_or_none(status.get('favorite_count')),
            'repost_count': int_or_none(status.get('retweet_count')),
            'comment_count': int_or_none(status.get('reply_count')),
            'age_limit': 18 if status.get('possibly_sensitive') else 0,
            'tags': tags,
        }

        media = try_get(status, lambda x: x['extended_entities']['media'][0])
        if media and media.get('type') != 'photo':
            video_info = media.get('video_info') or {}

            formats = []
            for variant in video_info.get('variants', []):
                formats.extend(self._extract_variant_formats(variant, twid))
            self._sort_formats(formats)

            thumbnails = []
            media_url = media.get('media_url_https') or media.get('media_url')
            if media_url:
                def add_thumbnail(name, size):
                    thumbnails.append({
                        'id': name,
                        'url': update_url_query(media_url, {'name': name}),
                        'width': int_or_none(size.get('w') or size.get('width')),
                        'height': int_or_none(size.get('h') or size.get('height')),
                    })
                for name, size in media.get('sizes', {}).items():
                    add_thumbnail(name, size)
                add_thumbnail('orig', media.get('original_info') or {})

            info.update({
                'formats': formats,
                'thumbnails': thumbnails,
                'duration': float_or_none(video_info.get('duration_millis'), 1000),
            })
        else:
            card = status.get('card')
            if card:
                binding_values = card['binding_values']

                def get_binding_value(k):
                    o = binding_values.get(k) or {}
                    return try_get(o, lambda x: x[x['type'].lower() + '_value'])

                card_name = card['name'].split(':')[-1]
                if card_name in ('amplify', 'promo_video_website'):
                    is_amplify = card_name == 'amplify'
                    vmap_url = get_binding_value('amplify_url_vmap') if is_amplify else get_binding_value('player_stream_url')
                    content_id = get_binding_value('%s_content_id' % (card_name if is_amplify else 'player'))
                    formats = self._extract_formats_from_vmap_url(vmap_url, content_id or twid)
                    self._sort_formats(formats)

                    thumbnails = []
                    for suffix in ('_small', '', '_large', '_x_large', '_original'):
                        image = get_binding_value('player_image' + suffix) or {}
                        image_url = image.get('url')
                        if not image_url or '/player-placeholder' in image_url:
                            continue
                        thumbnails.append({
                            'id': suffix[1:] if suffix else 'medium',
                            'url': image_url,
                            'width': int_or_none(image.get('width')),
                            'height': int_or_none(image.get('height')),
                        })

                    info.update({
                        'formats': formats,
                        'thumbnails': thumbnails,
                        'duration': int_or_none(get_binding_value(
                            'content_duration_seconds')),
                    })
                elif card_name == 'player':
                    info.update({
                        '_type': 'url',
                        'url': get_binding_value('player_url'),
                    })
                elif card_name == 'periscope_broadcast':
                    info.update({
                        '_type': 'url',
                        'url': get_binding_value('url') or get_binding_value('player_url'),
                        'ie_key': PeriscopeIE.ie_key(),
                    })
                elif card_name == 'broadcast':
                    info.update({
                        '_type': 'url',
                        'url': get_binding_value('broadcast_url'),
                        'ie_key': TwitterBroadcastIE.ie_key(),
                    })
                else:
                    raise ExtractorError('Unsupported Twitter Card.')
            else:
                expanded_url = try_get(status, lambda x: x['entities']['urls'][0]['expanded_url'])
                if not expanded_url:
                    raise ExtractorError("There's no video in this tweet.")
                info.update({
                    '_type': 'url',
                    'url': expanded_url,
                })
        return info

class TwitterAmplifyIE(TwitterBaseIE):
    IE_NAME = 'twitter:amplify'
    _VALID_URL = r'https?://amp\.twimg\.com/v/(?P<id>[0-9a-f\-]{36})'

    _TEST = {
        'url': 'https://amp.twimg.com/v/0ba0c3c7-0af3-4c0a-bed5-7efd1ffa2951',
        'md5': '7df102d0b9fd7066b86f3159f8e81bf6',
        'info_dict': {
            'id': '0ba0c3c7-0af3-4c0a-bed5-7efd1ffa2951',
            'ext': 'mp4',
            'title': 'Twitter Video',
            'thumbnail': 're:^https?://.*',
        },
    }

    def _real_extract(self, url):
        video_id = self._match_id(url)
        webpage = self._download_webpage(url, video_id)

        vmap_url = self._html_search_meta(
            'twitter:amplify:vmap', webpage, 'vmap url')
        formats = self._extract_formats_from_vmap_url(vmap_url, video_id)

        thumbnails = []
        thumbnail = self._html_search_meta(
            'twitter:image:src', webpage, 'thumbnail', fatal=False)

        def _find_dimension(target):
            w = int_or_none(self._html_search_meta(
                'twitter:%s:width' % target, webpage, fatal=False))
            h = int_or_none(self._html_search_meta(
                'twitter:%s:height' % target, webpage, fatal=False))
            return w, h

        if thumbnail:
            thumbnail_w, thumbnail_h = _find_dimension('image')
            thumbnails.append({
                'url': thumbnail,
                'width': thumbnail_w,
                'height': thumbnail_h,
            })

        video_w, video_h = _find_dimension('player')
        formats[0].update({
            'width': video_w,
            'height': video_h,
        })

        return {
            'id': video_id,
            'title': 'Twitter Video',
            'formats': formats,
            'thumbnails': thumbnails,
        }

class TwitterBroadcastIE(TwitterBaseIE, PeriscopeBaseIE):
    IE_NAME = 'twitter:broadcast'
    _VALID_URL = TwitterBaseIE._BASE_REGEX + r'i/broadcasts/(?P<id>[0-9a-zA-Z]{13})'

    _TEST = {
        # untitled Periscope video
        'url': 'https://twitter.com/i/broadcasts/1yNGaQLWpejGj',
        'info_dict': {
            'id': '1yNGaQLWpejGj',
            'ext': 'mp4',
            'title': 'Andrea May Sahouri - Periscope Broadcast',
            'uploader': 'Andrea May Sahouri',
            'uploader_id': '1PXEdBZWpGwKe',
        },
    }

    def _real_extract(self, url):
        broadcast_id = self._match_id(url)
        broadcast = self._call_api(
            'broadcasts/show.json', broadcast_id,
            {'ids': broadcast_id})['broadcasts'][broadcast_id]
        info = self._parse_broadcast_data(broadcast, broadcast_id)
        media_key = broadcast['media_key']
        source = self._call_api(
            'live_video_stream/status/' + media_key, media_key)['source']
        m3u8_url = source.get('noRedirectPlaybackUrl') or source['location']
        if '/live_video_stream/geoblocked/' in m3u8_url:
            self.raise_geo_restricted()
        m3u8_id = compat_parse_qs(compat_urllib_parse_urlparse(
            m3u8_url).query).get('type', [None])[0]
        state, width, height = self._extract_common_format_info(broadcast)
        info['formats'] = self._extract_pscp_m3u8_formats(
            m3u8_url, broadcast_id, m3u8_id, state, width, height)
        return info


code:
from __future__ import unicode_literals

import re

from .common import InfoExtractor
from ..compat import (
    compat_HTTPError,
    compat_kwargs,
    compat_str,
    compat_urllib_request,
    compat_urlparse,
)
from ..utils import (
    determine_ext,
    extract_attributes,
    ExtractorError,
    float_or_none,
    int_or_none,
    js_to_json,
    sanitized_Request,
    try_get,
    unescapeHTML,
    url_or_none,
    urlencode_postdata,
)

class UdemyIE(InfoExtractor):
    IE_NAME = 'udemy'
    _VALID_URL = r'''(?x)
                    https?://
                        (?:[^/]+\.)?udemy\.com/
                        (?:
                            [^#]+\#/lecture/|
                            lecture/view/?\?lectureId=|
                            [^/]+/learn/v4/t/lecture/
                        )
                        (?P<id>\d+)
                    '''
    _LOGIN_URL = 'https://www.udemy.com/join/login-popup/?displayType=ajax&showSkipButton=1'
    _ORIGIN_URL = 'https://www.udemy.com'
    _NETRC_MACHINE = 'udemy'

    _TESTS = [{
        'url': 'https://www.udemy.com/java-tutorial/#/lecture/172757',
        'md5': '98eda5b657e752cf945d8445e261b5c5',
        'info_dict': {
            'id': '160614',
            'ext': 'mp4',
            'title': 'Introduction and Installation',
            'description': 'md5:c0d51f6f21ef4ec65f091055a5eef876',
            'duration': 579.29,
        },
        'skip': 'Requires udemy account credentials',
    }, {
        # new URL schema
        'url': 'https://www.udemy.com/electric-bass-right-from-the-start/learn/v4/t/lecture/4580906',
        'only_matching': True,
    }, {
        # no url in outputs format entry
        'url': 'https://www.udemy.com/learn-web-development-complete-step-by-step-guide-to-success/learn/v4/t/lecture/4125812',
        'only_matching': True,
    }, {
        # only outputs rendition
        'url': 'https://www.udemy.com/how-you-can-help-your-local-community-5-amazing-examples/learn/v4/t/lecture/3225750?start=0',
        'only_matching': True,
    }, {
        'url': 'https://wipro.udemy.com/java-tutorial/#/lecture/172757',
        'only_matching': True,
    }]

    def _extract_course_info(self, webpage, video_id):
        course = self._parse_json(
            unescapeHTML(self._search_regex(
                r'ng-init=["\'].*\bcourse=({.+?})[;"\']',
                webpage, 'course', default='{}')),
            video_id, fatal=False) or {}
        course_id = course.get('id') or self._search_regex(
            [
                r'data-course-id=["\'](\d+)',
                r'"courseId"\s*:\s*(\d+)'
            ], webpage, 'course id')
        return course_id, course.get('title')

    def _enroll_course(self, base_url, webpage, course_id):
        def combine_url(base_url, url):
            return compat_urlparse.urljoin(base_url, url) if not url.startswith('http') else url

        checkout_url = unescapeHTML(self._search_regex(
            r'href=(["\'])(?P<url>(?:https?://(?:www\.)?udemy\.com)?/(?:payment|cart)/checkout/.+?)\1',
            webpage, 'checkout url', group='url', default=None))
        if checkout_url:
            raise ExtractorError(
                'Course %s is not free. You have to pay for it before you can download. '
                'Use this URL to confirm purchase: %s'
                % (course_id, combine_url(base_url, checkout_url)),
                expected=True)

        enroll_url = unescapeHTML(self._search_regex(
            r'href=(["\'])(?P<url>(?:https?://(?:www\.)?udemy\.com)?/course/subscribe/.+?)\1',
            webpage, 'enroll url', group='url', default=None))
        if enroll_url:
            webpage = self._download_webpage(
                combine_url(base_url, enroll_url),
                course_id, 'Enrolling in the course',
                headers={'Referer': base_url})
            if '>You have enrolled in' in webpage:
                self.to_screen('%s: Successfully enrolled in the course' % course_id)

    def _download_lecture(self, course_id, lecture_id):
        return self._download_json(
            'https://www.udemy.com/api-2.0/users/me/subscribed-courses/%s/lectures/%s?'
            % (course_id, lecture_id),
            lecture_id, 'Downloading lecture JSON', query={
                'fields[lecture]': 'title,description,view_html,asset',
                'fields[asset]': 'asset_type,stream_url,thumbnail_url,download_urls,stream_urls,captions,data',
            })

    def _handle_error(self, response):
        if not isinstance(response, dict):
            return
        error = response.get('error')
        if error:
            error_str = 'Udemy returned error #%s: %s' % (error.get('code'), error.get('message'))
            error_data = error.get('data')
            if error_data:
                error_str += ' - %s' % error_data.get('formErrors')
            raise ExtractorError(error_str, expected=True)

    def _download_webpage_handle(self, *args, **kwargs):
        headers = kwargs.get('headers', {}).copy()
        headers['User-Agent'] = 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/72.0.3626.109 Safari/537.36'
        kwargs['headers'] = headers
        ret = super(UdemyIE, self)._download_webpage_handle(
            *args, **compat_kwargs(kwargs))
        if not ret:
            return ret
        webpage, _ = ret
        if any(p in webpage for p in (
                '>Please verify you are a human',
                'Access to this page has been denied because we believe you are using automation tools to browse the website',
                '"_pxCaptcha"')):
            raise ExtractorError(
                'Udemy asks you to solve a CAPTCHA. Login with browser, '
                'solve CAPTCHA, then export cookies and pass cookie file to '
                'youtube-dl with --cookies.', expected=True)
        return ret

    def _download_json(self, url_or_request, *args, **kwargs):
        headers = {
            'X-Udemy-Snail-Case': 'true',
            'X-Requested-With': 'XMLHttpRequest',
        }
        for cookie in self._downloader.cookiejar:
            if cookie.name == 'client_id':
                headers['X-Udemy-Client-Id'] = cookie.value
            elif cookie.name == 'access_token':
                headers['X-Udemy-Bearer-Token'] = cookie.value
                headers['X-Udemy-Authorization'] = 'Bearer %s' % cookie.value

        if isinstance(url_or_request, compat_urllib_request.Request):
            for header, value in headers.items():
                url_or_request.add_header(header, value)
        else:
            url_or_request = sanitized_Request(url_or_request, headers=headers)

        response = super(UdemyIE, self)._download_json(url_or_request, *args, **kwargs)
        self._handle_error(response)
        return response

    def _real_initialize(self):
        self._login()

    def _login(self):
        username, password = self._get_login_info()
        if username is None:
            return

        login_popup = self._download_webpage(
            self._LOGIN_URL, None, 'Downloading login popup')

        def is_logged(webpage):
            return any(re.search(p, webpage) for p in (
                r'href=["\'](?:https://www\.udemy\.com)?/user/logout/',
                r'>Logout<'))

        # already logged in
        if is_logged(login_popup):
            return

        login_form = self._form_hidden_inputs('login-form', login_popup)

        login_form.update({
            'email': username,
            'password': password,
        })

        response = self._download_webpage(
            self._LOGIN_URL, None, 'Logging in',
            data=urlencode_postdata(login_form),
            headers={
                'Referer': self._ORIGIN_URL,
                'Origin': self._ORIGIN_URL,
            })

        if not is_logged(response):
            error = self._html_search_regex(
                r'(?s)<div[^>]+class="form-errors[^"]*">(.+?)</div>',
                response, 'error message', default=None)
            if error:
                raise ExtractorError('Unable to login: %s' % error, expected=True)
            raise ExtractorError('Unable to log in')

    def _real_extract(self, url):
        lecture_id = self._match_id(url)

        webpage = self._download_webpage(url, lecture_id)

        course_id, _ = self._extract_course_info(webpage, lecture_id)

        try:
            lecture = self._download_lecture(course_id, lecture_id)
        except ExtractorError as e:
            # Error could possibly mean we are not enrolled in the course
            if isinstance(e.cause, compat_HTTPError) and e.cause.code == 403:
                self._enroll_course(url, webpage, course_id)
                lecture = self._download_lecture(course_id, lecture_id)
            else:
                raise

        title = lecture['title']
        description = lecture.get('description')

        asset = lecture['asset']

        asset_type = asset.get('asset_type') or asset.get('assetType')
        if asset_type != 'Video':
            raise ExtractorError(
                'Lecture %s is not a video' % lecture_id, expected=True)

        stream_url = asset.get('stream_url') or asset.get('streamUrl')
        if stream_url:
            youtube_url = self._search_regex(
                r'(https?://www\.youtube\.com/watch\?v=.*)', stream_url, 'youtube URL', default=None)
            if youtube_url:
                return self.url_result(youtube_url, 'Youtube')

        video_id = compat_str(asset['id'])
        thumbnail = asset.get('thumbnail_url') or asset.get('thumbnailUrl')
        duration = float_or_none(asset.get('data', {}).get('duration'))

        subtitles = {}
        automatic_captions = {}

        formats = []

        def extract_output_format(src, f_id):
            return {
                'url': src.get('url'),
                'format_id': '%sp' % (src.get('height') or f_id),
                'width': int_or_none(src.get('width')),
                'height': int_or_none(src.get('height')),
                'vbr': int_or_none(src.get('video_bitrate_in_kbps')),
                'vcodec': src.get('video_codec'),
                'fps': int_or_none(src.get('frame_rate')),
                'abr': int_or_none(src.get('audio_bitrate_in_kbps')),
                'acodec': src.get('audio_codec'),
                'asr': int_or_none(src.get('audio_sample_rate')),
                'tbr': int_or_none(src.get('total_bitrate_in_kbps')),
                'filesize': int_or_none(src.get('file_size_in_bytes')),
            }

        outputs = asset.get('data', {}).get('outputs')
        if not isinstance(outputs, dict):
            outputs = {}

        def add_output_format_meta(f, key):
            output = outputs.get(key)
            if isinstance(output, dict):
                output_format = extract_output_format(output, key)
                output_format.update(f)
                return output_format
            return f

        def extract_formats(source_list):
            if not isinstance(source_list, list):
                return
            for source in source_list:
                video_url = url_or_none(source.get('file') or source.get('src'))
                if not video_url:
                    continue
                if source.get('type') == 'application/x-mpegURL' or determine_ext(video_url) == 'm3u8':
                    formats.extend(self._extract_m3u8_formats(
                        video_url, video_id, 'mp4', entry_protocol='m3u8_native',
                        m3u8_id='hls', fatal=False))
                    continue
                format_id = source.get('label')
                f = {
                    'url': video_url,
                    'format_id': '%sp' % format_id,
                    'height': int_or_none(format_id),
                }
                if format_id:
                    # Some videos contain additional metadata (e.g.
                    # https://www.udemy.com/ios9-swift/learn/#/lecture/3383208)
                    f = add_output_format_meta(f, format_id)
                formats.append(f)

        def extract_subtitles(track_list):
            if not isinstance(track_list, list):
                return
            for track in track_list:
                if not isinstance(track, dict):
                    continue
                if track.get('kind') != 'captions':
                    continue
                src = url_or_none(track.get('src'))
                if not src:
                    continue
                lang = track.get('language') or track.get(
                    'srclang') or track.get('label')
                sub_dict = automatic_captions if track.get(
                    'autogenerated') is True else subtitles
                sub_dict.setdefault(lang, []).append({
                    'url': src,
                })

        for url_kind in ('download', 'stream'):
            urls = asset.get('%s_urls' % url_kind)
            if isinstance(urls, dict):
                extract_formats(urls.get('Video'))

        captions = asset.get('captions')
        if isinstance(captions, list):
            for cc in captions:
                if not isinstance(cc, dict):
                    continue
                cc_url = url_or_none(cc.get('url'))
                if not cc_url:
                    continue
                lang = try_get(cc, lambda x: x['locale']['locale'], compat_str)
                sub_dict = (automatic_captions if cc.get('source') == 'auto'
                            else subtitles)
                sub_dict.setdefault(lang or 'en', []).append({
                    'url': cc_url,
                })

        view_html = lecture.get('view_html')
        if view_html:
            view_html_urls = set()
            for source in re.findall(r'<source[^>]+>', view_html):
                attributes = extract_attributes(source)
                src = attributes.get('src')
                if not src:
                    continue
                res = attributes.get('data-res')
                height = int_or_none(res)
                if src in view_html_urls:
                    continue
                view_html_urls.add(src)
                if attributes.get('type') == 'application/x-mpegURL' or determine_ext(src) == 'm3u8':
                    m3u8_formats = self._extract_m3u8_formats(
                        src, video_id, 'mp4', entry_protocol='m3u8_native',
                        m3u8_id='hls', fatal=False)
                    for f in m3u8_formats:
                        m = re.search(r'/hls_(?P<height>\d{3,4})_(?P<tbr>\d{2,})/', f['url'])
                        if m:
                            if not f.get('height'):
                                f['height'] = int(m.group('height'))
                            if not f.get('tbr'):
                                f['tbr'] = int(m.group('tbr'))
                    formats.extend(m3u8_formats)
                else:
                    formats.append(add_output_format_meta({
                        'url': src,
                        'format_id': '%dp' % height if height else None,
                        'height': height,
                    }, res))

            # react rendition since 2017.04.15 (see
            # https://github.com/ytdl-org/youtube-dl/issues/12744)
            data = self._parse_json(
                self._search_regex(
                    r'videojs-setup-data=(["\'])(?P<data>{.+?})\1', view_html,
                    'setup data', default='{}', group='data'), video_id,
                transform_source=unescapeHTML, fatal=False)
            if data and isinstance(data, dict):
                extract_formats(data.get('sources'))
                if not duration:
                    duration = int_or_none(data.get('duration'))
                extract_subtitles(data.get('tracks'))

            if not subtitles and not automatic_captions:
                text_tracks = self._parse_json(
                    self._search_regex(
                        r'text-tracks=(["\'])(?P<data>\[.+?\])\1', view_html,
                        'text tracks', default='{}', group='data'), video_id,
                    transform_source=lambda s: js_to_json(unescapeHTML(s)),
                    fatal=False)
                extract_subtitles(text_tracks)

        if not formats and outputs:
            for format_id, output in outputs.items():
                f = extract_output_format(output, format_id)
                if f.get('url'):
                    formats.append(f)

        self._sort_formats(formats, field_preference=('height', 'width', 'tbr', 'format_id'))

        return {
            'id': video_id,
            'title': title,
            'description': description,
            'thumbnail': thumbnail,
            'duration': duration,
            'formats': formats,
            'subtitles': subtitles,
            'automatic_captions': automatic_captions,
        }

class UdemyCourseIE(UdemyIE):
    IE_NAME = 'udemy:course'
    _VALID_URL = r'https?://(?:[^/]+\.)?udemy\.com/(?P<id>[^/?#&]+)'
    _TESTS = [{
        'url': 'https://www.udemy.com/java-tutorial/',
        'only_matching': True,
    }, {
        'url': 'https://wipro.udemy.com/java-tutorial/',
        'only_matching': True,
    }]

    @classmethod
    def suitable(cls, url):
        return False if UdemyIE.suitable(url) else super(UdemyCourseIE, cls).suitable(url)

    def _real_extract(self, url):
        course_path = self._match_id(url)

        webpage = self._download_webpage(url, course_path)

        course_id, title = self._extract_course_info(webpage, course_path)

        self._enroll_course(url, webpage, course_id)

        response = self._download_json(
            'https://www.udemy.com/api-2.0/courses/%s/cached-subscriber-curriculum-items' % course_id,
            course_id, 'Downloading course curriculum', query={
                'fields[chapter]': 'title,object_index',
                'fields[lecture]': 'title,asset',
                'page_size': '1000',
            })

        entries = []
        chapter, chapter_number = [None] * 2
        for entry in response['results']:
            clazz = entry.get('_class')
            if clazz == 'lecture':
                asset = entry.get('asset')
                if isinstance(asset, dict):
                    asset_type = asset.get('asset_type') or asset.get('assetType')
                    if asset_type != 'Video':
                        continue
                lecture_id = entry.get('id')
                if lecture_id:
                    entry = {
                        '_type': 'url_transparent',
                        'url': 'https://www.udemy.com/%s/learn/v4/t/lecture/%s' % (course_path, entry['id']),
                        'title': entry.get('title'),
                        'ie_key': UdemyIE.ie_key(),
                    }
                    if chapter_number:
                        entry['chapter_number'] = chapter_number
                    if chapter:
                        entry['chapter'] = chapter
                    entries.append(entry)
            elif clazz == 'chapter':
                chapter_number = entry.get('object_index')
                chapter = entry.get('title')

        return self.playlist_result(entries, course_id, title)


code:
# coding: utf-8
from __future__ import unicode_literals

import re

from .common import InfoExtractor
from ..utils import (
    determine_ext,
    int_or_none,
    js_to_json,
)
from ..compat import compat_urlparse

class UDNEmbedIE(InfoExtractor):
    IE_DESC = '聯合影音'
    _PROTOCOL_RELATIVE_VALID_URL = r'//video\.udn\.com/(?:embed|play)/news/(?P<id>\d+)'
    _VALID_URL = r'https?:' + _PROTOCOL_RELATIVE_VALID_URL
    _TESTS = [{
        'url': 'http://video.udn.com/embed/news/300040',
        'info_dict': {
            'id': '300040',
            'ext': 'mp4',
            'title': '生物老師男變女 全校挺"做自己"',
            'thumbnail': r're:^https?://.*\.jpg$',
        },
        'params': {
            # m3u8 download
            'skip_download': True,
        },
        'expected_warnings': ['Failed to parse JSON Expecting value'],
    }, {
        'url': 'https://video.udn.com/embed/news/300040',
        'only_matching': True,
    }, {
        # From https://video.udn.com/news/303776
        'url': 'https://video.udn.com/play/news/303776',
        'only_matching': True,
    }]

    def _real_extract(self, url):
        video_id = self._match_id(url)

        page = self._download_webpage(url, video_id)

        options_str = self._html_search_regex(
            r'var\s+options\s*=\s*([^;]+);', page, 'options')
        trans_options_str = js_to_json(options_str)
        options = self._parse_json(trans_options_str, 'options', fatal=False) or {}
        if options:
            video_urls = options['video']
            title = options['title']
            poster = options.get('poster')
        else:
            video_urls = self._parse_json(self._html_search_regex(
                r'"video"\s*:\s*({.+?})\s*,', trans_options_str, 'video urls'), 'video urls')
            title = self._html_search_regex(
                r"title\s*:\s*'(.+?)'\s*,", options_str, 'title')
            poster = self._html_search_regex(
                r"poster\s*:\s*'(.+?)'\s*,", options_str, 'poster', default=None)

        if video_urls.get('youtube'):
            return self.url_result(video_urls.get('youtube'), 'Youtube')

        formats = []
        for video_type, api_url in video_urls.items():
            if not api_url:
                continue

            video_url = self._download_webpage(
                compat_urlparse.urljoin(url, api_url), video_id,
                note='retrieve url for %s video' % video_type)

            ext = determine_ext(video_url)
            if ext == 'm3u8':
                formats.extend(self._extract_m3u8_formats(
                    video_url, video_id, ext='mp4', m3u8_id='hls'))
            elif ext == 'f4m':
                formats.extend(self._extract_f4m_formats(
                    video_url, video_id, f4m_id='hds'))
            else:
                mobj = re.search(r'_(?P<height>\d+)p_(?P<tbr>\d+)\.mp4', video_url)
                a_format = {
                    'url': video_url,
                    # video_type may be 'mp4', which confuses YoutubeDL
                    'format_id': 'http-' + video_type,
                }
                if mobj:
                    a_format.update({
                        'height': int_or_none(mobj.group('height')),
                        'tbr': int_or_none(mobj.group('tbr')),
                    })
                formats.append(a_format)

        self._sort_formats(formats)

        return {
            'id': video_id,
            'formats': formats,
            'title': title,
            'thumbnail': poster,
        }


code:
# coding: utf-8
from __future__ import unicode_literals

from .imggaming import ImgGamingBaseIE

class UFCTVIE(ImgGamingBaseIE):
    _VALID_URL = ImgGamingBaseIE._VALID_URL_TEMPL % r'(?:(?:app|www)\.)?(?:ufc\.tv|(?:ufc)?fightpass\.com)|ufcfightpass\.img(?:dge|gaming)\.com'
    _NETRC_MACHINE = 'ufctv'
    _REALM = 'ufc'

class UFCArabiaIE(ImgGamingBaseIE):
    _VALID_URL = ImgGamingBaseIE._VALID_URL_TEMPL % r'(?:(?:app|www)\.)?ufcarabia\.(?:ae|com)'
    _NETRC_MACHINE = 'ufcarabia'
    _REALM = 'admufc'