code:
# coding: utf-8
from __future__ import unicode_literals

import json
import re

from .common import InfoExtractor
from .facebook import FacebookIE

class BuzzFeedIE(InfoExtractor):
    _VALID_URL = r'https?://(?:www\.)?buzzfeed\.com/[^?#]*?/(?P<id>[^?#]+)'
    _TESTS = [{
        'url': 'http://www.buzzfeed.com/abagg/this-angry-ram-destroys-a-punching-bag-like-a-boss?utm_term=4ldqpia',
        'info_dict': {
            'id': 'this-angry-ram-destroys-a-punching-bag-like-a-boss',
            'title': 'This Angry Ram Destroys A Punching Bag Like A Boss',
            'description': 'Rambro!',
        },
        'playlist': [{
            'info_dict': {
                'id': 'aVCR29aE_OQ',
                'ext': 'mp4',
                'title': 'Angry Ram destroys a punching bag..',
                'description': 'md5:c59533190ef23fd4458a5e8c8c872345',
                'upload_date': '20141024',
                'uploader_id': 'Buddhanz1',
                'uploader': 'Angry Ram',
            }
        }]
    }, {
        'url': 'http://www.buzzfeed.com/sheridanwatson/look-at-this-cute-dog-omg?utm_term=4ldqpia',
        'params': {
            'skip_download': True,  # Got enough YouTube download tests
        },
        'info_dict': {
            'id': 'look-at-this-cute-dog-omg',
            'description': 're:Munchkin the Teddy Bear is back ?!',
            'title': 'You Need To Stop What You\'re Doing And Watching This Dog Walk On A Treadmill',
        },
        'playlist': [{
            'info_dict': {
                'id': 'mVmBL8B-In0',
                'ext': 'mp4',
                'title': 're:Munchkin the Teddy Bear gets her exercise',
                'description': 'md5:28faab95cda6e361bcff06ec12fc21d8',
                'upload_date': '20141124',
                'uploader_id': 'CindysMunchkin',
                'uploader': 're:^Munchkin the',
            },
        }]
    }, {
        'url': 'http://www.buzzfeed.com/craigsilverman/the-most-adorable-crash-landing-ever#.eq7pX0BAmK',
        'info_dict': {
            'id': 'the-most-adorable-crash-landing-ever',
            'title': 'Watch This Baby Goose Make The Most Adorable Crash Landing',
            'description': 'This gosling knows how to stick a landing.',
        },
        'playlist': [{
            'md5': '763ca415512f91ca62e4621086900a23',
            'info_dict': {
                'id': '971793786185728',
                'ext': 'mp4',
                'title': 'We set up crash pads so that the goslings on our roof would have a safe landi...',
                'uploader': 'Calgary Outdoor Centre-University of Calgary',
            },
        }],
        'add_ie': ['Facebook'],
    }]

    def _real_extract(self, url):
        playlist_id = self._match_id(url)
        webpage = self._download_webpage(url, playlist_id)

        all_buckets = re.findall(
            r'(?s)<div class="video-embed[^"]*"..*?rel:bf_bucket_data=\'([^\']+)\'',
            webpage)

        entries = []
        for bd_json in all_buckets:
            bd = json.loads(bd_json)
            video = bd.get('video') or bd.get('progload_video')
            if not video:
                continue
            entries.append(self.url_result(video['url']))

        facebook_urls = FacebookIE._extract_urls(webpage)
        entries.extend([
            self.url_result(facebook_url)
            for facebook_url in facebook_urls])

        return {
            '_type': 'playlist',
            'id': playlist_id,
            'title': self._og_search_title(webpage),
            'description': self._og_search_description(webpage),
            'entries': entries,
        }


code:
from __future__ import unicode_literals

import re

from .common import InfoExtractor
from ..utils import (
    determine_ext,
    merge_dicts,
    parse_duration,
    url_or_none,
)

class BYUtvIE(InfoExtractor):
    _VALID_URL = r'https?://(?:www\.)?byutv\.org/(?:watch|player)/(?!event/)(?P<id>[0-9a-f-]+)(?:/(?P<display_id>[^/?#&]+))?'
    _TESTS = [{
        # ooyalaVOD
        'url': 'http://www.byutv.org/watch/6587b9a3-89d2-42a6-a7f7-fd2f81840a7d/studio-c-season-5-episode-5',
        'info_dict': {
            'id': 'ZvanRocTpW-G5_yZFeltTAMv6jxOU9KH',
            'display_id': 'studio-c-season-5-episode-5',
            'ext': 'mp4',
            'title': 'Season 5 Episode 5',
            'description': 'md5:1d31dc18ef4f075b28f6a65937d22c65',
            'thumbnail': r're:^https?://.*',
            'duration': 1486.486,
        },
        'params': {
            'skip_download': True,
        },
        'add_ie': ['Ooyala'],
    }, {
        # dvr
        'url': 'https://www.byutv.org/player/8f1dab9b-b243-47c8-b525-3e2d021a3451/byu-softball-pacific-vs-byu-41219---game-2',
        'info_dict': {
            'id': '8f1dab9b-b243-47c8-b525-3e2d021a3451',
            'display_id': 'byu-softball-pacific-vs-byu-41219---game-2',
            'ext': 'mp4',
            'title': 'Pacific vs. BYU (4/12/19)',
            'description': 'md5:1ac7b57cb9a78015910a4834790ce1f3',
            'duration': 11645,
        },
        'params': {
            'skip_download': True
        },
    }, {
        'url': 'http://www.byutv.org/watch/6587b9a3-89d2-42a6-a7f7-fd2f81840a7d',
        'only_matching': True,
    }, {
        'url': 'https://www.byutv.org/player/27741493-dc83-40b0-8420-e7ae38a2ae98/byu-football-toledo-vs-byu-93016?listid=4fe0fee5-0d3c-4a29-b725-e4948627f472&listindex=0&q=toledo',
        'only_matching': True,
    }]

    def _real_extract(self, url):
        mobj = re.match(self._VALID_URL, url)
        video_id = mobj.group('id')
        display_id = mobj.group('display_id') or video_id

        video = self._download_json(
            'https://api.byutv.org/api3/catalog/getvideosforcontent',
            display_id, query={
                'contentid': video_id,
                'channel': 'byutv',
                'x-byutv-context': 'web$US',
            }, headers={
                'x-byutv-context': 'web$US',
                'x-byutv-platformkey': 'xsaaw9c7y5',
            })

        ep = video.get('ooyalaVOD')
        if ep:
            return {
                '_type': 'url_transparent',
                'ie_key': 'Ooyala',
                'url': 'ooyala:%s' % ep['providerId'],
                'id': video_id,
                'display_id': display_id,
                'title': ep.get('title'),
                'description': ep.get('description'),
                'thumbnail': ep.get('imageThumbnail'),
            }

        info = {}
        formats = []
        for format_id, ep in video.items():
            if not isinstance(ep, dict):
                continue
            video_url = url_or_none(ep.get('videoUrl'))
            if not video_url:
                continue
            ext = determine_ext(video_url)
            if ext == 'm3u8':
                formats.extend(self._extract_m3u8_formats(
                    video_url, video_id, 'mp4', entry_protocol='m3u8_native',
                    m3u8_id='hls', fatal=False))
            elif ext == 'mpd':
                formats.extend(self._extract_mpd_formats(
                    video_url, video_id, mpd_id='dash', fatal=False))
            else:
                formats.append({
                    'url': video_url,
                    'format_id': format_id,
                })
            merge_dicts(info, {
                'title': ep.get('title'),
                'description': ep.get('description'),
                'thumbnail': ep.get('imageThumbnail'),
                'duration': parse_duration(ep.get('length')),
            })
        self._sort_formats(formats)

        return merge_dicts(info, {
            'id': video_id,
            'display_id': display_id,
            'title': display_id,
            'formats': formats,
        })

repo_name: rrosajp/youtube


code:
# coding: utf-8
from __future__ import unicode_literals

import re

from .common import InfoExtractor
from ..utils import js_to_json

class C56IE(InfoExtractor):
    _VALID_URL = r'https?://(?:(?:www|player)\.)?56\.com/(?:.+?/)?(?:v_|(?:play_album.+-))(?P<textid>.+?)\.(?:html|swf)'
    IE_NAME = '56.com'
    _TESTS = [{
        'url': 'http://www.56.com/u39/v_OTM0NDA3MTY.html',
        'md5': 'e59995ac63d0457783ea05f93f12a866',
        'info_dict': {
            'id': '93440716',
            'ext': 'flv',
            'title': '网事知多少 第32期：车怒',
            'duration': 283.813,
        },
    }, {
        'url': 'http://www.56.com/u47/v_MTM5NjQ5ODc2.html',
        'md5': '',
        'info_dict': {
            'id': '82247482',
            'title': '爱的诅咒之杜鹃花开',
        },
        'playlist_count': 7,
        'add_ie': ['Sohu'],
    }]

    def _real_extract(self, url):
        mobj = re.match(self._VALID_URL, url, flags=re.VERBOSE)
        text_id = mobj.group('textid')

        webpage = self._download_webpage(url, text_id)
        sohu_video_info_str = self._search_regex(
            r'var\s+sohuVideoInfo\s*=\s*({[^}]+});', webpage, 'Sohu video info', default=None)
        if sohu_video_info_str:
            sohu_video_info = self._parse_json(
                sohu_video_info_str, text_id, transform_source=js_to_json)
            return self.url_result(sohu_video_info['url'], 'Sohu')

        page = self._download_json(
            'http://vxml.56.com/json/%s/' % text_id, text_id, 'Downloading video info')

        info = page['info']

        formats = [
            {
                'format_id': f['type'],
                'filesize': int(f['filesize']),
                'url': f['url']
            } for f in info['rfiles']
        ]
        self._sort_formats(formats)

        return {
            'id': info['vid'],
            'title': info['Subject'],
            'duration': int(info['duration']) / 1000.0,
            'formats': formats,
            'thumbnail': info.get('bimg') or info.get('img'),
        }


code:
# coding: utf-8
from __future__ import unicode_literals

import re

from .common import InfoExtractor
from ..compat import (
    compat_urllib_parse_urlencode,
    compat_urlparse,
)
from ..utils import (
    clean_html,
    parse_duration,
    str_to_int,
    unified_strdate,
)

class CamdemyIE(InfoExtractor):
    _VALID_URL = r'https?://(?:www\.)?camdemy\.com/media/(?P<id>\d+)'
    _TESTS = [{
        # single file
        'url': 'http://www.camdemy.com/media/5181/',
        'md5': '5a5562b6a98b37873119102e052e311b',
        'info_dict': {
            'id': '5181',
            'ext': 'mp4',
            'title': 'Ch1-1 Introduction, Signals (02-23-2012)',
            'thumbnail': r're:^https?://.*\.jpg$',
            'creator': 'ss11spring',
            'duration': 1591,
            'upload_date': '20130114',
            'view_count': int,
        }
    }, {
        # With non-empty description
        # webpage returns "No permission or not login"
        'url': 'http://www.camdemy.com/media/13885',
        'md5': '4576a3bb2581f86c61044822adbd1249',
        'info_dict': {
            'id': '13885',
            'ext': 'mp4',
            'title': 'EverCam + Camdemy QuickStart',
            'thumbnail': r're:^https?://.*\.jpg$',
            'description': 'md5:2a9f989c2b153a2342acee579c6e7db6',
            'creator': 'evercam',
            'duration': 318,
        }
    }, {
        # External source (YouTube)
        'url': 'http://www.camdemy.com/media/14842',
        'info_dict': {
            'id': '2vsYQzNIsJo',
            'ext': 'mp4',
            'title': 'Excel 2013 Tutorial - How to add Password Protection',
            'description': 'Excel 2013 Tutorial for Beginners - How to add Password Protection',
            'upload_date': '20130211',
            'uploader': 'Hun Kim',
            'uploader_id': 'hunkimtutorials',
        },
        'params': {
            'skip_download': True,
        },
    }]

    def _real_extract(self, url):
        video_id = self._match_id(url)

        webpage = self._download_webpage(url, video_id)

        src_from = self._html_search_regex(
            r"class=['\"]srcFrom['\"][^>]*>Sources?(?:\s+from)?\s*:\s*<a[^>]+(?:href|title)=(['\"])(?P<url>(?:(?!\1).)+)\1",
            webpage, 'external source', default=None, group='url')
        if src_from:
            return self.url_result(src_from)

        oembed_obj = self._download_json(
            'http://www.camdemy.com/oembed/?format=json&url=' + url, video_id)

        title = oembed_obj['title']
        thumb_url = oembed_obj['thumbnail_url']
        video_folder = compat_urlparse.urljoin(thumb_url, 'video/')
        file_list_doc = self._download_xml(
            compat_urlparse.urljoin(video_folder, 'fileList.xml'),
            video_id, 'Downloading filelist XML')
        file_name = file_list_doc.find('./video/item/fileName').text
        video_url = compat_urlparse.urljoin(video_folder, file_name)

        # Some URLs return "No permission or not login" in a webpage despite being
        # freely available via oembed JSON URL (e.g. http://www.camdemy.com/media/13885)
        upload_date = unified_strdate(self._search_regex(
            r'>published on ([^<]+)<', webpage,
            'upload date', default=None))
        view_count = str_to_int(self._search_regex(
            r'role=["\']viewCnt["\'][^>]*>([\d,.]+) views',
            webpage, 'view count', default=None))
        description = self._html_search_meta(
            'description', webpage, default=None) or clean_html(
            oembed_obj.get('description'))

        return {
            'id': video_id,
            'url': video_url,
            'title': title,
            'thumbnail': thumb_url,
            'description': description,
            'creator': oembed_obj.get('author_name'),
            'duration': parse_duration(oembed_obj.get('duration')),
            'upload_date': upload_date,
            'view_count': view_count,
        }

class CamdemyFolderIE(InfoExtractor):
    _VALID_URL = r'https?://(?:www\.)?camdemy\.com/folder/(?P<id>\d+)'
    _TESTS = [{
        # links with trailing slash
        'url': 'http://www.camdemy.com/folder/450',
        'info_dict': {
            'id': '450',
            'title': '信號與系統 2012 & 2011 (Signals and Systems)',
        },
        'playlist_mincount': 145
    }, {
        # links without trailing slash
        # and multi-page
        'url': 'http://www.camdemy.com/folder/853',
        'info_dict': {
            'id': '853',
            'title': '科學計算 - 使用 Matlab'
        },
        'playlist_mincount': 20
    }, {
        # with displayMode parameter. For testing the codes to add parameters
        'url': 'http://www.camdemy.com/folder/853/?displayMode=defaultOrderByOrg',
        'info_dict': {
            'id': '853',
            'title': '科學計算 - 使用 Matlab'
        },
        'playlist_mincount': 20
    }]

    def _real_extract(self, url):
        folder_id = self._match_id(url)

        # Add displayMode=list so that all links are displayed in a single page
        parsed_url = list(compat_urlparse.urlparse(url))
        query = dict(compat_urlparse.parse_qsl(parsed_url[4]))
        query.update({'displayMode': 'list'})
        parsed_url[4] = compat_urllib_parse_urlencode(query)
        final_url = compat_urlparse.urlunparse(parsed_url)

        page = self._download_webpage(final_url, folder_id)
        matches = re.findall(r"href='(/media/\d+/?)'", page)

        entries = [self.url_result('http://www.camdemy.com' + media_path)
                   for media_path in matches]

        folder_title = self._html_search_meta('keywords', page)

        return self.playlist_result(entries, folder_id, folder_title)


code:
# coding: utf-8
from __future__ import unicode_literals

from .common import InfoExtractor
from ..utils import (
    ExtractorError,
    int_or_none,
    url_or_none,
)

class CamModelsIE(InfoExtractor):
    _VALID_URL = r'https?://(?:www\.)?cammodels\.com/cam/(?P<id>[^/?#&]+)'
    _TESTS = [{
        'url': 'https://www.cammodels.com/cam/AutumnKnight/',
        'only_matching': True,
        'age_limit': 18
    }]

    def _real_extract(self, url):
        user_id = self._match_id(url)

        webpage = self._download_webpage(
            url, user_id, headers=self.geo_verification_headers())

        manifest_root = self._html_search_regex(
            r'manifestUrlRoot=([^&\']+)', webpage, 'manifest', default=None)

        if not manifest_root:
            ERRORS = (
                ("I'm offline, but let's stay connected", 'This user is currently offline'),
                ('in a private show', 'This user is in a private show'),
                ('is currently performing LIVE', 'This model is currently performing live'),
            )
            for pattern, message in ERRORS:
                if pattern in webpage:
                    error = message
                    expected = True
                    break
            else:
                error = 'Unable to find manifest URL root'
                expected = False
            raise ExtractorError(error, expected=expected)

        manifest = self._download_json(
            '%s%s.json' % (manifest_root, user_id), user_id)

        formats = []
        for format_id, format_dict in manifest['formats'].items():
            if not isinstance(format_dict, dict):
                continue
            encodings = format_dict.get('encodings')
            if not isinstance(encodings, list):
                continue
            vcodec = format_dict.get('videoCodec')
            acodec = format_dict.get('audioCodec')
            for media in encodings:
                if not isinstance(media, dict):
                    continue
                media_url = url_or_none(media.get('location'))
                if not media_url:
                    continue

                format_id_list = [format_id]
                height = int_or_none(media.get('videoHeight'))
                if height is not None:
                    format_id_list.append('%dp' % height)
                f = {
                    'url': media_url,
                    'format_id': '-'.join(format_id_list),
                    'width': int_or_none(media.get('videoWidth')),
                    'height': height,
                    'vbr': int_or_none(media.get('videoKbps')),
                    'abr': int_or_none(media.get('audioKbps')),
                    'fps': int_or_none(media.get('fps')),
                    'vcodec': vcodec,
                    'acodec': acodec,
                }
                if 'rtmp' in format_id:
                    f['ext'] = 'flv'
                elif 'hls' in format_id:
                    f.update({
                        'ext': 'mp4',
                        # hls skips fragments, preferring rtmp
                        'preference': -1,
                    })
                else:
                    continue
                formats.append(f)
        self._sort_formats(formats)

        return {
            'id': user_id,
            'title': self._live_title(user_id),
            'is_live': True,
            'formats': formats,
            'age_limit': 18
        }


code:
# coding: utf-8
from __future__ import unicode_literals

from .common import InfoExtractor
from ..utils import (
    int_or_none,
    unified_timestamp,
)

class CamTubeIE(InfoExtractor):
    _VALID_URL = r'https?://(?:(?:www|api)\.)?camtube\.co/recordings?/(?P<id>[^/?#&]+)'
    _TESTS = [{
        'url': 'https://camtube.co/recording/minafay-030618-1136-chaturbate-female',
        'info_dict': {
            'id': '42ad3956-dd5b-445a-8313-803ea6079fac',
            'display_id': 'minafay-030618-1136-chaturbate-female',
            'ext': 'mp4',
            'title': 'minafay-030618-1136-chaturbate-female',
            'duration': 1274,
            'timestamp': 1528018608,
            'upload_date': '20180603',
            'age_limit': 18
        },
        'params': {
            'skip_download': True,
        },
    }]

    _API_BASE = 'https://api.camtube.co'

    def _real_extract(self, url):
        display_id = self._match_id(url)

        token = self._download_json(
            '%s/rpc/session/new' % self._API_BASE, display_id,
            'Downloading session token')['token']

        self._set_cookie('api.camtube.co', 'session', token)

        video = self._download_json(
            '%s/recordings/%s' % (self._API_BASE, display_id), display_id,
            headers={'Referer': url})

        video_id = video['uuid']
        timestamp = unified_timestamp(video.get('createdAt'))
        duration = int_or_none(video.get('duration'))
        view_count = int_or_none(video.get('viewCount'))
        like_count = int_or_none(video.get('likeCount'))
        creator = video.get('stageName')

        formats = [{
            'url': '%s/recordings/%s/manifest.m3u8'
                   % (self._API_BASE, video_id),
            'format_id': 'hls',
            'ext': 'mp4',
            'protocol': 'm3u8_native',
        }]

        return {
            'id': video_id,
            'display_id': display_id,
            'title': display_id,
            'timestamp': timestamp,
            'duration': duration,
            'view_count': view_count,
            'like_count': like_count,
            'creator': creator,
            'formats': formats,
            'age_limit': 18
        }


code:
from __future__ import unicode_literals

import re

from .common import InfoExtractor
from ..utils import (
    int_or_none,
    parse_duration,
    unified_strdate,
)

class CamWithHerIE(InfoExtractor):
    _VALID_URL = r'https?://(?:www\.)?camwithher\.tv/view_video\.php\?.*\bviewkey=(?P<id>\w+)'

    _TESTS = [{
        'url': 'http://camwithher.tv/view_video.php?viewkey=6e9a24e2c0e842e1f177&page=&viewtype=&category=',
        'info_dict': {
            'id': '5644',
            'ext': 'flv',
            'title': 'Periscope Tease',
            'description': 'In the clouds teasing on periscope to my favorite song',
            'duration': 240,
            'view_count': int,
            'comment_count': int,
            'uploader': 'MileenaK',
            'upload_date': '20160322',
            'age_limit': 18,
        },
        'params': {
            'skip_download': True,
        }
    }, {
        'url': 'http://camwithher.tv/view_video.php?viewkey=6dfd8b7c97531a459937',
        'only_matching': True,
    }, {
        'url': 'http://camwithher.tv/view_video.php?page=&viewkey=6e9a24e2c0e842e1f177&viewtype=&category=',
        'only_matching': True,
    }, {
        'url': 'http://camwithher.tv/view_video.php?viewkey=b6c3b5bea9515d1a1fc4&page=&viewtype=&category=mv',
        'only_matching': True,
    }]

    def _real_extract(self, url):
        video_id = self._match_id(url)

        webpage = self._download_webpage(url, video_id)

        flv_id = self._html_search_regex(
            r']+href=["\']/download/\?v=(\d+)', webpage, 'video id')

        # Video URL construction algorithm is reverse-engineered from cwhplayer.swf
        rtmp_url = 'rtmp://camwithher.tv/clipshare/%s' % (
            ('mp4:%s.mp4' % flv_id) if int(flv_id) > 2010 else flv_id)

        title = self._html_search_regex(
            r'
]+style="float:left"[^>]*>\s*
(.+?)
', webpage, 'title')
        description = self._html_search_regex(
            r'>Description:(.+?)
', webpage, 'description', default=None)

        runtime = self._search_regex(
            r'Runtime\s*:\s*(.+?) \|', webpage, 'duration', default=None)
        if runtime:
            runtime = re.sub(r'[\s-]', '', runtime)
        duration = parse_duration(runtime)
        view_count = int_or_none(self._search_regex(
            r'Views\s*:\s*(\d+)', webpage, 'view count', default=None))
        comment_count = int_or_none(self._search_regex(
            r'Comments\s*:\s*(\d+)', webpage, 'comment count', default=None))

        uploader = self._search_regex(
            r'Added by\s*:\s*]+>([^<]+)', webpage, 'uploader', default=None)
        upload_date = unified_strdate(self._search_regex(
            r'Added on\s*:\s*([\d-]+)', webpage, 'upload date', default=None))

        return {
            'id': flv_id,
            'url': rtmp_url,
            'ext': 'flv',
            'no_resume': True,
            'title': title,
            'description': description,
            'duration': duration,
            'view_count': view_count,
            'comment_count': comment_count,
            'uploader': uploader,
            'upload_date': upload_date,
            'age_limit': 18
        }


code:
# coding: utf-8
from __future__ import unicode_literals

import re

from .common import InfoExtractor
from ..utils import parse_duration

class Canalc2IE(InfoExtractor):
    IE_NAME = 'canalc2.tv'
    _VALID_URL = r'https?://(?:(?:www\.)?canalc2\.tv/video/|archives-canalc2\.u-strasbg\.fr/video\.asp\?.*\bidVideo=)(?P<id>\d+)'

    _TESTS = [{
        'url': 'http://www.canalc2.tv/video/12163',
        'md5': '060158428b650f896c542dfbb3d6487f',
        'info_dict': {
            'id': '12163',
            'ext': 'mp4',
            'title': 'Terrasses du Numérique',
            'duration': 122,
        },
    }, {
        'url': 'http://archives-canalc2.u-strasbg.fr/video.asp?idVideo=11427&voir=oui',
        'only_matching': True,
    }]

    def _real_extract(self, url):
        video_id = self._match_id(url)

        webpage = self._download_webpage(
            'http://www.canalc2.tv/video/%s' % video_id, video_id)

        title = self._html_search_regex(
            r'(?s)class="[^"]*col_description[^"]*">.*?<h3>(.+?)</h3>',
            webpage, 'title')

        formats = []
        for _, video_url in re.findall(r'file\s*=\s*(["\'])(.+?)\1', webpage):
            if video_url.startswith('rtmp://'):
                rtmp = re.search(
                    r'^(?P<url>rtmp://[^/]+/(?P<app>.+/))(?P<play_path>mp4:.+)$', video_url)
                formats.append({
                    'url': rtmp.group('url'),
                    'format_id': 'rtmp',
                    'ext': 'flv',
                    'app': rtmp.group('app'),
                    'play_path': rtmp.group('play_path'),
                    'page_url': url,
                })
            else:
                formats.append({
                    'url': video_url,
                    'format_id': 'http',
                })

        if formats:
            info = {
                'formats': formats,
            }
        else:
            info = self._parse_html5_media_entries(url, webpage, url)[0]

        self._sort_formats(info['formats'])

        info.update({
            'id': video_id,
            'title': title,
            'duration': parse_duration(self._search_regex(
                r'id=["\']video_duree["\'][^>]*>([^<]+)',
                webpage, 'duration', fatal=False)),
        })
        return info


code:
# coding: utf-8
from __future__ import unicode_literals

import re

from .common import InfoExtractor
from ..utils import (
    # ExtractorError,
    # HEADRequest,
    int_or_none,
    qualities,
    unified_strdate,
)

class CanalplusIE(InfoExtractor):
    IE_DESC = 'mycanal.fr and piwiplus.fr'
    _VALID_URL = r'https?://(?:www\.)?(?P<site>mycanal|piwiplus)\.fr/(?:[^/]+/)*(?P<display_id>[^?/]+)(?:\.html\?.*\bvid=|/p/)(?P<id>\d+)'
    _VIDEO_INFO_TEMPLATE = 'http://service.canal-plus.com/video/rest/getVideosLiees/%s/%s?format=json'
    _SITE_ID_MAP = {
        'mycanal': 'cplus',
        'piwiplus': 'teletoon',
    }

    # Only works for direct mp4 URLs
    _GEO_COUNTRIES = ['FR']

    _TESTS = [{
        'url': 'https://www.mycanal.fr/d17-emissions/lolywood/p/1397061',
        'info_dict': {
            'id': '1397061',
            'display_id': 'lolywood',
            'ext': 'mp4',
            'title': 'Euro 2016 : Je préfère te prévenir - Lolywood - Episode 34',
            'description': 'md5:7d97039d455cb29cdba0d652a0efaa5e',
            'upload_date': '20160602',
        },
    }, {
        # geo restricted, bypassed
        'url': 'http://www.piwiplus.fr/videos-piwi/pid1405-le-labyrinthe-boing-super-ranger.html?vid=1108190',
        'info_dict': {
            'id': '1108190',
            'display_id': 'pid1405-le-labyrinthe-boing-super-ranger',
            'ext': 'mp4',
            'title': 'BOING SUPER RANGER - Ep : Le labyrinthe',
            'description': 'md5:4cea7a37153be42c1ba2c1d3064376ff',
            'upload_date': '20140724',
        },
        'expected_warnings': ['HTTP Error 403: Forbidden'],
    }]

    def _real_extract(self, url):
        site, display_id, video_id = re.match(self._VALID_URL, url).groups()

        site_id = self._SITE_ID_MAP[site]

        info_url = self._VIDEO_INFO_TEMPLATE % (site_id, video_id)
        video_data = self._download_json(info_url, video_id, 'Downloading video JSON')

        if isinstance(video_data, list):
            video_data = [video for video in video_data if video.get('ID') == video_id][0]
        media = video_data['MEDIA']
        infos = video_data['INFOS']

        preference = qualities(['MOBILE', 'BAS_DEBIT', 'HAUT_DEBIT', 'HD'])

        # _, fmt_url = next(iter(media['VIDEOS'].items()))
        # if '/geo' in fmt_url.lower():
        #     response = self._request_webpage(
        #         HEADRequest(fmt_url), video_id,
        #         'Checking if the video is georestricted')
        #     if '/blocage' in response.geturl():
        #         raise ExtractorError(
        #             'The video is not available in your country',
        #             expected=True)

        formats = []
        for format_id, format_url in media['VIDEOS'].items():
            if not format_url:
                continue
            if format_id == 'HLS':
                formats.extend(self._extract_m3u8_formats(
                    format_url, video_id, 'mp4', 'm3u8_native', m3u8_id=format_id, fatal=False))
            elif format_id == 'HDS':
                formats.extend(self._extract_f4m_formats(
                    format_url + '?hdcore=2.11.3', video_id, f4m_id=format_id, fatal=False))
            else:
                formats.append({
                    # the secret extracted from ya function in http://player.canalplus.fr/common/js/canalPlayer.js
                    'url': format_url + '?secret=pqzerjlsmdkjfoiuerhsdlfknaes',
                    'format_id': format_id,
                    'preference': preference(format_id),
                })
        self._sort_formats(formats)

        thumbnails = [{
            'id': image_id,
            'url': image_url,
        } for image_id, image_url in media.get('images', {}).items()]

        titrage = infos['TITRAGE']

        return {
            'id': video_id,
            'display_id': display_id,
            'title': '%s - %s' % (titrage['TITRE'],
                                  titrage['SOUS_TITRE']),
            'upload_date': unified_strdate(infos.get('PUBLICATION', {}).get('DATE')),
            'thumbnails': thumbnails,
            'description': infos.get('DESCRIPTION'),
            'duration': int_or_none(infos.get('DURATION')),
            'view_count': int_or_none(infos.get('NB_VUES')),
            'like_count': int_or_none(infos.get('NB_LIKES')),
            'comment_count': int_or_none(infos.get('NB_COMMENTS')),
            'formats': formats,
        }


code:
from __future__ import unicode_literals

import re
import json

from .common import InfoExtractor
from .gigya import GigyaBaseIE
from ..compat import compat_HTTPError
from ..utils import (
    ExtractorError,
    strip_or_none,
    float_or_none,
    int_or_none,
    merge_dicts,
    parse_iso8601,
    str_or_none,
    url_or_none,
)

class CanvasIE(InfoExtractor):
    _VALID_URL = r'https?://mediazone\.vrt\.be/api/v1/(?P<site_id>canvas|een|ketnet|vrt(?:video|nieuws)|sporza)/assets/(?P<id>[^/?#&]+)'
    _TESTS = [{
        'url': 'https://mediazone.vrt.be/api/v1/ketnet/assets/md-ast-4ac54990-ce66-4d00-a8ca-9eac86f4c475',
        'md5': '68993eda72ef62386a15ea2cf3c93107',
        'info_dict': {
            'id': 'md-ast-4ac54990-ce66-4d00-a8ca-9eac86f4c475',
            'display_id': 'md-ast-4ac54990-ce66-4d00-a8ca-9eac86f4c475',
            'ext': 'mp4',
            'title': 'Nachtwacht: De Greystook',
            'description': 'Nachtwacht: De Greystook',
            'thumbnail': r're:^https?://.*\.jpg$',
            'duration': 1468.04,
        },
        'expected_warnings': ['is not a supported codec', 'Unknown MIME type'],
    }, {
        'url': 'https://mediazone.vrt.be/api/v1/canvas/assets/mz-ast-5e5f90b6-2d72-4c40-82c2-e134f884e93e',
        'only_matching': True,
    }]
    _HLS_ENTRY_PROTOCOLS_MAP = {
        'HLS': 'm3u8_native',
        'HLS_AES': 'm3u8',
    }
    _REST_API_BASE = 'https://media-services-public.vrt.be/vualto-video-aggregator-web/rest/external/v1'

    def _real_extract(self, url):
        mobj = re.match(self._VALID_URL, url)
        site_id, video_id = mobj.group('site_id'), mobj.group('id')

        # Old API endpoint, serves more formats but may fail for some videos
        data = self._download_json(
            'https://mediazone.vrt.be/api/v1/%s/assets/%s'
            % (site_id, video_id), video_id, 'Downloading asset JSON',
            'Unable to download asset JSON', fatal=False)

        # New API endpoint
        if not data:
            token = self._download_json(
                '%s/tokens' % self._REST_API_BASE, video_id,
                'Downloading token', data=b'',
                headers={'Content-Type': 'application/json'})['vrtPlayerToken']
            data = self._download_json(
                '%s/videos/%s' % (self._REST_API_BASE, video_id),
                video_id, 'Downloading video JSON', fatal=False, query={
                    'vrtPlayerToken': token,
                    'client': '%s@PROD' % site_id,
                }, expected_status=400)
            message = data.get('message')
            if message and not data.get('title'):
                if data.get('code') == 'AUTHENTICATION_REQUIRED':
                    self.raise_login_required(message)
                raise ExtractorError(message, expected=True)

        title = data['title']
        description = data.get('description')

        formats = []
        for target in data['targetUrls']:
            format_url, format_type = url_or_none(target.get('url')), str_or_none(target.get('type'))
            if not format_url or not format_type:
                continue
            format_type = format_type.upper()
            if format_type in self._HLS_ENTRY_PROTOCOLS_MAP:
                formats.extend(self._extract_m3u8_formats(
                    format_url, video_id, 'mp4', self._HLS_ENTRY_PROTOCOLS_MAP[format_type],
                    m3u8_id=format_type, fatal=False))
            elif format_type == 'HDS':
                formats.extend(self._extract_f4m_formats(
                    format_url, video_id, f4m_id=format_type, fatal=False))
            elif format_type == 'MPEG_DASH':
                formats.extend(self._extract_mpd_formats(
                    format_url, video_id, mpd_id=format_type, fatal=False))
            elif format_type == 'HSS':
                formats.extend(self._extract_ism_formats(
                    format_url, video_id, ism_id='mss', fatal=False))
            else:
                formats.append({
                    'format_id': format_type,
                    'url': format_url,
                })
        self._sort_formats(formats)

        subtitles = {}
        subtitle_urls = data.get('subtitleUrls')
        if isinstance(subtitle_urls, list):
            for subtitle in subtitle_urls:
                subtitle_url = subtitle.get('url')
                if subtitle_url and subtitle.get('type') == 'CLOSED':
                    subtitles.setdefault('nl', []).append({'url': subtitle_url})

        return {
            'id': video_id,
            'display_id': video_id,
            'title': title,
            'description': description,
            'formats': formats,
            'duration': float_or_none(data.get('duration'), 1000),
            'thumbnail': data.get('posterImageUrl'),
            'subtitles': subtitles,
        }

class CanvasEenIE(InfoExtractor):
    IE_DESC = 'canvas.be and een.be'
    _VALID_URL = r'https?://(?:www\.)?(?P<site_id>canvas|een)\.be/(?:[^/]+/)*(?P<id>[^/?#&]+)'
    _TESTS = [{
        'url': 'http://www.canvas.be/video/de-afspraak/najaar-2015/de-afspraak-veilt-voor-de-warmste-week',
        'md5': 'ed66976748d12350b118455979cca293',
        'info_dict': {
            'id': 'mz-ast-5e5f90b6-2d72-4c40-82c2-e134f884e93e',
            'display_id': 'de-afspraak-veilt-voor-de-warmste-week',
            'ext': 'flv',
            'title': 'De afspraak veilt voor de Warmste Week',
            'description': 'md5:24cb860c320dc2be7358e0e5aa317ba6',
            'thumbnail': r're:^https?://.*\.jpg$',
            'duration': 49.02,
        },
        'expected_warnings': ['is not a supported codec'],
    }, {
        # with subtitles
        'url': 'http://www.canvas.be/video/panorama/2016/pieter-0167',
        'info_dict': {
            'id': 'mz-ast-5240ff21-2d30-4101-bba6-92b5ec67c625',
            'display_id': 'pieter-0167',
            'ext': 'mp4',
            'title': 'Pieter 0167',
            'description': 'md5:943cd30f48a5d29ba02c3a104dc4ec4e',
            'thumbnail': r're:^https?://.*\.jpg$',
            'duration': 2553.08,
            'subtitles': {
                'nl': [{
                    'ext': 'vtt',
                }],
            },
        },
        'params': {
            'skip_download': True,
        },
        'skip': 'Pagina niet gevonden',
    }, {
        'url': 'https://www.een.be/thuis/emma-pakt-thilly-aan',
        'info_dict': {
            'id': 'md-ast-3a24ced2-64d7-44fb-b4ed-ed1aafbf90b8',
            'display_id': 'emma-pakt-thilly-aan',
            'ext': 'mp4',
            'title': 'Emma pakt Thilly aan',
            'description': 'md5:c5c9b572388a99b2690030afa3f3bad7',
            'thumbnail': r're:^https?://.*\.jpg$',
            'duration': 118.24,
        },
        'params': {
            'skip_download': True,
        },
        'expected_warnings': ['is not a supported codec'],
    }, {
        'url': 'https://www.canvas.be/check-point/najaar-2016/de-politie-uw-vriend',
        'only_matching': True,
    }]

    def _real_extract(self, url):
        mobj = re.match(self._VALID_URL, url)
        site_id, display_id = mobj.group('site_id'), mobj.group('id')

        webpage = self._download_webpage(url, display_id)

        title = strip_or_none(self._search_regex(
            r'
]+class="video__body__header__title"[^>]*>(.+?)
',
            webpage, 'title', default=None) or self._og_search_title(
            webpage, default=None))

        video_id = self._html_search_regex(
            r'data-video=(["\'])(?P<id>(?:(?!\1).)+)\1', webpage, 'video id',
            group='id')

        return {
            '_type': 'url_transparent',
            'url': 'https://mediazone.vrt.be/api/v1/%s/assets/%s' % (site_id, video_id),
            'ie_key': CanvasIE.ie_key(),
            'id': video_id,
            'display_id': display_id,
            'title': title,
            'description': self._og_search_description(webpage),
        }

class VrtNUIE(GigyaBaseIE):
    IE_DESC = 'VrtNU.be'
    _VALID_URL = r'https?://(?:www\.)?vrt\.be/(?P<site_id>vrtnu)/(?:[^/]+/)*(?P<id>[^/?#&]+)'
    _TESTS = [{
        # Available via old API endpoint
        'url': 'https://www.vrt.be/vrtnu/a-z/postbus-x/1/postbus-x-s1a1/',
        'info_dict': {
            'id': 'pbs-pub-2e2d8c27-df26-45c9-9dc6-90c78153044d$vid-90c932b1-e21d-4fb8-99b1-db7b49cf74de',
            'ext': 'mp4',
            'title': 'De zwarte weduwe',
            'description': 'md5:db1227b0f318c849ba5eab1fef895ee4',
            'duration': 1457.04,
            'thumbnail': r're:^https?://.*\.jpg$',
            'season': 'Season 1',
            'season_number': 1,
            'episode_number': 1,
        },
        'skip': 'This video is only available for registered users',
        'params': {
            'username': '<snip>',
            'password': '<snip>',
        },
        'expected_warnings': ['is not a supported codec'],
    }, {
        # Only available via new API endpoint
        'url': 'https://www.vrt.be/vrtnu/a-z/kamp-waes/1/kamp-waes-s1a5/',
        'info_dict': {
            'id': 'pbs-pub-0763b56c-64fb-4d38-b95b-af60bf433c71$vid-ad36a73c-4735-4f1f-b2c0-a38e6e6aa7e1',
            'ext': 'mp4',
            'title': 'Aflevering 5',
            'description': 'Wie valt door de mand tijdens een missie?',
            'duration': 2967.06,
            'season': 'Season 1',
            'season_number': 1,
            'episode_number': 5,
        },
        'skip': 'This video is only available for registered users',
        'params': {
            'username': '<snip>',
            'password': '<snip>',
        },
        'expected_warnings': ['Unable to download asset JSON', 'is not a supported codec', 'Unknown MIME type'],
    }]
    _NETRC_MACHINE = 'vrtnu'
    _APIKEY = '3_0Z2HujMtiWq_pkAjgnS2Md2E11a1AwZjYiBETtwNE-EoEHDINgtnvcAOpNgmrVGy'
    _CONTEXT_ID = 'R3595707040'

    def _real_initialize(self):
        self._login()

    def _login(self):
        username, password = self._get_login_info()
        if username is None:
            return

        auth_data = {
            'APIKey': self._APIKEY,
            'targetEnv': 'jssdk',
            'loginID': username,
            'password': password,
            'authMode': 'cookie',
        }

        auth_info = self._gigya_login(auth_data)

        # Sometimes authentication fails for no good reason, retry
        login_attempt = 1
        while login_attempt <= 3:
            try:
                # When requesting a token, no actual token is returned, but the
                # necessary cookies are set.
                self._request_webpage(
                    'https://token.vrt.be',
                    None, note='Requesting a token', errnote='Could not get a token',
                    headers={
                        'Content-Type': 'application/json',
                        'Referer': 'https://www.vrt.be/vrtnu/',
                    },
                    data=json.dumps({
                        'uid': auth_info['UID'],
                        'uidsig': auth_info['UIDSignature'],
                        'ts': auth_info['signatureTimestamp'],
                        'email': auth_info['profile']['email'],
                    }).encode('utf-8'))
            except ExtractorError as e:
                if isinstance(e.cause, compat_HTTPError) and e.cause.code == 401:
                    login_attempt += 1
                    self.report_warning('Authentication failed')
                    self._sleep(1, None, msg_template='Waiting for %(timeout)s seconds before trying again')
                else:
                    raise e
            else:
                break

    def _real_extract(self, url):
        display_id = self._match_id(url)

        webpage, urlh = self._download_webpage_handle(url, display_id)

        info = self._search_json_ld(webpage, display_id, default={})

        # title is optional here since it may be extracted by extractor
        # that is delegated from here
        title = strip_or_none(self._html_search_regex(
            r'(?ms)
(.+?)
',
            webpage, 'title', default=None))

        description = self._html_search_regex(
            r'(?ms)
(.+?)
',
            webpage, 'description', default=None)

        season = self._html_search_regex(
            [r'''(?xms)
\s*
                    seizoen\ (.+?)\s*

''',
             r'<option value="seizoen (\d{1,3})" data-href="[^"]+?" selected>'],
            webpage, 'season', default=None)

        season_number = int_or_none(season)

        episode_number = int_or_none(self._html_search_regex(
            r'''(?xms)
\s*
                    afl\s*(\d+)

''',
            webpage, 'episode_number', default=None))

        release_date = parse_iso8601(self._html_search_regex(
            r'(?ms)
\s*<time\ datetime="(.+?)"',
            webpage, 'release_date', default=None))

        # If there's a ? or a # in the URL, remove them and everything after
        clean_url = urlh.geturl().split('?')[0].split('#')[0].strip('/')
        securevideo_url = clean_url + '.mssecurevideo.json'

        try:
            video = self._download_json(securevideo_url, display_id)
        except ExtractorError as e:
            if isinstance(e.cause, compat_HTTPError) and e.cause.code == 401:
                self.raise_login_required()
            raise

        # We are dealing with a '../<show>.relevant' URL
        redirect_url = video.get('url')
        if redirect_url:
            return self.url_result(self._proto_relative_url(redirect_url, 'https:'))

        # There is only one entry, but with an unknown key, so just get
        # the first one
        video_id = list(video.values())[0].get('videoid')

        return merge_dicts(info, {
            '_type': 'url_transparent',
            'url': 'https://mediazone.vrt.be/api/v1/vrtvideo/assets/%s' % video_id,
            'ie_key': CanvasIE.ie_key(),
            'id': video_id,
            'display_id': display_id,
            'title': title,
            'description': description,
            'season': season,
            'season_number': season_number,
            'episode_number': episode_number,
            'release_date': release_date,
        })


code:
# coding: utf-8
from __future__ import unicode_literals

from .common import InfoExtractor
from ..compat import compat_str
from ..utils import (
    float_or_none,
    int_or_none,
    try_get,
)

from .videomore import VideomoreIE

class CarambaTVIE(InfoExtractor):
    _VALID_URL = r'(?:carambatv:|https?://video1\.carambatv\.ru/v/)(?P<id>\d+)'
    _TESTS = [{
        'url': 'http://video1.carambatv.ru/v/191910501',
        'md5': '2f4a81b7cfd5ab866ee2d7270cb34a2a',
        'info_dict': {
            'id': '191910501',
            'ext': 'mp4',
            'title': '[BadComedian] - Разборка в Маниле (Абсолютный обзор)',
            'thumbnail': r're:^https?://.*\.jpg',
            'duration': 2678.31,
        },
    }, {
        'url': 'carambatv:191910501',
        'only_matching': True,
    }]

    def _real_extract(self, url):
        video_id = self._match_id(url)

        video = self._download_json(
            'http://video1.carambatv.ru/v/%s/videoinfo.js' % video_id,
            video_id)

        title = video['title']

        base_url = video.get('video') or 'http://video1.carambatv.ru/v/%s/' % video_id

        formats = [{
            'url': base_url + f['fn'],
            'height': int_or_none(f.get('height')),
            'format_id': '%sp' % f['height'] if f.get('height') else None,
        } for f in video['qualities'] if f.get('fn')]
        self._sort_formats(formats)

        thumbnail = video.get('splash')
        duration = float_or_none(try_get(
            video, lambda x: x['annotations'][0]['end_time'], compat_str))

        return {
            'id': video_id,
            'title': title,
            'thumbnail': thumbnail,
            'duration': duration,
            'formats': formats,
        }

class CarambaTVPageIE(InfoExtractor):
    _VALID_URL = r'https?://carambatv\.ru/(?:[^/]+/)+(?P<id>[^/?#&]+)'
    _TEST = {
        'url': 'http://carambatv.ru/movie/bad-comedian/razborka-v-manile/',
        'md5': 'a49fb0ec2ad66503eeb46aac237d3c86',
        'info_dict': {
            'id': '475222',
            'ext': 'flv',
            'title': '[BadComedian] - Разборка в Маниле (Абсолютный обзор)',
            'thumbnail': r're:^https?://.*\.jpg',
            # duration reported by videomore is incorrect
            'duration': int,
        },
        'add_ie': [VideomoreIE.ie_key()],
    }

    def _real_extract(self, url):
        video_id = self._match_id(url)

        webpage = self._download_webpage(url, video_id)

        videomore_url = VideomoreIE._extract_url(webpage)
        if not videomore_url:
            videomore_id = self._search_regex(
                r'getVMCode\s*\(\s*["\']?(\d+)', webpage, 'videomore id',
                default=None)
            if videomore_id:
                videomore_url = 'videomore:%s' % videomore_id
        if videomore_url:
            title = self._og_search_title(webpage)
            return {
                '_type': 'url_transparent',
                'url': videomore_url,
                'ie_key': VideomoreIE.ie_key(),
                'title': title,
            }

        video_url = self._og_search_property('video:iframe', webpage, default=None)

        if not video_url:
            video_id = self._search_regex(
                r'(?:video_id|crmb_vuid)\s*[:=]\s*["\']?(\d+)',
                webpage, 'video id')
            video_url = 'carambatv:%s' % video_id

        return self.url_result(video_url, CarambaTVIE.ie_key())


code:
# coding: utf-8
from __future__ import unicode_literals

from .turner import TurnerBaseIE
from ..utils import int_or_none

class CartoonNetworkIE(TurnerBaseIE):
    _VALID_URL = r'https?://(?:www\.)?cartoonnetwork\.com/video/(?:[^/]+/)+(?P<id>[^/?#]+)-(?:clip|episode)\.html'
    _TEST = {
        'url': 'https://www.cartoonnetwork.com/video/ben-10/how-to-draw-upgrade-episode.html',
        'info_dict': {
            'id': '6e3375097f63874ebccec7ef677c1c3845fa850e',
            'ext': 'mp4',
            'title': 'How to Draw Upgrade',
            'description': 'md5:2061d83776db7e8be4879684eefe8c0f',
        },
        'params': {
            # m3u8 download
            'skip_download': True,
        },
    }

    def _real_extract(self, url):
        display_id = self._match_id(url)
        webpage = self._download_webpage(url, display_id)

        def find_field(global_re, name, content_re=None, value_re='[^"]+', fatal=False):
            metadata_re = ''
            if content_re:
                metadata_re = r'|video_metadata\.content_' + content_re
            return self._search_regex(
                r'(?:_cnglobal\.currentVideo\.%s%s)\s*=\s*"(%s)";' % (global_re, metadata_re, value_re),
                webpage, name, fatal=fatal)

        media_id = find_field('mediaId', 'media id', 'id', '[0-9a-f]{40}', True)
        title = find_field('episodeTitle', 'title', '(?:episodeName|name)', fatal=True)

        info = self._extract_ngtv_info(
            media_id, {'networkId': 'cartoonnetwork'}, {
                'url': url,
                'site_name': 'CartoonNetwork',
                'auth_required': find_field('authType', 'auth type') != 'unauth',
            })

        series = find_field(
            'propertyName', 'series', 'showName') or self._html_search_meta('partOfSeries', webpage)
        info.update({
            'id': media_id,
            'display_id': display_id,
            'title': title,
            'description': self._html_search_meta('description', webpage),
            'series': series,
            'episode': title,
        })

        for field in ('season', 'episode'):
            field_name = field + 'Number'
            info[field + '_number'] = int_or_none(find_field(
                field_name, field + ' number', value_re=r'\d+') or self._html_search_meta(field_name, webpage))

        return info


code:
# coding: utf-8
from __future__ import unicode_literals

import hashlib
import json
import re
from xml.sax.saxutils import escape

from .common import InfoExtractor
from ..compat import (
    compat_str,
    compat_HTTPError,
)
from ..utils import (
    js_to_json,
    smuggle_url,
    try_get,
    xpath_text,
    xpath_element,
    xpath_with_ns,
    find_xpath_attr,
    orderedSet,
    parse_duration,
    parse_iso8601,
    parse_age_limit,
    strip_or_none,
    int_or_none,
    ExtractorError,
)

class CBCIE(InfoExtractor):
    IE_NAME = 'cbc.ca'
    _VALID_URL = r'https?://(?:www\.)?cbc\.ca/(?!player/)(?:[^/]+/)+(?P<id>[^/?#]+)'
    _TESTS = [{
        # with mediaId
        'url': 'http://www.cbc.ca/22minutes/videos/clips-season-23/don-cherry-play-offs',
        'md5': '97e24d09672fc4cf56256d6faa6c25bc',
        'info_dict': {
            'id': '2682904050',
            'ext': 'mp4',
            'title': 'Don Cherry – All-Stars',
            'description': 'Don Cherry has a bee in his bonnet about AHL player John Scott because that guy’s got heart.',
            'timestamp': 1454463000,
            'upload_date': '20160203',
            'uploader': 'CBCC-NEW',
        },
        'skip': 'Geo-restricted to Canada',
    }, {
        # with clipId, feed available via tpfeed.cbc.ca and feed.theplatform.com
        'url': 'http://www.cbc.ca/22minutes/videos/22-minutes-update/22-minutes-update-episode-4',
        'md5': '162adfa070274b144f4fdc3c3b8207db',
        'info_dict': {
            'id': '2414435309',
            'ext': 'mp4',
            'title': '22 Minutes Update: What Not To Wear Quebec',
            'description': "This week's latest Canadian top political story is What Not To Wear Quebec.",
            'upload_date': '20131025',
            'uploader': 'CBCC-NEW',
            'timestamp': 1382717907,
        },
    }, {
        # with clipId, feed only available via tpfeed.cbc.ca
        'url': 'http://www.cbc.ca/archives/entry/1978-robin-williams-freestyles-on-90-minutes-live',
        'md5': '0274a90b51a9b4971fe005c63f592f12',
        'info_dict': {
            'id': '2487345465',
            'ext': 'mp4',
            'title': 'Robin Williams freestyles on 90 Minutes Live',
            'description': 'Wacky American comedian Robin Williams shows off his infamous "freestyle" comedic talents while being interviewed on CBC\'s 90 Minutes Live.',
            'upload_date': '19780210',
            'uploader': 'CBCC-NEW',
            'timestamp': 255977160,
        },
    }, {
        # multiple iframes
        'url': 'http://www.cbc.ca/natureofthings/blog/birds-eye-view-from-vancouvers-burrard-street-bridge-how-we-got-the-shot',
        'playlist': [{
            'md5': '377572d0b49c4ce0c9ad77470e0b96b4',
            'info_dict': {
                'id': '2680832926',
                'ext': 'mp4',
                'title': 'An Eagle\'s-Eye View Off Burrard Bridge',
                'description': 'Hercules the eagle flies from Vancouver\'s Burrard Bridge down to a nearby park with a mini-camera strapped to his back.',
                'upload_date': '20160201',
                'timestamp': 1454342820,
                'uploader': 'CBCC-NEW',
            },
        }, {
            'md5': '415a0e3f586113894174dfb31aa5bb1a',
            'info_dict': {
                'id': '2658915080',
                'ext': 'mp4',
                'title': 'Fly like an eagle!',
                'description': 'Eagle equipped with a mini camera flies from the world\'s tallest tower',
                'upload_date': '20150315',
                'timestamp': 1426443984,
                'uploader': 'CBCC-NEW',
            },
        }],
        'skip': 'Geo-restricted to Canada',
    }, {
        # multiple CBC.APP.Caffeine.initInstance(...)
        'url': 'http://www.cbc.ca/news/canada/calgary/dog-indoor-exercise-winter-1.3928238',
        'info_dict': {
            'title': 'Keep Rover active during the deep freeze with doggie pushups and other fun indoor tasks',
            'id': 'dog-indoor-exercise-winter-1.3928238',
            'description': 'md5:c18552e41726ee95bd75210d1ca9194c',
        },
        'playlist_mincount': 6,
    }]

    @classmethod
    def suitable(cls, url):
        return False if CBCPlayerIE.suitable(url) else super(CBCIE, cls).suitable(url)

    def _extract_player_init(self, player_init, display_id):
        player_info = self._parse_json(player_init, display_id, js_to_json)
        media_id = player_info.get('mediaId')
        if not media_id:
            clip_id = player_info['clipId']
            feed = self._download_json(
                'http://tpfeed.cbc.ca/f/ExhSPC/vms_5akSXx4Ng_Zn?byCustomValue={:mpsReleases}{%s}' % clip_id,
                clip_id, fatal=False)
            if feed:
                media_id = try_get(feed, lambda x: x['entries'][0]['guid'], compat_str)
            if not media_id:
                media_id = self._download_json(
                    'http://feed.theplatform.com/f/h9dtGB/punlNGjMlc1F?fields=id&byContent=byReleases%3DbyId%253D' + clip_id,
                    clip_id)['entries'][0]['id'].split('/')[-1]
        return self.url_result('cbcplayer:%s' % media_id, 'CBCPlayer', media_id)

    def _real_extract(self, url):
        display_id = self._match_id(url)
        webpage = self._download_webpage(url, display_id)
        title = self._og_search_title(webpage, default=None) or self._html_search_meta(
            'twitter:title', webpage, 'title', default=None) or self._html_search_regex(
                r'<title>([^<]+)</title>', webpage, 'title', fatal=False)
        entries = [
            self._extract_player_init(player_init, display_id)
            for player_init in re.findall(r'CBC\.APP\.Caffeine\.initInstance\(({.+?})\);', webpage)]
        media_ids = []
        for media_id_re in (
                r'<iframe[^>]+src="[^"]+?mediaId=(\d+)"',
                r'<div[^>]+\bid=["\']player-(\d+)',
                r'guid["\']\s*:\s*["\'](\d+)'):
            media_ids.extend(re.findall(media_id_re, webpage))
        entries.extend([
            self.url_result('cbcplayer:%s' % media_id, 'CBCPlayer', media_id)
            for media_id in orderedSet(media_ids)])
        return self.playlist_result(
            entries, display_id, strip_or_none(title),
            self._og_search_description(webpage))

class CBCPlayerIE(InfoExtractor):
    IE_NAME = 'cbc.ca:player'
    _VALID_URL = r'(?:cbcplayer:|https?://(?:www\.)?cbc\.ca/(?:player/play/|i/caffeine/syndicate/\?mediaId=))(?P<id>\d+)'
    _TESTS = [{
        'url': 'http://www.cbc.ca/player/play/2683190193',
        'md5': '64d25f841ddf4ddb28a235338af32e2c',
        'info_dict': {
            'id': '2683190193',
            'ext': 'mp4',
            'title': 'Gerry Runs a Sweat Shop',
            'description': 'md5:b457e1c01e8ff408d9d801c1c2cd29b0',
            'timestamp': 1455071400,
            'upload_date': '20160210',
            'uploader': 'CBCC-NEW',
        },
        'skip': 'Geo-restricted to Canada',
    }, {
        # Redirected from http://www.cbc.ca/player/AudioMobile/All%20in%20a%20Weekend%20Montreal/ID/2657632011/
        'url': 'http://www.cbc.ca/player/play/2657631896',
        'md5': 'e5e708c34ae6fca156aafe17c43e8b75',
        'info_dict': {
            'id': '2657631896',
            'ext': 'mp3',
            'title': 'CBC Montreal is organizing its first ever community hackathon!',
            'description': 'The modern technology we tend to depend on so heavily, is never without it\'s share of hiccups and headaches. Next weekend - CBC Montreal will be getting members of the public for its first Hackathon.',
            'timestamp': 1425704400,
            'upload_date': '20150307',
            'uploader': 'CBCC-NEW',
        },
    }, {
        'url': 'http://www.cbc.ca/player/play/2164402062',
        'md5': '33fcd8f6719b9dd60a5e73adcb83b9f6',
        'info_dict': {
            'id': '2164402062',
            'ext': 'mp4',
            'title': 'Cancer survivor four times over',
            'description': 'Tim Mayer has beaten three different forms of cancer four times in five years.',
            'timestamp': 1320410746,
            'upload_date': '20111104',
            'uploader': 'CBCC-NEW',
        },
    }]

    def _real_extract(self, url):
        video_id = self._match_id(url)
        return {
            '_type': 'url_transparent',
            'ie_key': 'ThePlatform',
            'url': smuggle_url(
                'http://link.theplatform.com/s/ExhSPC/media/guid/2655402169/%s?mbr=true&formats=MPEG4,FLV,MP3' % video_id, {
                    'force_smil_url': True
                }),
            'id': video_id,
        }

class CBCWatchBaseIE(InfoExtractor):
    _device_id = None
    _device_token = None
    _API_BASE_URL = 'https://api-cbc.cloud.clearleap.com/cloffice/client/'
    _NS_MAP = {
        'media': 'http://search.yahoo.com/mrss/',
        'clearleap': 'http://www.clearleap.com/namespace/clearleap/1.0/',
    }
    _GEO_COUNTRIES = ['CA']
    _LOGIN_URL = 'https://api.loginradius.com/identity/v2/auth/login'
    _TOKEN_URL = 'https://cloud-api.loginradius.com/sso/jwt/api/token'
    _API_KEY = '3f4beddd-2061-49b0-ae80-6f1f2ed65b37'
    _NETRC_MACHINE = 'cbcwatch'

    def _signature(self, email, password):
        data = json.dumps({
            'email': email,
            'password': password,
        }).encode()
        headers = {'content-type': 'application/json'}
        query = {'apikey': self._API_KEY}
        resp = self._download_json(self._LOGIN_URL, None, data=data, headers=headers, query=query)
        access_token = resp['access_token']

        # token
        query = {
            'access_token': access_token,
            'apikey': self._API_KEY,
            'jwtapp': 'jwt',
        }
        resp = self._download_json(self._TOKEN_URL, None, headers=headers, query=query)
        return resp['signature']

    def _call_api(self, path, video_id):
        url = path if path.startswith('http') else self._API_BASE_URL + path
        for _ in range(2):
            try:
                result = self._download_xml(url, video_id, headers={
                    'X-Clearleap-DeviceId': self._device_id,
                    'X-Clearleap-DeviceToken': self._device_token,
                })
            except ExtractorError as e:
                if isinstance(e.cause, compat_HTTPError) and e.cause.code == 401:
                    # Device token has expired, re-acquiring device token
                    self._register_device()
                    continue
                raise
        error_message = xpath_text(result, 'userMessage') or xpath_text(result, 'systemMessage')
        if error_message:
            raise ExtractorError('%s said: %s' % (self.IE_NAME, error_message))
        return result

    def _real_initialize(self):
        if self._valid_device_token():
            return
        device = self._downloader.cache.load(
            'cbcwatch', self._cache_device_key()) or {}
        self._device_id, self._device_token = device.get('id'), device.get('token')
        if self._valid_device_token():
            return
        self._register_device()

    def _valid_device_token(self):
        return self._device_id and self._device_token

    def _cache_device_key(self):
        email, _ = self._get_login_info()
        return '%s_device' % hashlib.sha256(email.encode()).hexdigest() if email else 'device'

    def _register_device(self):
        result = self._download_xml(
            self._API_BASE_URL + 'device/register',
            None, 'Acquiring device token',
            data=b'<device><type>web</type></device>')
        self._device_id = xpath_text(result, 'deviceId', fatal=True)
        email, password = self._get_login_info()
        if email and password:
            signature = self._signature(email, password)
            data = '<login><token>{0}</token><device><deviceId>{1}</deviceId><type>web</type></device></login>'.format(
                escape(signature), escape(self._device_id)).encode()
            url = self._API_BASE_URL + 'device/login'
            result = self._download_xml(
                url, None, data=data,
                headers={'content-type': 'application/xml'})
            self._device_token = xpath_text(result, 'token', fatal=True)
        else:
            self._device_token = xpath_text(result, 'deviceToken', fatal=True)
        self._downloader.cache.store(
            'cbcwatch', self._cache_device_key(), {
                'id': self._device_id,
                'token': self._device_token,
            })

    def _parse_rss_feed(self, rss):
        channel = xpath_element(rss, 'channel', fatal=True)

        def _add_ns(path):
            return xpath_with_ns(path, self._NS_MAP)

        entries = []
        for item in channel.findall('item'):
            guid = xpath_text(item, 'guid', fatal=True)
            title = xpath_text(item, 'title', fatal=True)

            media_group = xpath_element(item, _add_ns('media:group'), fatal=True)
            content = xpath_element(media_group, _add_ns('media:content'), fatal=True)
            content_url = content.attrib['url']

            thumbnails = []
            for thumbnail in media_group.findall(_add_ns('media:thumbnail')):
                thumbnail_url = thumbnail.get('url')
                if not thumbnail_url:
                    continue
                thumbnails.append({
                    'id': thumbnail.get('profile'),
                    'url': thumbnail_url,
                    'width': int_or_none(thumbnail.get('width')),
                    'height': int_or_none(thumbnail.get('height')),
                })

            timestamp = None
            release_date = find_xpath_attr(
                item, _add_ns('media:credit'), 'role', 'releaseDate')
            if release_date is not None:
                timestamp = parse_iso8601(release_date.text)

            entries.append({
                '_type': 'url_transparent',
                'url': content_url,
                'id': guid,
                'title': title,
                'description': xpath_text(item, 'description'),
                'timestamp': timestamp,
                'duration': int_or_none(content.get('duration')),
                'age_limit': parse_age_limit(xpath_text(item, _add_ns('media:rating'))),
                'episode': xpath_text(item, _add_ns('clearleap:episode')),
                'episode_number': int_or_none(xpath_text(item, _add_ns('clearleap:episodeInSeason'))),
                'series': xpath_text(item, _add_ns('clearleap:series')),
                'season_number': int_or_none(xpath_text(item, _add_ns('clearleap:season'))),
                'thumbnails': thumbnails,
                'ie_key': 'CBCWatchVideo',
            })

        return self.playlist_result(
            entries, xpath_text(channel, 'guid'),
            xpath_text(channel, 'title'),
            xpath_text(channel, 'description'))

class CBCWatchVideoIE(CBCWatchBaseIE):
    IE_NAME = 'cbc.ca:watch:video'
    _VALID_URL = r'https?://api-cbc\.cloud\.clearleap\.com/cloffice/client/web/play/?\?.*?\bcontentId=(?P<id>[\da-f]{8}-[\da-f]{4}-[\da-f]{4}-[\da-f]{4}-[\da-f]{12})'
    _TEST = {
        # geo-restricted to Canada, bypassable
        'url': 'https://api-cbc.cloud.clearleap.com/cloffice/client/web/play/?contentId=3c84472a-1eea-4dee-9267-2655d5055dcf&categoryId=ebc258f5-ee40-4cca-b66b-ba6bd55b7235',
        'only_matching': True,
    }

    def _real_extract(self, url):
        video_id = self._match_id(url)
        result = self._call_api(url, video_id)

        m3u8_url = xpath_text(result, 'url', fatal=True)
        formats = self._extract_m3u8_formats(re.sub(r'/([^/]+)/[^/?]+\.m3u8', r'/\1/\1.m3u8', m3u8_url), video_id, 'mp4', fatal=False)
        if len(formats) < 2:
            formats = self._extract_m3u8_formats(m3u8_url, video_id, 'mp4')
        for f in formats:
            format_id = f.get('format_id')
            if format_id.startswith('AAC'):
                f['acodec'] = 'aac'
            elif format_id.startswith('AC3'):
                f['acodec'] = 'ac-3'
        self._sort_formats(formats)

        info = {
            'id': video_id,
            'title': video_id,
            'formats': formats,
        }

        rss = xpath_element(result, 'rss')
        if rss:
            info.update(self._parse_rss_feed(rss)['entries'][0])
            del info['url']
            del info['_type']
            del info['ie_key']
        return info

class CBCWatchIE(CBCWatchBaseIE):
    IE_NAME = 'cbc.ca:watch'
    _VALID_URL = r'https?://(?:gem|watch)\.cbc\.ca/(?:[^/]+/)+(?P<id>[0-9a-f-]+)'
    _TESTS = [{
        # geo-restricted to Canada, bypassable
        'url': 'http://watch.cbc.ca/doc-zone/season-6/customer-disservice/38e815a-009e3ab12e4',
        'info_dict': {
            'id': '9673749a-5e77-484c-8b62-a1092a6b5168',
            'ext': 'mp4',
            'title': 'Customer (Dis)Service',
            'description': 'md5:8bdd6913a0fe03d4b2a17ebe169c7c87',
            'upload_date': '20160219',
            'timestamp': 1455840000,
        },
        'params': {
            # m3u8 download
            'skip_download': True,
            'format': 'bestvideo',
        },
    }, {
        # geo-restricted to Canada, bypassable
        'url': 'http://watch.cbc.ca/arthur/all/1ed4b385-cd84-49cf-95f0-80f004680057',
        'info_dict': {
            'id': '1ed4b385-cd84-49cf-95f0-80f004680057',
            'title': 'Arthur',
            'description': 'Arthur, the sweetest 8-year-old aardvark, and his pals solve all kinds of problems with humour, kindness and teamwork.',
        },
        'playlist_mincount': 30,
    }, {
        'url': 'https://gem.cbc.ca/media/this-hour-has-22-minutes/season-26/episode-20/38e815a-0108c6c6a42',
        'only_matching': True,
    }]

    def _real_extract(self, url):
        video_id = self._match_id(url)
        rss = self._call_api('web/browse/' + video_id, video_id)
        return self._parse_rss_feed(rss)

class CBCOlympicsIE(InfoExtractor):
    IE_NAME = 'cbc.ca:olympics'
    _VALID_URL = r'https?://olympics\.cbc\.ca/video/[^/]+/(?P<id>[^/?#]+)'
    _TESTS = [{
        'url': 'https://olympics.cbc.ca/video/whats-on-tv/olympic-morning-featuring-the-opening-ceremony/',
        'only_matching': True,
    }]

    def _real_extract(self, url):
        display_id = self._match_id(url)
        webpage = self._download_webpage(url, display_id)
        video_id = self._hidden_inputs(webpage)['videoId']
        video_doc = self._download_xml(
            'https://olympics.cbc.ca/videodata/%s.xml' % video_id, video_id)
        title = xpath_text(video_doc, 'title', fatal=True)
        is_live = xpath_text(video_doc, 'kind') == 'Live'
        if is_live:
            title = self._live_title(title)

        formats = []
        for video_source in video_doc.findall('videoSources/videoSource'):
            uri = xpath_text(video_source, 'uri')
            if not uri:
                continue
            tokenize = self._download_json(
                'https://olympics.cbc.ca/api/api-akamai/tokenize',
                video_id, data=json.dumps({
                    'VideoSource': uri,
                }).encode(), headers={
                    'Content-Type': 'application/json',
                    'Referer': url,
                    # d3.VideoPlayer._init in https://olympics.cbc.ca/components/script/base.js
                    'Cookie': '_dvp=TK:C0ObxjerU',  # AKAMAI CDN cookie
                }, fatal=False)
            if not tokenize:
                continue
            content_url = tokenize['ContentUrl']
            video_source_format = video_source.get('format')
            if video_source_format == 'IIS':
                formats.extend(self._extract_ism_formats(
                    content_url, video_id, ism_id=video_source_format, fatal=False))
            else:
                formats.extend(self._extract_m3u8_formats(
                    content_url, video_id, 'mp4',
                    'm3u8' if is_live else 'm3u8_native',
                    m3u8_id=video_source_format, fatal=False))
        self._sort_formats(formats)

        return {
            'id': video_id,
            'display_id': display_id,
            'title': title,
            'description': xpath_text(video_doc, 'description'),
            'thumbnail': xpath_text(video_doc, 'thumbnailUrl'),
            'duration': parse_duration(xpath_text(video_doc, 'duration')),
            'formats': formats,
            'is_live': is_live,
        }


code:
from __future__ import unicode_literals

from .theplatform import ThePlatformFeedIE
from ..utils import (
    ExtractorError,
    int_or_none,
    find_xpath_attr,
    xpath_element,
    xpath_text,
    update_url_query,
)

class CBSBaseIE(ThePlatformFeedIE):
    def _parse_smil_subtitles(self, smil, namespace=None, subtitles_lang='en'):
        subtitles = {}
        for k, ext in [('sMPTE-TTCCURL', 'tt'), ('ClosedCaptionURL', 'ttml'), ('webVTTCaptionURL', 'vtt')]:
            cc_e = find_xpath_attr(smil, self._xpath_ns('.//param', namespace), 'name', k)
            if cc_e is not None:
                cc_url = cc_e.get('value')
                if cc_url:
                    subtitles.setdefault(subtitles_lang, []).append({
                        'ext': ext,
                        'url': cc_url,
                    })
        return subtitles

class CBSIE(CBSBaseIE):
    _VALID_URL = r'(?:cbs:|https?://(?:www\.)?(?:cbs\.com/shows/[^/]+/video|colbertlateshow\.com/(?:video|podcasts))/)(?P<id>[\w-]+)'

    _TESTS = [{
        'url': 'http://www.cbs.com/shows/garth-brooks/video/_u7W953k6la293J7EPTd9oHkSPs6Xn6_/connect-chat-feat-garth-brooks/',
        'info_dict': {
            'id': '_u7W953k6la293J7EPTd9oHkSPs6Xn6_',
            'ext': 'mp4',
            'title': 'Connect Chat feat. Garth Brooks',
            'description': 'Connect with country music singer Garth Brooks, as he chats with fans on Wednesday November 27, 2013. Be sure to tune in to Garth Brooks: Live from Las Vegas, Friday November 29, at 9/8c on CBS!',
            'duration': 1495,
            'timestamp': 1385585425,
            'upload_date': '20131127',
            'uploader': 'CBSI-NEW',
        },
        'params': {
            # m3u8 download
            'skip_download': True,
        },
        '_skip': 'Blocked outside the US',
    }, {
        'url': 'http://colbertlateshow.com/video/8GmB0oY0McANFvp2aEffk9jZZZ2YyXxy/the-colbeard/',
        'only_matching': True,
    }, {
        'url': 'http://www.colbertlateshow.com/podcasts/dYSwjqPs_X1tvbV_P2FcPWRa_qT6akTC/in-the-bad-room-with-stephen/',
        'only_matching': True,
    }]

    def _extract_video_info(self, content_id, site='cbs', mpx_acc=2198311517):
        items_data = self._download_xml(
            'http://can.cbs.com/thunder/player/videoPlayerService.php',
            content_id, query={'partner': site, 'contentId': content_id})
        video_data = xpath_element(items_data, './/item')
        title = xpath_text(video_data, 'videoTitle', 'title', True)
        tp_path = 'dJ5BDC/media/guid/%d/%s' % (mpx_acc, content_id)
        tp_release_url = 'http://link.theplatform.com/s/' + tp_path

        asset_types = []
        subtitles = {}
        formats = []
        last_e = None
        for item in items_data.findall('.//item'):
            asset_type = xpath_text(item, 'assetType')
            if not asset_type or asset_type in asset_types or 'HLS_FPS' in asset_type or 'DASH_CENC' in asset_type:
                continue
            asset_types.append(asset_type)
            query = {
                'mbr': 'true',
                'assetTypes': asset_type,
            }
            if asset_type.startswith('HLS') or asset_type in ('OnceURL', 'StreamPack'):
                query['formats'] = 'MPEG4,M3U'
            elif asset_type in ('RTMP', 'WIFI', '3G'):
                query['formats'] = 'MPEG4,FLV'
            try:
                tp_formats, tp_subtitles = self._extract_theplatform_smil(
                    update_url_query(tp_release_url, query), content_id,
                    'Downloading %s SMIL data' % asset_type)
            except ExtractorError as e:
                last_e = e
                continue
            formats.extend(tp_formats)
            subtitles = self._merge_subtitles(subtitles, tp_subtitles)
        if last_e and not formats:
            raise last_e
        self._sort_formats(formats)

        info = self._extract_theplatform_metadata(tp_path, content_id)
        info.update({
            'id': content_id,
            'title': title,
            'series': xpath_text(video_data, 'seriesTitle'),
            'season_number': int_or_none(xpath_text(video_data, 'seasonNumber')),
            'episode_number': int_or_none(xpath_text(video_data, 'episodeNumber')),
            'duration': int_or_none(xpath_text(video_data, 'videoLength'), 1000),
            'thumbnail': xpath_text(video_data, 'previewImageURL'),
            'formats': formats,
            'subtitles': subtitles,
        })
        return info

    def _real_extract(self, url):
        content_id = self._match_id(url)
        return self._extract_video_info(content_id)


code:
# coding: utf-8
from __future__ import unicode_literals

import re

from .cbs import CBSIE
from ..utils import int_or_none

class CBSInteractiveIE(CBSIE):
    _VALID_URL = r'https?://(?:www\.)?(?P<site>cnet|zdnet)\.com/(?:videos|video(?:/share)?)/(?P<id>[^/?]+)'
    _TESTS = [{
        'url': 'http://www.cnet.com/videos/hands-on-with-microsofts-windows-8-1-update/',
        'info_dict': {
            'id': 'R49SYt__yAfmlXR85z4f7gNmCBDcN_00',
            'display_id': 'hands-on-with-microsofts-windows-8-1-update',
            'ext': 'mp4',
            'title': 'Hands-on with Microsoft Windows 8.1 Update',
            'description': 'The new update to the Windows 8 OS brings improved performance for mouse and keyboard users.',
            'uploader_id': '6085384d-619e-11e3-b231-14feb5ca9861',
            'uploader': 'Sarah Mitroff',
            'duration': 70,
            'timestamp': 1396479627,
            'upload_date': '20140402',
        },
        'params': {
            # m3u8 download
            'skip_download': True,
        },
    }, {
        'url': 'http://www.cnet.com/videos/whiny-pothole-tweets-at-local-government-when-hit-by-cars-tomorrow-daily-187/',
        'md5': 'f11d27b2fa18597fbf92444d2a9ed386',
        'info_dict': {
            'id': 'kjOJd_OoVJqbg_ZD8MZCOk8Wekb9QccK',
            'display_id': 'whiny-pothole-tweets-at-local-government-when-hit-by-cars-tomorrow-daily-187',
            'ext': 'mp4',
            'title': 'Whiny potholes tweet at local government when hit by cars (Tomorrow Daily 187)',
            'description': 'md5:d2b9a95a5ffe978ae6fbd4cf944d618f',
            'uploader_id': 'b163284d-6b73-44fc-b3e6-3da66c392d40',
            'uploader': 'Ashley Esqueda',
            'duration': 1482,
            'timestamp': 1433289889,
            'upload_date': '20150603',
        },
    }, {
        'url': 'http://www.zdnet.com/video/share/video-keeping-android-smartphones-and-tablets-secure/',
        'info_dict': {
            'id': 'k0r4T_ehht4xW_hAOqiVQPuBDPZ8SRjt',
            'display_id': 'video-keeping-android-smartphones-and-tablets-secure',
            'ext': 'mp4',
            'title': 'Video: Keeping Android smartphones and tablets secure',
            'description': 'Here\'s the best way to keep Android devices secure, and what you do when they\'ve come to the end of their lives.',
            'uploader_id': 'f2d97ea2-8175-11e2-9d12-0018fe8a00b0',
            'uploader': 'Adrian Kingsley-Hughes',
            'duration': 731,
            'timestamp': 1449129925,
            'upload_date': '20151203',
        },
        'params': {
            # m3u8 download
            'skip_download': True,
        },
    }, {
        'url': 'http://www.zdnet.com/video/huawei-matebook-x-video/',
        'only_matching': True,
    }]

    MPX_ACCOUNTS = {
        'cnet': 2198311517,
        'zdnet': 2387448114,
    }

    def _real_extract(self, url):
        site, display_id = re.match(self._VALID_URL, url).groups()
        webpage = self._download_webpage(url, display_id)

        data_json = self._html_search_regex(
            r"data(?:-(?:cnet|zdnet))?-video(?:-(?:uvp(?:js)?|player))?-options='([^']+)'",
            webpage, 'data json')
        data = self._parse_json(data_json, display_id)
        vdata = data.get('video') or (data.get('videos') or data.get('playlist'))[0]

        video_id = vdata['mpxRefId']

        title = vdata['title']
        author = vdata.get('author')
        if author:
            uploader = '%s %s' % (author['firstName'], author['lastName'])
            uploader_id = author.get('id')
        else:
            uploader = None
            uploader_id = None

        info = self._extract_video_info(video_id, site, self.MPX_ACCOUNTS[site])
        info.update({
            'id': video_id,
            'display_id': display_id,
            'title': title,
            'duration': int_or_none(vdata.get('duration')),
            'uploader': uploader,
            'uploader_id': uploader_id,
        })
        return info


code:
# coding: utf-8
from __future__ import unicode_literals

from .anvato import AnvatoIE
from .sendtonews import SendtoNewsIE
from ..compat import compat_urlparse
from ..utils import (
    parse_iso8601,
    unified_timestamp,
)

class CBSLocalIE(AnvatoIE):
    _VALID_URL = r'https?://[a-z]+\.cbslocal\.com/(?:\d+/\d+/\d+|video)/(?P<id>[0-9a-z-]+)'

    _TESTS = [{
        # Anvato backend
        'url': 'http://losangeles.cbslocal.com/2016/05/16/safety-advocates-say-fatal-car-seat-failures-are-public-health-crisis',
        'md5': 'f0ee3081e3843f575fccef901199b212',
        'info_dict': {
            'id': '3401037',
            'ext': 'mp4',
            'title': 'Safety Advocates Say Fatal Car Seat Failures Are \'Public Health Crisis\'',
            'description': 'Collapsing seats have been the focus of scrutiny for decades, though experts say remarkably little has been done to address the issue. Randy Paige reports.',
            'thumbnail': 're:^https?://.*',
            'timestamp': 1463440500,
            'upload_date': '20160516',
            'uploader': 'CBS',
            'subtitles': {
                'en': 'mincount:5',
            },
            'categories': [
                'Stations\\Spoken Word\\KCBSTV',
                'Syndication\\MSN',
                'Syndication\\NDN',
                'Syndication\\AOL',
                'Syndication\\Yahoo',
                'Syndication\\Tribune',
                'Syndication\\Curb.tv',
                'Content\\News'
            ],
            'tags': ['CBS 2 News Evening'],
        },
    }, {
        # SendtoNews embed
        'url': 'http://cleveland.cbslocal.com/2016/05/16/indians-score-season-high-15-runs-in-blowout-win-over-reds-rapid-reaction/',
        'info_dict': {
            'id': 'GxfCe0Zo7D-175909-5588',
        },
        'playlist_count': 9,
        'params': {
            # m3u8 download
            'skip_download': True,
        },
    }, {
        'url': 'http://newyork.cbslocal.com/video/3580809-a-very-blue-anniversary/',
        'info_dict': {
            'id': '3580809',
            'ext': 'mp4',
            'title': 'A Very Blue Anniversary',
            'description': 'CBS2’s Cindy Hsu has more.',
            'thumbnail': 're:^https?://.*',
            'timestamp': int,
            'upload_date': r're:^\d{8}$',
            'uploader': 'CBS',
            'subtitles': {
                'en': 'mincount:5',
            },
            'categories': [
                'Stations\\Spoken Word\\WCBSTV',
                'Syndication\\AOL',
                'Syndication\\MSN',
                'Syndication\\NDN',
                'Syndication\\Yahoo',
                'Content\\News',
                'Content\\News\\Local News',
            ],
            'tags': ['CBS 2 News Weekends', 'Cindy Hsu', 'Blue Man Group'],
        },
    }]

    def _real_extract(self, url):
        display_id = self._match_id(url)
        webpage = self._download_webpage(url, display_id)

        sendtonews_url = SendtoNewsIE._extract_url(webpage)
        if sendtonews_url:
            return self.url_result(
                compat_urlparse.urljoin(url, sendtonews_url),
                ie=SendtoNewsIE.ie_key())

        info_dict = self._extract_anvato_videos(webpage, display_id)

        timestamp = unified_timestamp(self._html_search_regex(
            r'class="(?:entry|post)-date"[^>]*>([^<]+)', webpage,
            'released date', default=None)) or parse_iso8601(
            self._html_search_meta('uploadDate', webpage))

        info_dict.update({
            'display_id': display_id,
            'timestamp': timestamp,
        })

        return info_dict


code:
from __future__ import unicode_literals

from .cbs import CBSBaseIE

class CBSSportsIE(CBSBaseIE):
    _VALID_URL = r'https?://(?:www\.)?cbssports\.com/[^/]+/(?:video|news)/(?P<id>[^/?#&]+)'

    _TESTS = [{
        'url': 'https://www.cbssports.com/nba/video/donovan-mitchell-flashes-star-potential-in-game-2-victory-over-thunder/',
        'info_dict': {
            'id': '1214315075735',
            'ext': 'mp4',
            'title': 'Donovan Mitchell flashes star potential in Game 2 victory over Thunder',
            'description': 'md5:df6f48622612c2d6bd2e295ddef58def',
            'timestamp': 1524111457,
            'upload_date': '20180419',
            'uploader': 'CBSI-NEW',
        },
        'params': {
            # m3u8 download
            'skip_download': True,
        }
    }, {
        'url': 'https://www.cbssports.com/nba/news/nba-playoffs-2018-watch-76ers-vs-heat-game-3-series-schedule-tv-channel-online-stream/',
        'only_matching': True,
    }]

    def _extract_video_info(self, filter_query, video_id):
        return self._extract_feed_info('dJ5BDC', 'VxxJg8Ymh8sE', filter_query, video_id)

    def _real_extract(self, url):
        display_id = self._match_id(url)
        webpage = self._download_webpage(url, display_id)
        video_id = self._search_regex(
            [r'(?:=|%26)pcid%3D(\d+)', r'embedVideo(?:Container)?_(\d+)'],
            webpage, 'video id')
        return self._extract_video_info('byId=%s' % video_id, video_id)


code:
# coding: utf-8
from __future__ import unicode_literals

from .common import InfoExtractor
from ..utils import (
    int_or_none,
    parse_iso8601,
    try_get,
    url_or_none,
)

class CCCIE(InfoExtractor):
    IE_NAME = 'media.ccc.de'
    _VALID_URL = r'https?://(?:www\.)?media\.ccc\.de/v/(?P<id>[^/?#&]+)'

    _TESTS = [{
        'url': 'https://media.ccc.de/v/30C3_-_5443_-_en_-_saal_g_-_201312281830_-_introduction_to_processor_design_-_byterazor#video',
        'md5': '3a1eda8f3a29515d27f5adb967d7e740',
        'info_dict': {
            'id': '1839',
            'ext': 'mp4',
            'title': 'Introduction to Processor Design',
            'creator': 'byterazor',
            'description': 'md5:df55f6d073d4ceae55aae6f2fd98a0ac',
            'thumbnail': r're:^https?://.*\.jpg$',
            'upload_date': '20131228',
            'timestamp': 1388188800,
            'duration': 3710,
            'tags': list,
        }
    }, {
        'url': 'https://media.ccc.de/v/32c3-7368-shopshifting#download',
        'only_matching': True,
    }]

    def _real_extract(self, url):
        display_id = self._match_id(url)
        webpage = self._download_webpage(url, display_id)
        event_id = self._search_regex(r"data-id='(\d+)'", webpage, 'event id')
        event_data = self._download_json('https://media.ccc.de/public/events/%s' % event_id, event_id)

        formats = []
        for recording in event_data.get('recordings', []):
            recording_url = recording.get('recording_url')
            if not recording_url:
                continue
            language = recording.get('language')
            folder = recording.get('folder')
            format_id = None
            if language:
                format_id = language
            if folder:
                if language:
                    format_id += '-' + folder
                else:
                    format_id = folder
            vcodec = 'h264' if 'h264' in folder else (
                'none' if folder in ('mp3', 'opus') else None
            )
            formats.append({
                'format_id': format_id,
                'url': recording_url,
                'width': int_or_none(recording.get('width')),
                'height': int_or_none(recording.get('height')),
                'filesize': int_or_none(recording.get('size'), invscale=1024 * 1024),
                'language': language,
                'vcodec': vcodec,
            })
        self._sort_formats(formats)

        return {
            'id': event_id,
            'display_id': display_id,
            'title': event_data['title'],
            'creator': try_get(event_data, lambda x: ', '.join(x['persons'])),
            'description': event_data.get('description'),
            'thumbnail': event_data.get('thumb_url'),
            'timestamp': parse_iso8601(event_data.get('date')),
            'duration': int_or_none(event_data.get('length')),
            'tags': event_data.get('tags'),
            'formats': formats,
        }

class CCCPlaylistIE(InfoExtractor):
    IE_NAME = 'media.ccc.de:lists'
    _VALID_URL = r'https?://(?:www\.)?media\.ccc\.de/c/(?P<id>[^/?#&]+)'
    _TESTS = [{
        'url': 'https://media.ccc.de/c/30c3',
        'info_dict': {
            'title': '30C3',
            'id': '30c3',
        },
        'playlist_count': 135,
    }]

    def _real_extract(self, url):
        playlist_id = self._match_id(url).lower()

        conf = self._download_json(
            'https://media.ccc.de/public/conferences/' + playlist_id,
            playlist_id)

        entries = []
        for e in conf['events']:
            event_url = url_or_none(e.get('frontend_link'))
            if event_url:
                entries.append(self.url_result(event_url, ie=CCCIE.ie_key()))

        return self.playlist_result(entries, playlist_id, conf.get('title'))


code:
# coding: utf-8
from __future__ import unicode_literals

import re

from .common import InfoExtractor
from ..utils import (
    clean_html,
    int_or_none,
    parse_duration,
    parse_iso8601,
    parse_resolution,
    url_or_none,
)

class CCMAIE(InfoExtractor):
    _VALID_URL = r'https?://(?:www\.)?ccma\.cat/(?:[^/]+/)*?(?P<type>video|audio)/(?P<id>\d+)'
    _TESTS = [{
        'url': 'http://www.ccma.cat/tv3/alacarta/lespot-de-la-marato-de-tv3/lespot-de-la-marato-de-tv3/video/5630208/',
        'md5': '7296ca43977c8ea4469e719c609b0871',
        'info_dict': {
            'id': '5630208',
            'ext': 'mp4',
            'title': 'L\'espot de La Marató de TV3',
            'description': 'md5:f12987f320e2f6e988e9908e4fe97765',
            'timestamp': 1470918540,
            'upload_date': '20160811',
        }
    }, {
        'url': 'http://www.ccma.cat/catradio/alacarta/programa/el-consell-de-savis-analitza-el-derbi/audio/943685/',
        'md5': 'fa3e38f269329a278271276330261425',
        'info_dict': {
            'id': '943685',
            'ext': 'mp3',
            'title': 'El Consell de Savis analitza el derbi',
            'description': 'md5:e2a3648145f3241cb9c6b4b624033e53',
            'upload_date': '20171205',
            'timestamp': 1512507300,
        }
    }]

    def _real_extract(self, url):
        media_type, media_id = re.match(self._VALID_URL, url).groups()

        media = self._download_json(
            'http://dinamics.ccma.cat/pvideo/media.jsp', media_id, query={
                'media': media_type,
                'idint': media_id,
            })

        formats = []
        media_url = media['media']['url']
        if isinstance(media_url, list):
            for format_ in media_url:
                format_url = url_or_none(format_.get('file'))
                if not format_url:
                    continue
                label = format_.get('label')
                f = parse_resolution(label)
                f.update({
                    'url': format_url,
                    'format_id': label,
                })
                formats.append(f)
        else:
            formats.append({
                'url': media_url,
                'vcodec': 'none' if media_type == 'audio' else None,
            })
        self._sort_formats(formats)

        informacio = media['informacio']
        title = informacio['titol']
        durada = informacio.get('durada', {})
        duration = int_or_none(durada.get('milisegons'), 1000) or parse_duration(durada.get('text'))
        timestamp = parse_iso8601(informacio.get('data_emissio', {}).get('utc'))

        subtitles = {}
        subtitols = media.get('subtitols', {})
        if subtitols:
            sub_url = subtitols.get('url')
            if sub_url:
                subtitles.setdefault(
                    subtitols.get('iso') or subtitols.get('text') or 'ca', []).append({
                        'url': sub_url,
                    })

        thumbnails = []
        imatges = media.get('imatges', {})
        if imatges:
            thumbnail_url = imatges.get('url')
            if thumbnail_url:
                thumbnails = [{
                    'url': thumbnail_url,
                    'width': int_or_none(imatges.get('amplada')),
                    'height': int_or_none(imatges.get('alcada')),
                }]

        return {
            'id': media_id,
            'title': title,
            'description': clean_html(informacio.get('descripcio')),
            'duration': duration,
            'timestamp': timestamp,
            'thumbnails': thumbnails,
            'subtitles': subtitles,
            'formats': formats,
        }