'''
Parse links in html and css pages.
'''

import logging
import re
import hashlib
import urllib.parse
from functools import partial
import multidict
import html

from bs4 import BeautifulSoup

from . import stats
from .urls import URL
from . import facet
from . import config

LOGGER = logging.getLogger(__name__)


async def do_parser(body, body_bytes, resp_headers, url, crawler):
    if len(body) > int(config.read('Multiprocess', 'ParseInBurnerSize')):
        stats.stats_sum('parser in burner thread', 1)
        # headers is a multidict.CIMultiDictProxy case-blind dict
        # and the Proxy form of it doesn't pickle, so convert to one that does
        resp_headers = multidict.CIMultiDict(resp_headers)
        links, embeds, sha1, facets, base = await crawler.burner.burn(
            partial(do_burner_work_html, body, body_bytes, resp_headers,
                    burn_prefix='burner ', url=url),
            url=url)
    else:
        stats.stats_sum('parser in main thread', 1)
        # no coroutine state because this is a burn, not an await
        links, embeds, sha1, facets, base = do_burner_work_html(
            body, body_bytes, resp_headers, burn_prefix='main ', url=url)

    return links, embeds, sha1, facets, base


def do_burner_work_html(html, html_bytes, headers, burn_prefix='', url=None):
    stats.stats_sum('parser html bytes', len(html_bytes))

    # This embodies a minimal parsing policy; it needs to be made pluggable/configurable
    #  split head/body
    #  soup the head so we can accurately get base and other details
    #  regex the body for links and embeds, for speed

    with stats.record_burn(burn_prefix+'split_head_body', url=url):
        head, body = split_head_body(html, url=url)

    '''
    beautiful soup + lxml2 parses only about 4-16 MB/s
    '''
    stats.stats_sum('head soup bytes', len(head))
    with stats.record_burn(burn_prefix+'head soup', url=url):
        try:
            head_soup = BeautifulSoup(head, 'lxml')
        except Exception as e:
            LOGGER.info('url %s threw the %r exception in BeautifulSoup', url, e)
            stats.stats_sum('head soup exception '+str(e), 1)
            raise

    base = head_soup.find('base') or {}
    base = base.get('href')
    if base:
        # base can be relative, e.g. 'subdir/' or '.'
        base = urllib.parse.urljoin(url.url, base)
    base_or_url = base or url

    with stats.record_burn(burn_prefix+'find_head_links_soup', url=url):
        links, embeds = find_head_links_soup(head_soup)

    with stats.record_burn(burn_prefix+'find_body_links_re', url=url):
        lbody, ebody = find_body_links_re(body)
        links += lbody
        embeds += ebody

    embeds = clean_link_objects(embeds, ('javascript:', 'data:'))
    links = clean_link_objects(links, ('javascript:',))

    with stats.record_burn(burn_prefix+'url_clean_join', url=url):
        links = url_clean_join(links, url=base_or_url)
        embeds = url_clean_join(embeds, url=base_or_url)

    with stats.record_burn(burn_prefix+'sha1 html', url=url):
        sha1 = 'sha1:' + hashlib.sha1(html_bytes).hexdigest()

    with stats.record_burn(burn_prefix+'facets', url=url):
        facets = facet.compute_all(html, head, body, headers, links, embeds, head_soup=head_soup, url=url)

    links = collapse_links(links)
    embeds = collapse_links(embeds)

    return links, embeds, sha1, facets, base


def collapse_links(links):
    ret = []
    for link in links:
        l = link.get('href')
        if not l:
            l = link.get('src')
        if l:
            ret.append(l)
    return ret


def clean_link_objects(link_objects, schemes):
    '''
    Drop all elements of the link_objects that are in schemes.
    '''
    schemes = tuple(schemes)
    ret = []
    for link_object in link_objects:
        if link_object is None:
            continue
        u = link_object.get('href') or link_object.get('src')
        if u and u.startswith(schemes):
            continue
        ret.append(link_object)
    return ret


def find_html_links_re(html):
    '''
    Find the outgoing links and embeds in html, both head and body.
    This can't tell the difference between links and embeds, so we
    call them all links.

    On a 3.4ghz x86 core, runs at ~ 50 megabytes/sec.
    '''
    stats.stats_sum('html_links_re parser bytes', len(html))

    delims = set(
        [m[1] for m in re.findall(r'''\s(?:href|src)\s{,3}=\s{,3}(?P<delim>['"])(.*?)(?P=delim)''', html, re.I | re.S)]
    )
    no_delims = set(re.findall(r'''\s(?:href|src)\s{,3}=\s{,3}([^\s'"<>]+)''', html, re.I))

    links = delims.union(no_delims)

    links = [{'href': h} for h in links]
    return links, []


def find_body_links_re(body):
    '''
    Find links in an html body, divided among links and embeds.

    On a 3.4 ghz x86 core, runs at ~ 25 megabyte/sec.
    '''
    stats.stats_sum('body_links_re parser bytes', len(body))

    embeds_delims = set(
        [m[1] for m in re.findall(r'''\ssrc\s{,3}=\s{,3}(?P<delim>['"])(.*?)(?P=delim)''', body, re.I | re.S)]
    )
    embeds_no_delims = set(re.findall(r'''\ssrc\s{,3}=\s{,3}([^\s'"<>]+)''', body, re.I))
    embeds = embeds_delims.union(embeds_no_delims)

    links_delims = set(
        [m[1] for m in re.findall(r'''\shref\s{,3}=\s{,3}(?P<delim>['"])(.*?)(?P=delim)''', body, re.I | re.S)]
    )
    links_no_delims = set(re.findall(r'''\shref\s{,3}=\s{,3}([^\s'"<>]+)''', body, re.I))
    links = links_delims.union(links_no_delims)

    embeds = [{'src': s} for s in embeds]
    links = [{'href': h} for h in links]

    return links, embeds


def find_body_links_anchors_re(body):
    '''
    Find links and anchors in an html body, divided among links and embeds.

    On a 3.4 ghz x86 core, runs at ~ NN megabyte/sec.
    '''
    stats.stats_sum('body_links_anchors_re parser bytes', len(body))

    embeds_delims = set(
        [m[1] for m in re.findall(r'''\ssrc\s{,3}=\s{,3}(?P<delim>['"])(.*?)(?P=delim)''', body, re.I | re.S | re.X)]
    )
    embeds_no_delims = set(re.findall(r'''\ssrc\s{,3}=\s{,3}([^\s'"<>]+)''', body, re.I | re.X))
    embeds = embeds_delims.union(embeds_no_delims)

    links_delims = set()
    for m in re.finditer(r'''\shref\s{,3}=\s{,3}(?P<delim>['"])(.*?)(?P=delim) [^>]{,400} >''', body, re.I | re.S | re.X):
        delim = m.group(1)
        href = m.group(2)

        if delim in href:
            # this happens when the size above isn't big enough.
            href = href.split(delim, 1)[0]
            stats.stats_sum('body_links_anchors_re parser extra delim split needed', 1)

        if href.startswith('#'):
            continue
        end = m.end(0)
        anchor = body[end:]
        mm = re.match(r'(.{,101}?)</a>', anchor, re.I | re.S)
        if mm:
            anchor = mm.group(1)
        else:
            anchor = anchor.split('<', 1)[0]
        links_delims.add((href, anchor))

    links_no_delims = set()
    for m in re.finditer(r'''\shref\s{,3}=\s{,3}([^'">\s]+) [^>]{,200} >''', body, re.I | re.S | re.X):
        href = m.group(1)
        if href == '#':
            continue
        end = m.end(0)
        anchor = body[end:]
        mm = re.match(r'(.{,101}?)</a>', anchor, re.I | re.S)
        if mm:
            anchor = mm.group(1)
        else:
            anchor = anchor.split('<', 1)[0]
        links_no_delims.add((href, anchor))

    links = links_delims.union(links_no_delims)

    embeds = [{'src': s} for s in embeds]
    links = [dict((('href', h[0]), *trim_anchor(h[1]))) for h in links]

    return links, embeds


def find_css_links_re(css):
    '''
    Finds the links embedded in css files
    '''
    stats.stats_sum('css_links_re parser bytes', len(css))

    embeds_delims = set(
        [m[1] for m in re.findall(r'''\surl\(\s?(?P<delim>['"])(.*?)(?P=delim)''', css, re.I | re.S)]
    )
    embeds_no_delims = set(re.findall(r'''\surl\(\s?([^\s'"<>()]+)''', css, re.I))

    return [], list(embeds_delims.union(embeds_no_delims))


def find_head_links_soup(head_soup):
    embeds = []
    for tag in head_soup.find_all(src=True):
        lo = build_link_object(tag)
        if lo:
            embeds.append(lo)
    for tag in head_soup.find_all(href=True):
        lo = build_link_object(tag)
        if lo:
            embeds.append(lo)
    return [], embeds


def trim_anchor(anchor):
    ret = []
    # pull out <img alt=""> here?
    anchor = re.sub(r'<.*?>', '', anchor, re.S).strip()
    anchor = html.unescape(anchor)

    if len(anchor) > 100:
        anchor = anchor[:100]
        ret.append(('anchor_truncated', True))
    if anchor:
        ret.append(('anchor', anchor))
    return ret


def build_link_object(tag):
    ret = {'tag': tag.name}

    if tag.get('href'):
        ret['href'] = tag.get('href')
    if tag.get('src'):
        ret['src'] = tag.get('src')
    if 'href' not in ret and 'src' not in ret:
        # href or src was present but false
        return

    if tag.name == 'a':
        try:
            parts = tag.itertext(with_tail=False)
        except TypeError:
            parts = None
        if parts:
            anchor = ' '.join(parts)
            anchor = re.sub(r'\s+', ' ', anchor).strip()
            ret.update(*trim_anchor(anchor))
        if tag.get('target'):
            ret['target'] = tag.get('target')

    if tag.name == 'iframe':
        if tag.get('name'):
            ret['name'] = tag.get('name')

    return ret


def find_body_links_soup(body_soup):
    embeds = []
    links = []
    for tag in body_soup.find_all(src=True):
        if tag.name == 'iframe':
            lo = build_link_object(tag)
            if lo:
                links.append(lo)
        else:
            lo = build_link_object(tag)
            if lo:
                embeds.append(lo)
    for tag in body_soup.find_all(href=True):
        if tag.name == 'link':
            rel = tag.get('rel', [None])[0]
            if rel == 'stylesheet':
                lo = build_link_object(tag)
                if lo:
                    embeds.append(lo)
            else:
                pass  # discard other body-ok like 'prefetch'
        else:
            lo = build_link_object(tag)
            if lo:
                links.append(lo)
    return links, embeds


def url_clean_join(links, url=None):
    ret = []
    for link in links:
        # you can have both, for example <link> tags occasionally incorectly have both in the wild
        if 'href' in link:
            link['href'] = URL(link['href'], urljoin=url)
        if 'src' in link:
            link['src'] = URL(link['src'], urljoin=url)
        ret.append(link)
    return ret


def url_dedup(link_objects):
    ret = []
    dedup = set()
    for link_object in link_objects:
        link = link_object.get('href') or link_object.get('src')
        if link:
            if link in dedup:
                continue
            dedup.add(link)
            ret.append(link_object)
    return ret


def report():
    # XXX fix these names
    # XXX how does this get just the burner thread? use the prefix
    b = stats.stat_value('parser html bytes')
    c = stats.stat_value('find_html_links re')
    LOGGER.info('Burner thread report:')
    if c is not None and c > 0:
        LOGGER.info('  Burner thread parsed %.1f MB/cpu-second', b / c / 1000000)
    d = stats.stat_value('sha1 html')
    if d is not None and d > 0:
        


    t, c = stats.burn_values('find_html_links url_clean_join')
    if c is not None and c > 0 and t is not None and t > 0:
        LOGGER.info('  Burner thread cleaned %.1f kilo-urls/cpu-second', c / t / 1000)


def split_head_body(html, url=None):
    '''
    Efficiently split the head from the body, so we can use different
    parsers on each.  There's no point doing this split if it's
    expensive.

    It's legal for webpages to leave off <head> and <body>; the HTML5
    standard requires browsers to figure it out based on the html
    tags. We can't do that efficiently, so we punt for such webpages,
    and return the entire page as body.
    '''

    # heuristic: if there's a <head> tag at all, it's early in the document
    m = re.search(r'<head[\s>]', html[:2000], re.I)
    if not m:
        stats.stats_sum('parser split short fail', 1)
        # well darn. try the same re as below, but with limited size
        m = re.search(r'<(?:/head>|body[\s>])', html[:50000], re.I)
        if m:
            stats.stats_sum('parser split short fail save', 1)
            return html[:m.start()], html[m.end():]
        else:
            return '', html

    # having seen <head>, we're willing to parse for a long time for </head or <body
    m = re.search(r'<(?:/head>|body[\s>])', html[:1000000], re.I)
    if not m:
        stats.stats_sum('parser split long fail', 1)
        return '', html

    return html[:m.start()], html[m.end():]  # matched text is not included in either


def parse_refresh(s):
    '''
    https://www.w3.org/TR/html5/document-metadata.html#statedef-http-equiv-refresh

    See in real life and not standard-conforming, in order of popularity:
      whitespace after t before the ';'
      starting with a decimal point
      starting with a minus sign
      empty time, starts with ';'
      url= but missing the ';'
    None of these actually work in modern FF, Chrome, or Safari
    '''
    t = None
    refresh = r'\s* (\d+) (?:\.[\d\.]*)? [;,] \s* ([Uu][Rr][Ll] \s* = \s* ["\']?)? (.*)'
    m = re.match(refresh, s, re.X)
    if m:
        t, sep, url = m.groups()
        if sep and sep.endswith('"') and '"' in url:
            url = url[:url.index('"')]
        if sep and sep.endswith("'") and "'" in url:
            url = url[:url.index("'")]
        try:
            t = int(t)
        except ValueError:
            t = None
    else:
        if s.isdigit():
            t = int(s)
        url = None
    return t, url


'''
Helpers to minimize how many bytes we have to html parse.
Of course, these are all dangerous, but they might be useful
if the <head> of a webpage is abnormally large
'''


def regex_out_comments(html):
    # I think whitespace is allowed: < \s* !-- .* -- \s* > XXX
    return re.sub(r'<!--.*?-->', '', html, flags=re.S)


def regex_out_some_scripts(html):
    '''
    This nukes <script>...</script>, but does not nuke <script type="...
    '''
    return re.sub(r'<script>.*?</script>', '', html, flags=re.S)


def regex_out_all_scripts(html):
    return re.sub(r'<script[\s>].*?</script>', '', html, flags=re.S)

LOGGER.info('  Burner thread sha1 %.1f MB/cpu-second', b / d / 1000000)