Article Extraction from HTML
769808
public function newspaper_html( $html )
	{
		$data =  $this->post_call('/newspaper', ['html' => $html ] );
		
		return ( !empty($data['newspaper']) ) ? $data['newspaper'] : null;
	}
230785
function scrapeHtml(document, robots)
{
	var link,links,preliminaries,rootNode;
	
	rootNode = findRootNode(document);
	
	if (rootNode != null)
	{
		preliminaries = findPreliminaries(rootNode, robots);
		links = [];
		
		findLinks(rootNode, function(node, attrName, url)
		{
			link = linkObj(url);

			link.html.attrs = node.attrMap;
			link.html.attrName = attrName;
			link.html.base = preliminaries.base;
			link.html.index = links.length;
			link.html.selector = getSelector(node);
			link.html.tag = stringifyNode(node);
			link.html.tagName = node.nodeName;
			link.html.text = getText(node);

			// If not a "fake" (duplicated) element, as a result of adoption
			if (node.__location !== undefined)
			{
				link.html.location = node.__location.attrs[attrName];
			}
			
			links.push(link);
		});
	}
	
	return links;
}
1031852
def _extract_url_and_title(self, text, start):
        """"""
        # text[start] equals the opening parenthesis
        idx = self._whitespace.match(text, start + 1).end()
        if idx == len(text):
            return None, None
        end_idx = idx
        has_anglebrackets = text[idx] == "<"
        if has_anglebrackets:
            end_idx = self._find_balanced(text, end_idx+1, "<", ">")
        end_idx = self._find_balanced(text, end_idx, "(", ")")
        match = self._inline_link_title.search(text, idx, end_idx)
        if not match:
            return None, None
        url = text[idx:match.start()]
        if has_anglebrackets:
            url = self._strip_anglebrackets.sub(r'\1', url)
        return url, end_idx
1014892
def extract_urls(url, data, unescape=HTMLParser.HTMLParser().unescape):
    """"""
    parts = urlparse.urlparse(url)
    prefix = '%s://%s' % (parts.scheme, parts.netloc)

    accessed_dir = os.path.dirname(parts.path)
    if not accessed_dir.endswith('/'):
        accessed_dir += '/'

    for pattern, replacement in REPLACEMENT_REGEXES:
        fixed = replacement % {
            'base': prefix,
            'accessed_dir': accessed_dir,
        }
        data = re.sub(pattern, fixed, data)

    result = set()
    for match in re.finditer(MAYBE_HTML_URL_REGEX, data):
        found_url = unescape(match.groupdict()['absurl'])
        found_url = clean_url(
            found_url,
            force_scheme=parts[0])  # Use the main page's scheme
        result.add(found_url)

    return result
757050
def get_sentence_xpath_tuples(filename_url_or_filelike,
                              xpath_to_text=TEXT_FINDER_XPATH):
    

    parsed_html = get_html_tree(filename_url_or_filelike)

    try:
        xpath_finder = parsed_html.getroot().getroottree().getpath
    except(AttributeError):
        xpath_finder = parsed_html.getroottree().getpath

    nodes_with_text = parsed_html.xpath(xpath_to_text)

    sent_xpath_pairs = [
        # hard-code paragraph breaks (there has to be a better way)
        ('\n\n' + s, xpath_finder(n)) if e == 0
        else (s, xpath_finder(n))
        for n in nodes_with_text
        for e, s in enumerate(SENTENCE_TOKEN_PATTERN.split(
            BRACKET_PATTERN.sub('', ''.join(n.xpath('.//text()')))))
        if s.endswith(tuple(SENTENCE_ENDING))
        ]

    return sent_xpath_pairs
1023336
def _extract_links(self, selector, response_url, response_encoding, base_url):
        '''
        
        '''
        links = []
        # hacky way to get the underlying lxml parsed document
        for el, attr, attr_val in self._iter_links(selector.root):
            # pseudo lxml.html.HtmlElement.make_links_absolute(base_url)
            try:
                attr_val = urljoin(base_url, attr_val)
            except ValueError:
                continue # skipping bogus links
            else:
                url = self.process_attr(attr_val)
                if url is None:
                    continue
            # added 'ignore' to encoding errors
            url = to_native_str(url, encoding=response_encoding,
                                errors='ignore')
            # to fix relative links after process_value
            url = urljoin(response_url, url)
            link = Link(url, _collect_string_content(el) or u'',
                        nofollow=rel_has_nofollow(el.get('rel')))
            links.append(link)
        return self._deduplicate_if_needed(links)
608750
def _extract_annotations_from_span(span):
    """"""
    if span.time_events is None:
        return []

    annotations = []
    for time_event in span.time_events:
        annotation = time_event.annotation
        if not annotation:
            continue

        event_timestamp_mus = timestamp_to_microseconds(time_event.timestamp)
        annotations.append({'timestamp': int(round(event_timestamp_mus)),
                            'value': annotation.description})

    return annotations
1119029
def extract(self, item):
        

        article_candidate = ArticleCandidate()
        article_candidate.extractor = self._name()
        article_candidate.title = self._title(item)
        article_candidate.description = self._description(item)
        article_candidate.text = self._text(item)
        article_candidate.topimage = self._topimage(item)
        article_candidate.author = self._author(item)
        article_candidate.publish_date = self._publish_date(item)
        article_candidate.language = self._language(item)

        return article_candidate
875998
function extractComments (contents) {
  let context = 'idle'
  const lines = []
  contents.split('\n').forEach((line) => {
    if (line.trim() === '/**') {
      context = 'in'
      return
    }

    if (line.trim() === '*/') {
      context = 'out'
      return
    }

    if (context === 'in') {
      lines.push(line.replace(/\*/g, '').replace(/^\s\s/, ''))
    }
  })
  return lines.join('\n')
}
1319798
def get_article_from_slug(tree, slug, preview=False, draft=False):
    
    from ..models import Title

    titles = Title.objects.select_related('article').filter(article__tree=tree)
    published_only = (not draft and not preview)

    if draft:
        titles = titles.filter(publisher_is_draft=True)
    elif preview:
        titles = titles.filter(publisher_is_draft=False)
    else:
        titles = titles.filter(published=True, publisher_is_draft=False)
    titles = titles.filter(slug=slug)

    for title in titles.iterator():
        if published_only and not _page_is_published(title.article):
            continue

        title.article.title_cache = {title.language: title}
        return title.article
    return
959487
def scrape_links(self, text, context=False):
        ''''''
        return self.iter_processed_links(io.StringIO(text), context=context)
1183986
function extractContent(desc) {
  /*
  // search all possible matches, returning first find
  var founds = sites.findMatch(desc.uri);

  if (founds.length) {
    var $ = cheerio.load(desc.content);
  // use first found extractor
    founds.forEach(function(found) {
      if ($(found.selector)) {
        var content = $(found.selector).html();
        return { selector: found.selector, content: content };
      }
    });
  }
  // ok, let's try unfluff
  var uf = unfluff(desc.content);
  return { selector: 'body', content: desc.content, text: uf.text, title: uf.title };
  */
  var title = getTitle(desc.content, '<title') || getTitle(desc.content, '<TITLE');

  return { selector: 'body', content: desc.content, text: 'NOTEXTRACTED', title: title };
}
1111960
def get_news_content(self, url):
        
        res = WyuNews.__wyu_news_content(url)
        soup = BeautifulSoup(res, from_encoding='utf-8')
        tag_table = soup.find(self.__get_tag_table)

        tr = tag_table.findAll('tr')[4]

        return tr.decode()
623729
def extract_nodes(soup, nodename, attr = None, value = None):
    
    tags = soup.find_all(nodename)
    if attr != None and value != None:
        return list(filter(lambda tag: tag.get(attr) == value, tags))
    return list(tags)
173838
def extract_mentions(self):
        
        if not hasattr(self, "raw_content"):
            return set()
        mentions = re.findall(r'@{[^;]+; [\w.-]+@[^}]+}', self.raw_content)
        if not mentions:
            return set()
        mentions = {s.split(';')[1].strip(' }') for s in mentions}
        mentions = {s for s in mentions}
        return mentions
33569
def _get_html_contents(html):
    """"""
    parser = MyHTMLParser()
    parser.feed(html)
    if parser.is_code:
        return ('code', parser.data.strip())
    elif parser.is_math:
        return ('math', parser.data.strip())
    else:
        return '', ''
1361313
def extract(self, text: str) -> List[Extraction]:

        
        
        content = BeautifulSoup(text, 'html5lib')
        subject = content.find('h1').text.strip()
        recip = self.mailing_list_name
        
        navbar = content.find(id='navbar')
        if navbar == None:
            info = self.old_format(content)
        else:
            info = self.new_format(navbar, content)
        for i in info[0:3]:
            if i == 'None':
                print('missed something important')
        sender = info[0]
        date   = info[1]
        body   = info[2]
        nxt    = info[3]
        rep_to = info[4]
        pub = 'SeeSat_Obs'
        dRec = datetime.datetime.now().isoformat()
        
        msg_obj = { 
            'url' : self.email_url,
            '@context' : {
                '@vocab' : 'schema.org'
            },
            'subject' : subject,
            'recip' : recip,
            'sender' : sender
        }
        if date != 'None':
            msg_obj['dateReceived'] = date
        if body != 'None':
            msg_obj['body'] = body
        if nxt != 'None':
            msg_obj['nxt'] = nxt
        if rep_to != 'None':
            msg_obj['replyToMessage'] = rep_to
        return Extraction(value=msg_obj, extractor_name=self.name)