// Scrape the document's content to gather all links
351207
func (w *worker) processLinks(doc *goquery.Document) (result []*url.URL) {
	baseURL, _ := doc.Find("base[href]").Attr("href")
	urls := doc.Find("a[href]").Map(func(_ int, s *goquery.Selection) string {
		val, _ := s.Attr("href")
		if baseURL != "" {
			val = handleBaseTag(doc.Url, baseURL, val)
		}
		return val
	})
	for _, s := range urls {
		// If href starts with "#", then it points to this same exact URL, ignore (will fail to parse anyway)
		if len(s) > 0 && !strings.HasPrefix(s, "#") {
			if parsed, e := url.Parse(s); e == nil {
				parsed = doc.Url.ResolveReference(parsed)
				result = append(result, parsed)
			} else {
				w.logFunc(LogIgnored, "ignore on unparsable policy %s: %s", s, e.Error())
			}
		}
	}
	return
}
205178
func FindAllMentions(content string) []string {
	mentions := MentionPattern.FindAllString(content, -1)
	for i := range mentions {
		mentions[i] = mentions[i][strings.Index(mentions[i], "@")+1:] // Strip @ character
	}
	return mentions
}
263588
func (extr *ContentExtractor) isHighLinkDensity(node *goquery.Selection) bool {
	links := node.Find("a")
	if links == nil || links.Size() == 0 {
		return false
	}
	text := node.Text()
	words := strings.Split(text, " ")
	nwords := len(words)
	var sb []string
	links.Each(func(i int, s *goquery.Selection) {
		linkText := s.Text()
		sb = append(sb, linkText)
	})
	linkText := strings.Join(sb, "")
	linkWords := strings.Split(linkText, " ")
	nlinkWords := len(linkWords)
	nlinks := links.Size()
	linkDivisor := float64(nlinkWords) / float64(nwords)
	score := linkDivisor * float64(nlinks)

	if extr.config.debug {
		var logText string
		if len(node.Text()) >= 51 {
			logText = node.Text()[0:50]
		} else {
			logText = node.Text()
		}
		log.Printf("Calculated link density score as %1.5f for node %s\n", score, logText)
	}
	if score > 1.0 {
		return true
	}
	return false
}
696205
func (l Links) Next(href string, args ...interface{}) Links {
	return l.Link("next", href, args...)
}
564991
func (h *handler) fetchDocument(pn blob.Ref) (*document, error) {
	return h.fetchDocumentFunc([]blob.Ref{pn}, func(blobs []blob.Ref) (*search.SearchResult, error) {
		return h.searchDocument(context.TODO(), blobs[0])
	})
}
56116
func (t *Tab) GetDocumentElementsBySelector(docNodeId int, selector string) ([]*Element, error) {
	docNode, ok := t.getElement(docNodeId)
	if !ok {
		return nil, &ElementNotFoundErr{Message: fmt.Sprintf("docNodeId %d not found", docNodeId)}
	}
	nodeIds, errQuery := t.DOM.QuerySelectorAll(docNode.id, selector)
	if errQuery != nil {
		return nil, errQuery
	}

	elements := make([]*Element, len(nodeIds))

	for k, nodeId := range nodeIds {
		elements[k], _ = t.GetElementByNodeId(nodeId)
	}

	return elements, nil
}
842288
func (g Goose) ExtractFromURL(url string) (*Article, error) {
	cc := NewCrawler(g.config, url, "")
	return cc.Crawl()
}
81166
func crawlTitleFromURL(link string) string {
	_title := strings.Split(link, "/")
	title := strings.Split(_title[4], "-ft-")

	title[0] = strings.Replace(title[0], "-", " ", -1)

	return title[0]
}
471655
func (g Goose) ExtractFromRawHTML(url string, RawHTML string) (*Article, error) {
	cc := NewCrawler(g.config, url, RawHTML)
	return cc.Crawl()
}
75099
func (db *DB) fetchAllLinkNodes(tx *bbolt.Tx) ([]*LinkNode, error) {
	nodeMetaBucket := tx.Bucket(nodeInfoBucket)
	if nodeMetaBucket == nil {
		return nil, ErrLinkNodesNotFound
	}

	var linkNodes []*LinkNode
	err := nodeMetaBucket.ForEach(func(k, v []byte) error {
		if v == nil {
			return nil
		}

		nodeReader := bytes.NewReader(v)
		linkNode, err := deserializeLinkNode(nodeReader)
		if err != nil {
			return err
		}

		linkNodes = append(linkNodes, linkNode)
		return nil
	})
	if err != nil {
		return nil, err
	}

	return linkNodes, nil
}
748945
func (s *Selection) Html() (ret string, e error) {
	// Since there is no .innerHtml, the HTML content must be re-created from
	// the nodes using html.Render.
	var buf bytes.Buffer

	if len(s.Nodes) > 0 {
		for c := s.Nodes[0].FirstChild; c != nil; c = c.NextSibling {
			e = html.Render(&buf, c)
			if e != nil {
				return
			}
		}
		ret = buf.String()
	}

	return
}
322633
func (l LinkList) Find(predicate LinkPredicate) *Link {
	for _, lnk := range l {
		if predicate(lnk) {
			return lnk
		}
	}
	return nil
}
139029
func (db *DB) FetchAllLinkNodes() ([]*LinkNode, error) {
	var linkNodes []*LinkNode
	err := db.View(func(tx *bbolt.Tx) error {
		nodes, err := db.fetchAllLinkNodes(tx)
		if err != nil {
			return err
		}

		linkNodes = nodes
		return nil
	})
	if err != nil {
		return nil, err
	}

	return linkNodes, nil
}
1182896
func (g *HttpGetter) Get(href string) (*bytes.Buffer, error) {
	return g.get(href)
}
139792
func yaml_parser_parse_document_content(parser *yaml_parser_t, event *yaml_event_t) bool {
	token := peek_token(parser)
	if token == nil {
		return false
	}
	if token.typ == yaml_VERSION_DIRECTIVE_TOKEN ||
		token.typ == yaml_TAG_DIRECTIVE_TOKEN ||
		token.typ == yaml_DOCUMENT_START_TOKEN ||
		token.typ == yaml_DOCUMENT_END_TOKEN ||
		token.typ == yaml_STREAM_END_TOKEN {
		parser.state = parser.states[len(parser.states)-1]
		parser.states = parser.states[:len(parser.states)-1]
		return yaml_parser_process_empty_scalar(parser, event,
			token.start_mark)
	}
	return yaml_parser_parse_node(parser, event, true, false)
}
501876
func (f *Feed) GetEventURLs() ([]string, error) {
	s := make([]string, len(f.Entry))
	for i := 0; i < len(f.Entry); i++ {
		e := f.Entry[i]
		s[i] = strings.TrimRight(e.Link[1].Href, "/")
	}
	return s, nil
}
379602
func (h *handler) fetchDocumentByPages(pages []blob.Ref) (*document, error) {
	return h.fetchDocumentFunc(pages, h.searchDocumentByPages)
}