//crawler://google.com?schema=https"#"
770888
func crawlerConnector(e *ExchangeMessage, u URI, params ...interface{}) error {
	schema := "http"
	if len(params) == 0 {
		err := errors.New("Wrong argument, you need to pass document query")
		e.body = err
		e.SetHeader("error", err.Error())
		return err
	}
	if u.GetHost() != "" {
		if len(params) == 2 {
			schema = params[1].(string)
		}
	}
	url := strings.Replace(u.GetRaw(), "crawler", schema, -1)
	doc, err := goquery.NewDocument(url)
	if err != nil {
		e.SetHeader("status", "500")
		e.SetHeader("error", err.Error())
		return err
	}
	e.SetHeader("status", "200")
	selection := doc.Find(params[0].(string))
	find := make([]string, selection.Size(), selection.Size())
	selection.Each(func(i int, s *goquery.Selection) {
		find[i] = s.Text()
	})
	e.SetBody(find)
	return nil
}
207976
func (s *physicalSchemaProducer) Schema() *expression.Schema {
	if s.schema == nil {
		s.schema = expression.NewSchema()
	}
	return s.schema
}
1340850
func (s *baseSchemaProducer) Schema() *expression.Schema {
	if s.schema == nil {
		s.schema = expression.NewSchema()
	}
	return s.schema
}
740109
func (g *GistsService) All(uri *Hyperlink, uriParams M) (gists []Gist, result *Result) {
	url, err := ExpandWithDefault(uri, &GistsURL, uriParams)
	if err != nil {
		return nil, &Result{Err: err}
	}

	result = g.client.get(url, &gists)
	return
}
1081105
func (s *TermvectorsService) buildURL() (string, url.Values, error) {
	var pathParam = map[string]string{
		"index": s.index,
	}
	path := "/{index}"
	var err error

	if s.typ != "" {
		pathParam["type"] = s.typ
		path += "/{type}"
	} else {
		path += "/_termvectors"
	}
	if s.id != "" {
		pathParam["id"] = s.id
		path += "/{id}"
	}
	if s.typ != "" {
		path += "/_termvectors"
	}

	path, err = uritemplates.Expand(path, pathParam)
	if err != nil {
		return "", url.Values{}, err
	}

	// Add query string parameters
	params := url.Values{}
	if s.pretty {
		params.Set("pretty", "true")
	}
	if s.dfs != nil {
		params.Set("dfs", fmt.Sprintf("%v", *s.dfs))
	}
	if s.fieldStatistics != nil {
		params.Set("field_statistics", fmt.Sprintf("%v", *s.fieldStatistics))
	}
	if len(s.fields) > 0 {
		params.Set("fields", strings.Join(s.fields, ","))
	}
	if s.offsets != nil {
		params.Set("offsets", fmt.Sprintf("%v", *s.offsets))
	}
	if s.parent != "" {
		params.Set("parent", s.parent)
	}
	if s.payloads != nil {
		params.Set("payloads", fmt.Sprintf("%v", *s.payloads))
	}
	if s.positions != nil {
		params.Set("positions", fmt.Sprintf("%v", *s.positions))
	}
	if s.preference != "" {
		params.Set("preference", s.preference)
	}
	if s.realtime != nil {
		params.Set("realtime", fmt.Sprintf("%v", *s.realtime))
	}
	if s.routing != "" {
		params.Set("routing", s.routing)
	}
	if s.termStatistics != nil {
		params.Set("term_statistics", fmt.Sprintf("%v", *s.termStatistics))
	}
	if s.version != nil {
		params.Set("version", fmt.Sprintf("%v", s.version))
	}
	if s.versionType != "" {
		params.Set("version_type", s.versionType)
	}
	return path, params, nil
}
34378
func (api *API) Schema(ctx context.Context) []*IndexInfo {
	span, _ := tracing.StartSpanFromContext(ctx, "API.Schema")
	defer span.Finish()
	return api.holder.limitedSchema()
}
692425
func (d differ) crawl(diffs [][2]string, idx1, idx2 int32) [][2]string {
	var (
		u1 = d[0].Ugens[idx1]
		u2 = d[1].Ugens[idx2]
	)
	if u1.Name != u2.Name {
		return append(diffs, [2]string{
			fmt.Sprintf("ugen %d is a(n) %s", idx1, u1.Name),
			fmt.Sprintf("ugen %d is a(n) %s", idx2, u2.Name),
		})
	}
	if u1.NumOutputs != u2.NumOutputs {
		return append(diffs, [2]string{
			fmt.Sprintf("ugen %d has %d output(s)", idx1, u1.NumOutputs),
			fmt.Sprintf("ugen %d has %d output(s)", idx2, u2.NumOutputs),
		})
	}
	if u1.Rate != u2.Rate {
		return append(diffs, [2]string{
			fmt.Sprintf("ugen %d has rate %d", idx1, u1.Rate),
			fmt.Sprintf("ugen %d has rate %d", idx2, u2.Rate),
		})
	}
	if u1.SpecialIndex != u2.SpecialIndex {
		return append(diffs, [2]string{
			fmt.Sprintf("ugen %d has special index %d", idx1, u1.SpecialIndex),
			fmt.Sprintf("ugen %d has special index %d", idx2, u2.SpecialIndex),
		})
	}
	if l1, l2 := len(u1.Inputs), len(u2.Inputs); l1 != l2 {
		return append(diffs, [2]string{
			fmt.Sprintf("%s has %d inputs", u1.Name, l1),
			fmt.Sprintf("%s has %d inputs", u2.Name, l2),
		})
	}
	// They have the same number of inputs.
	for i, in1 := range u1.Inputs {
		var (
			in2 = u2.Inputs[i]
			oi1 = in1.OutputIndex
			oi2 = in2.OutputIndex
			ui1 = in1.UgenIndex
			ui2 = in2.UgenIndex
		)
		if in1.IsConstant() && in2.IsConstant() {
			if v1, v2 := d[0].Constants[oi1], d[1].Constants[oi2]; v1 != v2 {
				diffs = append(diffs, [2]string{
					fmt.Sprintf("%s (ugen %d), input %d has constant value %f", u1.Name, idx1, i, v1),
					fmt.Sprintf("%s (ugen %d), input %d has constant value %f", u2.Name, idx2, i, v2),
				})
			}
			continue
		}
		if in1.IsConstant() && !in2.IsConstant() {
			diffs = append(diffs, [2]string{
				fmt.Sprintf("%s(%d), input %d is constant (%f)", u1.Name, idx1, i, d[0].Constants[oi1]),
				fmt.Sprintf("%s(%d), input %d points to %s(%d)", u2.Name, idx2, i, d[1].Ugens[ui2].Name, ui2),
			})
			continue
		}
		if !in1.IsConstant() && in2.IsConstant() {
			diffs = append(diffs, [2]string{
				fmt.Sprintf("%s(%d), input %d points to %s(%d)", u1.Name, idx1, i, d[0].Ugens[ui1].Name, ui1),
				fmt.Sprintf("%s(%d), input %d is constant (%f)", u2.Name, idx2, i, d[1].Constants[oi2]),
			})
			continue
		}
		// They are both not constant.
		// TODO: detect cycles
		diffs = append(diffs, d.crawl(diffs, ui1, ui2)...)
	}
	return diffs
}
619623
func NormalizeGerritURL(gerritURL string) (string, error) {
	u, err := url.Parse(gerritURL)
	if err != nil {
		return "", err
	}
	if u.Scheme != "https" {
		return "", fmt.Errorf("%s should start with https://", gerritURL)
	}
	if !strings.HasSuffix(u.Host, "-review.googlesource.com") {
		return "", errors.New("only *-review.googlesource.com Gerrits supported")
	}
	if u.Fragment != "" {
		return "", errors.New("no fragments allowed in gerritURL")
	}
	if u.Path != "" && u.Path != "/" {
		return "", errors.New("Unexpected path in URL")
	}
	if u.Path != "/" {
		u.Path = "/"
	}
	return u.String(), nil
}
5436
func (s *String) Schema(schema interface{}) *String {
	checkSchema(&s.chain, s.value, schema)
	return s
}
923702
func (r *PEXReactor) crawlPeersRoutine() {
	// Do an initial crawl
	r.crawlPeers(r.book.GetSelection())

	// Fire periodically
	ticker := time.NewTicker(crawlPeerPeriod)

	for {
		select {
		case <-ticker.C:
			r.attemptDisconnects()
			r.crawlPeers(r.book.GetSelection())
			r.cleanupCrawlPeerInfos()
		case <-r.Quit():
			return
		}
	}
}
1020351
func (p *UserAgent) googleBot() bool {
	// This is a hackish way to detect Google's mobile bot (Googlebot, AdsBot-Google-Mobile, etc.).
	// See https://support.google.com/webmasters/answer/1061943
	if strings.Index(p.ua, "Google") != -1 {
		p.platform = ""
		p.undecided = true
	}
	return p.undecided
}
1077132
func (c *Crawler) enqueueUrls(ctxs []*URLContext) (cnt int) {
	for _, ctx := range ctxs {
		var isVisited, enqueue bool

		// Cannot directly enqueue a robots.txt URL, since it is managed as a special case
		// in the worker (doesn't return a response to crawler).
		if ctx.IsRobotsURL() {
			continue
		}
		// Check if it has been visited before, using the normalized URL
		_, isVisited = c.visited[ctx.normalizedURL.String()]

		// Filter the URL
		if enqueue = c.Options.Extender.Filter(ctx, isVisited); !enqueue {
			// Filter said NOT to use this url, so continue with next
			c.logFunc(LogIgnored, "ignore on filter policy: %s", ctx.normalizedURL)
			continue
		}

		// Even if filter said to use the URL, it still MUST be absolute, http(s)-prefixed,
		// and comply with the same host policy if requested.
		if !ctx.normalizedURL.IsAbs() {
			// Only absolute URLs are processed, so ignore
			c.logFunc(LogIgnored, "ignore on absolute policy: %s", ctx.normalizedURL)

		} else if !strings.HasPrefix(ctx.normalizedURL.Scheme, "http") {
			c.logFunc(LogIgnored, "ignore on scheme policy: %s", ctx.normalizedURL)

		} else if c.Options.SameHostOnly && !c.isSameHost(ctx) {
			// Only allow URLs coming from the same host
			c.logFunc(LogIgnored, "ignore on same host policy: %s", ctx.normalizedURL)

		} else {
			// All is good, visit this URL (robots.txt verification is done by worker)

			// Possible caveat: if the normalization changes the host, it is possible
			// that the robots.txt fetched for this host would differ from the one for
			// the unnormalized host. However, this should be rare, and is a weird
			// behaviour from the host (i.e. why would site.com differ in its rules
			// from www.site.com) and can be fixed by using a different normalization
			// flag. So this is an acceptable behaviour for gocrawl.

			// Launch worker if required, based on the host of the normalized URL
			w, ok := c.workers[ctx.normalizedURL.Host]
			if !ok {
				// No worker exists for this host, launch a new one
				w = c.launchWorker(ctx)
				// Automatically enqueue the robots.txt URL as first in line
				if robCtx, e := ctx.getRobotsURLCtx(); e != nil {
					c.Options.Extender.Error(newCrawlError(ctx, e, CekParseRobots))
					c.logFunc(LogError, "ERROR parsing robots.txt from %s: %s", ctx.normalizedURL, e)
				} else {
					c.logFunc(LogEnqueued, "enqueue: %s", robCtx.url)
					c.Options.Extender.Enqueued(robCtx)
					w.pop.stack(robCtx)
				}
			}

			cnt++
			c.logFunc(LogEnqueued, "enqueue: %s", ctx.url)
			c.Options.Extender.Enqueued(ctx)
			w.pop.stack(ctx)
			c.pushPopRefCount++

			// Once it is stacked, it WILL be visited eventually, so add it to the visited slice
			// (unless denied by robots.txt, but this is out of our hands, for all we
			// care, it is visited).
			if !isVisited {
				// The visited map works with the normalized URL
				c.visited[ctx.normalizedURL.String()] = struct{}{}
			}
		}
	}
	return
}
592647
func (c *Connection) Search(query interface{}, indexList []string, typeList []string, extraArgs url.Values) (*Response, error) {
	r := Request{
		Conn:      c,
		Query:     query,
		IndexList: indexList,
		TypeList:  typeList,
		method:    "POST",
		api:       "_search",
		ExtraArgs: extraArgs,
	}

	return r.Run()
}
1326826
func QuerySelector(o *js.Object, sel string) *js.Object {
	return o.Call("querySelector", sel)
}
572704
func NewSchema(solrUrl, core string) (*Schema, error) {
	u, err := url.ParseRequestURI(strings.TrimRight(solrUrl, "/"))
	if err != nil {
		return nil, err
	}

	return &Schema{url: u, core: core}, nil
}
560286
func MatchScheme(req *http.Request, ereq *Request) (bool, error) {
	return ereq.URLStruct.Scheme == "" || req.URL.Scheme == "" || ereq.URLStruct.Scheme == req.URL.Scheme, nil
}
1352353
func (i *Index) Search(query string, opts ...interface{}) (res QueryRes, err error) {
	body := searchReq{Params: transport.URLEncode(newSearchParams(query, opts...))}
	path := i.path("/query")
	err = i.transport.Request(&res, http.MethodPost, path, body, call.Read, opts...)
	return
}