// newScraper parses the given url and scrapes the site, returning
// a scraper object with all the site info and meta tags
341084
func newScraper(u *url.URL, timeout int) (*scraper, error) {
	var title string
	var language string
	var author string
	var description string
	var generator string
	var feed string
	charset := "utf-8"
	links := make([]string, 0)
	images := make([]string, 0)
	keywords := make([]string, 0)
	compatibility := make(map[string]string)

	scrpr := func(n *html.Node) {
		switch n.Data {
		case "html":
			language = findAttribute(n, "lang")
		case "title":
			title = n.FirstChild.Data
		case "a":
			links = addElement(links, u, n, "href")
		case "img":
			images = addElement(images, u, n, "src")
		case "link":
			typ := findAttribute(n, "type")
			switch typ {
			case "application/rss+xml":
				feed = findAttribute(n, "href")
			}
		case "meta":
			name := findAttribute(n, "name")
			switch name {
			case "author":
				author = findAttribute(n, "content")
			case "keywords":
				keywords = strings.Split(findAttribute(n, "content"), ", ")
			case "description":
				description = findAttribute(n, "content")
			case "generator":
				generator = findAttribute(n, "content")
			}

			httpEquiv := findAttribute(n, "http-equiv")
			switch httpEquiv {
			case "Content-Type":
				charset = findCharset(findAttribute(n, "content"))
			case "X-UA-Compatible":
				compatibility = mapifyStr(findAttribute(n, "content"))
			}
		}
	}

	cl := http.Client{
		Transport: &http.Transport{
			Dial: timeoutDialer(timeout),
		},
	}

	resp, err := cl.Get(u.String())
	if err != nil {
		return nil, err
	}
	defer resp.Body.Close()

	tree, err := h5.New(resp.Body)
	if err != nil {
		return nil, err
	}
	tree.Walk(scrpr)

	return &scraper{title,
		language,
		author,
		description,
		generator,
		feed,
		charset,
		links,
		images,
		keywords,
		compatibility}, nil
}
1329165
func NewSnapshotter(ctx context.Context, config *Config) (*Snapshotter, error) {
	// Make sure snapshotter configuration valid before running
	if err := config.parse(); err != nil {
		return nil, err
	}

	if err := config.Validate(); err != nil {
		return nil, err
	}

	var cleanupFn []closeFunc

	if err := os.MkdirAll(config.RootPath, 0750); err != nil && !os.IsExist(err) {
		return nil, errors.Wrapf(err, "failed to create root directory: %s", config.RootPath)
	}

	store, err := storage.NewMetaStore(filepath.Join(config.RootPath, metadataFileName))
	if err != nil {
		return nil, errors.Wrap(err, "failed to create metastore")
	}

	cleanupFn = append(cleanupFn, store.Close)

	poolDevice, err := NewPoolDevice(ctx, config)
	if err != nil {
		return nil, err
	}

	cleanupFn = append(cleanupFn, poolDevice.Close)

	return &Snapshotter{
		store:     store,
		config:    config,
		pool:      poolDevice,
		cleanupFn: cleanupFn,
	}, nil
}
1135294
func newS3Config(urlStr string, hostCfg *hostConfigV9) *Config {
	// We have a valid alias and hostConfig. We populate the
	// credentials from the match found in the config file.
	s3Config := new(Config)

	s3Config.AppName = "mc"
	s3Config.AppVersion = Version
	s3Config.AppComments = []string{os.Args[0], runtime.GOOS, runtime.GOARCH}
	s3Config.Debug = globalDebug
	s3Config.Insecure = globalInsecure

	s3Config.HostURL = urlStr
	if hostCfg != nil {
		s3Config.AccessKey = hostCfg.AccessKey
		s3Config.SecretKey = hostCfg.SecretKey
		s3Config.Signature = hostCfg.API
	}
	s3Config.Lookup = getLookupType(hostCfg.Lookup)
	return s3Config
}
35991
func NewSsoRemoveLogoutUrlType(Description string) *SsoRemoveLogoutUrlType {
	s := new(SsoRemoveLogoutUrlType)
	s.Description = Description
	return s
}
1372256
func NewSetXHRBreakpointArgs(url string) *SetXHRBreakpointArgs {
	args := new(SetXHRBreakpointArgs)
	args.URL = url
	return args
}
152145
func (errorformat *Errorformat) NewScanner(r io.Reader) *Scanner {
	return &Scanner{
		Errorformat: errorformat,
		source:      bufio.NewScanner(r),
		qi:          &qfinfo{},
		mlpoped:     true,
	}
}
440507
func NewServiceRenderer(model *surface.Model) (renderer *Renderer, err error) {
	renderer = &Renderer{}
	renderer.Model = model
	return renderer, nil
}
11127
func NewSnippetWriter(w io.Writer, c *Context, left, right string) *SnippetWriter {
	sw := &SnippetWriter{
		w:       w,
		context: c,
		left:    left,
		right:   right,
		funcMap: template.FuncMap{},
	}
	for name, namer := range c.Namers {
		sw.funcMap[name] = namer.Name
	}
	return sw
}
1007701
func (res *Resource) NewSlice() interface{} {
	if res.Value == nil {
		return nil
	}
	sliceType := reflect.SliceOf(reflect.TypeOf(res.Value))
	slice := reflect.MakeSlice(sliceType, 0, 0)
	slicePtr := reflect.New(sliceType)
	slicePtr.Elem().Set(slice)
	return slicePtr.Interface()
}
259128
func NewStrip(openStr, closeStr string) *Strip {
	return &Strip{
		Open:  (len(openStr) > 2) && openStr[2] == '~',
		Close: (len(closeStr) > 2) && closeStr[len(closeStr)-3] == '~',
	}
}
99762
func NewStringsWorker(handler StringsWatchHandler) worker.Worker {
	sw := &stringsWorker{
		handler: handler,
	}
	sw.tomb.Go(sw.loop)
	return sw
}
550729
func NewServer(addr string, group []*SiteConfig) (*Server, error) {
	s := &Server{
		Server:      makeHTTPServerWithTimeouts(addr, group),
		vhosts:      newVHostTrie(),
		sites:       group,
		connTimeout: GracefulTimeout,
	}
	s.vhosts.fallbackHosts = append(s.vhosts.fallbackHosts, getFallbacks(group)...)
	s.Server = makeHTTPServerWithHeaderLimit(s.Server, group)
	s.Server.Handler = s // this is weird, but whatever

	// extract TLS settings from each site config to build
	// a tls.Config, which will not be nil if TLS is enabled
	tlsConfig, err := makeTLSConfig(group)
	if err != nil {
		return nil, err
	}
	s.Server.TLSConfig = tlsConfig

	// if TLS is enabled, make sure we prepare the Server accordingly
	if s.Server.TLSConfig != nil {
		// enable QUIC if desired (requires HTTP/2)
		if HTTP2 && QUIC {
			s.quicServer = &h2quic.Server{Server: s.Server}
			s.Server.Handler = s.wrapWithSvcHeaders(s.Server.Handler)
		}

		// wrap the HTTP handler with a handler that does MITM detection
		tlsh := &tlsHandler{next: s.Server.Handler}
		s.Server.Handler = tlsh // this needs to be the "outer" handler when Serve() is called, for type assertion

		// when Serve() creates the TLS listener later, that listener should
		// be adding a reference the ClientHello info to a map; this callback
		// will be sure to clear out that entry when the connection closes.
		s.Server.ConnState = func(c net.Conn, cs http.ConnState) {
			// when a connection closes or is hijacked, delete its entry
			// in the map, because we are done with it.
			if tlsh.listener != nil {
				if cs == http.StateHijacked || cs == http.StateClosed {
					tlsh.listener.helloInfosMu.Lock()
					delete(tlsh.listener.helloInfos, c.RemoteAddr().String())
					tlsh.listener.helloInfosMu.Unlock()
				}
			}
		}

		// As of Go 1.7, if the Server's TLSConfig is not nil, HTTP/2 is enabled only
		// if TLSConfig.NextProtos includes the string "h2"
		if HTTP2 && len(s.Server.TLSConfig.NextProtos) == 0 {
			// some experimenting shows that this NextProtos must have at least
			// one value that overlaps with the NextProtos of any other tls.Config
			// that is returned from GetConfigForClient; if there is no overlap,
			// the connection will fail (as of Go 1.8, Feb. 2017).
			s.Server.TLSConfig.NextProtos = defaultALPN
		}
	}

	// Compile custom middleware for every site (enables virtual hosting)
	for _, site := range group {
		stack := Handler(staticfiles.FileServer{Root: http.Dir(site.Root), Hide: site.HiddenFiles, IndexPages: site.IndexPages})
		for i := len(site.middleware) - 1; i >= 0; i-- {
			stack = site.middleware[i](stack)
		}
		site.middlewareChain = stack
		s.vhosts.Insert(site.Addr.VHost(), site)
	}

	return s, nil
}
271348
func NewSelector(client RESTClient, mapping *meta.RESTMapping, namespace string, selector labels.Selector, export bool) *Selector {
	return &Selector{
		Client:    client,
		Mapping:   mapping,
		Namespace: namespace,
		Selector:  selector,
		Export:    export,
	}
}
812137
func NewScanner(name string, d EncodingHint) (UTFScanCloser, error) {
	f, err := OpenFile(name, d)
	if err != nil {
		return nil, err
	}

	return scanCloser{
		scanner: bufio.NewScanner(f),
		file:    f,
	}, nil
}
440874
func NewSeriesParser(handler *MetricHandler) *Parser {
	return &Parser{
		machine: NewSeriesMachine(handler),
		handler: handler,
	}
}
682295
func NewSharedLinkMetadata(Url string, Name string, LinkPermissions *LinkPermissions) *SharedLinkMetadata {
	s := new(SharedLinkMetadata)
	s.Url = Url
	s.Name = Name
	s.LinkPermissions = LinkPermissions
	return s
}
1031493
func NewScoping(s Sender, delimiter string, scopes ...string) XStater {
	var xs *xstats
	if DisablePooling {
		xs = &xstats{}
	} else {
		xs = xstatsPool.Get().(*xstats)
	}
	xs.s = s
	if len(scopes) > 0 {
		xs.prefix = strings.Join(scopes, delimiter) + delimiter
	} else {
		xs.prefix = ""
	}
	xs.delimiter = delimiter
	return xs
}