// HTMLBasicTokenize returns a list of HTML tags and literal elements. This is
// done with regular expressions, so the result is only marginally better than
// useless.
353209
func HTMLBasicTokenize(htmlStr string) (list []HTMLBasicSegmentType) {
	// This routine is adapted from http://www.fpdf.org/
	list = make([]HTMLBasicSegmentType, 0, 16)
	htmlStr = strings.Replace(htmlStr, "\n", " ", -1)
	htmlStr = strings.Replace(htmlStr, "\r", "", -1)
	tagRe, _ := regexp.Compile(`(?U)<.*>`)
	attrRe, _ := regexp.Compile(`([^=]+)=["']?([^"']+)`)
	capList := tagRe.FindAllStringIndex(htmlStr, -1)
	if capList != nil {
		var seg HTMLBasicSegmentType
		var parts []string
		pos := 0
		for _, cap := range capList {
			if pos < cap[0] {
				seg.Cat = 'T'
				seg.Str = htmlStr[pos:cap[0]]
				seg.Attr = nil
				list = append(list, seg)
			}
			if htmlStr[cap[0]+1] == '/' {
				seg.Cat = 'C'
				seg.Str = strings.ToLower(htmlStr[cap[0]+2 : cap[1]-1])
				seg.Attr = nil
				list = append(list, seg)
			} else {
				// Extract attributes
				parts = strings.Split(htmlStr[cap[0]+1:cap[1]-1], " ")
				if len(parts) > 0 {
					for j, part := range parts {
						if j == 0 {
							seg.Cat = 'O'
							seg.Str = strings.ToLower(parts[0])
							seg.Attr = make(map[string]string)
						} else {
							attrList := attrRe.FindAllStringSubmatch(part, -1)
							if attrList != nil {
								for _, attr := range attrList {
									seg.Attr[strings.ToLower(attr[1])] = attr[2]
								}
							}
						}
					}
					list = append(list, seg)
				}
			}
			pos = cap[1]
		}
		if len(htmlStr) > pos {
			seg.Cat = 'T'
			seg.Str = htmlStr[pos:]
			seg.Attr = nil
			list = append(list, seg)
		}
	} else {
		list = append(list, HTMLBasicSegmentType{Cat: 'T', Str: htmlStr, Attr: nil})
	}
	return
}
938184
func (r RegexpTokenizer) Tokenize(text string) []string {
	var tokens []string
	if r.gaps {
		temp := r.regex.Split(text, -1)
		if r.discard {
			for _, s := range temp {
				if s != "" {
					tokens = append(tokens, s)
				}
			}
		} else {
			tokens = temp
		}
	} else {
		tokens = r.regex.FindAllString(text, -1)
	}
	return tokens
}
1153716
func (p *Proposal) ValidateBasic() error {
	if p.Type != ProposalType {
		return errors.New("Invalid Type")
	}
	if p.Height < 0 {
		return errors.New("Negative Height")
	}
	if p.Round < 0 {
		return errors.New("Negative Round")
	}
	if p.POLRound < -1 {
		return errors.New("Negative POLRound (exception: -1)")
	}
	if err := p.BlockID.ValidateBasic(); err != nil {
		return fmt.Errorf("Wrong BlockID: %v", err)
	}
	// ValidateBasic above would pass even if the BlockID was empty:
	if !p.BlockID.IsComplete() {
		return fmt.Errorf("Expected a complete, non-empty BlockID, got: %v", p.BlockID)
	}

	// NOTE: Timestamp validation is subtle and handled elsewhere.

	if len(p.Signature) == 0 {
		return errors.New("Signature is missing")
	}
	if len(p.Signature) > MaxSignatureSize {
		return fmt.Errorf("Signature is too big (max: %d)", MaxSignatureSize)
	}
	return nil
}
304861
func tokenize(s string) []string {
	/*
		TODO: Properly detect if the string starts with a ( and ends with a )
		and that those two form a matching pair.

		Eg: (foo) -> true; (foo)(bar) -> false;
	*/
	if s[0] == '(' && s[len(s)-1] == ')' {
		s = s[1 : len(s)-1]
	}
	ret := []string{}
	for chunk := range split(s) {
		if len(chunk) != 0 {
			if chunk[0] == '(' && chunk[len(chunk)-1] == ')' {
				ret = append(ret, chunk)
			} else {
				for _, token := range pat.FindAllStringSubmatch(chunk, -1) {
					ret = append(ret, token[0])
				}
			}
		} else {
			fmt.Printf("Empty chunk in string '%s'\n", s)
		}
	}
	return ret
}
1103409
func renderMarkdown(t string) (results template.HTML) {
	// We don't want auto punctuation, which changes "foo" into “foo”
	r := blackfriday.NewHTMLRenderer(blackfriday.HTMLRendererParameters{
		Flags: blackfriday.CommonHTMLFlags &^ blackfriday.Smartypants,
	})
	buf := bytes.NewBuffer(blackfriday.Run(
		[]byte(t),
		blackfriday.WithRenderer(r),
		blackfriday.WithExtensions(blackfriday.CommonExtensions)))
	out := bytes.NewBuffer(nil)
	if err := sanitizehtml.Sanitize(out, buf); err != nil {
		return template.HTML(fmt.Sprintf("Failed to render markdown: %s", template.HTMLEscapeString(err.Error())))
	}
	return template.HTML(out.String())
}
430193
func NewTokenBuffer(l *html.Lexer) *TokenBuffer {
	return &TokenBuffer{
		l:   l,
		buf: make([]Token, 0, 8),
	}
}
679766
func HTMLAllowing(s string, args ...[]string) (string, error) {

	allowedTags := defaultTags
	if len(args) > 0 {
		allowedTags = args[0]
	}
	allowedAttributes := defaultAttributes
	if len(args) > 1 {
		allowedAttributes = args[1]
	}

	// Parse the html
	tokenizer := parser.NewTokenizer(strings.NewReader(s))

	buffer := bytes.NewBufferString("")
	ignore := ""

	for {
		tokenType := tokenizer.Next()
		token := tokenizer.Token()

		switch tokenType {

		case parser.ErrorToken:
			err := tokenizer.Err()
			if err == io.EOF {
				return buffer.String(), nil
			}
			return "", err

		case parser.StartTagToken:

			if len(ignore) == 0 && includes(allowedTags, token.Data) {
				token.Attr = cleanAttributes(token.Attr, allowedAttributes)
				buffer.WriteString(token.String())
			} else if includes(ignoreTags, token.Data) {
				ignore = token.Data
			}

		case parser.SelfClosingTagToken:

			if len(ignore) == 0 && includes(allowedTags, token.Data) {
				token.Attr = cleanAttributes(token.Attr, allowedAttributes)
				buffer.WriteString(token.String())
			} else if token.Data == ignore {
				ignore = ""
			}

		case parser.EndTagToken:
			if len(ignore) == 0 && includes(allowedTags, token.Data) {
				token.Attr = []parser.Attribute{}
				buffer.WriteString(token.String())
			} else if token.Data == ignore {
				ignore = ""
			}

		case parser.TextToken:
			// We allow text content through, unless ignoring this entire tag and its contents (including other tags)
			if ignore == "" {
				buffer.WriteString(token.String())
			}
		case parser.CommentToken:
			// We ignore comments by default
		case parser.DoctypeToken:
			// We ignore doctypes by default - html5 does not require them and this is intended for sanitizing snippets of text
		default:
			// We ignore unknown token types by default

		}

	}

}
269939
func validateClientGenTags(values map[string][]string) error {
	for _, k := range supportedTags {
		delete(values, k)
	}
	for key := range values {
		if strings.HasPrefix(key, strings.TrimSuffix(genClientPrefix, ":")) {
			return errors.New("unknown tag detected: " + key)
		}
	}
	return nil
}
158646
func (g *globTokeniser) PeekToken() (token string, tokenType tc) {
	return g.peekToken, g.peekTokenType
}
144257
func NeuterAccents(s string) string {
	t := transform.Chain(norm.NFD, transform.RemoveFunc(isMn), norm.NFC)
	result, _, _ := transform.String(t, string(s))

	return result
}
32308
func SeriesKeySize(name []byte, tags models.Tags) int {
	var n int
	n += 2 + len(name)
	n += binaryutil.UvarintSize(uint64(len(tags)))
	for _, tag := range tags {
		n += 2 + len(tag.Key)
		n += 2 + len(tag.Value)
	}
	n += binaryutil.UvarintSize(uint64(n))
	return n
}
505868
func DeisIfy(msg string) string {
	var t = struct {
		Msg string
		C   map[string]string
	}{
		Msg: msg,
		C:   Colors,
	}
	tpl := "{{.C.Deis1}}\n{{.C.Deis2}} {{.Msg}}\n{{.C.Deis3}}\n"
	var buf bytes.Buffer
	template.Must(template.New("deis").Parse(tpl)).Execute(&buf, t)
	return buf.String()
}
915788
func sanitize(b []byte) []byte {
	// Replace ` with `+"`"+`
	b = bytes.Replace(b, []byte("`"), []byte("`+\"`\"+`"), -1)

	// Replace BOM with `+"\xEF\xBB\xBF"+`
	// (A BOM is valid UTF-8 but not permitted in Go source files.
	// I wouldn't bother handling this, but for some insane reason
	// jquery.js has a BOM somewhere in the middle.)
	return bytes.Replace(b, []byte("\xEF\xBB\xBF"), []byte("`+\"\\xEF\\xBB\\xBF\"+`"), -1)
}
396931
func (ns *Namespace) Emojify(s interface{}) (template.HTML, error) {
	ss, err := cast.ToStringE(s)
	if err != nil {
		return "", err
	}

	return template.HTML(helpers.Emojify([]byte(ss))), nil
}
1364263
func templateCleanAMIName(s string) string {
	allowed := []byte{'(', ')', '[', ']', ' ', '.', '/', '-', '\'', '@', '_'}
	b := []byte(s)
	newb := make([]byte, len(b))
	for i, c := range b {
		if isalphanumeric(c) || bytes.IndexByte(allowed, c) != -1 {
			newb[i] = c
		} else {
			newb[i] = '-'
		}
	}
	return string(newb[:])
}
752509
func (m *VoteSetBitsMessage) ValidateBasic() error {
	if m.Height < 0 {
		return errors.New("Negative Height")
	}
	if m.Round < 0 {
		return errors.New("Negative Round")
	}
	if !types.IsVoteTypeValid(m.Type) {
		return errors.New("Invalid Type")
	}
	if err := m.BlockID.ValidateBasic(); err != nil {
		return fmt.Errorf("Wrong BlockID: %v", err)
	}
	// NOTE: Votes.Size() can be zero if the node does not have any
	return nil
}
762728
func (m *NewValidBlockMessage) ValidateBasic() error {
	if m.Height < 0 {
		return errors.New("Negative Height")
	}
	if m.Round < 0 {
		return errors.New("Negative Round")
	}
	if err := m.BlockPartsHeader.ValidateBasic(); err != nil {
		return fmt.Errorf("Wrong BlockPartsHeader: %v", err)
	}
	if m.BlockParts.Size() != m.BlockPartsHeader.Total {
		return fmt.Errorf("BlockParts bit array size %d not equal to BlockPartsHeader.Total %d",
			m.BlockParts.Size(),
			m.BlockPartsHeader.Total)
	}
	return nil
}