// Stringify returns the raw string for a set of tokens.
12485
func Stringify(tokens ...Token) string {
	out := []string{}
	for _, t := range tokens {
		out = append(out, t.Value)
	}
	return strings.Join(out, "")
}
868774
func tokenize(src string, tc chan *sep) {
	var s scanner.Scanner
	fset := token.NewFileSet() // boilerplate stuff for scanner...
	file := fset.AddFile("", fset.Base(), len(src))
	s.Init(file, []byte(src), nil, 1)
	for {
		filePos, tok, _ := s.Scan()
		tc <- &sep{int(filePos) - file.Base(), tok}
		if tok == token.EOF {
			close(tc)
			break
		}
	}
}
190159
func (l Lexer) TokenizeString(s string) ([]Token, error) {
	r := bufio.NewReader(strings.NewReader(s))
	tokens := []Token{}
	err := l.Tokenize(r, func(t Token) error {
		tokens = append(tokens, t)
		return nil
	})
	return tokens, err
}
496911
func Tokenise(expr string) []string {

	spaced := unary_minus_rx.ReplaceAllString(expr, "$1 @")
	spaced = fp_rx.ReplaceAllString(spaced, " ${1} ")
	spaced = functions_rx.ReplaceAllString(spaced, " ${1} ")

	if symbols_rx != nil {
		spaced = symbols_rx.ReplaceAllString(spaced, " ${1} ")
	}

	symbols := []string{"(", ")"}
	for _, symbol := range symbols {
		spaced = strings.Replace(spaced, symbol, fmt.Sprintf(" %s ", symbol), -1)
	}

	stripped := whitespace_rx.ReplaceAllString(strings.TrimSpace(spaced), "|")
	result := strings.Split(stripped, "|")
	return result
}
1122976
func stringValue(e ast.Expr) (string, error) {
	lit, ok := e.(*ast.BasicLit)
	if !ok {
		return "", fmt.Errorf("not a string, but %T", e)
	}
	if lit.Kind != token.STRING {
		return "", fmt.Errorf("not a string, but %v", lit.Kind)
	}
	return strconv.Unquote(lit.Value)
}
1100233
func (p *point) StringSize() int {
	size := len(p.key) + len(p.fields) + 1

	if !p.time.IsZero() {
		digits := 1 // even "0" has one digit
		t := p.UnixNano()
		if t < 0 {
			// account for negative sign, then negate
			digits++
			t = -t
		}
		for t > 9 { // already accounted for one digit
			digits++
			t /= 10
		}
		size += digits + 1 // digits and a space
	}

	return size
}
974668
func Jsonify(s string) string {
	if s == "" {
		return ""
	}

	if commonInitialisms[s] {
		return strings.ToLower(s)
	}

	rs := []rune(s)
	rs[0] = unicode.ToLower(rs[0])

	return string(rs)
}
835245
func (l *iniLexer) Tokenize(r io.Reader) ([]Token, error) {
	b, err := ioutil.ReadAll(r)
	if err != nil {
		return nil, awserr.New(ErrCodeUnableToReadFile, "unable to read file", err)
	}

	return l.tokenize(b)
}
1351906
func (e *Serializer) Serialize(val interface{}, options ...map[string]interface{}) ([]byte, error) {

	var result []byte
	var err error

	if e.config.Indent {
		result, err = xml.MarshalIndent(val, "", "  ")
		result = append(result, '\n')
	} else {
		result, err = xml.Marshal(val)
	}
	if err != nil {
		return nil, err
	}
	if len(e.config.Prefix) > 0 {
		result = append(e.config.Prefix, result...)
	}
	return result, nil
}
565345
func stringifyEntries(data map[string]interface{}) map[string]string {
	entries := make(map[string]string)
	for k, v := range data {

		key := stringifyKey(k)
		entries[key] = fmt.Sprint(v)
	}
	return entries
}
407786
func pathify(parts []string, separator string) string {
	return separator + strings.Join(parts, separator)
}
975483
func (p *DefaultWordTokenizer) Tokenize(text string, onlyPeriodContext bool) []*Token {
	textLength := len(text)

	if textLength == 0 {
		return nil
	}

	tokens := make([]*Token, 0, 50)
	lastSpace := 0
	lineStart := false
	paragraphStart := false
	getNextWord := false

	for i, char := range text {
		if !unicode.IsSpace(char) && i != textLength - 1 {
			continue
		}

		if char == '\n' {
			if lineStart {
				paragraphStart = true
			}
			lineStart = true
		}

		var cursor int
		if i == textLength - 1 {
			cursor = textLength
		} else {
			cursor = i
		}

		word := strings.TrimSpace(text[lastSpace:cursor])

		if word == "" {
			continue
		}

		hasSentencePunct := p.PunctStrings.HasSentencePunct(word)
		if onlyPeriodContext && !hasSentencePunct && !getNextWord {
			lastSpace = cursor
			continue
		}

		token := NewToken(word)
		token.Position = cursor
		token.ParaStart = paragraphStart
		token.LineStart = lineStart
		tokens = append(tokens, token)

		lastSpace = cursor
		lineStart = false
		paragraphStart = false

		if hasSentencePunct {
			getNextWord = true
		} else {
			getNextWord = false
		}
	}

	if len(tokens) == 0 {
		token := NewToken(text)
		token.Position = textLength
		tokens = append(tokens, token)
	}

	return tokens
}
432131
func (i tokenType) String() string {
	s := tokenName[i]
	if s == "" {
		return fmt.Sprintf("t_unknown_%d", int(i))
	}
	return s
}
1184564
func (tok Token) String() string {
	if _, ok := tokens[tok]; ok {
		return tokens[tok]
	}
	return ""
}
316280
func (i token) String() string {
	return fmt.Sprintf("%s:%q", i.typ, i.val)
}
1035398
func (t *Token) String() string {
	return fmt.Sprintf("%s.%s", t.ID, t.Secret)
}
569447
func (t *SimpleTokenizer) Tokenize(sentence string) []string {
	// is the tokenizer really the best place to be making
	// everything lowercase? is this really a sanitizaion func?
	return strings.Split(strings.ToLower(sentence), t.SplitOn)
}