// Given a slice of token_item, reassembles them into the original literal
// expression.
778658
func joinTokens(tokens []tokenItem) string {
	var buf bytes.Buffer
	for i, tok := range tokens {
		if i > 0 {
			buf.WriteByte(' ')
		}
		buf.WriteString(tok.String())
	}
	return buf.String()
}
853111
func substitute(
	obj map[string]interface{}, tokenMatcher *regexp.Regexp, replacer func(key string) string,
) map[string]interface{} {
	newObj := make(map[string]interface{})
	for k, v := range obj {
		switch v := v.(type) {
		case string:
			newObj[k] = tokenMatcher.ReplaceAllStringFunc(v, replacer)
		case []string:
			newV := make([]string, len(v))
			for i := 0; i < len(v); i++ {
				newV[i] = tokenMatcher.ReplaceAllStringFunc(v[i], replacer)
			}
			newObj[k] = newV
		default:
			newObj[k] = v
		}
	}
	return newObj
}
134964
func (f Index) itemFromIterator(it iterator.Iterator, totalPrefix []byte) (i Item, err error) {
	key := it.Key()
	if !bytes.HasPrefix(key, totalPrefix) {
		return i, leveldb.ErrNotFound
	}
	// create a copy of key byte slice not to share leveldb underlaying slice array
	keyItem, err := f.decodeKeyFunc(append([]byte(nil), key...))
	if err != nil {
		return i, err
	}
	// create a copy of value byte slice not to share leveldb underlaying slice array
	valueItem, err := f.decodeValueFunc(keyItem, append([]byte(nil), it.Value()...))
	if err != nil {
		return i, err
	}
	return keyItem.Merge(valueItem), it.Error()
}
190159
func (l Lexer) TokenizeString(s string) ([]Token, error) {
	r := bufio.NewReader(strings.NewReader(s))
	tokens := []Token{}
	err := l.Tokenize(r, func(t Token) error {
		tokens = append(tokens, t)
		return nil
	})
	return tokens, err
}
703955
func (l *lexer) tokenize() {
	for l.state = lexData; l.state != nil; {
		l.state = l.state(l)
	}
}
1088895
func (dec *Decoder) tokenPrepareForDecode() error {
	// Note: Not calling peek before switch, to avoid
	// putting peek into the standard Decode path.
	// peek is only called when using the Token API.
	switch dec.tokenState {
	case tokenArrayComma:
		c, err := dec.peek()
		if err != nil {
			return err
		}
		if c != ',' {
			return &SyntaxError{"expected comma after array element", dec.offset()}
		}
		dec.scanp++
		dec.tokenState = tokenArrayValue
	case tokenObjectColon:
		c, err := dec.peek()
		if err != nil {
			return err
		}
		if c != ':' {
			return &SyntaxError{"expected colon after object key", dec.offset()}
		}
		dec.scanp++
		dec.tokenState = tokenObjectValue
	}
	return nil
}
526469
func decodeReferenceToken(token string) string {
	step1 := strings.Replace(token, `~1`, `/`, -1)
	step2 := strings.Replace(step1, `~0`, `~`, -1)
	return step2
}
627596
func (t *tree) parseLet(token item) ast.Node {
	var name = t.expect(itemDollarIdent, "let")
	if t.peek().typ == itemColon {
		t.next()
		var node = &ast.LetValueNode{token.pos, name.val[1:], t.parseExpr(0)}
		t.expect(itemRightDelimEnd, "let")
		return node
	}
	t.parseAttrs("kind")
	switch next := t.next(); next.typ {
	case itemRightDelim:
		var node = &ast.LetContentNode{token.pos, name.val[1:], t.itemList(itemLetEnd)}
		t.expect(itemRightDelim, "let")
		return node
	default:
		t.unexpected(next, "{let}")
	}
	panic("unreachable")
}
420866
func Tokenize(input []rune) ([]Token, error) {
	tokens := []Token{}
	for i, t, err := nextToken(0, input); err != errEndOfInput; i, t, err = nextToken(i, input) {
		if err != nil {
			return nil, err
		}

		tokens = append(tokens, t)
	}

	return tokens, nil
}
954629
func (t *Tree) nextNonSpace() (token item) {
	for {
		token = t.next()
		if token.typ != itemIdent && token.typ != itemEndL && token.typ != itemEmptyLine {
			break
		}
	}
	// fmt.Println("\t\tnextNonSpace", token.val)
	return token
}
306899
func (p *Parser) lookahead(context ParserContext) (Token, string) {
	tok, lit := p.scannedItems[p.position].tok, p.scannedItems[p.position].literal
	if context == Values {
		switch tok {
		case InToken, NotInToken:
			tok = IdentifierToken
		}
	}
	return tok, lit
}
240857
func SplitTokensIntoLines(tokens []Token) (out [][]Token) {
	var line []Token // nolint: prealloc
	for _, token := range tokens {
		for strings.Contains(token.Value, "\n") {
			parts := strings.SplitAfterN(token.Value, "\n", 2)
			// Token becomes the tail.
			token.Value = parts[1]

			// Append the head to the line and flush the line.
			clone := token.Clone()
			clone.Value = parts[0]
			line = append(line, clone)
			out = append(out, line)
			line = nil
		}
		line = append(line, token)
	}
	if len(line) > 0 {
		out = append(out, line)
	}
	// Strip empty trailing token line.
	if len(out) > 0 {
		last := out[len(out)-1]
		if len(last) == 1 && last[0].Value == "" {
			out = out[:len(out)-1]
		}
	}
	return
}
868774
func tokenize(src string, tc chan *sep) {
	var s scanner.Scanner
	fset := token.NewFileSet() // boilerplate stuff for scanner...
	file := fset.AddFile("", fset.Base(), len(src))
	s.Init(file, []byte(src), nil, 1)
	for {
		filePos, tok, _ := s.Scan()
		tc <- &sep{int(filePos) - file.Base(), tok}
		if tok == token.EOF {
			close(tc)
			break
		}
	}
}
1299004
func CopyToken(t Token) Token {
	switch v := t.(type) {
	case CharData:
		return v.Copy()
	case Comment:
		return v.Copy()
	case Directive:
		return v.Copy()
	case ProcInst:
		return v.Copy()
	case StartElement:
		return v.Copy()
	}
	return t
}
193246
func (t *tree) parseMapLiteral(first item, expr ast.Node) ast.Node {
	firstKey, ok := expr.(*ast.StringNode)
	if !ok {
		t.errorf("expected a string as map key, got: %T", expr)
	}

	var items = make(map[string]ast.Node)
	var key = firstKey.Value
	for {
		items[key] = t.parseExpr(0)
		next := t.next()
		if next.typ == itemRightBracket {
			return &ast.MapLiteralNode{first.pos, items}
		}
		if next.typ != itemComma {
			t.unexpected(next, "map literal")
		}
		tok := t.expect(itemString, "map literal")
		var err error
		key, err = unquoteString(tok.val)
		if err != nil {
			t.error(err)
		}
		t.expect(itemColon, "map literal")
	}
}
336938
func (t *tree) expect(expected itemType, context string) item {
	token := t.next()
	if token.typ != expected {
		t.unexpected(token, fmt.Sprintf("%v (expected %v)", context, expected.String()))
	}
	return token
}
1030412
func (s *DefaultSentenceTokenizer) Tokenize(text string) []*Sentence {
	annotatedTokens := s.AnnotatedTokens(text)

	lastBreak := 0
	sentences := make([]*Sentence, 0, len(annotatedTokens))
	for _, token := range annotatedTokens {
		if !token.SentBreak {
			continue
		}

		sentence := &Sentence{lastBreak, token.Position, text[lastBreak:token.Position]}
		sentences = append(sentences, sentence)

		lastBreak = token.Position
	}

	if lastBreak != len(text) {
		lastChar := len(text)
		sentence := &Sentence{lastBreak, lastChar, text[lastBreak:lastChar]}
		sentences = append(sentences, sentence)
	}

	return sentences
}