Walk through delimiter list and replace text tokens with tags
513994
function postProcess(state) {
    var i,
        foundStart = false,
        foundEnd = false,
        delim,
        token,
        delimiters = state.delimiters,
        max = state.delimiters.length;

    for (i = 0; i < max; i++) {
      delim = delimiters[i];
      if (delim.marker === '->') {
        foundStart = true;
      } else if (delim.marker === '<-') {
        foundEnd = true;
      }
    }
    if (foundStart && foundEnd) {
      for (i = 0; i < max; i++) {
        delim = delimiters[i];

        if (delim.marker === '->') {
          foundStart = true;
          token         = state.tokens[delim.token];
          token.type    = 'centertext_open';
          token.tag     = 'div';
          token.nesting = 1;
          token.markup  = '->';
          token.content = '';
          token.attrs = [ [ 'class', 'text-align-center' ] ];
        } else if (delim.marker === '<-') {
          if (foundStart) {
            token         = state.tokens[delim.token];
            token.type    = 'centertext_close';
            token.tag     = 'div';
            token.nesting = -1;
            token.markup  = '<-';
            token.content = '';
          }
        }
      }
    }
  }
940881
function tokenize(text) {
    var tokens = [];
    if (text && text.length > 0) {
        var token = '';
        for (var i = 0; i < text.length; i++) {
            var chr = text[i];
            if (breakingChars.indexOf(chr) >= 0) {
                if (token.length > 0) {
                    tokens.push(token);
                }
                token = '';
            }
            else {
                token += chr;
            }
        }
        if (token.length > 0) {
            tokens.push(token);
        }
    }
    return tokens;
}
321770
def _parse_literal_list(self, indent):
        """"""
        if self._cur_token['type'] not in self._literals:
            raise Exception(
                "Parser failed, _parse_literal_list was called on non-literal"
                " {} on line {}.".format(
                    repr(self._cur_token['value']), self._cur_token['line']
                )
            )

        # find next token after whitespace without incrementing
        temp_position = self._cur_position
        while (
            temp_position < self.num_tokens-1 and (
                self.tokens[temp_position]['type'] is TT.ws or
                self.tokens[temp_position]['type'] in self._literals
            )
        ):
            temp_position += 1
        next_token = self.tokens[temp_position]

        # end of stream
        if next_token['type'] is TT.ws:
            return self._cur_token['value']
        elif next_token['type'] is TT.comma:
            return self._parse_comma_list()
        elif next_token['type'] is TT.lbreak:
            while (
                temp_position < self.num_tokens-1 and
                self.tokens[temp_position]['type'] in (TT.lbreak, TT.ws)
            ):
                temp_position += 1
            if self.tokens[temp_position]['type'] in self._literals:
                return self._parse_newline_list(indent)
            else:
                rval = self._cur_token['value']
                self._increment()
                return rval
        else:
            rval = self._cur_token['value']
            self._increment()
            return rval
287132
def tag(self, tokens):
        """"""
        tags = []
        for token in tokens:
            normalized = self.lexicon[token].normalized
            for regex, tag in self.regexes:
                if regex.match(normalized):
                    tags.append((token, tag))
                    break
            else:
                tags.append((token, None))
        return tags
394342
def tokenize(cls, text, mode='c'):
        
        if mode == 'c':
            return [ch for ch in text]
        else:
            return [w for w in text.split()]
533436
function tokenize(source, options){
  var result = [];
  var tagStack = [];
  var lastTag = { children: result };
  var parseTag = false;
  var token;
  var state = TEXT;
  var pos = 0;
  var textStateEndPos = 0;
  var textEndPos;
  var bufferPos;
  var startPos;
  var m;

  /** @cut */ var attrMap;  // map to detect attribute duplicates
  /** @cut */ result.source_ = source;
  /** @cut */ result.warns = [];

  // skip starting whitespaces
  if (!options || options.trim !== false)
  {
    pos = textStateEndPos = source.match(/^\s*/)[0].length;
    source = source.trimRight();  // TODO: replace for basis.string.trimRight
  }

  while (pos < source.length || state != TEXT)
  {
    state.lastIndex = pos;
    startPos = pos;

    m = state.exec(source);

    if (!m || m.index !== pos)
    {
      // treat broken comment reference as comment content
      if (state == REFERENCE && token && token.type == TYPE_COMMENT)
      {
        state = COMMENT;
        continue;
      }

      if (parseTag)
        lastTag = tagStack.pop();

      if (token)
        lastTag.children.pop();

      if (token = lastTag.children.pop())
      {
        if (token.type == TYPE_TEXT && !token.refs)
          textStateEndPos -= token.value.length;
        else
          lastTag.children.push(token);
      }

      parseTag = false;
      state = TEXT;
      continue;
    }

    pos = state.lastIndex;

    //stat[state] = (stat[state] || 0) + 1;
    switch (state)
    {
      case TEXT:

        textEndPos = startPos + m[1].length;

        if (textStateEndPos != textEndPos)
        {
          var sourceText = textStateEndPos == startPos
            ? m[1]
            : source.substring(textStateEndPos, textEndPos);

          sourceText = sourceText.replace(/\s*(\r\n?|\n\r?)\s*/g, '');

          if (sourceText)
            lastTag.children.push({
              type: TYPE_TEXT,
              value: sourceText,
              range: {
                start_: textStateEndPos,
                end_: textEndPos
              }
            });
        }

        textStateEndPos = textEndPos;

        if (m[3])
        {
          lastTag.children.push({
            type: TYPE_TEXT,
            refs: ['l10n:' + m[3]],
            value: '{l10n:' + m[3] + '}',
            range: {
              start_: textEndPos,
              end_: pos
            }
          });
        }
        else if (m[2] == '{')
        {
          bufferPos = pos - 1;
          lastTag.children.push(token = {
            type: TYPE_TEXT,
            range: {
              start_: textEndPos,
              end_: textEndPos
            }
          });
          state = REFERENCE;
        }
        else if (m[4])
        {
          if (m[4] == '/')
          {
            token = null;
            state = CLOSE_TAG;
          }
          else //if (m[3] == '!--')
          {
            lastTag.children.push(token = {
              type: TYPE_COMMENT,
              range: {
                start_: textEndPos,
                end_: textEndPos
              }
            });

            if (m[5])
            {
              bufferPos = pos - m[5].length;
              state = REFERENCE;
            }
            else
            {
              bufferPos = pos;
              state = COMMENT;
            }
          }
        }
        else if (m[2]) // m[2] == '<' open tag
        {
          parseTag = true;
          tagStack.push(lastTag);

          lastTag.children.push(token = {
            type: TYPE_ELEMENT,
            attrs: [],
            children: [],
            range: {
              start_: textEndPos,
              end_: textEndPos
            }
          });
          lastTag = token;

          state = TAG_NAME;

          // create attribute map
          /** @cut */ attrMap = {};
        }

        break;

      case CLOSE_TAG:
        if (m[1] !== (lastTag.prefix ? lastTag.prefix + ':' : '') + lastTag.name)
        {
          lastTag.children.push({
            type: TYPE_TEXT,
            value: '</' + m[0],
            range: {
              start_: startPos - 2,
              end_: startPos + m[0].length
            }
          });
          /** @cut */ result.warns.push(['Wrong close tag: ' + source.substr(startPos - 2, m[0].length + 2), lastTag.children[lastTag.children.length - 1]]);
        }
        else
        {
          lastTag.range.end_ = startPos + m[0].length;
          lastTag = tagStack.pop();
        }

        state = TEXT;
        break;

      case TAG_NAME:
      case ATTRIBUTE_NAME_OR_END:
        if (m[2] == ':')
        {
          if (token.prefix)      // prefix was before, break tag parse
            state = BREAK_TAG_PARSE;
          else
            token.prefix = m[1];

          break;
        }

        if (m[1])
        {
          // store name (it may be null when check for attribute and end)
          token.name = m[1];
          token.range.end_ = startPos + m[1].length;

          // store attribute
          if (token.type == TYPE_ATTRIBUTE)
          {
            // detect attribute duplicates
            /** @cut */ var fullName = (token.prefix ? token.prefix + ':' : '') + token.name;
            /** @cut */ if (Object.prototype.hasOwnProperty.call(attrMap, fullName))
            /** @cut */   result.warns.push(['Duplicate attribute: ' + fullName, token]);
            /** @cut */ attrMap[fullName] = true;

            lastTag.attrs.push(token);
          }
        }

        if (m[2] == '{')
        {
          if (token.type == TYPE_ELEMENT)
            state = REFERENCE;
          else
            state = BREAK_TAG_PARSE;

          break;
        }

        if (m[3]) // end tag declaration
        {
          parseTag = false;
          lastTag.range.end_ = pos;

          if (m[3] == '/>' ||
              (!lastTag.prefix && SINGLETON_TAG.test(lastTag.name)))
          {
            /** @cut */ if (m[3] != '/>')
            /** @cut */   result.warns.push(['Tag <' + lastTag.name + '> doesn\'t closed explicit (use `/>` as tag ending)', lastTag]);

            lastTag = tagStack.pop();
          }
          else
          {
            // otherwise m[3] == '>'
            if (lastTag.prefix == 'b' && lastTag.name in TAG_IGNORE_CONTENT)
            {
              state = TAG_IGNORE_CONTENT[lastTag.name];
              break;
            }
          }

          state = TEXT;
          break;
        }

        if (m[2] == '=') // ATTRIBUTE_NAME_OR_END only
        {
          state = ATTRIBUTE_VALUE;
          break;
        }

        // m[2] == '\s+' next attr, state doesn't change
        token = {
          type: TYPE_ATTRIBUTE,
          range: {
            start_: pos,
            end_: pos
          }
        };
        state = ATTRIBUTE_NAME_OR_END;
        break;

      case COMMENT:
        token.value = source.substring(bufferPos, pos - 3);
        token.range.end_ = pos;
        state = TEXT;
        break;

      case REFERENCE:
        // add reference to token list name
        if (token.refs)
          token.refs.push(m[1]);
        else
          token.refs = [m[1]];

        // go next
        if (m[2] != '|') // m[2] == '}\s*'
        {
          if (token.type == TYPE_TEXT)
          {
            pos -= m[2].length - 1;
            token.value = source.substring(bufferPos, pos);
            token.range.end_ = pos;
            state = TEXT;
          }
          else if (token.type == TYPE_COMMENT)
          {
            state = COMMENT;
          }
          else if (token.type == TYPE_ATTRIBUTE && source[pos] == '=')
          {
            pos++;
            state = ATTRIBUTE_VALUE;
          }
          else // ATTRIBUTE || ELEMENT
          {
            token = {
              type: TYPE_ATTRIBUTE,
              range: {
                start_: pos,
                end_: pos
              }
            };
            state = ATTRIBUTE_NAME_OR_END;
          }
        }

        // continue to collect references
        break;

      case ATTRIBUTE_VALUE:
        token.value = m[1].replace(QUOTE_UNESCAPE, '"');
        token.range.end_ = pos;
        token.valueRange = {
          start_: startPos + 1,
          end_: startPos + 1 + m[1].length
        };

        token = {
          type: TYPE_ATTRIBUTE,
          range: {
            start_: pos,
            end_: pos
          }
        };
        state = ATTRIBUTE_NAME_OR_END;

        break;

      case TAG_IGNORE_CONTENT.text:
      case TAG_IGNORE_CONTENT.style:
        lastTag.children.push({
          type: TYPE_TEXT,
          value: m[1],
          range: {
            start_: startPos,
            end_: startPos + m[1].length
          }
        });

        lastTag = tagStack.pop();

        state = TEXT;
        break;

      default:
        throw SYNTAX_ERROR; // Must never to be here; bug in parser otherwise
    }

    if (state == TEXT)
      textStateEndPos = pos;
  }

  if (textStateEndPos != pos)
    lastTag.children.push({
      type: TYPE_TEXT,
      value: source.substring(textStateEndPos, pos),
      range: {
        start_: textStateEndPos,
        end_: pos
      }
    });

  // process attributes binding, join text tokens, normalize ranges and add location info
  postProcessing(result, options || {}, source);

  /** @cut */ if (lastTag.name)
  /** @cut */   result.warns.push(['No close tag for <' + (lastTag.prefix ? lastTag.prefix + ':' : '') + lastTag.name + '>', lastTag]);
  /** @cut */
  /** @cut */ if (!result.warns.length)
  /** @cut */   result.warns = false;

  result.templateTokens = true;

  return result;
}
771994
def generate_tokens(self, text):
        """"""
        if text != self.last_text:
            string_io = io.StringIO(text)
            self.last_tokens = list(
                tokenize.generate_tokens(string_io.readline)
            )
            self.last_text = text
        return self.last_tokens
453912
public static String[] tokenize(String str, String delimiters, boolean trim) {
        if (str == null) {
            return new String[0];
        }
        StringTokenizer st = new StringTokenizer(str, delimiters);
        List<String> tokens = new ArrayList<>();
        while (st.hasMoreTokens()) {
            tokens.add(trim ? st.nextToken().trim() : st.nextToken());
        }
        return tokens.toArray(new String[0]);
    }
1195745
function splitTag(editor, node, cursor) {
	const open = getText(editor, node.open);
	const m = open.match(/(\s*\/)?>$/);
	const start = node.open.end.translate([0, m ? -m[0].length : 0]);
	editor.setTextInBufferRange(new Range(start, node.open.end), `></${node.name}>`);

	if (cursor) {
		cursor.setBufferPosition(start.translate([0, 1]));
	}
}
125438
def _compile_template_re(delimiters):
    
    # The possible tag type characters following the opening tag,
    # excluding "=" and "{".
    tag_types = "!>&/#^"

    # TODO: are we following this in the spec?
    #
    #   The tag's content MUST be a non-whitespace character sequence
    #   NOT containing the current closing delimiter.
    #
    tag = r"""
        (?P<whitespace>[\ \t]*)
        %(otag)s \s*
        (?:
          (?P<change>=) \s* (?P<delims>.+?)   \s* = |
          (?P<raw>{)    \s* (?P<raw_name>.+?) \s* } |
          (?P<tag>[%(tag_types)s]?)  \s* (?P<tag_key>[\s\S]+?)
        )
        \s* %(ctag)s
    """ % {'tag_types': tag_types, 'otag': re.escape(delimiters[0]), 'ctag': re.escape(delimiters[1])}

    return re.compile(tag, re.VERBOSE)
1082567
def _scan_for_tokens(contents):
    """"""
    # Regexes are in priority order. Changing the order may alter the
    # behavior of the lexer
    scanner = re.Scanner([
        # Things inside quotes
        (r"(?<![^\s\(])([\"\'])(?:(?=(\\?))\2.)*?\1(?![^\s\)])",
         lambda s, t: (TokenType.QuotedLiteral, t)),
        # Numbers on their own
        (r"(?<![^\s\(])-?[0-9]+(?![^\s\)\(])", lambda s, t: (TokenType.Number,
                                                             t)),
        # Left Paren
        (r"\(", lambda s, t: (TokenType.LeftParen, t)),
        # Right Paren
        (r"\)", lambda s, t: (TokenType.RightParen, t)),
        # Either a valid function name or variable name.
        (r"(?<![^\s\(])[a-zA-z_][a-zA-Z0-9_]*(?![^\s\)\(])",
         lambda s, t: (TokenType.Word, t)),
        # Variable dereference.
        (r"(?<![^\s\(])\${[a-zA-z_][a-zA-Z0-9_]*}(?![^\s\)])",
         lambda s, t: (TokenType.Deref, t)),
        # Newline
        (r"\n", lambda s, t: (TokenType.Newline, t)),
        # Whitespace
        (r"\s+", lambda s, t: (TokenType.Whitespace, t)),
        # The beginning of a double-quoted string, terminating at end of line
        (r"(?<![^\s\(\\])[\"]([^\"]|\\[\"])*$",
         lambda s, t: (TokenType.BeginDoubleQuotedLiteral, t)),
        # The end of a double-quoted string
        (r"[^\s]*(?<!\\)[\"](?![^\s\)])",
         lambda s, t: (TokenType.EndDoubleQuotedLiteral, t)),
        # The beginning of a single-quoted string, terminating at end of line
        (r"(?<![^\s\(\\])[\']([^\']|\\[\'])*$",
         lambda s, t: (TokenType.BeginSingleQuotedLiteral, t)),
        # The end of a single-quoted string
        (r"[^\s]*(?<!\\)[\'](?![^\s\)])",
         lambda s, t: (TokenType.EndSingleQuotedLiteral, t)),
        # Begin-RST Comment Block
        (r"#.rst:$", lambda s, t: (TokenType.BeginRSTComment, t)),
        # Begin Inline RST
        (r"#\[=*\[.rst:$", lambda s, t: (TokenType.BeginInlineRST, t)),
        # End Inline RST
        (r"#\]=*\]$", lambda s, t: (TokenType.EndInlineRST, t)),
        # Comment
        (r"#", lambda s, t: (TokenType.Comment, t)),
        # Catch-all for literals which are compound statements.
        (r"([^\s\(\)]+|[^\s\(]*[^\)]|[^\(][^\s\)]*)",
         lambda s, t: (TokenType.UnquotedLiteral, t))
    ])

    tokens_return = []

    lines = contents.splitlines(True)
    lineno = 0
    for line in lines:
        lineno += 1
        col = 1

        tokens, remaining = scanner.scan(line)
        if remaining != "":
            msg = "Unknown tokens found on line {0}: {1}".format(lineno,
                                                                 remaining)
            raise RuntimeError(msg)

        for token_type, token_contents in tokens:
            tokens_return.append(Token(type=token_type,
                                       content=token_contents,
                                       line=lineno,
                                       col=col))

            col += len(token_contents)

    return tokens_return
47173
def replace_simple_tags(s, from_tag='italic', to_tag='i', to_open_tag=None):
    
    if to_open_tag:
        s = s.replace('<' + from_tag + '>', to_open_tag)
    elif to_tag:
        s = s.replace('<' + from_tag + '>', '<' + to_tag + '>')
        s = s.replace('<' + from_tag + '/>', '<' + to_tag + '/>')
    else:
        s = s.replace('<' + from_tag + '>', '')
        s = s.replace('<' + from_tag + '/>', '')

    if to_tag:
        s = s.replace('</' + from_tag + '>', '</' + to_tag + '>')
    else:
        s = s.replace('</' + from_tag + '>', '')

    return s
757037
def strip(self, data, strip_newlines=False):
        
        text = []
        for token_type, tag_name, tag_opts, token_text in self.tokenize(data):
            if token_type == self.TOKEN_DATA:
                text.append(token_text)
            elif token_type == self.TOKEN_NEWLINE and not strip_newlines:
                text.append(token_text)
        return ''.join(text)
459682
def _normalize_tags(chunk):
    
    ret = []
    for word, tag in chunk:
        if tag == 'NP-TL' or tag == 'NP':
            ret.append((word, 'NNP'))
            continue
        if tag.endswith('-TL'):
            ret.append((word, tag[:-3]))
            continue
        if tag.endswith('S'):
            ret.append((word, tag[:-1]))
            continue
        ret.append((word, tag))
    return ret
478019
def _tokenize(self, text, token_class=None):
        
        token_class = token_class or Token
        tokens = {}

        for i, match in enumerate(self.regex.finditer(text)):
            value = match.group(0)

            try:
                token = tokens[value]
            except KeyError:
                type = match.lastgroup
                token = token_class(value, type=type)
                tokens[value] = token

            yield token
476930
function tagReplace(original, tag, value) {
  var fullTag = `[[${  tag  }]]`;
  return original.split(fullTag).join(value ? value : '');
}
841625
def _convert_to_list(self, value, delimiters):
        
        if not value:
            return []
        if delimiters:
            return [l.strip() for l in value.split(delimiters)]
        return [l.strip() for l in value.split()]