this splits on whitespace, but keeps quoted and parened parts together
14797
function(str) {
  var deliminator_stack = [];
  var length = str.length;
  var i;
  var parts = [];
  var current_part = '';
  var opening_index;
  var closing_index;
  for (i = 0; i < length; i++) {
    opening_index = opening_deliminators.indexOf(str[i]);
    closing_index = closing_deliminators.indexOf(str[i]);
    if (is_space.test(str[i])) {
      if (deliminator_stack.length === 0) {
        if (current_part !== '') {
          parts.push(current_part);
        }
        current_part = '';
      } else {
        current_part += str[i];
      }
    } else {
      if (str[i] === '\\') {
        i++;
        current_part += str[i];
      } else {
        current_part += str[i];
        if (
          closing_index !== -1 &&
          closing_index === deliminator_stack[deliminator_stack.length - 1]
        ) {
          deliminator_stack.pop();
        } else if (opening_index !== -1) {
          deliminator_stack.push(opening_index);
        }
      }
    }
  }
  if (current_part !== '') {
    parts.push(current_part);
  }
  return parts;
}
869705
def t_heredocvar_ENCAPSED_AND_WHITESPACE(t):
    '
    t.lexer.lineno += t.value.count("\n")
    t.lexer.pop_state()
    return t
146146
def strip_spaces_and_quotes(value):
    
    value = value.strip() if value else ""
    if value and len(value) > 1 and (value[0] == value[-1] == '"'):
        value = value[1:-1]
    if not value:
        value = ""
    return value
1112962
def whitespace_tokenize(text):
    """"""
    text = text.strip()
    if not text:
        return []
    tokens = text.split()
    return tokens
125818
def fix_whitespace(line, offset, replacement):
    """"""
    # Replace escaped newlines too
    left = line[:offset].rstrip('\n\r \t\\')
    right = line[offset:].lstrip('\n\r \t\\')
    if right.startswith('#'):
        return line
    else:
        return left + replacement + right
154367
function splitEscaped(src, separator) {
    let escapeFlag = false,
        token = '',
        result = [];
    src.split('').forEach(letter => {
        if (escapeFlag) {
            token += letter;
            escapeFlag = false;
        } else if (letter === '\\') {
            escapeFlag = true;
        } else if (letter === separator) {
            result.push(token);
            token = '';
        } else {
            token += letter;
        }
    });
    if (token.length > 0) {
        result.push(token);
    }
    return result;
}
978052
def trimmed_split(s, seps=(";", ",")):
    """"""
    for sep in seps:
        if sep not in s:
            continue
        data = [item.strip() for item in s.strip().split(sep)]
        return data
    return [s]
919016
def whitespace_around_comma(logical_line):
    
    line = logical_line
    for separator in ',;:':
        found = line.find(separator + '  ')
        if found > -1:
            return found + 1, "E241 multiple spaces after '%s'" % separator
        found = line.find(separator + '\t')
        if found > -1:
            return found + 1, "E242 tab after '%s'" % separator
18356
@SuppressWarnings("fallthrough")
    protected void quotedString() {
        int pos = bp;
        nextChar();

        loop:
        while (bp < buflen) {
            switch (ch) {
                case '\n': case '\r': case '\f':
                    newline = true;
                    break;

                case ' ': case '\t':
                    break;

                case '"':
                    nextChar();
                    // trim trailing white-space?
                    return;

                case '@':
                    if (newline)
                        break loop;

            }
            nextChar();
        }
    }
672325
def split(self, text):
        """"""
        text = cleanup(text)
        return self.sent_detector.tokenize(text.strip())
1313818
def _prep_triple_quotes(s, mysplit=mysplit, replacements=replacements):
    

    s = mysplit(s)
    s[1::2] = (replacements[x] for x in s[1::2])
    return ''.join(s)
461861
private static String[] determineTokens(String from, byte separator, int single_quote) {
    ArrayList<String> tokens = new ArrayList();
    byte[] bits = from.getBytes();
    int offset = 0;
    int quotes = 0;
    while (offset < bits.length) {
      while ((offset < bits.length) && (bits[offset] == CHAR_SPACE)) ++offset; // skip first whitespace
      if(offset == bits.length)break;
      StringBuilder t = new StringBuilder();
      byte c = bits[offset];
      if ((c == CHAR_DOUBLE_QUOTE) || (c == single_quote)) {
        quotes = c;
        ++offset;
      }
      while (offset < bits.length) {
        c = bits[offset];
        if ((c == quotes)) {
          ++offset;
          if ((offset < bits.length) && (bits[offset] == c)) {
            t.append((char)c);
            ++offset;
            continue;
          }
          quotes = 0;
        } else if ((quotes == 0) && ((c == separator) || (c == CHAR_CR) || (c == CHAR_LF))) {
          break;
        } else {
          t.append((char)c);
          ++offset;
        }
      }
      c = (offset == bits.length) ? CHAR_LF : bits[offset];
      tokens.add(t.toString());
      if ((c == CHAR_CR) || (c == CHAR_LF) || (offset == bits.length))
        break;
      if (c != separator)
        return new String[0]; // an error
      ++offset;               // Skip separator
    }
    // If we have trailing empty columns (split by seperators) such as ",,\n"
    // then we did not add the final (empty) column, so the column count will
    // be down by 1.  Add an extra empty column here
    if( bits[bits.length-1] == separator  && bits[bits.length-1] != CHAR_SPACE)
      tokens.add("");
    return tokens.toArray(new String[tokens.size()]);
  }
524866
def _split_header(header):
        """"""
        params = {}
        parts = header.split(',')
        for param in parts:
            # Ignore realm parameter.
            if param.find('realm') > -1:
                continue
            # Remove whitespace.
            param = param.strip()
            # Split key-value.
            param_parts = param.split('=', 1)
            # Remove quotes and unescape the value.
            params[param_parts[0]] = unquote(param_parts[1].strip('\"'))
        return params
610079
function split (str) {
  if (str.indexOf(':') >= 0) {
    var parts = str.split(':')
    var m = parts.shift()
    parts = parts.join(':').split(FIELD_REGEX)
    parts.unshift(m)
    return parts
  } else {
    return str.split(FIELD_REGEX)
  }
}
1340700
def split_tokens(self):
        """"""
        if not self._split_tokens_calculated:
            # split into items (whitespace split)
            self._split_tokens = self._line_str.split()
            self._split_tokens_calculated = True

        return self._split_tokens
16997
def _split_after_delimiter(self, item, indent_amt):
        """"""
        self._delete_whitespace()

        if self.fits_on_current_line(item.size):
            return

        last_space = None
        for item in reversed(self._lines):
            if (
                last_space and
                (not isinstance(item, Atom) or not item.is_colon)
            ):
                break
            else:
                last_space = None
            if isinstance(item, self._Space):
                last_space = item
            if isinstance(item, (self._LineBreak, self._Indent)):
                return

        if not last_space:
            return

        self.add_line_break_at(self._lines.index(last_space), indent_amt)
104846
private void inlineText(WhitespaceRetentionPolicy whitespacePolicy) throws ParseException {
        switch (whitespacePolicy) {
            case REMOVE_ALL:
                skipWhitespace();
                break;
            case REMOVE_FIRST_SPACE:
                if (ch == ' ')
                    nextChar();
                break;
            case RETAIN_ALL:
            default:
                // do nothing
                break;

        }
        int pos = bp;
        int depth = 1;

        loop:
        while (bp < buflen) {
            switch (ch) {
                case '\n': case '\r': case '\f':
                    newline = true;
                    break;

                case ' ': case '\t':
                    break;

                case '{':
                    newline = false;
                    depth++;
                    break;

                case '}':
                    if (--depth == 0) {
                        return;
                    }
                    newline = false;
                    break;

                case '@':
                    if (newline)
                        break loop;
                    newline = false;
                    break;

                default:
                    newline = false;
                    break;
            }
            nextChar();
        }
        throw new ParseException("dc.unterminated.inline.tag");
    }