Inserts a breakline instead of a trailing space if the chunk is in CJK.
355674
def _insert_breaklines(self):
    
    target_chunks = ChunkList()
    for chunk in self:
      if chunk.word[-1] == ' ' and chunk.has_cjk():
        chunk.word = chunk.word[:-1]
        target_chunks.append(chunk)
        target_chunks.append(chunk.breakline())
      else:
        target_chunks.append(chunk)
    self.list = target_chunks
297419
private MarkdownDocStream pendingBreak() {
    if(newline == Newline.NONE) {
      return this;
    }
    out.append(newline == Newline.BREAK ? "\\\n" : newline == Newline.PAR ? "\n\n" : "\n");
    for(int i = indent, j = i; i > 0; i -= j) {
      out.append(WHITESPACES, 0, (j = i > WHITESPACES.length() ? WHITESPACES.length() : i));
    }
    newline = Newline.NONE;
    return this;
  }
1360155
function() {
			if ( !this._.inPre && this._.output.length > 0 )
				this._.output.push( this.lineBreakChars );
			this._.indent = 1;
		}
1167772
def replace_whitespace(text, SAFE_SPACE='|<^>|', insert=True):
    
    if insert:
        # replace whitespace with safe space chr
        args = (' ', SAFE_SPACE)
    else:
        # replace safe space chr with whitespace
        args = (SAFE_SPACE, ' ')

    return '\n'.join(line.strip().replace(*args)
                     for line in str.splitlines(text))
432775
@Override
    protected void handleTransliterate(Replaceable text,
                                       Position offsets, boolean isIncremental) {

        int maxLen = UCharacterName.INSTANCE.getMaxCharNameLength() + 1; // allow for temporary trailing space

        StringBuffer name = new StringBuffer(maxLen);

        // Get the legal character set
        UnicodeSet legal = new UnicodeSet();
        UCharacterName.INSTANCE.getCharNameCharacters(legal);

        int cursor = offsets.start;
        int limit = offsets.limit;

        // Modes:
        // 0 - looking for open delimiter
        // 1 - after open delimiter
        int mode = 0;
        int openPos = -1; // open delim candidate pos

        int c;
        while (cursor < limit) {
            c = text.char32At(cursor);

            switch (mode) {
            case 0: // looking for open delimiter
                if (c == OPEN_DELIM) { // quick check first
                    openPos = cursor;
                    int i = Utility.parsePattern(OPEN_PAT, text, cursor, limit);
                    if (i >= 0 && i < limit) {
                        mode = 1;
                        name.setLength(0);
                        cursor = i;
                        continue; // *** reprocess char32At(cursor)
                    }
                }
                break;

            case 1: // after open delimiter
                // Look for legal chars.  If \s+ is found, convert it
                // to a single space.  If closeDelimiter is found, exit
                // the loop.  If any other character is found, exit the
                // loop.  If the limit is reached, exit the loop.

                // Convert \s+ => SPACE.  This assumes there are no
                // runs of >1 space characters in names.
                if (PatternProps.isWhiteSpace(c)) {
                    // Ignore leading whitespace
                    if (name.length() > 0 &&
                        name.charAt(name.length()-1) != SPACE) {
                        name.append(SPACE);
                        // If we are too long then abort.  maxLen includes
                        // temporary trailing space, so use '>'.
                        if (name.length() > maxLen) {
                            mode = 0;
                        }
                    }
                    break;
                }

                if (c == CLOSE_DELIM) {

                    int len = name.length();

                    // Delete trailing space, if any
                    if (len > 0 &&
                        name.charAt(len-1) == SPACE) {
                        name.setLength(--len);
                    }

                    c = UCharacter.getCharFromExtendedName(name.toString());
                    if (c != -1) {
                        // Lookup succeeded

                        // assert(UTF16.getCharCount(CLOSE_DELIM) == 1);
                        cursor++; // advance over CLOSE_DELIM

                        String str = UTF16.valueOf(c);
                        text.replace(openPos, cursor, str);

                        // Adjust indices for the change in the length of
                        // the string.  Do not assume that str.length() ==
                        // 1, in case of surrogates.
                        int delta = cursor - openPos - str.length();
                        cursor -= delta;
                        limit -= delta;
                        // assert(cursor == openPos + str.length());
                    }
                    // If the lookup failed, we leave things as-is and
                    // still switch to mode 0 and continue.
                    mode = 0;
                    openPos = -1; // close off candidate
                    continue; // *** reprocess char32At(cursor)
                }

                if (legal.contains(c)) {
                    UTF16.append(name, c);
                    // If we go past the longest possible name then abort.
                    // maxLen includes temporary trailing space, so use '>='.
                    if (name.length() >= maxLen) {
                        mode = 0;
                    }
                }

                // Invalid character
                else {
                    --cursor; // Backup and reprocess this character
                    mode = 0;
                }

                break;
            }

            cursor += UTF16.getCharCount(c);
        }

        offsets.contextLimit += limit - offsets.limit;
        offsets.limit = limit;
        // In incremental mode, only advance the cursor up to the last
        // open delimiter candidate.
        offsets.start = (isIncremental && openPos >= 0) ? openPos : cursor;
    }
342347
def translate_logical_line(start, end, contents, indent_stack, endmarker=False):
    """"""
    # Remove leading blank lines.
    while contents[0] == '\n':
      start += 1
      contents.pop(0)
    # Remove trailing blank lines.
    while contents[-1] == '\n':
      end -= 1
      contents.pop()

    indent = len(indent_stack[-1]) if indent_stack else 0
    if endmarker:
      indent = len(contents[0])
    return start, end + 1, indent
855876
function(editor) {
			if (!this.insertLineBreakOnlyAction(editor)) {
				var curPadding = editorUtils.getCurrentLinePadding(editor);
				var content = String(editor.getContent());
				var caretPos = editor.getCaretPos();
				var len = content.length;
				var nl = '\n';
					
				// check out next line padding
				var lineRange = editor.getCurrentLineRange();
				var nextPadding = '';
					
				for (var i = lineRange.end, ch; i < len; i++) {
					ch = content.charAt(i);
					if (ch == ' ' || ch == '\t')
						nextPadding += ch;
					else
						break;
				}
				
				if (nextPadding.length > curPadding.length) {
					editor.replaceContent(nl + nextPadding, caretPos, caretPos, true);
				} else {
					editor.replaceContent(nl, caretPos);
				}
			}
			
			return true;
		}
10757
function breakLine (lineOfText, maxLength) {
  const lines = []
  let currText = lineOfText

  while (currText) {
    if (currText.length <= maxLength) {
      lines.push(currText)
      break
    } else {
      let line = ''
      const words = currText.split(/\b/g)
      while (words.length && line.length + words[0].length < maxLength) {
        line += words.shift()
      }
      if (line) lines.push(line.trim())
      currText = words.join('')
    }
  }

  return lines
}
593288
private boolean skipWhitespace() {
    boolean foundLineTerminator = false;
    while (!isAtEnd() && peekWhitespace()) {
      if (isLineTerminator(nextChar())) {
        foundLineTerminator = true;
      }
    }
    return foundLineTerminator;
  }
1164785
def insert_line_below(self, copy_margin=True):
        
        if copy_margin:
            insert = '\n' + self.document.leading_whitespace_in_current_line
        else:
            insert = '\n'

        self.cursor_position += self.document.get_end_of_line_position()
        self.insert_text(insert)
169439
public void newLineIfNotEmpty() {
		for (int i = segments.size() - 1; i >= 0; i--) {
			String segment = segments.get(i);
			if (lineDelimiter.equals(segment)) {
				segments.subList(i + 1, segments.size()).clear();
				cachedToString = null;
				return;
			}
			for (int j = 0; j < segment.length(); j++) {
				if (!WhitespaceMatcher.isWhitespace(segment.charAt(j))) {
					newLine();
					return;
				}
			}
		}
		segments.clear();
		cachedToString = null;
	}
931445
function fixSpaces(string) {
  if (string.charAt(0) == " ") string = nbsp + string.slice(1);
  return string.replace(/\t/g, function(){return makeWhiteSpace(indentUnit);})
    .replace(/[ \u00a0]{2,}/g, function(s) {return makeWhiteSpace(s.length);});
}
676764
def whitespace_token
      if md=WHITESPACE.match(@chunk)
        input = md.to_a[0]
        input.length
      else
        did_not_match
      end
    end
245847
private function tibetanLineBreaking()
	{
		for ($ptr = 0; $ptr < count($this->OTLdata); $ptr++) {
			// Break opportunities at U+0F0B Tsheg or U=0F0D
			if (isset($this->OTLdata[$ptr]['uni']) && ($this->OTLdata[$ptr]['uni'] == 0x0F0B || $this->OTLdata[$ptr]['uni'] == 0x0F0D)) {
				if (isset($this->OTLdata[$ptr + 1]['uni']) && ($this->OTLdata[$ptr + 1]['uni'] == 0x0F0D || $this->OTLdata[$ptr + 1]['uni'] == 0xF0E)) {
					continue;
				}
				// Set end of word marker in OTLdata at matchpos
				$this->OTLdata[$ptr]['wordend'] = true;
			}
		}
	}
1028770
function normalizeBlocks(input) {

    var numberOfLinesToIndent = 0;
    var thisLineContainsStuff = false;
    var thisLinesIndentation = '';
    var out = '';

    for (var i = 0; i < input.length; i++) {

        var chr = input[i];

        if (chr == '\r') {
            continue;
        }

        if (chr === '\n') {
            if (!thisLineContainsStuff) {
                numberOfLinesToIndent++;
            } else {
                out += '\n';
            }

            thisLineContainsStuff = false;
            thisLinesIndentation = '';
            continue;
        }

        if (!thisLineContainsStuff && (chr === ' ' || chr === '\t')) {
            thisLinesIndentation += chr;
            continue;
        }

        if (chr !== ' ' || chr !== '\t') {

            if (!thisLineContainsStuff) {
                for (var j = 0; j < numberOfLinesToIndent; j++) {
                    out += thisLinesIndentation + '\n';
                }

                out += thisLinesIndentation + chr;
            } else {
                out += chr;
            }

            numberOfLinesToIndent = 0;
            thisLineContainsStuff = true;

        }

    }

    return out;

}
421750
function nullCharacterInForeignContent(p, token) {
    token.chars = UNICODE.REPLACEMENT_CHARACTER;
    p._insertCharacters(token);
}
1140902
def _skip_whitespace(self):
        """"""
        i = 0
        while self._cur_token['type'] is TT.ws and not self._finished:
            self._increment()
            i += 1

        return i