Calculate character segments with non-interruptable chains of unicodes
170863
function getSegments(font, bounds) {
  bounds = bounds || Number.MAX_VALUE;

  var result = [];
  var segment;

  // prevEndCode only changes when a segment closes
  _.forEach(font.codePoints, function (glyph, unicode) {
    unicode = parseInt(unicode, 10);
    if (unicode >= bounds) {
      return false;
    }
    // Initialize first segment or add new segment if code "hole" is found
    if (!segment || unicode !== (segment.end + 1)) {
      if (segment) {
        result.push(segment);
      }
      segment = {
        start: unicode
      };
    }
    segment.end = unicode;
  });

  // Need to finish the last segment
  if (segment) {
    result.push(segment);
  }

  _.forEach(result, function (segment) {
    segment.length = segment.end - segment.start + 1;
  });

  return result;
}
894316
def char_groups(self, t, i):
        """"""

        current = []
        pos = i.index - 1
        found = 0
        sub_first = None
        escaped = False
        first = None

        try:
            while True:
                if not escaped and t == "\\":
                    escaped = True
                elif escaped:
                    escaped = False
                    current.extend(self.reference(t, i, True))
                elif t == "[" and not found:
                    found += 1
                    first = pos
                    current.append(t)
                elif t == "[" and found and self.version == _regex.V1:
                    # Start of sub char set found
                    posix = None if self.is_bytes else self.get_posix(i)
                    if posix:
                        current.append(posix)
                        pos = i.index - 2
                    else:
                        found += 1
                        sub_first = pos
                        current.append(t)
                elif t == "[":
                    posix = None if self.is_bytes else self.get_posix(i)
                    if posix:
                        current.append(posix)
                        pos = i.index - 2
                    else:
                        current.append(t)
                elif t == "^" and found == 1 and (pos == first + 1):
                    # Found ^ at start of first char set; adjust 1st char position
                    current.append(t)
                    first = pos
                elif self.version == _regex.V1 and t == "^" and found > 1 and (pos == sub_first + 1):
                    # Found ^ at start of sub char set; adjust 1st char sub position
                    current.append(t)
                    sub_first = pos
                elif t == "]" and found == 1 and (pos != first + 1):
                    # First char set closed; log range
                    current.append(t)
                    found = 0
                    break
                elif self.version == _regex.V1 and t == "]" and found > 1 and (pos != sub_first + 1):
                    # Sub char set closed; decrement depth counter
                    found -= 1
                    current.append(t)
                else:
                    current.append(t)
                pos += 1
                t = next(i)
        except StopIteration:
            pass

        if escaped:
            current.append(t)
        return current
1116292
def full_process(s, force_ascii=False):
    """"""

    if force_ascii:
        s = asciidammit(s)
    # Keep only Letters and Numbers (see Unicode docs).
    string_out = StringProcessor.replace_non_letters_non_numbers_with_whitespace(s)
    # Force into lowercase.
    string_out = StringProcessor.to_lower_case(string_out)
    # Remove leading and trailing whitespaces.
    string_out = StringProcessor.strip(string_out)
    return string_out
529948
def slugify(value, separator='-', max_length=0, word_boundary=False,
            entities=True, decimal=True, hexadecimal=True):
    '''
    '''
    value = normalize('NFKD', to_string(value, 'utf-8', 'ignore'))
    if unidecode:
        value = unidecode(value)

    # character entity reference
    if entities:
        value = CHAR_ENTITY_REXP.sub(
            lambda m: chr(name2codepoint[m.group(1)]), value)

    # decimal character reference
    if decimal:
        try:
            value = DECIMAL_REXP.sub(lambda m: chr(int(m.group(1))), value)
        except Exception:
            pass

    # hexadecimal character reference
    if hexadecimal:
        try:
            value = HEX_REXP.sub(lambda m: chr(int(m.group(1), 16)), value)
        except Exception:
            pass

    value = value.lower()

    value = REPLACE1_REXP.sub('', value)
    value = REPLACE2_REXP.sub('-', value)

    # remove redundant -
    value = REMOVE_REXP.sub('-', value).strip('-')

    # smart truncate if requested
    if max_length > 0:
        value = smart_truncate(value, max_length, word_boundary, '-')

    if separator != '-':
        value = value.replace('-', separator)

    return value
414458
def remove_line_breaks(text):
    
    return unicode(text, 'utf-8').replace('\f', '').replace('\n', '') \
        .replace('\r', '').replace(u'\xe2\x80\xa8', '') \
        .replace(u'\xe2\x80\xa9', '').replace(u'\xc2\x85', '') \
        .encode('utf-8')
832487
function pctCharAt(text, startIndex) {
        var chr = text.charAt(startIndex);
        if (!isPercentDigitDigit(text, startIndex)) {
            return chr;
        }
        var utf8CharCode = parseHex2(text, startIndex + 1);
        var numBytes = utf8.numBytes(utf8CharCode);
        if (numBytes === 0) {
            return chr;
        }
        for (var byteNumber = 1; byteNumber < numBytes; byteNumber += 1) {
            if (!isPercentDigitDigit(text, startIndex + 3 * byteNumber) || !utf8.isValidFollowingCharCode(parseHex2(text, startIndex + 3 * byteNumber + 1))) {
                return chr;
            }
        }
        return text.substr(startIndex, 3 * numBytes);
    }
728661
def _get_matchable_segments(segments):
    
    for subsegment in segments:
        if isinstance(subsegment, Token):
            break # No tokens allowed next to segments
        if isinstance(subsegment, Segment):
            if isinstance(subsegment, MatchableSegment):
                yield subsegment

            for matchable_subsegment in _get_matchable_segments(subsegment):
                yield matchable_subsegment
348982
def u2b(string):
    
    if ((PY2 and isinstance(string, unicode)) or
        ((not PY2) and isinstance(string, str))):
        return string.encode('utf-8')
    return string
1323759
def multicharacters(input)
      output = input.gsub(/\([Cc]\)/, "©")
      output = output.gsub(/\([Pp]\)/, "℗")
      output = output.gsub(/\([Rr]\)/, "®")
      output = output.gsub(/\((SM|sm|Sm)\)/, "℠")
      output = output.gsub(/\((TM|tm|Tm)\)/, "™")
      output = output.gsub(/\+-/, "±")
      output = output.gsub(/-\+/, "∓")
      output = output.gsub(/N[oO]\.?\s*(\d+)/, '№\1')
      output = output.gsub(/°C/, '℃')
      output = output.gsub(/°F/, '℉')
    end
217890
def _strslices(self):
        """"""
        pad = self.padding or 0
        for sli in self.slices():
            if sli.start + 1 == sli.stop:
                yield "%0*d" % (pad, sli.start)
            else:
                assert sli.step >= 0, "Internal error: sli.step < 0"
                if sli.step == 1:
                    yield "%0*d-%0*d" % (pad, sli.start, pad, sli.stop - 1)
                else:
                    yield "%0*d-%0*d/%d" % (pad, sli.start, pad, sli.stop - 1, \
                                            sli.step)
65745
def _native_to_unicode(s):
  """"""
  if six.PY2:
    return s if isinstance(s, unicode) else s.decode("utf-8")
  else:
    return s
1228093
def _switch_charset_list(characters, target=''):
    '''
    
    '''
    # Copy the list to avoid modifying the existing one.
    characters = characters[:]
    offset = block_offset * offsets[target]['direction']
    for n in range(len(characters)):
        chars = list(characters[n])

        for m in range(len(chars)):
            char = chars[m]
            char_offset = ord(char) + offset
            # Verify that the offset character is within the valid range.
            if in_range(char_offset, target):
                chars[m] = chr(char_offset)
            else:
                chars[m] = char

        characters[n] = ''.join(chars)

    return characters
49307
def _unicode_to_native(s):
  """"""
  if six.PY2:
    return s.encode("utf-8") if isinstance(s, unicode) else s
  else:
    return s
546394
def characters(self):
        
        chars = 0
        for fragment in self.fragments:
            chars += fragment.characters
        return chars
955101
function eatCharacter(stream) {
        var first = stream.next();
        // Read special literals: backspace, newline, space, return.
        // Just read all lowercase letters.
        if (first.match(/[a-z]/) && stream.match(/[a-z]+/, true)) {
            return;
        }
        // Read unicode character: \u1000 \uA0a1
        if (first === "u") {
            stream.match(/[0-9a-z]{4}/i, true);
        }
    }
991777
function ( text ){
		   var out = [text],
			   chars = ['-','+','z','Z'],
			   i = chars.length;

			while (i--) {
			  if(text.indexOf(chars[i]) > -1){
				  out[0] = text.slice( 0, text.indexOf(chars[i]) );
				  out.push( text.slice( text.indexOf(chars[i]) ) );
				  break;
			   }
			}
		   return out;
		}
885687
def _unicode(self):
        ''''''

        return u'\n'.join ([ u''.join(c) for c in self.w ])