Convert Unicode to Wylie: one tsekbar
7624
function toWylieOneTsekbar(str, len, i) {
	var orig_i = i;
	var warns = [];
	var stacks = [];// ArrayList<ToWylieStack>;
	ITER: 
	while (true) {
		var st = toWylieOneStack(str, len, i);
		stacks.push(st);
		warns = warns.concat(st.warns);
		i += st.tokens_used;
		if (st.visarga) break ITER;
		if (i >= len || tib_top(str.charAt(i)) == null) break ITER;
	}
	// figure out if some of these stacks can be prefixes or suffixes (in which case
	// they don't need their "a" vowels)
	var last = stacks.length - 1;
	if (stacks.length > 1 && stacks[0].single_cons != null) {
		// we don't count the wazur in the root stack, for prefix checking
		var cs = stacks[1].cons_str.replace(/\+w/g, "")
		if (prefix(stacks[0].single_cons, cs)) stacks[0].prefix = true;
	}
	if (stacks.length > 1 && stacks[last].single_cons != null 
	&& isSuffix(stacks[last].single_cons)) {
		stacks[last].suffix = true;
	}
	if (stacks.length > 2 && stacks[last].single_cons != null 
	&& stacks[last - 1].single_cons != null
	&& isSuffix(stacks[last - 1].single_cons)
	&& suff2(stacks[last].single_cons, stacks[last - 1].single_cons)) {
		stacks[last].suff2 = true;
		stacks[last - 1].suffix = true;
	}
	// if there are two stacks and both can be prefix-suffix, then 1st is root
	if (stacks.length == 2 && stacks[0].prefix && stacks[1].suffix) {
	    stacks[0].prefix = false;
	}
	// if there are three stacks and they can be prefix, suffix and suff2, then check w/ a table
	if (stacks.length == 3 && stacks[0].prefix && stacks[1].suffix && stacks[2].suff2) {
		var strb = []
		for (var si = 0; si < stacks.length; si++) strb.push(stacks[si].single_cons)
		var ztr = strb.join('')
		var root = ambiguous_key(ztr)
		if (root == null) {
			warns.push("Ambiguous syllable found: root consonant not known for \"" + ztr + "\".")
			// make it up...  (ex. "mgas" for ma, ga, sa)
			root = 1
		}
		stacks[root].prefix = stacks[root].suffix = false
		stacks[root + 1].suff2 = false
	}
	// if the prefix together with the main stack could be mistaken for a single stack, add a "."
	if (stacks[0].prefix && tib_stack(stacks[0].single_cons + "+" + stacks[1].cons_str)) 
		stacks[0].dot = true;
	// put it all together
	var out = ''
	for (var si = 0; si < stacks.length; si++) out += putStackTogether(stacks[si])
	var ret = new ToWylieTsekbar();
	ret.wylie = out;
	ret.tokens_used = i - orig_i;
	ret.warns = warns;
	return ret;
}
910466
def toUnicode(data, encoding=DEFAULT_ENCODING):
    
    if isinstance(data, unicode_type):
        return data

    if isinstance(data, bytes_type):
        return unicode_type(data, encoding=encoding)

    if hasattr(data, '__iter__'):
        try:
            dict(data)
        except TypeError:
            pass
        except ValueError:
            return (toUnicode(i, encoding) for i in data)
        else:
            if hasattr(data, 'items'):
                data = data.items()

            return dict(((toUnicode(k, encoding), toUnicode(v, encoding)) for k, v in data))
    return data
263084
def as_unicode(obj):
    
    if sys.version_info.major < 3 and isinstance(obj, str):
        obj = obj.decode('utf-8')
    return unicode(obj)
707829
def _peek_unicode(
        self, is_long
    ):  # type: (bool) -> Tuple[Optional[str], Optional[str]]
        
        # we always want to restore after exiting this scope
        with self._state(save_marker=True, restore=True):
            if self._current not in {"u", "U"}:
                raise self.parse_error(
                    InternalParserError, "_peek_unicode() entered on non-unicode value"
                )

            self.inc()  # Dropping prefix
            self.mark()

            if is_long:
                chars = 8
            else:
                chars = 4

            if not self.inc_n(chars):
                value, extracted = None, None
            else:
                extracted = self.extract()

                if extracted[0].lower() == "d" and extracted[1].strip("01234567"):
                    return None, None

                try:
                    value = chr(int(extracted, 16))
                except ValueError:
                    value = None

            return value, extracted
1235800
def rand_unicode(min_char=MIN_UNICHR, max_char=MAX_UNICHR, min_len=MIN_STRLEN,
                 max_len=MAX_STRLEN, **kwargs):
    '''
    '''
    from syn.five import unichr
    return unicode(rand_str(min_char, max_char, min_len, max_len, unichr))
7622
def _to_unicode(self, data, encoding, errors='strict'):
    ''''''

    # strip Byte Order Mark (if present)
    if (len(data) >= 4) and (data[:2] == '\xfe\xff') and (data[2:4] != '\x00\x00'):
        encoding = 'utf-16be'
        data = data[2:]
    elif (len(data) >= 4) and (data[:2] == '\xff\xfe') and (data[2:4] != '\x00\x00'):
        encoding = 'utf-16le'
        data = data[2:]
    elif data[:3] == '\xef\xbb\xbf':
        encoding = 'utf-8'
        data = data[3:]
    elif data[:4] == '\x00\x00\xfe\xff':
        encoding = 'utf-32be'
        data = data[4:]
    elif data[:4] == '\xff\xfe\x00\x00':
        encoding = 'utf-32le'
        data = data[4:]
    newdata = unicode(data, encoding, errors)
    return newdata
1065149
def _convert_to_unicode(string):
    
    codepoints = []
    for character in string.split('-'):
        if character in BLACKLIST_UNICODE:
            next

        codepoints.append(
            '\U{0:0>8}'.format(character).decode('unicode-escape')
        )

    return codepoints
417655
def recode_unicode(s, encoding='utf-8'):
    
    if isinstance(s, str):
        return s.encode().decode(encoding)
    return s
214111
def encode(tsig, *l):
    

    fun = encodefuns()
    if tsig not in fun:
        raise "No encoder for type %r." % tsig
    v = fun[tsig](*l)
    # Padd tsig out with spaces.
    tsig = (tsig + '   ')[: 4]
    return tsig + ('\x00' * 4) + v
566441
def convert_from_unicode(data):
    
    # if isinstance(data, basestring):

    if isinstance(data, str):
        return str(data)
    elif isinstance(data, collectionsAbc.Mapping):
        return dict(map(convert_from_unicode, data.items()))
    elif isinstance(data, collectionsAbc.Iterable):
        return type(data)(map(convert_from_unicode, data))
    else:
        return data
447942
def decode_unicode_obj(obj):
    
    if isinstance(obj, dict):
        r = {}
        for k, v in iteritems(obj):
            r[decode_unicode_string(k)] = decode_unicode_obj(v)
        return r
    elif isinstance(obj, six.string_types):
        return decode_unicode_string(obj)
    elif isinstance(obj, (list, tuple)):
        return [decode_unicode_obj(x) for x in obj]
    else:
        return obj
903221
def convert_nonascii(self, lst):
        """"""
        for item in lst:
            if item.startswith('"') and item.endswith('"'):
                item = item[1:-1]
                yield item.encode('utf-8').decode('unicode-escape')
            else:
                yield item.encode('utf-8').decode('unicode-escape')
203564
def to_char(token):
    
    if ord(token) in _range(9216, 9229 + 1):
        token = _unichr(ord(token) - 9216)

    return token
703775
def strip_symbols(s, pass_symbols=(u'й', u'Й', u'\n')):
    
    result = []
    for char in s:
        # Pass these symbols without processing
        if char in pass_symbols:
            result.append(char)
            continue
        for c in unicodedata.normalize('NFKC', char):
            if unicodedata.category(c) == 'Zs':
                result.append(u' ')
                continue
            if unicodedata.category(c) not in ['So', 'Mn', 'Lo', 'Cn', 'Co', 'Cf', 'Cc']:
                result.append(c)
    return u"".join(result)
701204
function unicodeStringToTypedArray(s) {
    var escstr = encodeURIComponent(s);
    var binstr = escstr.replace(/%([0-9A-F]{2})/g, function(match, p1) {
        return String.fromCharCode('0x' + p1);
    });
    var ua = new Uint8Array(binstr.length);
    Array.prototype.forEach.call(binstr, function (ch, i) {
        ua[i] = ch.charCodeAt(0);
    });
    return ua;
}
523288
def literal_unicode_representer(dumper, data):
    
    if '\n' in data:
        return dumper.represent_scalar(u'tag:yaml.org,2002:str', data, style='|')
    else:
        return dumper.represent_scalar(u'tag:yaml.org,2002:str', data)
1035251
def consume_unicode_range
      value = @s.scan(RE_HEX) || String.new

      while value.length < 6
        break unless @s.peek == '?'
        value << @s.consume
      end

      range = {}

      if value.include?('?')
        range[:start] = value.gsub('?', '0').hex
        range[:end]   = value.gsub('?', 'F').hex
        return create_token(:unicode_range, range)
      end

      range[:start] = value.hex

      if @s.peek(2) =~ RE_UNICODE_RANGE_END
        @s.consume
        range[:end] = (@s.scan(RE_HEX) || '').hex
      else
        range[:end] = range[:start]
      end

      create_token(:unicode_range, range)
    end