Print Utf8 encoded alpha-numeric text
160112
def text(self,txt):
        
        if not txt:
            return
        try:
            txt = txt.decode('utf-8')
        except:
            try:
                txt = txt.decode('utf-16')
            except:
                pass

        self.extra_chars = 0
        
        def encode_char(char):  
            """ 
            Encodes a single utf-8 character into a sequence of 
            esc-pos code page change instructions and character declarations 
            """ 
            char_utf8 = char.encode('utf-8')
            encoded  = ''
            encoding = self.encoding # we reuse the last encoding to prevent code page switches at every character
            encodings = {
                    # TODO use ordering to prevent useless switches
                    # TODO Support other encodings not natively supported by python ( Thai, Khazakh, Kanjis )
                    'cp437': TXT_ENC_PC437,
                    'cp850': TXT_ENC_PC850,
                    'cp852': TXT_ENC_PC852,
                    'cp857': TXT_ENC_PC857,
                    'cp858': TXT_ENC_PC858,
                    'cp860': TXT_ENC_PC860,
                    'cp863': TXT_ENC_PC863,
                    'cp865': TXT_ENC_PC865,
                    'cp866': TXT_ENC_PC866,
                    'cp862': TXT_ENC_PC862,
                    'cp720': TXT_ENC_PC720,
                    'cp936': TXT_ENC_PC936,
                    'iso8859_2': TXT_ENC_8859_2,
                    'iso8859_7': TXT_ENC_8859_7,
                    'iso8859_9': TXT_ENC_8859_9,
                    'cp1254'   : TXT_ENC_WPC1254,
                    'cp1255'   : TXT_ENC_WPC1255,
                    'cp1256'   : TXT_ENC_WPC1256,
                    'cp1257'   : TXT_ENC_WPC1257,
                    'cp1258'   : TXT_ENC_WPC1258,
                    'katakana' : TXT_ENC_KATAKANA,
            }
            remaining = copy.copy(encodings)

            if not encoding :
                encoding = 'cp437'

            while True: # Trying all encoding until one succeeds
                try:
                    if encoding == 'katakana': # Japanese characters
                        if jcconv:
                            # try to convert japanese text to a half-katakanas 
                            kata = jcconv.kata2half(jcconv.hira2kata(char_utf8))
                            if kata != char_utf8:
                                self.extra_chars += len(kata.decode('utf-8')) - 1
                                # the conversion may result in multiple characters
                                return encode_str(kata.decode('utf-8')) 
                        else:
                             kata = char_utf8
                        
                        if kata in TXT_ENC_KATAKANA_MAP:
                            encoded = TXT_ENC_KATAKANA_MAP[kata]
                            break
                        else: 
                            raise ValueError()
                    else:
                        encoded = char.encode(encoding)
                        break

                except ValueError: #the encoding failed, select another one and retry
                    if encoding in remaining:
                        del remaining[encoding]
                    if len(remaining) >= 1:
                        encoding = remaining.items()[0][0]
                    else:
                        encoding = 'cp437'
                        encoded  = '\xb1'    # could not encode, output error character
                        break;

            if encoding != self.encoding:
                # if the encoding changed, remember it and prefix the character with
                # the esc-pos encoding change sequence
                self.encoding = encoding
                encoded = encodings[encoding] + encoded

            return encoded
        
        def encode_str(txt):
            buffer = ''
            for c in txt:
                buffer += encode_char(c)
            return buffer

        txt = encode_str(txt)

        # if the utf-8 -> codepage conversion inserted extra characters, 
        # remove double spaces to try to restore the original string length
        # and prevent printing alignment issues
        while self.extra_chars > 0: 
            dspace = txt.find('  ')
            if dspace > 0:
                txt = txt[:dspace] + txt[dspace+1:]
                self.extra_chars -= 1
            else:
                break

        self._raw(txt)
814768
def read_utf8(fh, byteorder, dtype, count, offsetsize):
    """"""
    return fh.read(count).decode('utf-8')
584712
def convert_to_ascii(statement):
    
    import unicodedata

    text = unicodedata.normalize('NFKD', statement.text)
    text = text.encode('ascii', 'ignore').decode('utf-8')

    statement.text = str(text)
    return statement
118301
def normal_print(raw):
    ''' '''
    lines = raw.split('\n')
    for line in lines:
        if line:
            print(line + '\n')
1218219
def print_prompt_values(values, message=None, sub_attr=None):
    """"""
    if message:
        prompt_message(message)

    for index, entry in enumerate(values):
        if sub_attr:
            line = '{:2d}: {}'.format(index, getattr(utf8(entry), sub_attr))
        else:
            line = '{:2d}: {}'.format(index, utf8(entry))

        with indent(3):
            print_message(line)
63489
def alphanumeric(text):
    """"""
    return "".join([c for c in text if re.match(r'\w', c)])
1185210
def alphanum_key(string):
    
    convert = lambda text: int(text) if text.isdigit() else text
    return [convert(segment) for segment in re.split('([0-9]+)', string)]
1046665
def decode_modified_utf8(s: bytes) -> str:
    
    s = bytearray(s)
    buff = []
    buffer_append = buff.append
    ix = 0
    while ix < len(s):
        x = s[ix]
        ix += 1

        if x >> 7 == 0:
            # Just an ASCII character, nothing else to do.
            pass
        elif x >> 6 == 6:
            y = s[ix]
            ix += 1
            x = ((x & 0x1F) << 6) + (y & 0x3F)
        elif x >> 4 == 14:
            y, z = s[ix:ix+2]
            ix += 2
            x = ((x & 0xF) << 12) + ((y & 0x3F) << 6) + (z & 0x3F)
        elif x == 0xED:
            v, w, x, y, z = s[ix:ix+6]
            ix += 5
            x = 0x10000 + (
                ((v & 0x0F) << 16) +
                ((w & 0x3F) << 10) +
                ((y & 0x0F) << 6) +
                (z & 0x3F)
            )
        elif x == 0xC0 and s[ix] == 0x80:
            ix += 1
            x = 0
        buffer_append(x)
    return u''.join(chr(b) for b in buff)
1108228
def _to_unicode(instr):
    '''
    
    '''
    if instr is None or isinstance(instr, six.text_type):
        return instr
    else:
        return six.text_type(instr, 'utf8')
1046189
def xmlstring(self, pretty_print=False):
        """"""
        s = ElementTree.tostring(self.xml(), xml_declaration=False, pretty_print=pretty_print, encoding='utf-8')
        if sys.version < '3':
            if isinstance(s, str):
                s = unicode(s,'utf-8') #pylint: disable=undefined-variable
        else:
            if isinstance(s,bytes):
                s = str(s,'utf-8')

        s = s.replace('ns0:','') #ugly patch to get rid of namespace prefix
        s = s.replace(':ns0','')
        return s
137022
def _detect_encoding(data=None):
    
    import locale
    enc_list = ['utf-8', 'latin-1', 'iso8859-1', 'iso8859-2',
                'utf-16', 'cp720']
    code = locale.getpreferredencoding(False)
    if data is None:
        return code
    if code.lower() not in enc_list:
        enc_list.insert(0, code.lower())
    for c in enc_list:
        try:
            for line in data:
                line.decode(c)
        except (UnicodeDecodeError, UnicodeError, AttributeError):
            continue
        return c
    print("Encoding not detected. Please pass encoding value manually")
1321221
def b64_print(s):
    
    if USING_PYTHON2:
        string = str(s)
    else:
        string = str(s, 'utf8')
    return '\n'.join(
        string[pos:pos + b64_intro] for pos in range(0, len(string), b64_intro)
    )
1280113
def _utf8_encode(self, d):
        
        for k, v in d.items():
            if isinstance(v, str):
                d[k] = v.encode('utf8').lower()
            if isinstance(v, list):
                for index,item in enumerate(v):
                    item = item.encode('utf8').lower()
                    v[index] = item
            if isinstance(v, dict):
                d[k] = self._utf8_encode(v)
        
        return d
1183414
function __encode_utf8(text) {

        // Fill buffer with UTF-8
        for (var i=0; i<text.length; i++) {
            var codepoint = text.charCodeAt(i);
            __append_utf8(codepoint);
        }

        // Flush buffer
        if (length > 0) {
            var out_buffer = buffer.subarray(0, length);
            length = 0;
            return out_buffer;
        }

    }
1233392
def utf8(s):
    
    if isinstance(s, mitogen.core.UnicodeType):
        s = s.encode('utf-8')
    return s
819645
def int_to_string(number, alphabet, padding=None):
    
    output = ""
    alpha_len = len(alphabet)
    while number:
        number, digit = divmod(number, alpha_len)
        output += alphabet[digit]
    if padding:
        remainder = max(padding - len(output), 0)
        output = output + alphabet[0] * remainder
    return output[::-1]
1000111
def UTF8Strsub(utf, start, len):
    
    ret = libxml2mod.xmlUTF8Strsub(utf, start, len)
    return ret