Public: Detect Unicode string.

 Attempts to parse Unicode BOM and falls back to UTF-8.

 str - ASCII-8BIT encoded String

 Returns encoded String.
14905
def detect_unicode(str)
      str = detect_unicode_bom(str)

      # Fallback to UTF-8
      if str.encoding == Encoding::BINARY
        str.force_encoding(Encoding::UTF_8)
      end

      str
    end
800427
def ucnstring_to_python(ucn_string):
    
    res = re.findall("U\+[0-9a-fA-F]*", ucn_string)
    for r in res:
        ucn_string = ucn_string.replace(text_type(r), text_type(ucn_to_unicode(r)))

    ucn_string = ucn_string.encode('utf-8')

    assert isinstance(ucn_string, bytes)
    return ucn_string
261622
def utf8(value: Union[None, str, bytes]) -> Optional[bytes]:  # noqa: F811
    
    if isinstance(value, _UTF8_TYPES):
        return value
    if not isinstance(value, unicode_type):
        raise TypeError("Expected bytes, unicode, or None; got %r" % type(value))
    return value.encode("utf-8")
1233392
def utf8(s):
    
    if isinstance(s, mitogen.core.UnicodeType):
        s = s.encode('utf-8')
    return s
717694
def _ensure_decoded(s):
    
    if isinstance(s, np.bytes_):
        s = s.decode('UTF-8')
    return s
1327997
def is_unicode_string(string):
    """"""
    try:
        if sys.version_info >= (3, 0):
            # isn't a python 3 str actually unicode
            if not isinstance(string, str):
                string.decode('utf-8')

        else:
            string.decode('utf-8')
    except UnicodeError:
        raise aomi.exceptions.Validation('Not a unicode string')
660725
def detect_charset(self):
        

        charset = None

        body_chunk = self.get_body_chunk()

        if body_chunk:
            # Try to extract charset from http-equiv meta tag
            match_charset = RE_META_CHARSET.search(body_chunk)
            if match_charset:
                charset = match_charset.group(1)
            else:
                match_charset_html5 = RE_META_CHARSET_HTML5.search(body_chunk)
                if match_charset_html5:
                    charset = match_charset_html5.group(1)

            # TODO: <meta charset="utf-8" />
            bom_enc, bom = read_bom(body_chunk)
            if bom_enc:
                charset = bom_enc
                self.bom = bom

            # Try to process XML declaration
            if not charset:
                if body_chunk.startswith(b'<?xml'):
                    match = RE_XML_DECLARATION.search(body_chunk)
                    if match:
                        enc_match = RE_DECLARATION_ENCODING.search(
                            match.group(0))
                        if enc_match:
                            charset = enc_match.group(1)

        if not charset:
            if 'Content-Type' in self.headers:
                pos = self.headers['Content-Type'].find('charset=')
                if pos > -1:
                    charset = self.headers['Content-Type'][(pos + 8):]

        if charset:
            charset = charset.lower()
            if not isinstance(charset, str):
                # Convert to unicode (py2.x) or string (py3.x)
                charset = charset.decode('utf-8')
            # Check that python knows such charset
            try:
                codecs.lookup(charset)
            except LookupError:
                logger.debug('Unknown charset found: %s.'
                             ' Using utf-8 istead.', charset)
                self.charset = 'utf-8'
            else:
                self.charset = charset
290018
def decode(data):
    

    if isinstance(data, unicode):
        return data

    # Detect standard unicode BOMs.
    for bom, encoding in UNICODE_BOMS:
        if data.startswith(bom):
            return data[len(bom):].decode(encoding, errors='ignore')

    # Try straight UTF-8.
    try:
        return data.decode('utf-8')
    except UnicodeDecodeError:
        pass

    # Test for various common encodings.
    for encoding in COMMON_ENCODINGS:
        try:
            return data.decode(encoding)
        except UnicodeDecodeError:
            pass

    # Anything else gets filtered.
    return NON_ASCII_FILTER.sub('', data).decode('ascii', errors='replace')
1368616
def get_text_contents(self):
        
        contents = self.get_contents()
        # The behavior of various decode() methods and functions
        # w.r.t. the initial BOM bytes is different for different
        # encodings and/or Python versions.  ('utf-8' does not strip
        # them, but has a 'utf-8-sig' which does; 'utf-16' seems to
        # strip them; etc.)  Just sidestep all the complication by
        # explicitly stripping the BOM before we decode().
        if contents[:len(codecs.BOM_UTF8)] == codecs.BOM_UTF8:
            return contents[len(codecs.BOM_UTF8):].decode('utf-8')
        if contents[:len(codecs.BOM_UTF16_LE)] == codecs.BOM_UTF16_LE:
            return contents[len(codecs.BOM_UTF16_LE):].decode('utf-16-le')
        if contents[:len(codecs.BOM_UTF16_BE)] == codecs.BOM_UTF16_BE:
            return contents[len(codecs.BOM_UTF16_BE):].decode('utf-16-be')
        try:
            return contents.decode('utf-8')
        except UnicodeDecodeError as e:
            try:
                return contents.decode('latin-1')
            except UnicodeDecodeError as e:
                return contents.decode('utf-8', error='backslashreplace')
872639
def detect_css(str)
      str = detect_unicode_bom(str)

      if name = scan_css_charset(str)
        encoding = Encoding.find(name)
        str = str.dup
        str.force_encoding(encoding)
        len = "@charset \"#{name}\";".encode(encoding).size
        str.slice!(0, len)
        str
      end

      # Fallback to UTF-8
      if str.encoding == Encoding::BINARY
        str.force_encoding(Encoding::UTF_8)
      end

      str
    end
1286754
def determine_encoding(buf):
    
    # The ultimate default is utf8; bravo, W3C
    bom_encoding = 'UTF-8'

    if not buf:
        # What
        return bom_encoding

    if isinstance(buf, six.text_type):
        # We got a file that, for whatever reason, produces already-decoded
        # text.  Check for the BOM (which is useless now) and believe
        # whatever's in the @charset.
        if buf[0] == '\ufeff':
            buf = buf[0:]

        # This is pretty similar to the code below, but without any encoding
        # double-checking.
        charset_start = '@charset "'
        charset_end = '";'
        if buf.startswith(charset_start):
            start = len(charset_start)
            end = buf.index(charset_end, start)
            return buf[start:end]
        else:
            return bom_encoding

    # BOMs
    if buf[:3] == b'\xef\xbb\xbf':
        bom_encoding = 'UTF-8'
        buf = buf[3:]
    if buf[:4] == b'\x00\x00\xfe\xff':
        bom_encoding = 'UTF-32BE'
        buf = buf[4:]
    elif buf[:4] == b'\xff\xfe\x00\x00':
        bom_encoding = 'UTF-32LE'
        buf = buf[4:]
    if buf[:4] == b'\x00\x00\xff\xfe':
        raise UnicodeError("UTF-32-2143 is not supported")
    elif buf[:4] == b'\xfe\xff\x00\x00':
        raise UnicodeError("UTF-32-2143 is not supported")
    elif buf[:2] == b'\xfe\xff':
        bom_encoding = 'UTF-16BE'
        buf = buf[2:]
    elif buf[:2] == b'\xff\xfe':
        bom_encoding = 'UTF-16LE'
        buf = buf[2:]

    # The spec requires exactly this syntax; no escapes or extra spaces or
    # other shenanigans, thank goodness.
    charset_start = '@charset "'.encode(bom_encoding)
    charset_end = '";'.encode(bom_encoding)
    if buf.startswith(charset_start):
        start = len(charset_start)
        end = buf.index(charset_end, start)
        encoded_encoding = buf[start:end]
        encoding = encoded_encoding.decode(bom_encoding)

        # Ensure that decoding with the specified encoding actually produces
        # the same @charset rule
        encoded_charset = buf[:end + len(charset_end)]
        if (encoded_charset.decode(encoding) !=
                encoded_charset.decode(bom_encoding)):
            raise UnicodeError(
                "@charset {0} is incompatible with detected encoding {1}"
                .format(bom_encoding, encoding))
    else:
        # With no @charset, believe the BOM
        encoding = bom_encoding

    return encoding
1058237
def to_unicode(value):
    
    if isinstance(value, ffi.CData):
        return ffi.string(value).decode('utf-8')
    elif isinstance(value, binary_type):
        return value.decode('utf-8')
    elif isinstance(value, text_type):
        return value
    else:
        raise ValueError('Value must be text, bytes, or char[]')
794870
def _decode(value):
        
        if value.isdigit():
            return int(value)
        if isinstance(value, bytes):
            return value.decode('utf-8')
        else:
            return value
709317
def to_unicode(value, encoding=None):
    
    if isinstance(value, text_type):
        return value
    elif isinstance(value, type(None)):
        return text_type(value)

    if not isinstance(value, bytes):
        raise TypeError(
            "Expected bytes, unicode, or None; got %r" % type(value)
        )

    if encoding is not None:
        return value.decode(encoding)
    else:
        try:
            value = value.decode('utf-8')
        except:
            try:
                value = value.decode('gbk')
            except:
                pass

        return value
771005
function stripBOM (str) {
  var bom = str.charCodeAt(0);

  // Check for the UTF-16 byte order mark (0xFEFF or 0xFFFE)
  if (bom === 0xFEFF || bom === 0xFFFE) {
    return str.slice(1);
  }

  return str;
}
221530
def to_utf8(str_or_unicode):
    
    if not isinstance(str_or_unicode, six.text_type):
        return str_or_unicode.encode("utf-8", "ignore")
    return str(str_or_unicode)
80427
def binary_encode_if_any_high_ascii(string)
      string = ensure_utf8(string)
      string.force_encoding('BINARY') if string.bytes.detect { |byte| byte > 127 }
      string
    end