given string/unicode or bytes/string, determine encoding
           from magic encoding comment, return body as unicode
           or raw if decode_raw=False
343018
def decode_raw_stream(self, text, decode_raw, known_encoding, filename):
        
        if isinstance(text, compat.text_type):
            m = self._coding_re.match(text)
            encoding = m and m.group(1) or known_encoding or 'ascii'
            return encoding, text

        if text.startswith(codecs.BOM_UTF8):
            text = text[len(codecs.BOM_UTF8):]
            parsed_encoding = 'utf-8'
            m = self._coding_re.match(text.decode('utf-8', 'ignore'))
            if m is not None and m.group(1) != 'utf-8':
                raise exceptions.CompileException(
                                "Found utf-8 BOM in file, with conflicting "
                                "magic encoding comment of '%s'" % m.group(1),
                                text.decode('utf-8', 'ignore'),
                                0, 0, filename)
        else:
            m = self._coding_re.match(text.decode('utf-8', 'ignore'))
            if m:
                parsed_encoding = m.group(1)
            else:
                parsed_encoding = known_encoding or 'ascii'

        if decode_raw:
            try:
                text = text.decode(parsed_encoding)
            except UnicodeDecodeError:
                raise exceptions.CompileException(
                        "Unicode decode operation of encoding '%s' failed" %
                        parsed_encoding,
                        text.decode('utf-8', 'ignore'),
                        0, 0, filename)

        return parsed_encoding, text
1032934
def cast_str(s, encoding='utf8', errors='strict'):
    """"""
    if isinstance(s, bytes):
        return s.decode(encoding, errors)
    elif isinstance(s, str):
        return s
    else:
        raise TypeError("Expected unicode or bytes, got %r" % s)
1185361
def encoded(text, encoding=DEFAULT_ENCODING):
    
    # already a string item
    if type(text) == bytes_type:
        return text

    elif type(text) != unicode_type:
        # convert a QString value
        if type(text).__name__ == 'QString':
            if encoding == 'utf-8':
                return unicode_type(text.toUtf8(), 'utf-8')
            elif encoding == 'latin-1':
                return unicode_type(text.toLatin1(), 'latin-1')
            elif encoding == 'ascii':
                return unicode_type(text.toAscii(), 'ascii')
            else:
                return unicode_type(text, encoding)

        # convert a standard item
        else:
            try:
                return bytes_type(text)
            except StandardError:
                return '????'

    if encoding:
        try:
            return text.encode(encoding)
        except StandardError:
            return text.encode(encoding, errors='ignore')

    else:
        for enc in SUPPORTED_ENCODINGS:
            try:
                return text.encode(enc)
            except StandardError:
                pass

    return '????'
679048
def unicodestr(s, encoding='utf-8', fallback='iso-8859-1'):
    """"""

    if isinstance(s, unicode):
        return s

    try:
        return s.decode(encoding)
    except UnicodeError:
        return s.decode(fallback)
178384
def is_unicode_string(string):
    
    if string is None:
        return None
    if PY2:
        return isinstance(string, unicode)
    return isinstance(string, str)
884602
def native_string_to_unicode(s, encoding="ascii", errors="strict"):
    
    if not isinstance(s, str):
        raise TypeError("{} must be type str, not {}".format(s, type(s)))
    if str is unicode:
        # Python 3
        return s
    else:
        # Python 2
        return s.decode(encoding=encoding, errors=errors)
1138764
def ensure_unicode(str_or_unicode):
    
    if isinstance(str_or_unicode, str):
        return str_or_unicode.decode('utf-8')
    elif isinstance(str_or_unicode, unicode):
        return str_or_unicode
    else:
        raise ValueError("Input '{0}' should be a string or unicode, "
                         "but its of type {1}".format(str_or_unicode,
                                                      type(str_or_unicode)))
557459
def _normalize_string(raw_str):
  
  return " ".join(
      token.strip()
      for token in tokenizer.encode(text_encoder.native_to_unicode(raw_str)))
199760
def decode(string, encoding=None, errors=None):
    
    if encoding is None:
        encoding = getpreferredencoding()
    if errors is None:
        errors = getpreferrederrors()
    return string.decode(encoding, errors)
1233823
def _get_unicode(data, force=False):
    """"""
    if isinstance(data, binary_type):
        return data.decode('utf-8')
    elif data is None:
        return ''
    elif force:
        if PY2:
            return unicode(data)
        else:
            return str(data)
    else:
        return data
540050
def ensure_text(s, encoding='utf-8', errors='strict'):
    
    if isinstance(s, binary_type):
        return s.decode(encoding, errors)
    elif isinstance(s, text_type):
        return s
    else:
        raise TypeError("not expecting type '%s'" % type(s))
654848
def decode_string(data, encoding='hex'):
    '''
    
    '''
    if six.PY2:
        return data.decode(encoding)
    else:
        return codecs.decode(data.encode('ascii'), encoding)
1192445
def safely_decode(unicode_or_str, encoding='utf-8'):
  '''  '''
  if isinstance(unicode_or_str, unicode):
    ustr = unicode_or_str
  elif isinstance(unicode_or_str, str):
    ustr = unicode_or_str.decode(encoding, 'ignore')
  else:
    raise Exception(u'Not of type unicode or str')

  return ustr
1091345
def try_decoding(data, encoding):
    ''''''
    try:
        data.decode(encoding, 'strict')
    except UnicodeError:
        # Data under 16 bytes is very unlikely to be truncated
        if len(data) > 16:
            for trim in (1, 2, 3):
                trimmed_data = data[:-trim]
                if trimmed_data:
                    try:
                        trimmed_data.decode(encoding, 'strict')
                    except UnicodeError:
                        continue
                    else:
                        return True
        return False
    else:
        return True
818939
def get_coding(text, force_chardet=False):
    
    if not force_chardet:
        for line in text.splitlines()[:2]:
            try:
                result = CODING_RE.search(to_text_string(line))
            except UnicodeDecodeError:
                # This could fail because to_text_string assume the text
                # is utf8-like and we don't know the encoding to give
                # it to to_text_string
                pass
            else:
                if result:
                    codec = result.group(1)
                    # sometimes we find a false encoding that can
                    # result in errors
                    if codec in CODECS:
                        return codec

    # Fallback using chardet
    if is_binary_string(text):
        detector = UniversalDetector()
        for line in text.splitlines()[:2]:
            detector.feed(line)
            if detector.done: break

        detector.close()
        return detector.result['encoding']

    return None
528307
function detectEncoding(label, input_stream) {
      if (input_stream.match([0xFF, 0xFE])) {
        input_stream.offset(2);
        return 'utf-16';
      }
      if (input_stream.match([0xFE, 0xFF])) {
        input_stream.offset(2);
        return 'utf-16be';
      }
      if (input_stream.match([0xEF, 0xBB, 0xBF])) {
        input_stream.offset(3);
        return 'utf-8';
      }
      return label;
    }
822622
def detect_encoding(data, encoding=None, fallback='latin1', is_html=False):
    '''
    '''
    if encoding:
        encoding = normalize_codec_name(encoding)

    bs4_detector = EncodingDetector(
        data,
        override_encodings=(encoding,) if encoding else (),
        is_html=is_html
    )
    candidates = itertools.chain(bs4_detector.encodings, (fallback,))

    for candidate in candidates:
        if not candidate:
            continue

        candidate = normalize_codec_name(candidate)

        if not candidate:
            continue

        if candidate == 'ascii' and fallback != 'ascii':
            # it's never ascii :)
            # Falling back on UTF-8/CP-1252/Latin-1 reduces chance of
            # failure
            continue

        if try_decoding(data, candidate):
            return candidate

    raise ValueError('Unable to detect encoding.')