UTF-16 (BOM) => ISO-8859-1
16723
public static function iconv_fallback_utf16_iso88591($string) {
		$bom = substr($string, 0, 2);
		if ($bom == "\xFE\xFF") {
			return self::iconv_fallback_utf16be_iso88591(substr($string, 2));
		} elseif ($bom == "\xFF\xFE") {
			return self::iconv_fallback_utf16le_iso88591(substr($string, 2));
		}
		return $string;
	}
78856
def utf82unicode(value):
    
    try:
        return value.decode("utf8")
    except Exception as e:
        if not _Log:
            _late_import()

        if not is_binary(value):
            _Log.error("Can not convert {{type}} to unicode because it's not bytes",  type= type(value).__name__)

        e = _Except.wrap(e)
        for i, c in enumerate(value):
            try:
                c.decode("utf8")
            except Exception as f:
                _Log.error("Can not convert charcode {{c}} in string index {{i}}", i=i, c=ord(c), cause=[e, _Except.wrap(f)])

        try:
            latin1 = text_type(value.decode("latin1"))
            _Log.error("Can not explain conversion failure, but seems to be latin1", e)
        except Exception:
            pass

        try:
            a = text_type(value.decode("latin1"))
            _Log.error("Can not explain conversion failure, but seems to be latin1", e)
        except Exception:
            pass

        _Log.error("Can not explain conversion failure of " + type(value).__name__ + "!", e)
89986
function( /*String*/ path, /*String?*/ encoding) {
					path = frontSlash(path);
					if (encoding === 'utf-8') {
						encoding = 'utf8';
					}
					if (!encoding) {
						encoding = 'utf8';
					}

					var text = cache[path].contents + "";
					//Hmm, would not expect to get A BOM, but it seems to happen,
					//remove it just in case.
					if (text.indexOf('\uFEFF') === 0) {
						text = text.substring(1, text.length);
					}


					return text;
				}
873533
def detect_unicode_bom(str)
      bom_bytes = str.byteslice(0, 4).bytes.to_a

      BOM.each do |encoding, bytes|
        if bom_bytes[0, bytes.size] == bytes
          str = str.dup
          str.force_encoding(Encoding::BINARY)
          str.slice!(0, bytes.size)
          str.force_encoding(encoding)
          return str
        end
      end

      return str
    end
899837
function _utf8ArrayToStr(data) {
    const extraByteMap = [1, 1, 1, 1, 2, 2, 3, 0];
    var count = data.length;
    var str = "";

    for (var index = 0; index < count;) {
        var ch = data[index++];
        if (ch & 0x80) {
            var extra = extraByteMap[(ch >> 3) & 0x07];
            if (!(ch & 0x40) || !extra || ((index + extra) > count))
                return null;

            ch = ch & (0x3F >> extra);
            for (; extra > 0; extra -= 1) {
                var chx = data[index++];
                if ((chx & 0xC0) != 0x80)
                    return null;

                ch = (ch << 6) | (chx & 0x3F);
            }
        }

        str += String.fromCharCode(ch);
    }

    return str;
}
65368
def utf(text):
    """"""
    try:
        output = unicode(text, encoding='utf-8')
    except UnicodeDecodeError:
        output = text
    except TypeError:
        output = text
    return output
290018
def decode(data):
    

    if isinstance(data, unicode):
        return data

    # Detect standard unicode BOMs.
    for bom, encoding in UNICODE_BOMS:
        if data.startswith(bom):
            return data[len(bom):].decode(encoding, errors='ignore')

    # Try straight UTF-8.
    try:
        return data.decode('utf-8')
    except UnicodeDecodeError:
        pass

    # Test for various common encodings.
    for encoding in COMMON_ENCODINGS:
        try:
            return data.decode(encoding)
        except UnicodeDecodeError:
            pass

    # Anything else gets filtered.
    return NON_ASCII_FILTER.sub('', data).decode('ascii', errors='replace')
1319896
static function iconv_fallback_utf16_iso88591($string) {
		$bom = substr($string, 0, 2);
		if ($bom == "\xFE\xFF") {
			return getid3_lib::iconv_fallback_utf16be_iso88591(substr($string, 2));
		} elseif ($bom == "\xFF\xFE") {
			return getid3_lib::iconv_fallback_utf16le_iso88591(substr($string, 2));
		}
		return $string;
	}
166858
def convert_to_codec_key(value):
    
    if not value:
        # fallback to utf-8
        value = 'UTF-8'
    # UTF-8 -> utf_8
    converted = value.replace('-', '_').lower()
    # fix some corner cases, see https://github.com/pyQode/pyQode/issues/11
    all_aliases = {
        'ascii': [
            'us_ascii',
            'us',
            'ansi_x3.4_1968',
            'cp367',
            'csascii',
            'ibm367',
            'iso_ir_6',
            'iso646_us',
            'iso_646.irv:1991'
        ],
        'utf-7': [
            'csunicode11utf7',
            'unicode_1_1_utf_7',
            'unicode_2_0_utf_7',
            'x_unicode_1_1_utf_7',
            'x_unicode_2_0_utf_7',
        ],
        'utf_8': [
            'unicode_1_1_utf_8',
            'unicode_2_0_utf_8',
            'x_unicode_1_1_utf_8',
            'x_unicode_2_0_utf_8',
        ],
        'utf_16': [
            'utf_16le',
            'ucs_2',
            'unicode',
            'iso_10646_ucs2'
        ],
        'latin_1': ['iso_8859_1']
    }

    for key, aliases in all_aliases.items():
        if converted in aliases:
            return key
    return converted
1206490
private String calculateRawEncoding(final String bomEnc, final String xmlGuessEnc, final String xmlEnc, final InputStream is) throws IOException {
        String encoding;
        if (bomEnc == null) {
            if (xmlGuessEnc == null || xmlEnc == null) {
                if (defaultEncoding == null) {
                    encoding = UTF_8;
                } else {
                    encoding = defaultEncoding;
                }
            } else if (xmlEnc.equals(UTF_16) && (xmlGuessEnc.equals(UTF_16BE) || xmlGuessEnc.equals(UTF_16LE))) {
                encoding = xmlGuessEnc;
            } else {
                encoding = xmlEnc;
            }
        } else if (bomEnc.equals(UTF_8)) {
            if (xmlGuessEnc != null && !xmlGuessEnc.equals(UTF_8)) {
                throw new XmlReaderException(RAW_EX_1.format(new Object[] { bomEnc, xmlGuessEnc, xmlEnc }), bomEnc, xmlGuessEnc, xmlEnc, is);
            }
            if (xmlEnc != null && !xmlEnc.equals(UTF_8)) {
                throw new XmlReaderException(RAW_EX_1.format(new Object[] { bomEnc, xmlGuessEnc, xmlEnc }), bomEnc, xmlGuessEnc, xmlEnc, is);
            }
            encoding = UTF_8;
        } else if (bomEnc.equals(UTF_16BE) || bomEnc.equals(UTF_16LE)) {
            if (xmlGuessEnc != null && !xmlGuessEnc.equals(bomEnc)) {
                throw new IOException(RAW_EX_1.format(new Object[] { bomEnc, xmlGuessEnc, xmlEnc }));
            }
            if (xmlEnc != null && !xmlEnc.equals(UTF_16) && !xmlEnc.equals(bomEnc)) {
                throw new XmlReaderException(RAW_EX_1.format(new Object[] { bomEnc, xmlGuessEnc, xmlEnc }), bomEnc, xmlGuessEnc, xmlEnc, is);
            }
            encoding = bomEnc;
        } else {
            throw new XmlReaderException(RAW_EX_2.format(new Object[] { bomEnc, xmlGuessEnc, xmlEnc }), bomEnc, xmlGuessEnc, xmlEnc, is);
        }
        return encoding;
    }
832507
private static function ucs4array_utf8($input)
    {
        $output = '';
        foreach ($input as $v) {
            if ($v < 128) { // 7bit are transferred literally
                $output .= chr($v);
            } elseif ($v < (1 << 11)) { // 2 bytes
                $output .= chr(192+($v >> 6)).chr(128+($v & 63));
            } elseif ($v < (1 << 16)) { // 3 bytes
                $output .= chr(224+($v >> 12)).chr(128+(($v >> 6) & 63)).chr(128+($v & 63));
            } elseif ($v < (1 << 21)) { // 4 bytes
                $output .= chr(240+($v >> 18)).chr(128+(($v >> 12) & 63)).chr(128+(($v >> 6) & 63)).chr(128+($v & 63));
            } elseif (self::$safe_mode) {
                $output .= self::$safe_char;
            } else {
                throw new Exception('Conversion from UCS-4 to UTF-8 failed: malformed input at byte '.$k);
            }
        }
        return $output;
    }
827086
def encode(text, orig_coding):
    
    if orig_coding == 'utf-8-bom':
        return BOM_UTF8 + text.encode("utf-8"), 'utf-8-bom'

    # Try declared coding spec
    coding = get_coding(text)
    if coding:
        try:
            return text.encode(coding), coding
        except (UnicodeError, LookupError):
            raise RuntimeError("Incorrect encoding (%s)" % coding)
    if (orig_coding and orig_coding.endswith('-default') or
       orig_coding.endswith('-guessed')):
        coding = orig_coding.replace("-default", "")
        coding = orig_coding.replace("-guessed", "")
        try:
            return text.encode(coding), coding
        except (UnicodeError, LookupError):
            pass

    # Try saving as ASCII
    try:
        return text.encode('ascii'), 'ascii'
    except UnicodeError:
        pass

    # Save as UTF-8 without BOM
    return text.encode('utf-8'), 'utf-8'
1347948
public static function iconv_fallback_utf8_utf16le($string, $bom=false) {
		$newcharstring = '';
		if ($bom) {
			$newcharstring .= "\xFF\xFE";
		}
		$offset = 0;
		$stringlength = strlen($string);
		while ($offset < $stringlength) {
			if ((ord($string{$offset}) | 0x07) == 0xF7) {
				// 11110bbb 10bbbbbb 10bbbbbb 10bbbbbb
				$charval = ((ord($string{($offset + 0)}) & 0x07) << 18) &
						   ((ord($string{($offset + 1)}) & 0x3F) << 12) &
						   ((ord($string{($offset + 2)}) & 0x3F) <<  6) &
							(ord($string{($offset + 3)}) & 0x3F);
				$offset += 4;
			} elseif ((ord($string{$offset}) | 0x0F) == 0xEF) {
				// 1110bbbb 10bbbbbb 10bbbbbb
				$charval = ((ord($string{($offset + 0)}) & 0x0F) << 12) &
						   ((ord($string{($offset + 1)}) & 0x3F) <<  6) &
							(ord($string{($offset + 2)}) & 0x3F);
				$offset += 3;
			} elseif ((ord($string{$offset}) | 0x1F) == 0xDF) {
				// 110bbbbb 10bbbbbb
				$charval = ((ord($string{($offset + 0)}) & 0x1F) <<  6) &
							(ord($string{($offset + 1)}) & 0x3F);
				$offset += 2;
			} elseif ((ord($string{$offset}) | 0x7F) == 0x7F) {
				// 0bbbbbbb
				$charval = ord($string{$offset});
				$offset += 1;
			} else {
				// error? maybe throw some warning here?
				$charval = false;
				$offset += 1;
			}
			if ($charval !== false) {
				$newcharstring .= (($charval < 65536) ? self::LittleEndian2String($charval, 2) : '?'."\x00");
			}
		}
		return $newcharstring;
	}
368326
def normalize_encoding(sample, encoding):
    
    encoding = codecs.lookup(encoding).name
    # Work around 'Incorrect detection of utf-8-sig encoding'
    # <https://github.com/PyYoshi/cChardet/issues/28>
    if encoding == 'utf-8':
        if sample.startswith(codecs.BOM_UTF8):
            encoding = 'utf-8-sig'
    # Use the BOM stripping name (without byte-order) for UTF-16 encodings
    elif encoding == 'utf-16-be':
        if sample.startswith(codecs.BOM_UTF16_BE):
            encoding = 'utf-16'
    elif encoding == 'utf-16-le':
        if sample.startswith(codecs.BOM_UTF16_LE):
            encoding = 'utf-16'
    return encoding
263164
static function dbCharsetToPhpCharset($in)
    {
        $out = $in;
        switch ($in) {
            case "None":$out = false;
                break;
            case "UTF8":$out = "UTF-8";
                break;
            case "LATIN1":$out = "ISO-8859-1";
                break;
            case "LATIN2":$out = "ISO-8859-2";
                break;
            case "LATIN3":$out = "ISO-8859-3";
                break;
            case "LATIN4":$out = "ISO-8859-4";
                break;
            case "LATIN4":$out = "ISO-8859-4";
                break;
            case "LATIN5":$out = "ISO-8859-9";
                break;
            case "LATIN6":$out = "ISO-8859-10";
                break;
            case "LATIN7":$out = "ISO-8859-13";
                break;
            case "LATIN8":$out = "ISO-8859-14";
                break;
            case "LATIN9":$out = "ISO-8859-15";
                break;
            case "LATIN9":$out = "ISO-8859-16";
                break;
            case "LATIN9":$out = "ISO-8859-16";
                break;
            case "ISO-8859-1":
            case "ISO-8859-2":
            case "ISO-8859-3":
            case "ISO-8859-4":
            case "ISO-8859-5":
            case "ISO-8859-6":
            case "ISO-8859-7":
            case "ISO-8859-8":
            case "ISO-8859-8-I":
            case "ISO-8859-9":
            case "ISO-8859-10":
            case "ISO-8859-11":
            case "ISO-8859-12":
            case "ISO-8859-13":
            case "ISO-8859-14":
            case "ISO-8859-15":
            case "ISO-8859-16":$out = $in;
                break;
            case "GB18030":
            case "GB2312":
            case "GBK":
            case "BIG5":$out = strtolower($in);
                break;
            case "WIN1250":$out = "Windows-1250";
                break;
            case "WIN1251":$out = "Windows-1251";
                break;
            case "WIN1252":$out = "Windows-1252";
                break;
            case "WIN1253":$out = "Windows-1253";
                break;
            case "WIN1254":$out = "Windows-1254";
                break;
            case "WIN1255":$out = "Windows-1255";
                break;
            case "WIN1256":$out = "Windows-1256";
                break;
            case "WIN1257":$out = "Windows-1257";
                break;
            case "WIN1258":$out = "Windows-1258";
                break;
            case "TIS620":$out = "tis-620";
                break;
            case "SJIS":$out = "shift-jis";
                break;
            default:$out = false;
        }

        return $out;
    }
483067
function _utf8ToChar(&$chars, &$idx)
	{
		if(($chars[$idx] >= 240) && ($chars[$idx] <= 255))
			/* 4 bytes */
			$utf =    ((int)($chars[$idx]-240)   << 18) +
				((int)($chars[++$idx]-128) << 12) +
				((int)($chars[++$idx]-128) << 6) +
				((int)($chars[++$idx]-128) << 0);
		else if (($chars[$idx] >= 224) && ($chars[$idx] <= 239))
			/* 3 bytes */
			$utf =    ((int)($chars[$idx]-224)   << 12) +
				((int)($chars[++$idx]-128) << 6) +
				((int)($chars[++$idx]-128) << 0);
		else if (($chars[$idx] >= 192) && ($chars[$idx] <= 223))
			/* 2 bytes */
			$utf =    ((int)($chars[$idx]-192)   << 6) +
				((int)($chars[++$idx]-128) << 0);
		else
			/* 1 byte */
			$utf = $chars[$idx];
		if(array_key_exists($utf,$this->utfMap))
			return \chr($this->utfMap[$utf]);
		else
			return "?";
	}
1255064
def UTF8ToUTF16BE(instr, setbom=True):
    ""
    outstr = "".encode()
    if (setbom):
        outstr += "\xFE\xFF".encode("latin1")
    if not isinstance(instr, unicode):
        instr = instr.decode('UTF-8')
    outstr += instr.encode('UTF-16BE')
    # convert bytes back to fake unicode string until PEP461-like is implemented
    if PY3K:
        outstr = outstr.decode("latin1")
    return outstr