Method called to try to figure out physical encoding the underlying
input stream uses.
163473
protected void resolveStreamEncoding()
        throws IOException, WstxException
    {
        // Let's first set defaults:
        mBytesPerChar = 0;
        mBigEndian = true;

        /* Ok; first just need 4 bytes for determining bytes-per-char from
         * BOM or first char(s) of likely xml declaration:
         */
        if (ensureLoaded(4)) {
            bomblock:
            do { // BOM/auto-detection block
                int quartet = (mByteBuffer[0] << 24)
                    | ((mByteBuffer[1] & 0xFF) << 16)
                    | ((mByteBuffer[2] & 0xFF) << 8)
                    | (mByteBuffer[3] & 0xFF);

                /* Handling of (usually) optional BOM (required for
                 * multi-byte formats); first 32-bit charsets:
                 */
                switch (quartet) {
                case 0x0000FEFF:
                    mBigEndian = true;
                    mInputPtr = mBytesPerChar = 4;
                    break bomblock;
                case 0xFFFE0000: // UCS-4, LE?
                    mInputPtr = mBytesPerChar = 4;
                    mBigEndian = false;
                    break bomblock;
                case 0x0000FFFE: // UCS-4, in-order...
                    reportWeirdUCS4("2143");
                    break bomblock;
                case 0x0FEFF0000: // UCS-4, in-order...
                    reportWeirdUCS4("3412");
                    break bomblock;
                }

                // Ok, if not, how about 16-bit encoding BOMs?
                int msw = quartet >>> 16;
                if (msw == 0xFEFF) { // UTF-16, BE
                    mInputPtr = mBytesPerChar = 2;
                    mBigEndian = true;
                    break;
                }
                if (msw == 0xFFFE) { // UTF-16, LE
                    mInputPtr = mBytesPerChar = 2;
                    mBigEndian = false;
                    break;
                }

                // And if not, then UTF-8 BOM?
                if ((quartet >>> 8) == 0xEFBBBF) { // UTF-8
                    mInputPtr = 3;
                    mBytesPerChar = 1;
                    mBigEndian = true; // doesn't really matter
                    break;
                }

                /* And if that wasn't succesful, how about auto-detection
                 * for '<?xm' (or subset for multi-byte encodings) marker?
                 */
                // Note: none of these consume bytes... so ptr remains at 0

                switch (quartet) {
                case 0x0000003c: // UCS-4, BE?
                    mBigEndian = true;
                    mBytesPerChar = 4;
                    break bomblock;
                case 0x3c000000: // UCS-4, LE?
                    mBytesPerChar = 4;
                    mBigEndian = false;
                    break bomblock;
                case 0x00003c00: // UCS-4, in-order...
                    reportWeirdUCS4("2143");
                    break bomblock;
                case 0x003c0000: // UCS-4, in-order...
                    reportWeirdUCS4("3412");
                    break bomblock;
                case 0x003c003f: // UTF-16, BE
                    mBytesPerChar = 2;
                    mBigEndian = true;
                    break bomblock;
                case 0x3c003f00: // UTF-16, LE
                    mBytesPerChar = 2;
                    mBigEndian = false;
                    break bomblock;
                case 0x3c3f786d: // UTF-8, Ascii, ISO-Latin
                    mBytesPerChar = 1;
                    mBigEndian = true; // doesn't really matter
                    break bomblock;

                case 0x4c6fa794:
                    mBytesPerChar = -1;
                    mEBCDIC = true;

                    /* For xml declaration handling we can basically
                     * use any of EBCDIC variants, since declaration
                     * must not contain control or punctuation characters
                     * that would differ
                     */
                    mSingleByteTranslation = EBCDICCodec.getCp037Mapping();
                    break bomblock;
                }
                
                /* Otherwise it's either single-byte doc without xml
                 * declaration, or corrupt input...
                 */
            } while (false); // BOM/auto-detection block
            
            mHadBOM = (mInputPtr > 0);

            // Let's update location markers to ignore BOM.
            mInputProcessed = -mInputPtr;
            mInputRowStart = mInputPtr;
        }

        /* Hmmh. If we haven't figured it out, let's just assume
         * UTF-8 as per XML specs:
         */
        mByteSizeFound = (mBytesPerChar != 0);
        if (!mByteSizeFound) {
            mBytesPerChar = 1;
            mBigEndian = true; // doesn't matter
        }
    }
1106634
function validateEncoding(encoding) {
		if (encoding === null || encoding === 'buffer' || Buffer.isEncoding(encoding)) {
			return true;
		}
		throw new TypeError('Unknown encoding: ' + encoding);
	}
1265375
def decode(s):
    
    if isinstance(s, bytes):
        return s.decode(sys.getdefaultencoding())
    else:
        return s
95314
def guess_mimetype(filename):
    
    m, encoding = mimetypes.guess_type(filename)
    if encoding:
        m = ENCODING_MIMETYPES.get(encoding, None)
    return m or 'application/octet-stream'
379488
private static String getBOMEncoding(final BufferedInputStream is) throws IOException {
        String encoding = null;
        final int[] bytes = new int[3];
        is.mark(3);
        bytes[0] = is.read();
        bytes[1] = is.read();
        bytes[2] = is.read();

        if (bytes[0] == 0xFE && bytes[1] == 0xFF) {
            encoding = UTF_16BE;
            is.reset();
            is.read();
            is.read();
        } else if (bytes[0] == 0xFF && bytes[1] == 0xFE) {
            encoding = UTF_16LE;
            is.reset();
            is.read();
            is.read();
        } else if (bytes[0] == 0xEF && bytes[1] == 0xBB && bytes[2] == 0xBF) {
            encoding = UTF_8;
        } else {
            is.reset();
        }
        return encoding;
    }
1130501
private static int calculateBinaryString(int[] inputData, StringBuilder binaryString) {

        EncodeMode last_mode = EncodeMode.NUMERIC;
        int encoding_method, i, j, read_posn;
        boolean latch;
        int remainder, d1, d2, value;
        String padstring;
        double weight;
        int group_val;
        int current_length;

        read_posn = 0;

        /* Decide whether a compressed data field is required and if so what method to use - method 2 = no compressed data field */

        if (inputData.length >= 16 && inputData[0] == '0' && inputData[1] == '1') {
            /* (01) and other AIs */
            encoding_method = 1;
        } else {
            /* any AIs */
            encoding_method = 2;
        }

        if (inputData.length >= 20 && encoding_method == 1 && inputData[2] == '9' && inputData[16] == '3') {
            /* Possibly encoding method > 2 */

            if (inputData.length >= 26 && inputData[17] == '1') {
                /* Methods 3, 7, 9, 11 and 13 */

                if (inputData[18] == '0') {
                    /* (01) and (310x) */
                    /* In kilos */

                    weight = 0.0;
                    for (i = 0; i < 6; i++) {
                        weight *= 10;
                        weight += (inputData[20 + i] - '0');
                    }

                    if (weight < 99999.0) { /* Maximum weight = 99999 */

                        if (inputData[19] == '3' && inputData.length == 26) {
                            /* (01) and (3103) */
                            weight /= 1000.0;
                            if (weight <= 32.767) {
                                encoding_method = 3;
                            }
                        }

                        if (inputData.length == 34) {
                            if (inputData[26] == '1' && inputData[27] == '1') {
                                /* (01), (310x) and (11) - metric weight and production date */
                                encoding_method = 7;
                            }
                            if (inputData[26] == '1' && inputData[27] == '3') {
                                /* (01), (310x) and (13) - metric weight and packaging date */
                                encoding_method = 9;
                            }
                            if (inputData[26] == '1' && inputData[27] == '5') {
                                /* (01), (310x) and (15) - metric weight and "best before" date */
                                encoding_method = 11;
                            }
                            if (inputData[26] == '1' && inputData[27] == '7') {
                                /* (01), (310x) and (17) - metric weight and expiration date */
                                encoding_method = 13;
                            }
                        }
                    }
                }
            }

            if (inputData.length >= 26 && inputData[17] == '2') {
                /* Methods 4, 8, 10, 12 and 14 */

                if (inputData[18] == '0') {
                    /* (01) and (320x) */
                    /* In pounds */

                    weight = 0.0;
                    for (i = 0; i < 6; i++) {
                        weight *= 10;
                        weight += (inputData[20 + i] - '0');
                    }

                    if (weight < 99999.0) { /* Maximum weight = 99999 */

                        if (((inputData[19] == '2') || (inputData[19] == '3'))
                                && (inputData.length == 26)) {
                            /* (01) and (3202)/(3203) */

                            if (inputData[19] == '3') {
                                weight /= 1000.0;
                                if (weight <= 22.767) {
                                    encoding_method = 4;
                                }
                            } else {
                                weight /= 100.0;
                                if (weight <= 99.99) {
                                    encoding_method = 4;
                                }
                            }

                        }

                        if (inputData.length == 34) {
                            if (inputData[26] == '1' && inputData[27] == '1') {
                                /* (01), (320x) and (11) - English weight and production date */
                                encoding_method = 8;
                            }
                            if (inputData[26] == '1' && inputData[27] == '3') {
                                /* (01), (320x) and (13) - English weight and packaging date */
                                encoding_method = 10;
                            }
                            if (inputData[26] == '1' && inputData[27] == '5') {
                                /* (01), (320x) and (15) - English weight and "best before" date */
                                encoding_method = 12;
                            }
                            if (inputData[26] == '1' && inputData[27] == '7') {
                                /* (01), (320x) and (17) - English weight and expiration date */
                                encoding_method = 14;
                            }
                        }
                    }
                }
            }

            if (inputData[17] == '9') {
                /* Methods 5 and 6 */
                if (inputData[18] == '2' && inputData[19] >= '0' && inputData[19] <= '3') {
                    /* (01) and (392x) */
                    encoding_method = 5;
                }
                if (inputData[18] == '3' && inputData[19] >= '0' && inputData[19] <= '3') {
                    /* (01) and (393x) */
                    encoding_method = 6;
                }
            }
        }

        switch (encoding_method) { /* Encoding method - Table 10 */
        case 1:
            binaryString.append("1XX");
            read_posn = 16;
            break;
        case 2:
            binaryString.append("00XX");
            read_posn = 0;
            break;
        case 3:
            binaryString.append("0100");
            read_posn = inputData.length;
            break;
        case 4:
            binaryString.append("0101");
            read_posn = inputData.length;
            break;
        case 5:
            binaryString.append("01100XX");
            read_posn = 20;
            break;
        case 6:
            binaryString.append("01101XX");
            read_posn = 23;
            break;
        case 7:
            binaryString.append("0111000");
            read_posn = inputData.length;
            break;
        case 8:
            binaryString.append("0111001");
            read_posn = inputData.length;
            break;
        case 9:
            binaryString.append("0111010");
            read_posn = inputData.length;
            break;
        case 10:
            binaryString.append("0111011");
            read_posn = inputData.length;
            break;
        case 11:
            binaryString.append("0111100");
            read_posn = inputData.length;
            break;
        case 12:
            binaryString.append("0111101");
            read_posn = inputData.length;
            break;
        case 13:
            binaryString.append("0111110");
            read_posn = inputData.length;
            break;
        case 14:
            binaryString.append("0111111");
            read_posn = inputData.length;
            break;
        }

        /* Variable length symbol bit field is just given a place holder (XX) for the time being */

        /* Verify that the data to be placed in the compressed data field is all numeric data before carrying out compression */
        for (i = 0; i < read_posn; i++) {
            if (inputData[i] < '0' || inputData[i] > '9') {
                /* Something is wrong */
                throw new OkapiException("Invalid characters in input data");
            }
        }

        /* Now encode the compressed data field */
        if (encoding_method == 1) {
            /* Encoding method field "1" - general item identification data */
            group_val = inputData[2] - '0';

            for (j = 0; j < 4; j++) {
                if ((group_val & (0x08 >> j)) == 0) {
                    binaryString.append('0');
                } else {
                    binaryString.append('1');
                }
            }

            for (i = 1; i < 5; i++) {
                group_val = 100 * (inputData[i * 3] - '0');
                group_val += 10 * (inputData[(i * 3) + 1] - '0');
                group_val += inputData[(i * 3) + 2] - '0';

                for (j = 0; j < 10; j++) {
                    if ((group_val & (0x200 >> j)) == 0) {
                        binaryString.append('0');
                    } else {
                        binaryString.append('1');
                    }
                }
            }
        }

        if (encoding_method == 3) {
            /* Encoding method field "0100" - variable weight item (0,001 kilogram increments) */

            for (i = 1; i < 5; i++) {
                group_val = 100 * (inputData[i * 3] - '0');
                group_val += 10 * (inputData[(i * 3) + 1] - '0');
                group_val += (inputData[(i * 3) + 2] - '0');

                for (j = 0; j < 10; j++) {
                    if ((group_val & (0x200 >> j)) == 0) {
                        binaryString.append('0');
                    } else {
                        binaryString.append('1');
                    }
                }
            }

            group_val = 0;
            for (i = 0; i < 6; i++) {
                group_val *= 10;
                group_val += inputData[20 + i] - '0';
            }

            for (j = 0; j < 15; j++) {
                if ((group_val & (0x4000 >> j)) == 0) {
                    binaryString.append('0');
                } else {
                    binaryString.append('1');
                }
            }
        }

        if (encoding_method == 4) {
            /* Encoding method field "0101" - variable weight item (0,01 or 0,001 pound increment) */

            for (i = 1; i < 5; i++) {
                group_val = 100 * (inputData[i * 3] - '0');
                group_val += 10 * (inputData[(i * 3) + 1] - '0');
                group_val += (inputData[(i * 3) + 2] - '0');

                for (j = 0; j < 10; j++) {
                    if ((group_val & (0x200 >> j)) == 0) {
                        binaryString.append('0');
                    } else {
                        binaryString.append('1');
                    }
                }
            }


            group_val = 0;
            for (i = 0; i < 6; i++) {
                group_val *= 10;
                group_val += inputData[20 + i] - '0';
            }

            if (inputData[19] == '3') {
                group_val = group_val + 10000;
            }

            for (j = 0; j < 15; j++) {
                if ((group_val & (0x4000 >> j)) == 0) {
                    binaryString.append('0');
                } else {
                    binaryString.append('1');
                }
            }
        }

        if (encoding_method >= 7 && encoding_method <= 14) {
            /* Encoding method fields "0111000" through "0111111" - variable weight item plus date */

            for (i = 1; i < 5; i++) {
                group_val = 100 * (inputData[i * 3] - '0');
                group_val += 10 * (inputData[(i * 3) + 1] - '0');
                group_val += (inputData[(i * 3) + 2] - '0');

                for (j = 0; j < 10; j++) {
                    if ((group_val & (0x200 >> j)) == 0) {
                        binaryString.append('0');
                    } else {
                        binaryString.append('1');
                    }
                }
            }

            group_val = inputData[19] - '0';

            for (i = 0; i < 5; i++) {
                group_val *= 10;
                group_val += inputData[21 + i] - '0';
            }

            for (j = 0; j < 20; j++) {
                if ((group_val & (0x80000 >> j)) == 0) {
                    binaryString.append('0');
                } else {
                    binaryString.append('1');
                }
            }

            if (inputData.length == 34) {
                /* Date information is included */
                group_val = ((10 * (inputData[28] - '0')) + (inputData[29] - '0')) * 384;
                group_val += (((10 * (inputData[30] - '0')) + (inputData[31] - '0')) - 1) * 32;
                group_val += (10 * (inputData[32] - '0')) + (inputData[33] - '0');
            } else {
                group_val = 38400;
            }

            for (j = 0; j < 16; j++) {
                if ((group_val & (0x8000 >> j)) == 0) {
                    binaryString.append('0');
                } else {
                    binaryString.append('1');
                }
            }
        }

        if (encoding_method == 5) {
            /* Encoding method field "01100" - variable measure item and price */

            for (i = 1; i < 5; i++) {
                group_val = 100 * (inputData[i * 3] - '0');
                group_val += 10 * (inputData[(i * 3) + 1] - '0');
                group_val += (inputData[(i * 3) + 2] - '0');

                for (j = 0; j < 10; j++) {
                    if ((group_val & (0x200 >> j)) == 0) {
                        binaryString.append('0');
                    } else {
                        binaryString.append('1');
                    }
                }
            }

            switch (inputData[19]) {
                case '0':
                    binaryString.append("00");
                    break;
                case '1':
                    binaryString.append("01");
                    break;
                case '2':
                    binaryString.append("10");
                    break;
                case '3':
                    binaryString.append("11");
                    break;
            }
        }

        if (encoding_method == 6) {
            /* Encoding method "01101" - variable measure item and price with ISO 4217 currency code */

            for (i = 1; i < 5; i++) {
                group_val = 100 * (inputData[i * 3] - '0');
                group_val += 10 * (inputData[(i * 3) + 1] - '0');
                group_val += (inputData[(i * 3) + 2] - '0');

                for (j = 0; j < 10; j++) {
                    if ((group_val & (0x200 >> j)) == 0) {
                        binaryString.append('0');
                    } else {
                        binaryString.append('1');
                    }
                }
            }

            switch (inputData[19]) {
                case '0':
                    binaryString.append("00");
                    break;
                case '1':
                    binaryString.append("01");
                    break;
                case '2':
                    binaryString.append("10");
                    break;
                case '3':
                    binaryString.append("11");
                    break;
            }

            group_val = 0;
            for (i = 0; i < 3; i++) {
                group_val *= 10;
                group_val += inputData[20 + i] - '0';
            }

            for (j = 0; j < 10; j++) {
                if ((group_val & (0x200 >> j)) == 0) {
                    binaryString.append('0');
                } else {
                    binaryString.append('1');
                }
            }
        }

        /* The compressed data field has been processed if appropriate - the rest of the data (if any) goes into a general-purpose data compaction field */

        int[] generalField = Arrays.copyOfRange(inputData, read_posn, inputData.length);
        EncodeMode[] generalFieldType = new EncodeMode[generalField.length];

        if (generalField.length != 0) {

            latch = false;
            for (i = 0; i < generalField.length; i++) {
                /* Tables 11, 12, 13 - ISO/IEC 646 encodation */
                int c = generalField[i];
                EncodeMode mode;
                if (c == FNC1) {
                    // FNC1 can be encoded in any system
                    mode = EncodeMode.ANY_ENC;
                } else if (c >= '0' && c <= '9') {
                    // numbers can be encoded in any system, but will usually narrow down to numeric encodation
                    mode = EncodeMode.ANY_ENC;
                } else if ((c >= 'A' && c <= 'Z') || c == '*' || c == ',' || c == '-' || c == '.' || c == '/') {
                    // alphanumeric encodation or ISO/IEC encodation
                    mode = EncodeMode.ALPHA_OR_ISO;
                } else if ((c >= 'a' && c <= 'z') || c == '!' || c == '"' || c == '%' || c == '&' || c == '\'' ||
                            c == '(' || c == ')' || c == '+' || c == ':' || c == ';' || c == '<' || c == '=' ||
                            c == '>' || c == '?' || c == '_' || c == ' ') {
                    // ISO/IEC encodation
                    mode = EncodeMode.ISOIEC;
                } else {
                    // unable to encode this character
                    mode = EncodeMode.INVALID_CHAR;
                    latch = true;
                }
                generalFieldType[i] = mode;
            }

            if (latch) {
                throw new OkapiException("Invalid characters in input data");
            }

            for (i = 0; i < generalField.length - 1; i++) {
                if (generalFieldType[i] == EncodeMode.ISOIEC && generalField[i + 1] == FNC1) {
                    generalFieldType[i + 1] = EncodeMode.ISOIEC;
                }
            }

            for (i = 0; i < generalField.length - 1; i++) {
                if (generalFieldType[i] == EncodeMode.ALPHA_OR_ISO && generalField[i + 1] == FNC1) {
                    generalFieldType[i + 1] = EncodeMode.ALPHA_OR_ISO;
                }
            }

            latch = applyGeneralFieldRules(generalFieldType); // modifies generalFieldType

            /* Set initial mode if not NUMERIC */
            if (generalFieldType[0] == EncodeMode.ALPHA) {
                binaryString.append("0000"); /* Alphanumeric latch */
                last_mode = EncodeMode.ALPHA;
            }
            if (generalFieldType[0] == EncodeMode.ISOIEC) {
                binaryString.append("0000"); /* Alphanumeric latch */
                binaryString.append("00100"); /* ISO/IEC 646 latch */
                last_mode = EncodeMode.ISOIEC;
            }

            i = 0;
            do {
                switch (generalFieldType[i]) {
                case NUMERIC:
                    if (last_mode != EncodeMode.NUMERIC) {
                        binaryString.append("000"); /* Numeric latch */
                    }

                    if (generalField[i] != FNC1) {
                        d1 = generalField[i] - '0';
                    } else {
                        d1 = 10;
                    }

                    if (generalField[i + 1] != FNC1) {
                        d2 = generalField[i + 1] - '0';
                    } else {
                        d2 = 10;
                    }

                    value = (11 * d1) + d2 + 8;

                    for (j = 0; j < 7; j++) {
                        if ((value & (0x40 >> j)) != 0) {
                            binaryString.append('1');
                        } else {
                            binaryString.append('0');
                        }
                    }

                    i += 2;
                    last_mode = EncodeMode.NUMERIC;
                    break;

                case ALPHA:
                    if (i != 0) {
                        if (last_mode == EncodeMode.NUMERIC) {
                            binaryString.append("0000"); /* Alphanumeric latch */
                        }
                        if (last_mode == EncodeMode.ISOIEC) {
                            binaryString.append("00100"); /* Alphanumeric latch */
                        }
                    }

                    if (generalField[i] >= '0' && generalField[i] <= '9') {

                        value = generalField[i] - 43;

                        for (j = 0; j < 5; j++) {
                            if ((value & (0x10 >> j)) != 0) {
                                binaryString.append('1');
                            } else {
                                binaryString.append('0');
                            }
                        }
                    }

                    if (generalField[i] >= 'A' && generalField[i] <= 'Z') {

                        value = generalField[i] - 33;

                        for (j = 0; j < 6; j++) {
                            if ((value & (0x20 >> j)) != 0) {
                                binaryString.append('1');
                            } else {
                                binaryString.append('0');
                            }
                        }
                    }

                    last_mode = EncodeMode.ALPHA;
                    if (generalField[i] == FNC1) {
                        binaryString.append("01111");
                    } /* FNC1 / Numeric latch */

                    if (generalField[i] == '*') binaryString.append("111010"); /* asterisk */
                    if (generalField[i] == ',') binaryString.append("111011"); /* comma */
                    if (generalField[i] == '-') binaryString.append("111100"); /* minus or hyphen */
                    if (generalField[i] == '.') binaryString.append("111101"); /* period or full stop */
                    if (generalField[i] == '/') binaryString.append("111110"); /* slash or solidus */

                    i++;
                    break;

                case ISOIEC:
                    if (i != 0) {
                        if (last_mode == EncodeMode.NUMERIC) {
                            binaryString.append("0000"); /* Alphanumeric latch */
                            binaryString.append("00100"); /* ISO/IEC 646 latch */
                        }
                        if (last_mode == EncodeMode.ALPHA) {
                            binaryString.append("00100"); /* ISO/IEC 646 latch */
                        }
                    }

                    if (generalField[i] >= '0' && generalField[i] <= '9') {

                        value = generalField[i] - 43;

                        for (j = 0; j < 5; j++) {
                            if ((value & (0x10 >> j)) != 0) {
                                binaryString.append('1');
                            } else {
                                binaryString.append('0');
                            }
                        }
                    }

                    if (generalField[i] >= 'A' && generalField[i] <= 'Z') {

                        value = generalField[i] - 1;

                        for (j = 0; j < 7; j++) {
                            if ((value & (0x40 >> j)) != 0) {
                                binaryString.append('1');
                            } else {
                                binaryString.append('0');
                            }
                        }
                    }

                    if (generalField[i] >= 'a' && generalField[i] <= 'z') {

                        value = generalField[i] - 7;

                        for (j = 0; j < 7; j++) {
                            if ((value & (0x40 >> j)) != 0) {
                                binaryString.append('1');
                            } else {
                                binaryString.append('0');
                            }
                        }
                    }

                    last_mode = EncodeMode.ISOIEC;
                    if (generalField[i] == FNC1) {
                        binaryString.append("01111");
                    } /* FNC1 / Numeric latch */

                    if (generalField[i] == '!') binaryString.append("11101000"); /* exclamation mark */
                    if (generalField[i] == 34)  binaryString.append("11101001"); /* quotation mark */
                    if (generalField[i] == 37)  binaryString.append("11101010"); /* percent sign */
                    if (generalField[i] == '&') binaryString.append("11101011"); /* ampersand */
                    if (generalField[i] == 39)  binaryString.append("11101100"); /* apostrophe */
                    if (generalField[i] == '(') binaryString.append("11101101"); /* left parenthesis */
                    if (generalField[i] == ')') binaryString.append("11101110"); /* right parenthesis */
                    if (generalField[i] == '*') binaryString.append("11101111"); /* asterisk */
                    if (generalField[i] == '+') binaryString.append("11110000"); /* plus sign */
                    if (generalField[i] == ',') binaryString.append("11110001"); /* comma */
                    if (generalField[i] == '-') binaryString.append("11110010"); /* minus or hyphen */
                    if (generalField[i] == '.') binaryString.append("11110011"); /* period or full stop */
                    if (generalField[i] == '/') binaryString.append("11110100"); /* slash or solidus */
                    if (generalField[i] == ':') binaryString.append("11110101"); /* colon */
                    if (generalField[i] == ';') binaryString.append("11110110"); /* semicolon */
                    if (generalField[i] == '<') binaryString.append("11110111"); /* less-than sign */
                    if (generalField[i] == '=') binaryString.append("11111000"); /* equals sign */
                    if (generalField[i] == '>') binaryString.append("11111001"); /* greater-than sign */
                    if (generalField[i] == '?') binaryString.append("11111010"); /* question mark */
                    if (generalField[i] == '_') binaryString.append("11111011"); /* underline or low line */
                    if (generalField[i] == ' ') binaryString.append("11111100"); /* space */

                    i++;
                    break;
                }
                current_length = i;
                if (latch) {
                    current_length++;
                }
            } while (current_length < generalField.length);

            remainder = calculateRemainder(binaryString.length());

            if (latch) {
                /* There is still one more numeric digit to encode */
                if (last_mode == EncodeMode.NUMERIC) {
                    if ((remainder >= 4) && (remainder <= 6)) {
                        value = generalField[i] - '0';
                        value++;

                        for (j = 0; j < 4; j++) {
                            if ((value & (0x08 >> j)) != 0) {
                                binaryString.append('1');
                            } else {
                                binaryString.append('0');
                            }
                        }
                    } else {
                        d1 = generalField[i] - '0';
                        d2 = 10;

                        value = (11 * d1) + d2 + 8;

                        for (j = 0; j < 7; j++) {
                            if ((value & (0x40 >> j)) != 0) {
                                binaryString.append('1');
                            } else {
                                binaryString.append('0');
                            }
                        }
                    }
                } else {
                    value = generalField[i] - 43;

                    for (j = 0; j < 5; j++) {
                        if ((value & (0x10 >> j)) != 0) {
                            binaryString.append('1');
                        } else {
                            binaryString.append('0');
                        }
                    }
                }
            }
        }

        if (binaryString.length() > 252) {
            throw new OkapiException("Input too long");
        }

        remainder = calculateRemainder(binaryString.length());

        /* Now add padding to binary string (7.2.5.5.4) */
        i = remainder;
        if ((generalField.length != 0) && (last_mode == EncodeMode.NUMERIC)) {
            padstring = "0000";
            i -= 4;
        } else {
            padstring = "";
        }
        for (; i > 0; i -= 5) {
            padstring += "00100";
        }

        binaryString.append(padstring.substring(0, remainder));

        /* Patch variable length symbol bit field */
        char patchEvenOdd, patchSize;
        if ((((binaryString.length() / 12) + 1) & 1) == 0) {
            patchEvenOdd = '0';
        } else {
            patchEvenOdd = '1';
        }
        if (binaryString.length() <= 156) {
            patchSize = '0';
        } else {
            patchSize = '1';
        }

        if (encoding_method == 1) {
            binaryString.setCharAt(2, patchEvenOdd);
            binaryString.setCharAt(3, patchSize);
        }
        if (encoding_method == 2) {
            binaryString.setCharAt(3, patchEvenOdd);
            binaryString.setCharAt(4, patchSize);
        }
        if (encoding_method == 5 || encoding_method == 6) {
            binaryString.setCharAt(6, patchEvenOdd);
            binaryString.setCharAt(7, patchSize);
        }

        return encoding_method;
    }
64766
private int decodeNextBWTByte() {
        int mergedPointer = bwtCurrentMergedPointer;
        int nextDecodedByte =  mergedPointer & 0xff;
        bwtCurrentMergedPointer = bwtMergedPointers[mergedPointer >>> 8];

        if (blockRandomised) {
            if (--randomCount == 0) {
                nextDecodedByte ^= 1;
                randomIndex = (randomIndex + 1) % 512;
                randomCount = Bzip2Rand.rNums(randomIndex);
            }
        }
        bwtBytesDecoded++;

        return nextDecodedByte;
    }
372449
def force_bytes(s, encoding='utf-8', strings_only=False, errors='strict'):
    
    # Handle the common case first for performance reasons.
    if isinstance(s, bytes):
        if encoding == 'utf-8':
            return s
        else:
            return s.decode('utf-8', errors).encode(encoding, errors)
    if strings_only and is_protected_type(s):
        return s
    if isinstance(s, memoryview):
        return bytes(s)
    if not isinstance(s, str):
        return str(s).encode(encoding, errors)
    else:
        return s.encode(encoding, errors)
1173106
private byte[] fetchDecoderOutput() {
        final CompositeByteBuf decoded = Unpooled.compositeBuffer();
        for (;;) {
            final ByteBuf buf = decoder.readInbound();
            if (buf == null) {
                break;
            }
            if (!buf.isReadable()) {
                buf.release();
                continue;
            }
            decoded.addComponent(true, buf);
        }
        final byte[] ret = ByteBufUtil.getBytes(decoded);
        decoded.release();
        return ret;
    }
822622
def detect_encoding(data, encoding=None, fallback='latin1', is_html=False):
    '''
    '''
    if encoding:
        encoding = normalize_codec_name(encoding)

    bs4_detector = EncodingDetector(
        data,
        override_encodings=(encoding,) if encoding else (),
        is_html=is_html
    )
    candidates = itertools.chain(bs4_detector.encodings, (fallback,))

    for candidate in candidates:
        if not candidate:
            continue

        candidate = normalize_codec_name(candidate)

        if not candidate:
            continue

        if candidate == 'ascii' and fallback != 'ascii':
            # it's never ascii :)
            # Falling back on UTF-8/CP-1252/Latin-1 reduces chance of
            # failure
            continue

        if try_decoding(data, candidate):
            return candidate

    raise ValueError('Unable to detect encoding.')
973419
protected int readByte(InputStream is) throws IOException
   {
      byte[] data = new byte[1];
      if (is.read(data) != data.length)
      {
         throw new EOFException();
      }

      return (MPPUtility.getByte(data, 0));
   }
1286754
def determine_encoding(buf):
    
    # The ultimate default is utf8; bravo, W3C
    bom_encoding = 'UTF-8'

    if not buf:
        # What
        return bom_encoding

    if isinstance(buf, six.text_type):
        # We got a file that, for whatever reason, produces already-decoded
        # text.  Check for the BOM (which is useless now) and believe
        # whatever's in the @charset.
        if buf[0] == '\ufeff':
            buf = buf[0:]

        # This is pretty similar to the code below, but without any encoding
        # double-checking.
        charset_start = '@charset "'
        charset_end = '";'
        if buf.startswith(charset_start):
            start = len(charset_start)
            end = buf.index(charset_end, start)
            return buf[start:end]
        else:
            return bom_encoding

    # BOMs
    if buf[:3] == b'\xef\xbb\xbf':
        bom_encoding = 'UTF-8'
        buf = buf[3:]
    if buf[:4] == b'\x00\x00\xfe\xff':
        bom_encoding = 'UTF-32BE'
        buf = buf[4:]
    elif buf[:4] == b'\xff\xfe\x00\x00':
        bom_encoding = 'UTF-32LE'
        buf = buf[4:]
    if buf[:4] == b'\x00\x00\xff\xfe':
        raise UnicodeError("UTF-32-2143 is not supported")
    elif buf[:4] == b'\xfe\xff\x00\x00':
        raise UnicodeError("UTF-32-2143 is not supported")
    elif buf[:2] == b'\xfe\xff':
        bom_encoding = 'UTF-16BE'
        buf = buf[2:]
    elif buf[:2] == b'\xff\xfe':
        bom_encoding = 'UTF-16LE'
        buf = buf[2:]

    # The spec requires exactly this syntax; no escapes or extra spaces or
    # other shenanigans, thank goodness.
    charset_start = '@charset "'.encode(bom_encoding)
    charset_end = '";'.encode(bom_encoding)
    if buf.startswith(charset_start):
        start = len(charset_start)
        end = buf.index(charset_end, start)
        encoded_encoding = buf[start:end]
        encoding = encoded_encoding.decode(bom_encoding)

        # Ensure that decoding with the specified encoding actually produces
        # the same @charset rule
        encoded_charset = buf[:end + len(charset_end)]
        if (encoded_charset.decode(encoding) !=
                encoded_charset.decode(bom_encoding)):
            raise UnicodeError(
                "@charset {0} is incompatible with detected encoding {1}"
                .format(bom_encoding, encoding))
    else:
        # With no @charset, believe the BOM
        encoding = bom_encoding

    return encoding
621275
def decode(self, stream, encoding='utf-8'):
        
        data = stream.read().decode(encoding).strip("\n")
        if data != "":
            logger.debug(('{0}: decoded "{1}" with encoding '
                          '{2}'.format(self.target_address, data, encoding)))
        return data
178038
def decode(data):
    
    data = ensure_bytes(data, 'utf8')
    codec = get_codec(data)
    return codec.converter.decode(data[CODE_LENGTH:])
659247
public static int readSingleByte(InputStream in) throws IOException {
        byte[] buffer = new byte[1];
        int result = in.read(buffer, 0, 1);
        return (result != -1) ? buffer[0] & 0xff : -1;
    }
247204
function( input, charset ) {

    if( /iso-?2022-?jp/i.test( charset ) ) {
      charset = 'shift_jis'
    }

    if( charset ) {
      return MIME.Iconv.decode( Buffer.from( input, 'base64' ), charset )
    } else {
      return Buffer.from( input, 'base64' ).toString()
    }

  }
768090
public void decodeHeader() throws IOException {
        if(!headerDecoded) {
            headerDecoded = true;

            int m = getMarker();
            if(m != 0xD8) {
                throw new IOException("no SOI");
            }
            m = getMarker();
            while(m != 0xC0 && m != 0xC1) { // SOF
                processMarker(m);
                m = getMarker();
                while(m == MARKER_NONE) {
                    m = getMarker();
                }
            }

            processSOF();
        }
    }