this is just a list of characters with known problems and I'm trying not to strip out other languages
506643
function ldap_slashes($str){
        $illegal=array("(",")","#"); // the + character has problems too, but it's an illegal character
        
        $legal=array();
        foreach ($illegal as $id => $char){ $legal[$id]="\\".$char; } //make up the array of legal chars
        
        $str=str_replace($illegal,$legal,$str); //replace them
        return ($str);
    }
333962
private function sanitize($aName)
    {
        $characters = [
            // Latin
            'À' => 'A', 'Á' => 'A', 'Â' => 'A', 'Ã' => 'A', 'Ä' => 'A',
            'Å' => 'A', 'Æ' => 'AE', 'Ç' => 'C', 'È' => 'E', 'É' => 'E',
            'Ê' => 'E', 'Ë' => 'E', 'Ì' => 'I', 'Í' => 'I', 'Î' => 'I',
            'Ï' => 'I', 'Ð' => 'D', 'Ñ' => 'N', 'Ò' => 'O', 'Ó' => 'O',
            'Ô' => 'O', 'Õ' => 'O', 'Ö' => 'O', 'Ő' => 'O', 'Ø' => 'O',
            'Ù' => 'U', 'Ú' => 'U', 'Û' => 'U', 'Ü' => 'U', 'Ű' => 'U',
            'Ũ' => 'U', 'Ý' => 'Y', 'Þ' => 'TH', 'ß' => 'ss',
            'à' => 'a', 'á' => 'a', 'â' => 'a', 'ã' => 'a', 'ä' => 'a',
            'å' => 'a', 'æ' => 'ae', 'ç' => 'c', 'è' => 'e', 'é' => 'e',
            'ê' => 'e', 'ë' => 'e', 'ì' => 'i', 'í' => 'i', 'î' => 'i',
            'ï' => 'i', 'ð' => 'd', 'ñ' => 'n', 'ò' => 'o', 'ó' => 'o',
            'ô' => 'o', 'õ' => 'o', 'ö' => 'o', 'ő' => 'o', 'ø' => 'o',
            'ù' => 'u', 'ú' => 'u', 'û' => 'u', 'ü' => 'u', 'ű' => 'u',
            'ũ' => 'u', 'ý' => 'y', 'þ' => 'th', 'ÿ' => 'y',
            // Croatian
            'Đ' => 'D', 'đ' => 'd',
            // Czech
            'Č' => 'C', 'Ď' => 'D', 'Ě' => 'E', 'Ň' => 'N', 'Ř' => 'R',
            'Š' => 'S', 'Ť' => 'T', 'Ů' => 'U', 'Ž' => 'Z', 'č' => 'c',
            'ď' => 'd', 'ě' => 'e', 'ň' => 'n', 'ř' => 'r', 'š' => 's',
            'ť' => 't', 'ů' => 'u', 'ž' => 'z',
            // Esperanto
            'Ĉ' => 'C', 'ĉ' => 'c', 'Ĝ' => 'G', 'ĝ' => 'g', 'Ĥ' => 'H',
            'ĥ' => 'h', 'Ĵ' => 'J', 'ĵ' => 'j', 'Ŝ' => 'S', 'ŝ' => 's',
            'Ŭ' => 'U', 'ŭ' => 'u',
            // Greek
            'Α' => 'A', 'Β' => 'B', 'Γ' => 'G', 'Δ' => 'D', 'Ε' => 'E',
            'Ζ' => 'Z', 'Η' => 'H', 'Θ' => '8', 'Ι' => 'I', 'Κ' => 'K',
            'Λ' => 'L', 'Μ' => 'M', 'Ν' => 'N', 'Ξ' => '3', 'Ο' => 'O',
            'Π' => 'P', 'Ρ' => 'R', 'Σ' => 'S', 'Τ' => 'T', 'Υ' => 'Y',
            'Φ' => 'F', 'Χ' => 'X', 'Ψ' => 'PS', 'Ω' => 'W', 'Ά' => 'A',
            'Έ' => 'E', 'Ί' => 'I', 'Ό' => 'O', 'Ύ' => 'Y', 'Ή' => 'H',
            'Ώ' => 'W', 'Ϊ' => 'I', 'Ϋ' => 'Y', 'α' => 'a', 'β' => 'b',
            'γ' => 'g', 'δ' => 'd', 'ε' => 'e', 'ζ' => 'z', 'η' => 'h',
            'θ' => '8', 'ι' => 'i', 'κ' => 'k', 'λ' => 'l', 'μ' => 'm',
            'ν' => 'n', 'ξ' => '3', 'ο' => 'o', 'π' => 'p', 'ρ' => 'r',
            'σ' => 's', 'τ' => 't', 'υ' => 'y', 'φ' => 'f', 'χ' => 'x',
            'ψ' => 'ps', 'ω' => 'w', 'ά' => 'a', 'έ' => 'e', 'ί' => 'i',
            'ό' => 'o', 'ύ' => 'y', 'ή' => 'h', 'ώ' => 'w', 'ς' => 's',
            'ϊ' => 'i', 'ΰ' => 'y', 'ϋ' => 'y', 'ΐ' => 'i',
            // Latvian
            'Ā' => 'A', 'Č' => 'C', 'Ē' => 'E', 'Ģ' => 'G', 'Ī' => 'i',
            'Ķ' => 'k', 'Ļ' => 'L', 'Ņ' => 'N', 'Š' => 'S', 'Ū' => 'u',
            'Ž' => 'Z', 'ā' => 'a', 'č' => 'c', 'ē' => 'e', 'ģ' => 'g',
            'ī' => 'i', 'ķ' => 'k', 'ļ' => 'l', 'ņ' => 'n', 'Ŗ' => 'R',
            'ŗ' => 'r', 'š' => 's', 'ū' => 'u', 'ž' => 'z', 'Ō' => 'O',
            'ō' => 'o',
            // Lithuanian
            'Ė' => 'E', 'ė' => 'e', 'Ĩ' => 'I', 'ĩ' => 'i', 'Į' => 'I',
            'į' => 'i', 'Ų' => 'U', 'ų' => 'u',
            // Maltese
            'Ċ' => 'C', 'ċ' => 'c', 'Ġ' => 'G', 'ġ' => 'g', 'Ħ' => 'H',
            'ħ' => 'h',
            // Polish
            'Ą' => 'A', 'Ć' => 'C', 'Ę' => 'e', 'Ł' => 'L', 'Ń' => 'N',
            'Ó' => 'o', 'Ś' => 'S', 'Ź' => 'Z', 'Ż' => 'Z', 'ą' => 'a',
            'ć' => 'c', 'ę' => 'e', 'ł' => 'l', 'ń' => 'n', 'ó' => 'o',
            'ś' => 's', 'ź' => 'z', 'ż' => 'z',
            // Romanian
            'Ă' => 'A', 'ă' => 'a', 'ș' => 'S', 'ș' => 's', 'Ț' => 'T',
            'ț' => 't', 'Ţ' => 't', 'ţ' => 't',
            // Russian
            'А' => 'A', 'Б' => 'B', 'В' => 'V', 'Г' => 'G', 'Д' => 'D',
            'Е' => 'E', 'Ё' => 'Yo', 'Ж' => 'Zh', 'З' => 'Z', 'И' => 'I',
            'Й' => 'J', 'К' => 'K', 'Л' => 'L', 'М' => 'M', 'Н' => 'N',
            'О' => 'O', 'П' => 'P', 'Р' => 'R', 'С' => 'S', 'Т' => 'T',
            'У' => 'U', 'Ф' => 'F', 'Х' => 'H', 'Ц' => 'C', 'Ч' => 'Ch',
            'Ш' => 'Sh', 'Щ' => 'Sh', 'Ъ' => '', 'Ы' => 'Y', 'Ь' => '',
            'Э' => 'E', 'Ю' => 'Yu', 'Я' => 'Ya', 'а' => 'a', 'б' => 'b',
            'в' => 'v', 'г' => 'g', 'д' => 'd', 'е' => 'e', 'ё' => 'yo',
            'ж' => 'zh', 'з' => 'z', 'и' => 'i', 'й' => 'j', 'к' => 'k',
            'л' => 'l', 'м' => 'm', 'н' => 'n', 'о' => 'o', 'п' => 'p',
            'р' => 'r', 'с' => 's', 'т' => 't', 'у' => 'u', 'ф' => 'f',
            'х' => 'h', 'ц' => 'c', 'ч' => 'ch', 'ш' => 'sh', 'щ' => 'sh',
            'ъ' => '', 'ы' => 'y', 'ь' => '', 'э' => 'e', 'ю' => 'yu',
            'я' => 'ya',
            // Sami
            'Ŋ' => 'N', 'ŋ' => 'n', 'Ŧ' => 'T', 'ŧ' => 't', 'Ǧ' => 'G',
            'ǧ' => 'g', 'Ǩ' => 'K', 'ǩ' => 'k', 'Ʒ' => 'Z', 'ʒ' => 'z',
            'Ǯ' => 'Z', 'ǯ' => 'z',
            // Slovak
            'Ľ' => 'L', 'ľ' => 'l', 'Ĺ' => 'L', 'ĺ' => 'l', 'Ŕ' => 'R',
            'ŕ' => 'r',
            // Turkish
            'Ş' => 'S', 'İ' => 'I', 'Ç' => 'C', 'Ü' => 'U', 'Ö' => 'O',
            'Ğ' => 'G', 'ş' => 's', 'ı' => 'i', 'ç' => 'c', 'ü' => 'u',
            'ö' => 'o', 'ğ' => 'g',
            // Ukrainian
            'Є' => 'Ye', 'І' => 'I', 'Ї' => 'Yi', 'Ґ' => 'G', 'є' => 'ye',
            'і' => 'i', 'ї' => 'yi', 'ґ' => 'g',
        ];

        $normalized = strtolower(strtr($aName, $characters));
        $cleaned = preg_replace('/([^a-z0-9]|-)+/', '-', $normalized);

        return trim($cleaned, '-');
    }
892325
function makeCharacters(agent) {
    let characters = function (id, ssoToken) {
        if (ssoToken === undefined) {
            return new CharacterInfoImpl(agent, id);
        }
        else {
            return new CharacterImpl({ agent, id, ssoToken });
        }
    };
    characters.search = search_1.makeDefaultSearch(agent, "character" /* CHARACTER */);
    characters.affiliations = function (ids) {
        return agent.request('post_characters_affiliation', { body: ids });
    };
    characters.names = function (ids) {
        if (ids.length > 100) {
            // Use universe/names end point since the /characters one breaks if
            // the URL gets too long.
            return names_1.getNames(agent, "character" /* CHARACTER */, ids);
        }
        else {
            // Use character/names end point
            return agent.request('get_characters_names', { query: { character_ids: ids } })
                .then(result => {
                let map = new Map();
                for (let c of result) {
                    map.set(c.character_id, c.character_name);
                }
                return map;
            });
        }
    };
    return characters;
}
1228093
def _switch_charset_list(characters, target=''):
    '''
    
    '''
    # Copy the list to avoid modifying the existing one.
    characters = characters[:]
    offset = block_offset * offsets[target]['direction']
    for n in range(len(characters)):
        chars = list(characters[n])

        for m in range(len(chars)):
            char = chars[m]
            char_offset = ord(char) + offset
            # Verify that the offset character is within the valid range.
            if in_range(char_offset, target):
                chars[m] = chr(char_offset)
            else:
                chars[m] = char

        characters[n] = ''.join(chars)

    return characters
1160398
private static function _buildSYLKCharacters()
	{
		self::$_SYLKCharacters = array(
			"\x1B 0"  => chr(0),
			"\x1B 1"  => chr(1),
			"\x1B 2"  => chr(2),
			"\x1B 3"  => chr(3),
			"\x1B 4"  => chr(4),
			"\x1B 5"  => chr(5),
			"\x1B 6"  => chr(6),
			"\x1B 7"  => chr(7),
			"\x1B 8"  => chr(8),
			"\x1B 9"  => chr(9),
			"\x1B :"  => chr(10),
			"\x1B ;"  => chr(11),
			"\x1B <"  => chr(12),
			"\x1B :"  => chr(13),
			"\x1B >"  => chr(14),
			"\x1B ?"  => chr(15),
			"\x1B!0"  => chr(16),
			"\x1B!1"  => chr(17),
			"\x1B!2"  => chr(18),
			"\x1B!3"  => chr(19),
			"\x1B!4"  => chr(20),
			"\x1B!5"  => chr(21),
			"\x1B!6"  => chr(22),
			"\x1B!7"  => chr(23),
			"\x1B!8"  => chr(24),
			"\x1B!9"  => chr(25),
			"\x1B!:"  => chr(26),
			"\x1B!;"  => chr(27),
			"\x1B!<"  => chr(28),
			"\x1B!="  => chr(29),
			"\x1B!>"  => chr(30),
			"\x1B!?"  => chr(31),
			"\x1B'?"  => chr(127),
			"\x1B(0"  => '€', // 128 in CP1252
			"\x1B(2"  => '‚', // 130 in CP1252
			"\x1B(3"  => 'ƒ', // 131 in CP1252
			"\x1B(4"  => '„', // 132 in CP1252
			"\x1B(5"  => '…', // 133 in CP1252
			"\x1B(6"  => '†', // 134 in CP1252
			"\x1B(7"  => '‡', // 135 in CP1252
			"\x1B(8"  => 'ˆ', // 136 in CP1252
			"\x1B(9"  => '‰', // 137 in CP1252
			"\x1B(:"  => 'Š', // 138 in CP1252
			"\x1B(;"  => '‹', // 139 in CP1252
			"\x1BNj"  => 'Œ', // 140 in CP1252
			"\x1B(>"  => 'Ž', // 142 in CP1252
			"\x1B)1"  => '‘', // 145 in CP1252
			"\x1B)2"  => '’', // 146 in CP1252
			"\x1B)3"  => '“', // 147 in CP1252
			"\x1B)4"  => '”', // 148 in CP1252
			"\x1B)5"  => '•', // 149 in CP1252
			"\x1B)6"  => '–', // 150 in CP1252
			"\x1B)7"  => '—', // 151 in CP1252
			"\x1B)8"  => '˜', // 152 in CP1252
			"\x1B)9"  => '™', // 153 in CP1252
			"\x1B):"  => 'š', // 154 in CP1252
			"\x1B);"  => '›', // 155 in CP1252
			"\x1BNz"  => 'œ', // 156 in CP1252
			"\x1B)>"  => 'ž', // 158 in CP1252
			"\x1B)?"  => 'Ÿ', // 159 in CP1252
			"\x1B*0"  => ' ', // 160 in CP1252
			"\x1BN!"  => '¡', // 161 in CP1252
			"\x1BN\"" => '¢', // 162 in CP1252
			"\x1BN#"  => '£', // 163 in CP1252
			"\x1BN("  => '¤', // 164 in CP1252
			"\x1BN%"  => '¥', // 165 in CP1252
			"\x1B*6"  => '¦', // 166 in CP1252
			"\x1BN'"  => '§', // 167 in CP1252
			"\x1BNH " => '¨', // 168 in CP1252
			"\x1BNS"  => '©', // 169 in CP1252
			"\x1BNc"  => 'ª', // 170 in CP1252
			"\x1BN+"  => '«', // 171 in CP1252
			"\x1B*<"  => '¬', // 172 in CP1252
			"\x1B*="  => '­', // 173 in CP1252
			"\x1BNR"  => '®', // 174 in CP1252
			"\x1B*?"  => '¯', // 175 in CP1252
			"\x1BN0"  => '°', // 176 in CP1252
			"\x1BN1"  => '±', // 177 in CP1252
			"\x1BN2"  => '²', // 178 in CP1252
			"\x1BN3"  => '³', // 179 in CP1252
			"\x1BNB " => '´', // 180 in CP1252
			"\x1BN5"  => 'µ', // 181 in CP1252
			"\x1BN6"  => '¶', // 182 in CP1252
			"\x1BN7"  => '·', // 183 in CP1252
			"\x1B+8"  => '¸', // 184 in CP1252
			"\x1BNQ"  => '¹', // 185 in CP1252
			"\x1BNk"  => 'º', // 186 in CP1252
			"\x1BN;"  => '»', // 187 in CP1252
			"\x1BN<"  => '¼', // 188 in CP1252
			"\x1BN="  => '½', // 189 in CP1252
			"\x1BN>"  => '¾', // 190 in CP1252
			"\x1BN?"  => '¿', // 191 in CP1252
			"\x1BNAA" => 'À', // 192 in CP1252
			"\x1BNBA" => 'Á', // 193 in CP1252
			"\x1BNCA" => 'Â', // 194 in CP1252
			"\x1BNDA" => 'Ã', // 195 in CP1252
			"\x1BNHA" => 'Ä', // 196 in CP1252
			"\x1BNJA" => 'Å', // 197 in CP1252
			"\x1BNa"  => 'Æ', // 198 in CP1252
			"\x1BNKC" => 'Ç', // 199 in CP1252
			"\x1BNAE" => 'È', // 200 in CP1252
			"\x1BNBE" => 'É', // 201 in CP1252
			"\x1BNCE" => 'Ê', // 202 in CP1252
			"\x1BNHE" => 'Ë', // 203 in CP1252
			"\x1BNAI" => 'Ì', // 204 in CP1252
			"\x1BNBI" => 'Í', // 205 in CP1252
			"\x1BNCI" => 'Î', // 206 in CP1252
			"\x1BNHI" => 'Ï', // 207 in CP1252
			"\x1BNb"  => 'Ð', // 208 in CP1252
			"\x1BNDN" => 'Ñ', // 209 in CP1252
			"\x1BNAO" => 'Ò', // 210 in CP1252
			"\x1BNBO" => 'Ó', // 211 in CP1252
			"\x1BNCO" => 'Ô', // 212 in CP1252
			"\x1BNDO" => 'Õ', // 213 in CP1252
			"\x1BNHO" => 'Ö', // 214 in CP1252
			"\x1B-7"  => '×', // 215 in CP1252
			"\x1BNi"  => 'Ø', // 216 in CP1252
			"\x1BNAU" => 'Ù', // 217 in CP1252
			"\x1BNBU" => 'Ú', // 218 in CP1252
			"\x1BNCU" => 'Û', // 219 in CP1252
			"\x1BNHU" => 'Ü', // 220 in CP1252
			"\x1B-="  => 'Ý', // 221 in CP1252
			"\x1BNl"  => 'Þ', // 222 in CP1252
			"\x1BN{"  => 'ß', // 223 in CP1252
			"\x1BNAa" => 'à', // 224 in CP1252
			"\x1BNBa" => 'á', // 225 in CP1252
			"\x1BNCa" => 'â', // 226 in CP1252
			"\x1BNDa" => 'ã', // 227 in CP1252
			"\x1BNHa" => 'ä', // 228 in CP1252
			"\x1BNJa" => 'å', // 229 in CP1252
			"\x1BNq"  => 'æ', // 230 in CP1252
			"\x1BNKc" => 'ç', // 231 in CP1252
			"\x1BNAe" => 'è', // 232 in CP1252
			"\x1BNBe" => 'é', // 233 in CP1252
			"\x1BNCe" => 'ê', // 234 in CP1252
			"\x1BNHe" => 'ë', // 235 in CP1252
			"\x1BNAi" => 'ì', // 236 in CP1252
			"\x1BNBi" => 'í', // 237 in CP1252
			"\x1BNCi" => 'î', // 238 in CP1252
			"\x1BNHi" => 'ï', // 239 in CP1252
			"\x1BNs"  => 'ð', // 240 in CP1252
			"\x1BNDn" => 'ñ', // 241 in CP1252
			"\x1BNAo" => 'ò', // 242 in CP1252
			"\x1BNBo" => 'ó', // 243 in CP1252
			"\x1BNCo" => 'ô', // 244 in CP1252
			"\x1BNDo" => 'õ', // 245 in CP1252
			"\x1BNHo" => 'ö', // 246 in CP1252
			"\x1B/7"  => '÷', // 247 in CP1252
			"\x1BNy"  => 'ø', // 248 in CP1252
			"\x1BNAu" => 'ù', // 249 in CP1252
			"\x1BNBu" => 'ú', // 250 in CP1252
			"\x1BNCu" => 'û', // 251 in CP1252
			"\x1BNHu" => 'ü', // 252 in CP1252
			"\x1B/="  => 'ý', // 253 in CP1252
			"\x1BN|"  => 'þ', // 254 in CP1252
			"\x1BNHy" => 'ÿ', // 255 in CP1252
		);
	}
1000878
def correct_chrom_names(chroms):
    """"""
    chrom_list = []
    for chrom in chroms:
        # fix chrom numbering
        chrom = str(chrom)
        chrom = chrom.replace('23', 'X')
        chrom = chrom.replace('24', 'Y')
        chrom = chrom.replace('25', 'Mt')
        if not chrom.startswith('chr'):
            chrom = 'chr' + chrom
        chrom_list.append(chrom)
    return chrom_list
585874
def drop_bad_characters(text):
    
    # Strip all non-ascii and non-printable characters
    text = ''.join([c for c in text if c in ALLOWED_CHARS])
    return text
355236
def make_charlist(self):
        """"""

        def worker_output(worker, output, error):
            """Worker finished callback."""
            self._charlist = output
            if error is None and output:
                self._allow_highlight = True
                self.rehighlight()
            self._allow_highlight = False

        text = to_text_string(self.document().toPlainText())
        tokens = self._lexer.get_tokens(text)

        # Before starting a new worker process make sure to end previous
        # incarnations
        self._worker_manager.terminate_all()

        worker = self._worker_manager.create_python_worker(
            self._make_charlist,
            tokens,
            self._tokmap,
            self.formats,
        )
        worker.sig_finished.connect(worker_output)
        worker.start()
776488
def char_sets():
    
    if not hasattr(char_sets, 'setlist'):
        clist = []
        try:
            data = requests.get('http://www.iana.org/assignments/character-'
                                'sets/character-sets-1.csv')
        except requests.exceptions.RequestException:
            return []

        for line in data.iter_lines():
            if line:
                line = line.decode("utf-8")
                if line.count(',') > 0:
                    vals = line.split(',')
                    if vals[0]:
                        clist.append(vals[0])
                    else:
                        clist.append(vals[1])

        char_sets.setlist = clist
    return char_sets.setlist
1312698
def com_google_fonts_check_name_unwanted_chars(ttFont):
  """"""
  failed = False
  replacement_map = [("\u00a9", '(c)'),
                     ("\u00ae", '(r)'),
                     ("\u2122", '(tm)')]
  for name in ttFont['name'].names:
    string = str(name.string, encoding=name.getEncoding())
    for mark, ascii_repl in replacement_map:
      new_string = string.replace(mark, ascii_repl)
      if string != new_string:
        yield FAIL, ("NAMEID #{} contains symbol that should be"
                     " replaced by '{}'.").format(name.nameID,
                                                  ascii_repl)
        failed = True
  if not failed:
    yield PASS, ("No need to substitute copyright, registered and"
                 " trademark symbols in name table entries of this font.")
1230486
def consistency_check(text, word_pairs, err, msg, offset=0):
    """"""
    errors = []

    msg = " ".join(msg.split())

    for w in word_pairs:
        matches = [
            [m for m in re.finditer(w[0], text)],
            [m for m in re.finditer(w[1], text)]
        ]

        if len(matches[0]) > 0 and len(matches[1]) > 0:

            idx_minority = len(matches[0]) > len(matches[1])

            for m in matches[idx_minority]:
                errors.append((
                    m.start() + offset,
                    m.end() + offset,
                    err,
                    msg.format(w[~idx_minority], m.group(0)),
                    w[~idx_minority]))

    return errors
148632
private void normalise(char[] c) {
		for (int i = 0; i< c.length; i++) {
			if (c[i] == 12288) {
				c[i] = (char) 32;
				continue;
			}else if (c[i]> 65280&& c[i]< 65375){
				c[i] = (char) (c[i] - 65248);			
			}
			else if(c[i] > 64 && c[i] < 91){
				c[i] += 32;
			}
			else{
				char cc = l2s.get(c[i]);
				if (cc != 0) {   
					c[i] = cc;   
				} 
			} 
		}
	}
1280169
private void scanIdent() {
        boolean isJavaIdentifierPart;
        char high;
        reader.putChar(true);
        do {
            switch (reader.ch) {
            case 'A': case 'B': case 'C': case 'D': case 'E':
            case 'F': case 'G': case 'H': case 'I': case 'J':
            case 'K': case 'L': case 'M': case 'N': case 'O':
            case 'P': case 'Q': case 'R': case 'S': case 'T':
            case 'U': case 'V': case 'W': case 'X': case 'Y':
            case 'Z':
            case 'a': case 'b': case 'c': case 'd': case 'e':
            case 'f': case 'g': case 'h': case 'i': case 'j':
            case 'k': case 'l': case 'm': case 'n': case 'o':
            case 'p': case 'q': case 'r': case 's': case 't':
            case 'u': case 'v': case 'w': case 'x': case 'y':
            case 'z':
            case '$': case '_':
            case '0': case '1': case '2': case '3': case '4':
            case '5': case '6': case '7': case '8': case '9':
                break;
            case '\u0000': case '\u0001': case '\u0002': case '\u0003':
            case '\u0004': case '\u0005': case '\u0006': case '\u0007':
            case '\u0008': case '\u000E': case '\u000F': case '\u0010':
            case '\u0011': case '\u0012': case '\u0013': case '\u0014':
            case '\u0015': case '\u0016': case '\u0017':
            case '\u0018': case '\u0019': case '\u001B':
            case '\u007F':
                reader.scanChar();
                continue;
            case '\u001A': // EOI is also a legal identifier part
                if (reader.bp >= reader.buflen) {
                    name = reader.name();
                    tk = tokens.lookupKind(name);
                    return;
                }
                reader.scanChar();
                continue;
            default:
                if (reader.ch < '\u0080') {
                    // all ASCII range chars already handled, above
                    isJavaIdentifierPart = false;
                } else {
                    if (Character.isIdentifierIgnorable(reader.ch)) {
                        reader.scanChar();
                        continue;
                    } else {
                        int codePoint = reader.peekSurrogates();
                        if (codePoint >= 0) {
                            if (isJavaIdentifierPart = Character.isJavaIdentifierPart(codePoint)) {
                                reader.putChar(true);
                            }
                        } else {
                            isJavaIdentifierPart = Character.isJavaIdentifierPart(reader.ch);
                        }
                    }
                }
                if (!isJavaIdentifierPart) {
                    name = reader.name();
                    tk = tokens.lookupKind(name);
                    return;
                }
            }
            reader.putChar(true);
        } while (true);
    }
256830
def errors_string(self):
        
        output = []
        for e in self.errors:
            output.append('%s: %s in %s:' % (e[1], e[2], e[0]))
            output.append(''.join(traceback.format_tb(e[3])))
        return '\n'.join(output)
1001136
def _clean_characters(x):
    
    if not isinstance(x, six.string_types):
        x = str(x)
    else:
        if not all(ord(char) < 128 for char in x):
            msg = "Found unicode character in input YAML (%s)" % (x)
            raise ValueError(repr(msg))
    for problem in [" ", ".", "/", "\\", "[", "]", "&", ";", "#", "+", ":", ")", "("]:
        x = x.replace(problem, "_")
    return x
983435
def sanitize(meta, diagnostics=False):
    
    bad_encodings, bad_fields = set(), set()

    def sane_encoding(field, text):
        "Transcoding helper."
        for encoding in ('utf-8', meta.get('encoding', None), 'cp1252'):
            if encoding:
                try:
                    u8_text = text.decode(encoding).encode("utf-8")
                    if encoding != 'utf-8':
                        bad_encodings.add(encoding)
                        bad_fields.add(field)
                    return u8_text
                except UnicodeError:
                    continue
        else:
            # Broken beyond anything reasonable
            bad_encodings.add('UNKNOWN/EXOTIC')
            bad_fields.add(field)
            return str(text, 'utf-8', 'replace').replace('\ufffd', '_').encode("utf-8")

    # Go through all string fields and check them
    for field in ("comment", "created by"):
        if field in meta:
            meta[field] = sane_encoding(field, meta[field])

    meta["info"]["name"] = sane_encoding('info name', meta["info"]["name"])

    for entry in meta["info"].get("files", []):
        entry["path"] = [sane_encoding('file path', i) for i in entry["path"]]

    return (meta, bad_encodings, bad_fields) if diagnostics else meta
1317195
def translate_list(cls, val):
        """"""
        escaped = ', '.join([cls.translate(v) for v in val])
        return 'Seq({})'.format(escaped)