Transliterate the given text to the latin script.

    This attempts to convert a given text to latin script using the
    closest match of characters vis a vis the original script.
341704
def latinize_text(text, ascii=False):
    
    if text is None or not isinstance(text, six.string_types) or not len(text):
        return text

    if ascii:
        if not hasattr(latinize_text, '_ascii'):
            # Transform to latin, separate accents, decompose, remove
            # symbols, compose, push to ASCII
            latinize_text._ascii = Transliterator.createInstance('Any-Latin; NFKD; [:Symbol:] Remove; [:Nonspacing Mark:] Remove; NFKC; Accents-Any; Latin-ASCII')  # noqa
        return latinize_text._ascii.transliterate(text)

    if not hasattr(latinize_text, '_tr'):
        latinize_text._tr = Transliterator.createInstance('Any-Latin')
    return latinize_text._tr.transliterate(text)
283590
def fix_one_step_and_explain(text):
    
    if isinstance(text, bytes):
        raise UnicodeError(BYTES_ERROR_TEXT)
    if len(text) == 0:
        return text, []

    # The first plan is to return ASCII text unchanged.
    if possible_encoding(text, 'ascii'):
        return text, []

    # As we go through the next step, remember the possible encodings
    # that we encounter but don't successfully fix yet. We may need them
    # later.
    possible_1byte_encodings = []

    # Suppose the text was supposed to be UTF-8, but it was decoded using
    # a single-byte encoding instead. When these cases can be fixed, they
    # are usually the correct thing to do, so try them next.
    for encoding in CHARMAP_ENCODINGS:
        if possible_encoding(text, encoding):
            encoded_bytes = text.encode(encoding)
            encode_step = ('encode', encoding, ENCODING_COSTS.get(encoding, 0))
            transcode_steps = []

            # Now, find out if it's UTF-8 (or close enough). Otherwise,
            # remember the encoding for later.
            try:
                decoding = 'utf-8'
                # Check encoded_bytes for sequences that would be UTF-8,
                # except they have b' ' where b'\xa0' would belong.
                if ALTERED_UTF8_RE.search(encoded_bytes):
                    encoded_bytes = restore_byte_a0(encoded_bytes)
                    cost = encoded_bytes.count(0xa0) * 2
                    transcode_steps.append(('transcode', 'restore_byte_a0', cost))

                # Check for the byte 0x1a, which indicates where one of our
                # 'sloppy' codecs found a replacement character.
                if encoding.startswith('sloppy') and 0x1a in encoded_bytes:
                    encoded_bytes = replace_lossy_sequences(encoded_bytes)
                    transcode_steps.append(('transcode', 'replace_lossy_sequences', 0))

                if 0xed in encoded_bytes or 0xc0 in encoded_bytes:
                    decoding = 'utf-8-variants'

                decode_step = ('decode', decoding, 0)
                steps = [encode_step] + transcode_steps + [decode_step]
                fixed = encoded_bytes.decode(decoding)
                return fixed, steps

            except UnicodeDecodeError:
                possible_1byte_encodings.append(encoding)

    # Look for a-hat-euro sequences that remain, and fix them in isolation.
    if PARTIAL_UTF8_PUNCT_RE.search(text):
        steps = [('transcode', 'fix_partial_utf8_punct_in_1252', 1)]
        fixed = fix_partial_utf8_punct_in_1252(text)
        return fixed, steps

    # The next most likely case is that this is Latin-1 that was intended to
    # be read as Windows-1252, because those two encodings in particular are
    # easily confused.
    if 'latin-1' in possible_1byte_encodings:
        if 'windows-1252' in possible_1byte_encodings:
            # This text is in the intersection of Latin-1 and
            # Windows-1252, so it's probably legit.
            return text, []
        else:
            # Otherwise, it means we have characters that are in Latin-1 but
            # not in Windows-1252. Those are C1 control characters. Nobody
            # wants those. Assume they were meant to be Windows-1252. Don't
            # use the sloppy codec, because bad Windows-1252 characters are
            # a bad sign.
            encoded = text.encode('latin-1')
            try:
                fixed = encoded.decode('windows-1252')
                steps = []
                if fixed != text:
                    steps = [('encode', 'latin-1', 0),
                             ('decode', 'windows-1252', 1)]
                return fixed, steps
            except UnicodeDecodeError:
                # This text contained characters that don't even make sense
                # if you assume they were supposed to be Windows-1252. In
                # that case, let's not assume anything.
                pass

    # The cases that remain are mixups between two different single-byte
    # encodings, and not the common case of Latin-1 vs. Windows-1252.
    #
    # These cases may be unsolvable without adding false positives, though
    # I have vague ideas about how to optionally address them in the future.

    # Return the text unchanged; the plan is empty.
    return text, []
905105
def parse(self, text):
        
        tokens = self.lex(text)
        parser = Parser(tokens)
        return parser.parse()
1328138
def native_(s, encoding='latin-1', errors='strict'):
    '''
    
    '''
    if PY3:
        out = s if isinstance(s, text_type) else str(s, encoding, errors)
    else:
        out = s.encode(encoding, errors) if isinstance(s, text_type) else str(s)

    return out
203845
def to_unicode(text):
    
    if not text or isinstance(text, unicode if PY2 else str):
        return text

    try:
        # Try UTF-8 first
        return text.decode("UTF-8")
    except UnicodeError:
        try:
            # Then Windows Latin-1
            return text.decode("CP1252")
        except UnicodeError:
            # Give up, return byte string in the hope things work out
            return text
1105114
def process_text(self, text):
        
        # escape base char
        text = text.replace('&', '&amp;').replace('<', '&lt;').replace('>', '&gt;')
        # make xml request body
        soap_body = self.__create_xml_request(text)
        # make total request
        length = len(soap_body.encode('UTF-8')) if PY3 else len(soap_body)
        soap_request = self.SOAP_REQUEST.format(
            length=length, host=self.HOST, content=soap_body)

        if PY3:  # convert to bytes
            soap_request = soap_request.encode(self._encoding)

        # send request use soket
        connector = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
        connector.settimeout(self._timeout)
        connector.connect((self.HOST, 80))
        connector.sendall(soap_request)
        # call for response
        response = b''
        buf = '0'
        while len(buf):
            buf = connector.recv(8192)
            response += buf
        connector.close()

        if PY3:  # convert to str
            response = response.decode()

        # parse response
        text_result = self.__parse_xml_response(response)
        # back replace and return
        return text_result.replace('&amp;', '&').replace('&lt;', '<').replace('&gt;', '>')
640180
public function make(string $text): string
    {
        $map = $this->getMap();
        $transliterated = str_replace(array_keys($map), array_values($map), $text);

        if (true === config('transliterate.remove_accents', false)) {
            $transliterator = IntlTransliterator::create('Any-Latin; Latin-ASCII');
            $transliterated = $transliterator->transliterate($transliterated);
        }

        return self::applyTransformers($transliterated);
    }
498639
def transliterate(self, target_language="en"):
    """"""
    return WordList([w.transliterate(target_language) for w in self.words],
                     language=target_language, parent=self)
759959
public function transliterate($string)
	{
		if ($this->transliterator !== null)
		{
			return call_user_func($this->transliterator, $string);
		}

		$string = Latin::toAscii($string);
		$string = strtolower($string);

		return $string;
	}
727452
def detranslify(text):
    """"""
    try:
        res = translit.detranslify(text)
    except Exception as err:
        # because filter must die silently
        res = default_value % {'error': err, 'value': text}
    return res
204377
def transform(self, text):
        """"""
        return re.sub(self.regex, self.repl, text)
255956
def run(self, text):
        
        for regex in self.regexes:
            text = regex.sub(self.repl, text)
        return text
300788
def lexify(self, text='????', letters=string.ascii_letters):
        
        return _re_qm.sub(lambda x: self.random_element(letters), text)
472219
def type_converter(text):
    
    if text.isdigit():
        return int(text), int

    try:
        return float(text), float
    except ValueError:
        return text, STRING_TYPE
896482
def translate(self, text):
        

        # Reset substitution counter
        self.count = 0

        # Process text
        return self._make_regex().sub(self, text)
640042
@Override
    protected void handleTransliterate(Replaceable text,
                                       Position index, boolean incremental) {
        /* Call each transliterator with the same start value and
         * initial cursor index, but with the limit index as modified
         * by preceding transliterators.  The cursor index must be
         * reset for each transliterator to give each a chance to
         * transliterate the text.  The initial cursor index is known
         * to still point to the same place after each transliterator
         * is called because each transliterator will not change the
         * text between start and the initial value of cursor.
         *
         * IMPORTANT: After the first transliterator, each subsequent
         * transliterator only gets to transliterate text committed by
         * preceding transliterators; that is, the cursor (output
         * value) of transliterator i becomes the limit (input value)
         * of transliterator i+1.  Finally, the overall limit is fixed
         * up before we return.
         *
         * Assumptions we make here:
         * (1) contextStart <= start <= limit <= contextLimit <= text.length()
         * (2) start <= start' <= limit'  ;cursor doesn't move back
         * (3) start <= limit'            ;text before cursor unchanged
         * - start' is the value of start after calling handleKT
         * - limit' is the value of limit after calling handleKT
         */

        /**
         * Example: 3 transliterators.  This example illustrates the
         * mechanics we need to implement.  C, S, and L are the contextStart,
         * start, and limit.  gl is the globalLimit.  contextLimit is
         * equal to limit throughout.
         *
         * 1. h-u, changes hex to Unicode
         *
         *    4  7  a  d  0      4  7  a
         *    abc/u0061/u    =>  abca/u
         *    C  S       L       C   S L   gl=f->a
         *
         * 2. upup, changes "x" to "XX"
         *
         *    4  7  a       4  7  a
         *    abca/u    =>  abcAA/u
         *    C  SL         C    S
         *                       L    gl=a->b
         * 3. u-h, changes Unicode to hex
         *
         *    4  7  a        4  7  a  d  0  3
         *    abcAA/u    =>  abc/u0041/u0041/u
         *    C  S L         C              S
         *                                  L   gl=b->15
         * 4. return
         *
         *    4  7  a  d  0  3
         *    abc/u0041/u0041/u
         *    C S L
         */

        if (trans.length < 1) {
            index.start = index.limit;
            return; // Short circuit for empty compound transliterators
        }

        // compoundLimit is the limit value for the entire compound
        // operation.  We overwrite index.limit with the previous
        // index.start.  After each transliteration, we update
        // compoundLimit for insertions or deletions that have happened.
        int compoundLimit = index.limit;

        // compoundStart is the start for the entire compound
        // operation.
        int compoundStart = index.start;

        int delta = 0; // delta in length

        StringBuffer log = null;
        ///CLOVER:OFF
        if (DEBUG) {
            log = new StringBuffer("CompoundTransliterator{" + getID() +
                                   (incremental ? "}i: IN=" : "}: IN="));
            UtilityExtensions.formatInput(log, text, index);
            System.out.println(Utility.escape(log.toString()));
        }
        ///CLOVER:ON

        // Give each transliterator a crack at the run of characters.
        // See comments at the top of the method for more detail.
        for (int i=0; i<trans.length; ++i) {
            index.start = compoundStart; // Reset start
            int limit = index.limit;

            if (index.start == index.limit) {
                // Short circuit for empty range
                ///CLOVER:OFF
                if (DEBUG) {
                    System.out.println("CompoundTransliterator[" + i +
                                       ".." + (trans.length-1) +
                                       (incremental ? "]i: " : "]: ") +
                                       UtilityExtensions.formatInput(text, index) +
                                       " (NOTHING TO DO)");
                }
                ///CLOVER:ON
                break;
            }

            ///CLOVER:OFF
            if (DEBUG) {
                log.setLength(0);
                log.append("CompoundTransliterator[" + i + "=" +
                           trans[i].getID() +
                           (incremental ? "]i: " : "]: "));
                UtilityExtensions.formatInput(log, text, index);
            }
            ///CLOVER:ON

            trans[i].filteredTransliterate(text, index, incremental);

            // In a properly written transliterator, start == limit after
            // handleTransliterate() returns when incremental is false.
            // Catch cases where the subclass doesn't do this, and throw
            // an exception.  (Just pinning start to limit is a bad idea,
            // because what's probably happening is that the subclass
            // isn't transliterating all the way to the end, and it should
            // in non-incremental mode.)
            if (!incremental && index.start != index.limit) {
                throw new RuntimeException("ERROR: Incomplete non-incremental transliteration by " + trans[i].getID());
            }

            ///CLOVER:OFF
            if (DEBUG) {
                log.append(" => ");
                UtilityExtensions.formatInput(log, text, index);
                System.out.println(Utility.escape(log.toString()));
            }
            ///CLOVER:ON

            // Cumulative delta for insertions/deletions
            delta += index.limit - limit;

            if (incremental) {
                // In the incremental case, only allow subsequent
                // transliterators to modify what has already been
                // completely processed by prior transliterators.  In the
                // non-incrmental case, allow each transliterator to
                // process the entire text.
                index.limit = index.start;
            }
        }

        compoundLimit += delta;

        // Start is good where it is -- where the last transliterator left
        // it.  Limit needs to be put back where it was, modulo
        // adjustments for deletions/insertions.
        index.limit = compoundLimit;

        ///CLOVER:OFF
        if (DEBUG) {
            log.setLength(0);
            log.append("CompoundTransliterator{" + getID() +
                       (incremental ? "}i: OUT=" : "}: OUT="));
            UtilityExtensions.formatInput(log, text, index);
            System.out.println(Utility.escape(log.toString()));
        }
        ///CLOVER:ON
    }
234701
def from_text(text, origin = root):
    

    if not isinstance(text, str):
        if isinstance(text, unicode) and sys.hexversion >= 0x02030000:
            return from_unicode(text, origin)
        else:
            raise ValueError("input to from_text() must be a string")
    if not (origin is None or isinstance(origin, Name)):
        raise ValueError("origin must be a Name or None")
    labels = []
    label = ''
    escaping = False
    edigits = 0
    total = 0
    if text == '@':
        text = ''
    if text:
        if text == '.':
            return Name([''])
        for c in text:
            if escaping:
                if edigits == 0:
                    if c.isdigit():
                        total = int(c)
                        edigits += 1
                    else:
                        label += c
                        escaping = False
                else:
                    if not c.isdigit():
                        raise BadEscape
                    total *= 10
                    total += int(c)
                    edigits += 1
                    if edigits == 3:
                        escaping = False
                        label += chr(total)
            elif c == '.':
                if len(label) == 0:
                    raise EmptyLabel
                labels.append(label)
                label = ''
            elif c == '\\':
                escaping = True
                edigits = 0
                total = 0
            else:
                label += c
        if escaping:
            raise BadEscape
        if len(label) > 0:
            labels.append(label)
        else:
            labels.append('')
    if (len(labels) == 0 or labels[-1] != '') and not origin is None:
        labels.extend(list(origin.labels))
    return Name(labels)