Reduce the Tashkeel,  by deleting evident cases.

    @param text: the input text fully vocalized.
    @type text: unicode.
    @return : partially vocalized text.
    @rtype: unicode.
761232
def reduce_tashkeel(text):
    
    patterns = [
        # delete all fathat,   except on waw and yeh
        u"(?<!(%s|%s))(%s|%s)" % (WAW, YEH, SUKUN, FATHA),
        # delete damma if followed by waw.
        u"%s(?=%s)" % (DAMMA, WAW),
        # delete kasra if followed by yeh.
        u"%s(?=%s)" % (KASRA, YEH),
        # delete fatha if followed by alef to reduce yeh maftouha
        #  and waw maftouha before alef.
        u"%s(?=%s)" % (FATHA, ALEF),
        # delete fatha from yeh and waw if they are in the word begining.
        u"(?<=\s(%s|%s))%s" % (WAW, YEH, FATHA),
        # delete kasra if preceded by Hamza below alef.
        u"(?<=%s)%s" % (ALEF_HAMZA_BELOW, KASRA),
    ]
    reduced = text
    for pat in patterns:
        reduced = re.sub(pat, '', reduced)
    return reduced
1168610
def split(text: str) -> List[str]:
    
    return [word for word in SEPARATOR.split(text) if word.strip(' \t')]
1003055
public static String capitalise(String text) {
        if (empty(text)) {
            return "";
        }
        return text.substring(0, 1).toUpperCase() + text.substring(1);
    }
14333
protected String removeQuotes( String text ) {
        assert text != null;
        if (text.length() > 2) {
            char first = text.charAt(0);
            // Need to remove these only if they are paired ...
            if (first == '"' || first == '\'') {
                int indexOfLast = text.length() - 1;
                char last = text.charAt(indexOfLast);
                if (last == first) {
                    text = text.substring(1, indexOfLast);
                }
            }
        }
        return text;
    }
672325
def split(self, text):
        """"""
        text = cleanup(text)
        return self.sent_detector.tokenize(text.strip())
323326
def fmt_sentence(text):
    
    text = text.strip()
    if len(text) == 0:  # if empty string, return it
        return text
    else:
        text = text.lower()  # lower all char
        # delete redundant empty space
        chunks = [chunk for chunk in text.split(" ") if len(chunk) >= 1]
        chunks[0] = chunks[0][0].upper() + chunks[0][1:]
        return " ".join(chunks)
322688
def tokenize(self, text: str, customize=True, disable=[]) -> List[Token]:
        

        """Tokenize text"""
        if not self.keep_multi_space:
            text = re.sub(' +', ' ', text)
        # disable spacy parsing, tagging etc as it takes a long time if the text is short
        tokens = self.nlp(text, disable=disable)
        if customize:
            tokens = [self.custom_token(a_token) for a_token in tokens]

        return tokens
1293810
def lemmatize(self, text):
        
        #: Do not process empty strings (Issue #3)
        if text.strip() == "":
            return []
        parsed_sentences = self._parse_text(text)
        _lemmalist = []
        for s in parsed_sentences:
            tokens = s.split()
            for i, t in enumerate(tokens):
                #: Filter empty tokens from the parser output (Issue #5)
                #: This only happens if parser input is improperly tokenized
                #: e.g. if there are empty strings in the list of tokens ['A', '', '.']
                if t.startswith('/'):
                    continue
                w, tag, phrase, role, lemma = t.split('/')
                # The lexicon uses Swiss spelling: "ss" instead of "ß".
                lemma = lemma.replace(u"ß", "ss")
                # Reverse previous replacement
                lemma = lemma.strip().replace("forwardslash", "/")
                if w[0].isupper() and i > 0:
                    lemma = lemma.title()
                elif tag.startswith("N") and i == 0:
                    lemma = lemma.title()
                # Todo: Check if it makes sense to treat '/' as punctuation
                # (especially for sentiment analysis it might be interesting
                # to treat it as OR ('oder')).
                if w in string.punctuation or lemma == '/':
                    continue
                else:
                    lemma = lemma

                _lemmalist.append((lemma, tag))
        return _lemmalist
828377
def _prepare_text(self, text):
        
        # Remove characters that should be escaped for XML input (but
        # which cause problems when escaped, since they become
        # tokens).
        text = re.sub(r'[<>&]', '', text)
        pattern = r'({})'.format(self._tokenizer.pattern)
        return re.sub(pattern, self._base_token_markup, text)
65780
def clean(text, cls=None, **kwargs):
    
    cls = cls or Scrubber
    scrubber = cls()
    return scrubber.clean(text, **kwargs)
1345583
@SuppressWarnings("MethodMayBeStatic")
    public NerResultSet process(final String _text) {
        // tokenization
        final List<List<Token>> tokens = TOKENIZER.process(_text);
        // extract name phrase:
        final List<List<Phrase>> phrases = new ArrayList<>();

        for (final List<Token> tokenList : tokens) {
            // for each sentence
            final List<Phrase> sentencePhrase = new ArrayList<>();
            int wordPtr = 0;
            int phraseStubStartPtr = 0;
            while (wordPtr < tokenList.size()) {

                // iterate through every words in the sentence
                final List<Token> currentPhrase = new ArrayList<>();

                // get the phrase stub
                while (true) {
                    // we merge the names combined with the word 'and'
                    // if the second phrase is much shorter than the first phrase
                    // example: Canberra Institution of Science and Technology
                    boolean mergeAndFlag = false;
                    if ("and".equalsIgnoreCase(tokenList.get(wordPtr).text)
                        && !currentPhrase.isEmpty()
                        && wordPtr + 1 < tokenList.size()
                        && detectNameWordInSentenceByPosition(tokenList, wordPtr + 1)) {
                        if (wordPtr + currentPhrase.size() - 1 >= tokenList.size()) {
                            mergeAndFlag = true;
                        }
                        else {
                            for (int i = wordPtr + 1; i <= wordPtr + currentPhrase.size() - 1; i++) {
                                if (!detectNameWordInSentenceByPosition(tokenList, i)) {
                                    mergeAndFlag = true;
                                    break;
                                }
                            }
                        }
                    }

                    if (detectNameWordInSentenceByPosition(tokenList, wordPtr)
                        || (equalsIgnoreCase(tokenList.get(wordPtr).text, "of", "the", "'s")
                            && !currentPhrase.isEmpty()
                            && wordPtr + 1 < tokenList.size()
                            && detectNameWordInSentenceByPosition(tokenList, wordPtr + 1))
                        || mergeAndFlag) {
                        if (currentPhrase.isEmpty()) phraseStubStartPtr = wordPtr; // record the index of the first word
                        currentPhrase.add(tokenList.get(wordPtr));
                        wordPtr++;
                        if (wordPtr == tokenList.size()) break;
                    }
                    else break;
                }

                //get the attached 'the' or 'a'
                int phrasePos = phraseStubStartPtr;
                int phraseLen = currentPhrase.size();

                // if phrasePos is zero, then it is the first word, there won't be any 'the' or 'a' attached to this word.
                if (phrasePos > 0) {
                    final Token attachedArticle = tokenList.get(phrasePos - 1);
                    if ("the".equalsIgnoreCase(attachedArticle.text) || "a".equalsIgnoreCase(attachedArticle.text)) {
                        phrasePos--;
                        phraseLen++;
                    }
                }

                //handling non-name single words such as: Monday, January, etc.
                if (currentPhrase.size() == 1 && phraseStubStartPtr == phrasePos) {
                    if (NON_NAME_WORDS.contains(currentPhrase.get(0).text)) {
                        currentPhrase.clear();
                        wordPtr--;
                    }
                }

                if (currentPhrase.isEmpty()) {
                    // if the current pointed word is not a name phrase
                    // ESTIMATE IF IT IS A TIME/DATE
                    final List<Token> currentDatePhrase = new ArrayList<>();
                    final DatePhraseModel dpm = new DatePhraseModel();
                    int tempPtr = wordPtr;
                    while (true) {
                        final Token word = tokenList.get(tempPtr);
                        final DateType type = getDateType(word.text);
                        if (type != NONE
                            || equalsIgnoreCase(word.text, "of", ",", "the")
                               && !currentDatePhrase.isEmpty()
                               && tempPtr + 1 != tokenList.size()
                               && getDateType(tokenList.get(tempPtr + 1).text) != NONE
                               && getDateType(tokenList.get(tempPtr - 1).text) != NONE) {
                            // is date word:
                            currentDatePhrase.add(word);
                            dpm.addType(type);
                            tempPtr++;
                            if (tempPtr == tokenList.size()) break;
                        }
                        else break;
                    }
                    if (dpm.isDate()) {
                        // add the phrase to phrase array
                        sentencePhrase.add(new Date(currentDatePhrase, wordPtr, currentDatePhrase.size(), wordPtr));
                        wordPtr = tempPtr;
                    }
                }
                else if (!(currentPhrase.size() == 1
                           && (isPastTense(currentPhrase.get(0).text) || isPlural(currentPhrase.get(0).text)))) {
                    sentencePhrase.add(new Name(currentPhrase, phrasePos, phraseLen, phraseStubStartPtr));
                }

                wordPtr++;
            }

            phrases.add(sentencePhrase);
        }

        // find attached words(preps):
        for (int si = 0; si < tokens.size(); si++) {
            final List<Token> sentenceToken = tokens.get(si);
            final List<Phrase> sentencePhrase = phrases.get(si);
            for (int pi = 0; pi < sentencePhrase.size(); pi++) {
                getAttachedPrep(sentenceToken, sentencePhrase, pi);
            }
        }

        return new NerResultSet(phrases, tokens);
    }
848018
public static String fromCamelCaseToSnakeCase(String text) {
    if (isNullOrEmpty(text)) {
      return text;
    }
    StringBuilder result = new StringBuilder();
    CharBuffer buf = CharBuffer.wrap(text);
    while (buf.hasRemaining()) {
      char c = buf.get();
      result.append(Character.toLowerCase(c));
      buf.mark();
      if (buf.hasRemaining()) {
        char c2 = buf.get();
        if (Character.isLowerCase(c) && Character.isUpperCase(c2)) {
          result.append("_");
        }
        buf.reset();
      }
    }
    return result.toString();
  }
473284
@Override
    public List<String> tokenize(String text) {
        List<String> tokens = new ArrayList<>(Arrays.asList(text.split("[\\p{Z}\\p{C}]+")));
        return tokens;
    }
570982
def _clean_text(self, text):
        
        clean = []
        for char in text:
            if char in self.punc_stops:
                clean += '.'
            elif char not in self.punc:
                clean += char
            else:
                pass
        return (''.join(clean)).lower()
1039732
private String getJustifiedText(String text) {
        String[] words = text.split(NORMAL_SPACE);

        for (String word : words) {
            boolean containsNewLine = (word.contains("\n") || word.contains("\r"));

            if (fitsInSentence(word, currentSentence, true)) {
                addWord(word, containsNewLine);
            } else {
                sentences.add(fillSentenceWithSpaces(currentSentence));
                currentSentence.clear();
                addWord(word, containsNewLine);
            }
        }

        //Making sure we add the last sentence if needed.
        if (currentSentence.size() > 0) {
            sentences.add(getSentenceFromList(currentSentence, true));
        }

        //Returns the justified text.
        return getSentenceFromList(sentences, false);
    }
244400
public String process(String text){
		String ret = text;
		
		//Pipeline
		ret = convertor.convert(ret);
		ret = senSegment(ret);
		ret = senTokenize(ret);
		ret = wordSegment(ret);
		ret = postProcessing(ret);
		ret = posTagging(ret);
		return ret;
	}
996897
public List<AnalyzedSentence> analyzeText(String text) throws IOException {
    List<String> sentences = sentenceTokenize(text);
    return analyzeSentences(sentences);
  }