Apply registered filters to stopwords
    :param stopwords: list
    :param language: string
    :return: filtered stopwords
19123
def apply_filters(stopwords, language):
    
    if language in _filters:
        for func in _filters[language]:
            stopwords = func(stopwords)

    for func in _filters[None]:
        stopwords = func(stopwords, language)

    return stopwords
862125
def get_nltk_builder(languages):
    
    all_stemmers = []
    all_stopwords_filters = []
    all_word_characters = set()

    for language in languages:
        if language == "en":
            # use Lunr's defaults
            all_stemmers.append(lunr.stemmer.stemmer)
            all_stopwords_filters.append(stop_word_filter)
            all_word_characters.update({r"\w"})
        else:
            stopwords, word_characters = _get_stopwords_and_word_characters(language)
            all_stemmers.append(
                Pipeline.registered_functions["stemmer-{}".format(language)]
            )
            all_stopwords_filters.append(
                generate_stop_word_filter(stopwords, language=language)
            )
            all_word_characters.update(word_characters)

    builder = Builder()
    multi_trimmer = generate_trimmer("".join(sorted(all_word_characters)))
    Pipeline.register_function(
        multi_trimmer, "lunr-multi-trimmer-{}".format("-".join(languages))
    )
    builder.pipeline.reset()

    for fn in chain([multi_trimmer], all_stopwords_filters, all_stemmers):
        builder.pipeline.add(fn)
    for fn in all_stemmers:
        builder.search_pipeline.add(fn)

    return builder
1185424
private static CharArraySet getStopwords(String stopwords) {
        List<String> stopwordsList = new ArrayList<>();
        for (String stop : stopwords.split(",")) {
            stopwordsList.add(stop.trim());
        }
        return new CharArraySet(stopwordsList, true);
    }
478313
def sanitize_words(self, words):
    
    _words = []
    for w in words:
      if isinstance(w, string_types) and not isinstance(w, unicode):
        _words.append(unicode(w, encoding="utf-8"))
      else:
        _words.append(w)
    return _words
899787
def rank(words: List[str], exclude_stopwords: bool = False) -> Counter:
    
    if not words:
        return None

    if exclude_stopwords:
        words = [word for word in words if word not in _STOPWORDS]

    return Counter(words)
1222448
def supported_language(lang):
    
    try:
      self.get_collection(lang=lang)
      return True
    except LanguageNotSupported as e:
      return False
783008
def custom_stopwords(stopwords)
      unless stopwords.is_a?(Enumerable)
        if stopwords.strip.empty?
          stopwords = []
        elsif File.exist?(stopwords)
          stopwords = File.read(stopwords).force_encoding('utf-8').split
        else
          return # Do not overwrite the default
        end
      end
      TokenFilter::Stopword::STOPWORDS[@language] = Set.new stopwords
    end
1281136
def get_languages(self):
        
        if self._languages:
            return self._languages
        self._languages = cache.get(self.PAGE_LANGUAGES_KEY % (self.pk))
        if self._languages is not None:
            return self._languages

        languages = [c['language'] for
            c in Content.objects.filter(page=self,
            type="slug").values('language')]
        # remove duplicates
        languages = sorted(set(languages))
        cache.set(self.PAGE_LANGUAGES_KEY % (self.pk), languages)
        self._languages = languages
        return languages
556505
def eliminate_repeats(text):
    ''''''
    
    bannedwords = read_file('stopwords.txt')
    alphabet = 'abcdefghijklmnopqrstuvwxyz'
    words = text.split()
    standardwords = []
    for word in words:
        newstr = ''
        for char in word:
            if char in alphabet or char in alphabet.upper():
               newstr += char
        if newstr not in standardwords and newstr != '' and newstr not in bannedwords:
            standardwords.append(newstr)
        
    return map(lambda x: x.lower(),standardwords)
425894
def msg_filter(self):
        
        if self.query:
            msg_filter = None
        else:
            msg_filter = self._request_request('msg_filter', 'all')
            available_msg_filters = {'untranslated', 'translated', 'fuzzy', 'all'}
            if msg_filter not in available_msg_filters:
                msg_filter = 'all'
        return msg_filter
902690
def get_stoplisted_unigram_corpus_and_custom(self,
                                                 custom_stoplist):
        '''
        
        '''
        if type(custom_stoplist) == str:
            custom_stoplist = [custom_stoplist]
        return self._remove_terms_from_list(set(self.get_default_stoplist())
                                            | set(w.lower() for w in custom_stoplist))
1309629
def word_hash(str, enable_stemmer = true,
                  tokenizer: Tokenizer::Whitespace,
                  token_filters: [TokenFilter::Stopword])
      if token_filters.include?(TokenFilter::Stemmer)
        unless enable_stemmer
          token_filters.reject! do |token_filter|
            token_filter == TokenFilter::Stemmer
          end
        end
      else
        token_filters << TokenFilter::Stemmer if enable_stemmer
      end
      words = tokenizer.call(str)
      token_filters.each do |token_filter|
        words = token_filter.call(words)
      end
      d = Hash.new(0)
      words.each do |word|
        d[word.intern] += 1
      end
      d
    end
164190
public void addLanguages( Iterable<QueryParser> languages ) {
        CheckArg.isNotNull(languages, "languages");
        for (QueryParser language : languages) {
            addLanguage(language);
        }
    }
1101916
def filter_queryset(self, request, queryset, view):
        
        if 'is_main_language' in request.GET:
            # TODO investigate possible error cases where page
            # does not have language
            return queryset.filter(languages__language__is_main_language=True)
        else:
            return queryset
487877
def add_filter(func, language=None):
    
    if language not in _filters:
        _filters[language] = []
    _filters[language].append(func)
941182
private function loadLanguageData(string $language = 'de')
  {
    if (\in_array($language, self::$availableLanguages, true) === false) {
      throw new StopWordsLanguageNotExists('language not supported: ' . $language);
    }

    $this->stopWords[$language] = $this->getData($language);
  }
1008765
def _removeStopwords(text_list):
    

    output_list = []

    for word in text_list:
        if word.lower() not in _stopwords:
            output_list.append(word)

    return output_list