Detecting new language elements.
769621
public function actionScan()
    {
        $this->stdout("Scanning translations...\n", Console::BOLD);
        $scanner = new Scanner();

        $items = $scanner->run();
        $this->stdout("{$items} new item(s) inserted into database.\n");
    }
1055653
def get_known_topics(self, lang):
        """"""
        return [topic['title']
                for topic in self.user_data.language_data[lang]['skills']
                if topic['learned']]
552023
def input(self, _in, out, **kwargs):
        """"""
        language_code = _re_language_code.search(_in.read()).group(
            'language_code'
        )
        _in.seek(0)  # move at the begining after matching the language
        catalog = read_po(_in)
        out.write('gettextCatalog.setStrings("{0}", '.format(language_code))
        out.write(json.dumps({
            key: value.string for key, value in catalog._messages.items()
            if key and value.string
        }))
        out.write(');')
786613
function _getPatternsForLanguage(language) {
        var patterns = language_patterns[language] || [],
            default_patterns = language_patterns[DEFAULT_LANGUAGE] || [];

        return _bypassDefaultPatterns(language) ? patterns : patterns.concat(default_patterns);
    }
1364850
private static Object[] extractExtendedFeatures(Configuration configuration, int length)
    {
        Object[] featureMap = new Object[length];

        State state = configuration.state;
        Sentence sentence = configuration.sentence;

        int b0Position = 0;
        int b1Position = 0;
        int b2Position = 0;
        int s0Position = 0;

        long svr = 0; // stack right valency
        long svl = 0; // stack left valency
        long bvl = 0; // buffer left valency

        long b0w = 0;
        long b0p = 0;

        long b1w = 0;
        long b1p = 0;

        long b2w = 0;
        long b2p = 0;

        long s0w = 0;
        long s0p = 0;
        long s0l = 0;

        long bl0p = 0;
        long bl0w = 0;
        long bl0l = 0;

        long bl1w = 0;
        long bl1p = 0;
        long bl1l = 0;

        long sr0p = 0;
        long sr0w = 0;
        long sr0l = 0;

        long sh0w = 0;
        long sh0p = 0;
        long sh0l = 0;

        long sl0p = 0;
        long sl0w = 0;
        long sl0l = 0;

        long sr1w = 0;
        long sr1p = 0;
        long sr1l = 0;

        long sh1w = 0;
        long sh1p = 0;

        long sl1w = 0;
        long sl1p = 0;
        long sl1l = 0;

        long sdl = 0;
        long sdr = 0;
        long bdl = 0;

        int[] words = sentence.getWords();
        int[] tags = sentence.getTags();

        if (0 < state.bufferSize())
        {
            b0Position = state.bufferHead();
            b0w = b0Position == 0 ? 0 : words[b0Position - 1];
            b0w += 2;
            b0p = b0Position == 0 ? 0 : tags[b0Position - 1];
            b0p += 2;
            bvl = state.leftValency(b0Position);

            int leftMost = state.leftMostModifier(state.getBufferItem(0));
            if (leftMost >= 0)
            {
                bl0p = leftMost == 0 ? 0 : tags[leftMost - 1];
                bl0p += 2;
                bl0w = leftMost == 0 ? 0 : words[leftMost - 1];
                bl0w += 2;
                bl0l = state.getDependent(leftMost);
                bl0l += 2;

                int l2 = state.leftMostModifier(leftMost);
                if (l2 >= 0)
                {
                    bl1w = l2 == 0 ? 0 : words[l2 - 1];
                    bl1w += 2;
                    bl1p = l2 == 0 ? 0 : tags[l2 - 1];
                    bl1p += 2;
                    bl1l = state.getDependent(l2);
                    bl1l += 2;
                }
            }

            if (1 < state.bufferSize())
            {
                b1Position = state.getBufferItem(1);
                b1w = b1Position == 0 ? 0 : words[b1Position - 1];
                b1w += 2;
                b1p = b1Position == 0 ? 0 : tags[b1Position - 1];
                b1p += 2;

                if (2 < state.bufferSize())
                {
                    b2Position = state.getBufferItem(2);

                    b2w = b2Position == 0 ? 0 : words[b2Position - 1];
                    b2w += 2;
                    b2p = b2Position == 0 ? 0 : tags[b2Position - 1];
                    b2p += 2;
                }
            }
        }

        if (0 < state.stackSize())
        {
            s0Position = state.stackTop();
            s0w = s0Position == 0 ? 0 : words[s0Position - 1];
            s0w += 2;
            s0p = s0Position == 0 ? 0 : tags[s0Position - 1];
            s0p += 2;
            s0l = state.getDependent(s0Position);
            s0l += 2;

            svl = state.leftValency(s0Position);
            svr = state.rightValency(s0Position);

            int leftMost = state.leftMostModifier(s0Position);
            if (leftMost >= 0)
            {
                sl0p = leftMost == 0 ? 0 : tags[leftMost - 1];
                sl0p += 2;
                sl0w = leftMost == 0 ? 0 : words[leftMost - 1];
                sl0w += 2;
                sl0l = state.getDependent(leftMost);
                sl0l += 2;
            }

            int rightMost = state.rightMostModifier(s0Position);
            if (rightMost >= 0)
            {
                sr0p = rightMost == 0 ? 0 : tags[rightMost - 1];
                sr0p += 2;
                sr0w = rightMost == 0 ? 0 : words[rightMost - 1];
                sr0w += 2;
                sr0l = state.getDependent(rightMost);
                sr0l += 2;
            }

            int headIndex = state.getHead(s0Position);
            if (headIndex >= 0)
            {
                sh0w = headIndex == 0 ? 0 : words[headIndex - 1];
                sh0w += 2;
                sh0p = headIndex == 0 ? 0 : tags[headIndex - 1];
                sh0p += 2;
                sh0l = state.getDependent(headIndex);
                sh0l += 2;
            }

            if (leftMost >= 0)
            {
                int l2 = state.leftMostModifier(leftMost);
                if (l2 >= 0)
                {
                    sl1w = l2 == 0 ? 0 : words[l2 - 1];
                    sl1w += 2;
                    sl1p = l2 == 0 ? 0 : tags[l2 - 1];
                    sl1p += 2;
                    sl1l = state.getDependent(l2);
                    sl1l += 2;
                }
            }
            if (headIndex >= 0)
            {
                if (state.hasHead(headIndex))
                {
                    int h2 = state.getHead(headIndex);
                    sh1w = h2 == 0 ? 0 : words[h2 - 1];
                    sh1w += 2;
                    sh1p = h2 == 0 ? 0 : tags[h2 - 1];
                    sh1p += 2;
                }
            }
            if (rightMost >= 0)
            {
                int r2 = state.rightMostModifier(rightMost);
                if (r2 >= 0)
                {
                    sr1w = r2 == 0 ? 0 : words[r2 - 1];
                    sr1w += 2;
                    sr1p = r2 == 0 ? 0 : tags[r2 - 1];
                    sr1p += 2;
                    sr1l = state.getDependent(r2);
                    sr1l += 2;
                }
            }
        }
        int index = 0;

        long b0wp = b0p;
        b0wp |= (b0w << 8);
        long b1wp = b1p;
        b1wp |= (b1w << 8);
        long s0wp = s0p;
        s0wp |= (s0w << 8);
        long b2wp = b2p;
        b2wp |= (b2w << 8);

        /**
         * From single words
         */
        if (s0w != 1)
        {
            featureMap[index++] = s0wp;
            featureMap[index++] = s0w;
        }
        else
        {
            featureMap[index++] = null;
            featureMap[index++] = null;
        }
        featureMap[index++] = s0p;

        if (b0w != 1)
        {
            featureMap[index++] = b0wp;
            featureMap[index++] = b0w;
        }
        else
        {
            featureMap[index++] = null;
            featureMap[index++] = null;
        }
        featureMap[index++] = b0p;

        if (b1w != 1)
        {
            featureMap[index++] = b1wp;
            featureMap[index++] = b1w;
        }
        else
        {
            featureMap[index++] = null;
            featureMap[index++] = null;
        }
        featureMap[index++] = b1p;

        if (b2w != 1)
        {
            featureMap[index++] = b2wp;
            featureMap[index++] = b2w;
        }
        else
        {
            featureMap[index++] = null;
            featureMap[index++] = null;
        }
        featureMap[index++] = b2p;

        /**
         * from word pairs
         */
        if (s0w != 1 && b0w != 1)
        {
            featureMap[index++] = (s0wp << 28) | b0wp;
            featureMap[index++] = (s0wp << 20) | b0w;
            featureMap[index++] = (s0w << 28) | b0wp;
        }
        else
        {
            featureMap[index++] = null;
            featureMap[index++] = null;
            featureMap[index++] = null;
        }

        if (s0w != 1)
        {
            featureMap[index++] = (s0wp << 8) | b0p;
        }
        else
        {
            featureMap[index++] = null;
        }

        if (b0w != 1)
        {
            featureMap[index++] = (s0p << 28) | b0wp;
        }
        else
        {
            featureMap[index++] = null;
        }

        if (s0w != 1 && b0w != 1)
        {
            featureMap[index++] = (s0w << 20) | b0w;
        }
        else
        {
            featureMap[index++] = null;
        }
        featureMap[index++] = (s0p << 8) | b0p;
        featureMap[index++] = (b0p << 8) | b1p;

        /**
         * from three words
         */
        featureMap[index++] = (b0p << 16) | (b1p << 8) | b2p;
        featureMap[index++] = (s0p << 16) | (b0p << 8) | b1p;
        featureMap[index++] = (sh0p << 16) | (s0p << 8) | b0p;
        featureMap[index++] = (s0p << 16) | (sl0p << 8) | b0p;
        featureMap[index++] = (s0p << 16) | (sr0p << 8) | b0p;
        featureMap[index++] = (s0p << 16) | (b0p << 8) | bl0p;

        /**
         * distance
         */
        long distance = 0;
        if (s0Position > 0 && b0Position > 0)
            distance = Math.abs(b0Position - s0Position);
        if (s0w != 1)
        {
            featureMap[index++] = s0w | (distance << 20);
        }
        else
        {
            featureMap[index++] = null;
        }
        featureMap[index++] = s0p | (distance << 8);
        if (b0w != 1)
        {
            featureMap[index++] = b0w | (distance << 20);
        }
        else
        {
            featureMap[index++] = null;
        }
        featureMap[index++] = b0p | (distance << 8);
        if (s0w != 1 && b0w != 1)
        {
            featureMap[index++] = s0w | (b0w << 20) | (distance << 40);
        }
        else
        {
            featureMap[index++] = null;
        }
        featureMap[index++] = s0p | (b0p << 8) | (distance << 28);

        /**
         * Valency information
         */
        if (s0w != 1)
        {
            featureMap[index++] = s0w | (svr << 20);
        }
        else
        {
            featureMap[index++] = null;
        }
        featureMap[index++] = s0p | (svr << 8);
        if (s0w != 1)
        {
            featureMap[index++] = s0w | (svl << 20);
        }
        else
        {
            featureMap[index++] = null;
        }
        featureMap[index++] = s0p | (svl << 8);
        if (b0w != 1)
        {
            featureMap[index++] = b0w | (bvl << 20);
        }
        else
        {
            featureMap[index++] = null;
        }
        featureMap[index++] = b0p | (bvl << 8);

        /**
         * Unigrams
         */
        if (sh0w != 1)
        {
            featureMap[index++] = sh0w;
        }
        else
        {
            featureMap[index++] = null;
        }
        featureMap[index++] = sh0p;
        featureMap[index++] = s0l;
        if (sl0w != 1)
        {
            featureMap[index++] = sl0w;
        }
        else
        {
            featureMap[index++] = null;
        }
        featureMap[index++] = sl0p;
        featureMap[index++] = sl0l;
        if (sr0w != 1)
        {
            featureMap[index++] = sr0w;
        }
        else
        {
            featureMap[index++] = null;
        }
        featureMap[index++] = sr0p;
        featureMap[index++] = sr0l;
        if (bl0w != 1)
        {
            featureMap[index++] = bl0w;
        }
        else
        {
            featureMap[index++] = null;
        }
        featureMap[index++] = bl0p;
        featureMap[index++] = bl0l;

        /**
         * From third order features
         */
        if (sh1w != 1)
        {
            featureMap[index++] = sh1w;
        }
        else
        {
            featureMap[index++] = null;
        }
        featureMap[index++] = sh1p;
        featureMap[index++] = sh0l;
        if (sl1w != 1)
        {
            featureMap[index++] = sl1w;
        }
        else
        {
            featureMap[index++] = null;
        }
        featureMap[index++] = sl1p;
        featureMap[index++] = sl1l;
        if (sr1w != 1)
        {
            featureMap[index++] = sr1w;
        }
        else
        {
            featureMap[index++] = null;
        }
        featureMap[index++] = sr1p;
        featureMap[index++] = sr1l;
        if (bl1w != 1)
        {
            featureMap[index++] = bl1w;
        }
        else
        {
            featureMap[index++] = null;
        }
        featureMap[index++] = bl1p;
        featureMap[index++] = bl1l;
        featureMap[index++] = s0p | (sl0p << 8) | (sl1p << 16);
        featureMap[index++] = s0p | (sr0p << 8) | (sr1p << 16);
        featureMap[index++] = s0p | (sh0p << 8) | (sh1p << 16);
        featureMap[index++] = b0p | (bl0p << 8) | (bl1p << 16);

        /**
         * label set
         */
        if (s0Position >= 0)
        {
            sdl = state.leftDependentLabels(s0Position);
            sdr = state.rightDependentLabels(s0Position);
        }

        if (b0Position >= 0)
        {
            bdl = state.leftDependentLabels(b0Position);
        }

        if (s0w != 1)
        {
            featureMap[index++] = (s0w + "|" + sdr);
        }
        else
        {
            featureMap[index++] = null;
        }
        featureMap[index++] = (s0p + "|" + sdr);
        if (s0w != 1)
        {
            featureMap[index++] = s0w + "|" + sdl;
        }
        else
        {
            featureMap[index++] = null;
        }
        featureMap[index++] = (s0p + "|" + sdl);
        if (b0w != 1)
        {
            featureMap[index++] = (b0w + "|" + bdl);
        }
        else
        {
            featureMap[index++] = null;
        }
        featureMap[index++] = (b0p + "|" + bdl);
        return featureMap;
    }
855664
function highlight (code, lang) {
  if (!lang) {
    return code
  }

  var isDiff = DIFF_REGEXP.test(lang)

  // Remove the diff suffix. E.g. "javascript.diff".
  lang = lang.replace(DIFF_REGEXP, '')

  // Ignore unknown languages.
  if (!isDiff && !supported(lang)) {
    return code
  }

  return isDiff ? diff(code, lang) : hljs.highlight(lang, code).value
}
190759
function generateNewQueryString(language) {
    var url = parseURL(location.search);
    if (url.language) {
      url.language = language;
      return stringifyURL(url);
    }
    return language;
  }
519915
private function gatherTranslatableStrings()
    {
        $this->translatables = [];

        $this->scanTwigFiles();
        $this->scanPhpFiles();

        ksort($this->translatables);
    }
595295
def morphemes(args):
  """"""
  morfessor = load_morfessor_model(lang=args.lang)
  for l in args.input:
    words = l.strip().split()
    morphemes = [(w, u"_".join(morfessor.viterbi_segment(w)[0])) for w in words]
    line_annotations = [u"{:<16}{:<5}".format(w,p) for w, p in morphemes]
    _print(u"\n".join(line_annotations))
    _print(u"")
382373
def _language(self, item):
        """"""

        response = item['spider_response'].body
        root = html.fromstring(response)

        # Check for lang-attributes
        lang = root.get('lang')

        if lang is None:
            lang = root.get('xml:lang')

        # Check for general meta tags
        if lang is None:
            meta = root.cssselect('meta[name="language"]')
            if len(meta) > 0:
                lang = meta[0].get('content')

        # Check for open graph tags
        if lang is None:
            meta = root.cssselect('meta[property="og:locale"]')
            if len(meta) > 0:
                lang = meta[0].get('content')

        # Look for <article> elements and inspect the one with the largest payload with langdetect
        if lang is None:
            article_list = []
            for article in root.xpath('//article'):
                article_list.append(re.sub(r'\s+', ' ', article.text_content().strip()))
                if len(article_list) > 0:
                    lang = detect(max(article_list))

        # Analyze the whole body with langdetect
        if lang is None:
            try:
                lang = detect(root.text_content().strip())
            except LangDetectException:
                pass

        # Try to normalize output
        if lang is not None:
            # First search for suitable locale in the original output
            matches = self.langcode_pattern.search(lang)
            if matches is not None:
                lang = matches.group(0)
            else:
                # If no match was found, normalize the original output and search again
                normalized = locale.normalize(re.split(r'\s|;|,', lang.strip())[0])
                matches = self.langcode_pattern.search(normalized)
                if matches is not None:
                    lang = matches.group(0)

        return lang
523356
function (language) {
  for (var i in config.languages) {
    if (config.languages[i] === language) {return true;}
  }
  //
  return false;
}
24383
def title_translations(self, key, value):
    """"""
    return {
        'language': langdetect.detect(value.get('a')),
        'source': value.get('9'),
        'subtitle': value.get('b'),
        'title': value.get('a'),
    }
927240
def detect(self, text, proxies=None, format="plain"):
    
    data = {
      "text": text,
      "format": format,
      "key": self.api_key,
    }
    try:
      response = requests.post(self.url("detect"), data=data, proxies=proxies)
    except ConnectionError:
      raise YandexTranslateException(self.error_codes[503])
    except ValueError:
      raise YandexTranslateException(response)
    else:
      response = response.json()
    language = response.get("lang", None)
    status_code = response.get("code", 200)
    if status_code != 200:
      raise YandexTranslateException(status_code)
    elif not language:
      raise YandexTranslateException(501)
    return language
760533
def get_language():
    """"""

    global sensor_graph, statement

    if sensor_graph is not None:
        return sensor_graph

    _create_primitives()
    _create_simple_statements()
    _create_block_bnf()

    sensor_graph = ZeroOrMore(statement) + StringEnd()
    sensor_graph.ignore(comment)

    return sensor_graph
785326
def match_lang(self, el, langs):
        """"""

        match = False
        has_ns = self.supports_namespaces()
        root = self.root
        has_html_namespace = self.has_html_namespace

        # Walk parents looking for `lang` (HTML) or `xml:lang` XML property.
        parent = el
        found_lang = None
        last = None
        while not found_lang:
            has_html_ns = self.has_html_ns(parent)
            for k, v in self.iter_attributes(parent):
                attr_ns, attr = self.split_namespace(parent, k)
                if (
                    ((not has_ns or has_html_ns) and (util.lower(k) if not self.is_xml else k) == 'lang') or
                    (
                        has_ns and not has_html_ns and attr_ns == NS_XML and
                        (util.lower(attr) if not self.is_xml and attr is not None else attr) == 'lang'
                    )
                ):
                    found_lang = v
                    break
            last = parent
            parent = self.get_parent(parent, no_iframe=self.is_html)

            if parent is None:
                root = last
                has_html_namespace = self.has_html_ns(root)
                parent = last
                break

        # Use cached meta language.
        if not found_lang and self.cached_meta_lang:
            for cache in self.cached_meta_lang:
                if root is cache[0]:
                    found_lang = cache[1]

        # If we couldn't find a language, and the document is HTML, look to meta to determine language.
        if found_lang is None and (not self.is_xml or (has_html_namespace and root.name == 'html')):
            # Find head
            found = False
            for tag in ('html', 'head'):
                found = False
                for child in self.get_children(parent, no_iframe=self.is_html):
                    if self.get_tag(child) == tag and self.is_html_tag(child):
                        found = True
                        parent = child
                        break
                if not found:  # pragma: no cover
                    break

            # Search meta tags
            if found:
                for child in parent:
                    if self.is_tag(child) and self.get_tag(child) == 'meta' and self.is_html_tag(parent):
                        c_lang = False
                        content = None
                        for k, v in self.iter_attributes(child):
                            if util.lower(k) == 'http-equiv' and util.lower(v) == 'content-language':
                                c_lang = True
                            if util.lower(k) == 'content':
                                content = v
                            if c_lang and content:
                                found_lang = content
                                self.cached_meta_lang.append((root, found_lang))
                                break
                    if found_lang:
                        break
                if not found_lang:
                    self.cached_meta_lang.append((root, False))

        # If we determined a language, compare.
        if found_lang:
            for patterns in langs:
                match = False
                for pattern in patterns:
                    if pattern.match(found_lang):
                        match = True
                if not match:
                    break

        return match
683614
def _load_replacement_patterns(self):
        """"""
        if self.language == 'latin':
            warnings.warn(
                        "LemmaReplacer is deprecated and will soon be removed from CLTK. Please use the BackoffLatinLemmatizer at cltk.lemmatize.latin.backoff.",
                        DeprecationWarning,
                        stacklevel=2)
            rel_path = os.path.join('~','cltk_data',
                                    self.language,
                                    'model','latin_models_cltk',
                                    'lemmata','latin_lemmata_cltk.py')
            path = os.path.expanduser(rel_path)
            #logger.info('Loading lemmata. This may take a minute.')
            loader = importlib.machinery.SourceFileLoader('latin_lemmata_cltk', path)

        elif self.language == 'greek':
            rel_path = os.path.join('~','cltk_data',
                                    self.language,
                                    'model','greek_models_cltk',
                                    'lemmata','greek_lemmata_cltk.py')
            path = os.path.expanduser(rel_path)
            #logger.info('Loading lemmata. This may take a minute.')
            loader = importlib.machinery.SourceFileLoader('greek_lemmata_cltk', path)
        module = loader.load_module()
        lemmata = module.LEMMATA
        return lemmata
631481
def _extract_language(self):
        

        if self.language:
            return

        found = False

        for pattern in self.config.language:
            for item in self.parsed_tree.xpath(pattern):
                stripped_language = item.strip()

                if stripped_language:
                    self.language = stripped_language
                    LOGGER.info(u'Language extracted: %s.', stripped_language,
                                extra={'siteconfig': self.config.host})
                    found = True
                    break

            if found:
                break