If there is a syntactic classifier ('syntacticClassifierAbsent' is false), we will be more conservative in order to avoid conflicting with the syntactic classifier.
766533
function getEncodedLexicalClassifications(text, lexState, syntacticClassifierAbsent) {
            var offset = 0;
            var token = 0 /* Unknown */;
            var lastNonTriviaToken = 0 /* Unknown */;
            // Empty out the template stack for reuse.
            while (templateStack.length > 0) {
                templateStack.pop();
            }
            // If we're in a string literal, then prepend: "\
            // (and a newline).  That way when we lex we'll think we're still in a string literal.
            //
            // If we're in a multiline comment, then prepend: /*
            // (and a newline).  That way when we lex we'll think we're still in a multiline comment.
            switch (lexState) {
                case 3 /* InDoubleQuoteStringLiteral */:
                    text = "\"\\\n" + text;
                    offset = 3;
                    break;
                case 2 /* InSingleQuoteStringLiteral */:
                    text = "'\\\n" + text;
                    offset = 3;
                    break;
                case 1 /* InMultiLineCommentTrivia */:
                    text = "/*\n" + text;
                    offset = 3;
                    break;
                case 4 /* InTemplateHeadOrNoSubstitutionTemplate */:
                    text = "`\n" + text;
                    offset = 2;
                    break;
                case 5 /* InTemplateMiddleOrTail */:
                    text = "}\n" + text;
                    offset = 2;
                // fallthrough
                case 6 /* InTemplateSubstitutionPosition */:
                    templateStack.push(12 /* TemplateHead */);
                    break;
            }
            scanner.setText(text);
            var result = {
                endOfLineState: 0 /* None */,
                spans: []
            };
            // We can run into an unfortunate interaction between the lexical and syntactic classifier
            // when the user is typing something generic.  Consider the case where the user types:
            //
            //      Foo<number
            //
            // From the lexical classifier's perspective, 'number' is a keyword, and so the word will
            // be classified as such.  However, from the syntactic classifier's tree-based perspective
            // this is simply an expression with the identifier 'number' on the RHS of the less than
            // token.  So the classification will go back to being an identifier.  The moment the user
            // types again, number will become a keyword, then an identifier, etc. etc.
            //
            // To try to avoid this problem, we avoid classifying contextual keywords as keywords
            // when the user is potentially typing something generic.  We just can't do a good enough
            // job at the lexical level, and so well leave it up to the syntactic classifier to make
            // the determination.
            //
            // In order to determine if the user is potentially typing something generic, we use a
            // weak heuristic where we track < and > tokens.  It's a weak heuristic, but should
            // work well enough in practice.
            var angleBracketStack = 0;
            do {
                token = scanner.scan();
                if (!ts.isTrivia(token)) {
                    if ((token === 39 /* SlashToken */ || token === 61 /* SlashEqualsToken */) && !noRegexTable[lastNonTriviaToken]) {
                        if (scanner.reScanSlashToken() === 10 /* RegularExpressionLiteral */) {
                            token = 10 /* RegularExpressionLiteral */;
                        }
                    }
                    else if (lastNonTriviaToken === 21 /* DotToken */ && isKeyword(token)) {
                        token = 69 /* Identifier */;
                    }
                    else if (isKeyword(lastNonTriviaToken) && isKeyword(token) && !canFollow(lastNonTriviaToken, token)) {
                        // We have two keywords in a row.  Only treat the second as a keyword if
                        // it's a sequence that could legally occur in the language.  Otherwise
                        // treat it as an identifier.  This way, if someone writes "private var"
                        // we recognize that 'var' is actually an identifier here.
                        token = 69 /* Identifier */;
                    }
                    else if (lastNonTriviaToken === 69 /* Identifier */ &&
                        token === 25 /* LessThanToken */) {
                        // Could be the start of something generic.  Keep track of that by bumping
                        // up the current count of generic contexts we may be in.
                        angleBracketStack++;
                    }
                    else if (token === 27 /* GreaterThanToken */ && angleBracketStack > 0) {
                        // If we think we're currently in something generic, then mark that that
                        // generic entity is complete.
                        angleBracketStack--;
                    }
                    else if (token === 117 /* AnyKeyword */ ||
                        token === 132 /* StringKeyword */ ||
                        token === 130 /* NumberKeyword */ ||
                        token === 120 /* BooleanKeyword */ ||
                        token === 133 /* SymbolKeyword */) {
                        if (angleBracketStack > 0 && !syntacticClassifierAbsent) {
                            // If it looks like we're could be in something generic, don't classify this
                            // as a keyword.  We may just get overwritten by the syntactic classifier,
                            // causing a noisy experience for the user.
                            token = 69 /* Identifier */;
                        }
                    }
                    else if (token === 12 /* TemplateHead */) {
                        templateStack.push(token);
                    }
                    else if (token === 15 /* OpenBraceToken */) {
                        // If we don't have anything on the template stack,
                        // then we aren't trying to keep track of a previously scanned template head.
                        if (templateStack.length > 0) {
                            templateStack.push(token);
                        }
                    }
                    else if (token === 16 /* CloseBraceToken */) {
                        // If we don't have anything on the template stack,
                        // then we aren't trying to keep track of a previously scanned template head.
                        if (templateStack.length > 0) {
                            var lastTemplateStackToken = ts.lastOrUndefined(templateStack);
                            if (lastTemplateStackToken === 12 /* TemplateHead */) {
                                token = scanner.reScanTemplateToken();
                                // Only pop on a TemplateTail; a TemplateMiddle indicates there is more for us.
                                if (token === 14 /* TemplateTail */) {
                                    templateStack.pop();
                                }
                                else {
                                    ts.Debug.assert(token === 13 /* TemplateMiddle */, "Should have been a template middle. Was " + token);
                                }
                            }
                            else {
                                ts.Debug.assert(lastTemplateStackToken === 15 /* OpenBraceToken */, "Should have been an open brace. Was: " + token);
                                templateStack.pop();
                            }
                        }
                    }
                    lastNonTriviaToken = token;
                }
                processToken();
            } while (token !== 1 /* EndOfFileToken */);
            return result;
            function processToken() {
                var start = scanner.getTokenPos();
                var end = scanner.getTextPos();
                addResult(start, end, classFromKind(token));
                if (end >= text.length) {
                    if (token === 9 /* StringLiteral */) {
                        // Check to see if we finished up on a multiline string literal.
                        var tokenText = scanner.getTokenText();
                        if (scanner.isUnterminated()) {
                            var lastCharIndex = tokenText.length - 1;
                            var numBackslashes = 0;
                            while (tokenText.charCodeAt(lastCharIndex - numBackslashes) === 92 /* backslash */) {
                                numBackslashes++;
                            }
                            // If we have an odd number of backslashes, then the multiline string is unclosed
                            if (numBackslashes & 1) {
                                var quoteChar = tokenText.charCodeAt(0);
                                result.endOfLineState = quoteChar === 34 /* doubleQuote */
                                    ? 3 /* InDoubleQuoteStringLiteral */
                                    : 2 /* InSingleQuoteStringLiteral */;
                            }
                        }
                    }
                    else if (token === 3 /* MultiLineCommentTrivia */) {
                        // Check to see if the multiline comment was unclosed.
                        if (scanner.isUnterminated()) {
                            result.endOfLineState = 1 /* InMultiLineCommentTrivia */;
                        }
                    }
                    else if (ts.isTemplateLiteralKind(token)) {
                        if (scanner.isUnterminated()) {
                            if (token === 14 /* TemplateTail */) {
                                result.endOfLineState = 5 /* InTemplateMiddleOrTail */;
                            }
                            else if (token === 11 /* NoSubstitutionTemplateLiteral */) {
                                result.endOfLineState = 4 /* InTemplateHeadOrNoSubstitutionTemplate */;
                            }
                            else {
                                ts.Debug.fail("Only 'NoSubstitutionTemplateLiteral's and 'TemplateTail's can be unterminated; got SyntaxKind #" + token);
                            }
                        }
                    }
                    else if (templateStack.length > 0 && ts.lastOrUndefined(templateStack) === 12 /* TemplateHead */) {
                        result.endOfLineState = 6 /* InTemplateSubstitutionPosition */;
                    }
                }
            }
            function addResult(start, end, classification) {
                if (classification === 8 /* whiteSpace */) {
                    // Don't bother with whitespace classifications.  They're not needed.
                    return;
                }
                if (start === 0 && offset > 0) {
                    // We're classifying the first token, and this was a case where we prepended
                    // text.  We should consider the start of this token to be at the start of
                    // the original text.
                    start += offset;
                }
                // All our tokens are in relation to the augmented text.  Move them back to be
                // relative to the original text.
                start -= offset;
                end -= offset;
                var length = end - start;
                if (length > 0) {
                    result.spans.push(start);
                    result.spans.push(length);
                    result.spans.push(classification);
                }
            }
        }
1325896
def negated(input_words, include_nt=True):
    
    input_words = [str(w).lower() for w in input_words]
    neg_words = []
    neg_words.extend(NEGATE)
    for word in neg_words:
        if word in input_words:
            return True
    if include_nt:
        for word in input_words:
            if "n't" in word:
                return True
    if "least" in input_words:
        i = input_words.index("least")
        if i > 0 and input_words[i - 1] != "at":
            return True
    return False
440220
public final void ruleOpOther() throws RecognitionException {

        		int stackSize = keepStackSize();
        	
        try {
            // InternalXbase.g:396:2: ( ( ( rule__OpOther__Alternatives ) ) )
            // InternalXbase.g:397:2: ( ( rule__OpOther__Alternatives ) )
            {
            // InternalXbase.g:397:2: ( ( rule__OpOther__Alternatives ) )
            // InternalXbase.g:398:3: ( rule__OpOther__Alternatives )
            {
            if ( state.backtracking==0 ) {
               before(grammarAccess.getOpOtherAccess().getAlternatives()); 
            }
            // InternalXbase.g:399:3: ( rule__OpOther__Alternatives )
            // InternalXbase.g:399:4: rule__OpOther__Alternatives
            {
            pushFollow(FOLLOW_2);
            rule__OpOther__Alternatives();

            state._fsp--;
            if (state.failed) return ;

            }

            if ( state.backtracking==0 ) {
               after(grammarAccess.getOpOtherAccess().getAlternatives()); 
            }

            }


            }

        }
        catch (RecognitionException re) {
            reportError(re);
            recover(input,re);
        }
        finally {

            	restoreStackSize(stackSize);

        }
        return ;
    }
124752
def classify(self, token_type, value, lineno, column, line):
        """"""
        if token_type == self.grammar.KEYWORD_TOKEN:
            label_index = self.grammar.keyword_ids.get(value, -1)
            if label_index != -1:
                return label_index
        label_index = self.grammar.token_ids.get(token_type, -1)
        if label_index == -1:
            raise ParseError("invalid token", token_type, value, lineno, column,
                             line)
        return label_index
1269010
public void resetSynonymsNet()
    {
        synchronized (synBuffer) {
            if ( synBuffer.size() == 0 ) {
                return;
            }
            
            Iterator<String[]> it = synBuffer.iterator();
            while ( it.hasNext() ) {
                String[] synLine = it.next();
                
                ///if ( synLine[0].length() > config.MAX_LENGTH ) {
                ///    continue;
                ///}
                
                //check if the baseWord is exists or not
                IWord baseWord = get(ILexicon.CJK_WORD, synLine[0]);
                if ( baseWord == null ) {
                    continue;
                }
                
                /*
                 * first get the synonyms entry from the root map
                 * create a new one and map it with the root word if it not exists 
                */
                SynonymsEntry synEntry = rootMap.get(baseWord.getValue());
                if ( synEntry == null ) {
                    synEntry = new SynonymsEntry(baseWord);
                    rootMap.put(baseWord.getValue(), synEntry);
                    synEntry.add(baseWord); //add the base word first
                }
                
                for ( int i = 1; i < synLine.length; i++ ) {
                    String[] parts = synLine[i].split("\\s*/\\s*");
                    ///if ( parts[0].length() > config.MAX_LENGTH ) {
                    ///    continue;
                    ///}
                    
                    //check if the word is exists or not
                    //  or create a new one
                    IWord synWord = get(ILexicon.CJK_WORD, parts[0]);
                    if ( synWord == null ) {
                        synWord = new Word(parts[0], IWord.T_CJK_WORD);
                        add(ILexicon.CJK_WORD, synWord);
                    }
                    
                    //check and extends the part of speech from the baseWord
                    if ( synWord.getPartSpeech() == null ) {
                        synWord.setPartSpeech(baseWord.getPartSpeech());
                    }
                    
                    //check and extends the entity from the baseWord
                    if ( synWord.getEntity() == null ) {
                        synWord.setEntity(baseWord.getEntity());
                    }
                    
                    //check and set the pinyin
                    if ( parts.length > 1 ) {
                        synWord.setPinyin(parts[1]);
                    }
                    
                    synEntry.add(synWord);
                }
            }
            
            synBuffer.clear();
        }
    }
1177807
def tlp_classifiers(self, name_tlp, val_tlp):
        '''
        
        '''

        classifier = {
            "WHITE": 0,
            "GREEN": 1,
            "AMBER": 2,
            "RED": 3
        }

        valid = True

        if classifier[name_tlp] > val_tlp:
            valid = False

        return valid
1033055
def ensure_boolean(arg):
    
    if not isinstance(arg, bool):
        raise TypeError("expected a boolean, got %s" % type(arg).__name__)
    return arg
274723
function shouldBeIncluded(file) {
  let isIncluded = false
  let isIgnored = false
  if (hasIgnorePattern) {
    return !ignorePattern.test(file)
  }
  if (hasIncludePattern) {
    return includePattern.test(file)
  }
  return true
}
828171
private static boolean isObjectLiteralThatCanBeSkipped(JSType t) {
    t = t.restrictByNotNullOrUndefined();
    return t.isRecordType() || t.isLiteralObject();
  }
1265737
private void fixConstraints(IPattern p) {
        if (p instanceof FactPattern) {
            fixConstraints((FactPattern) p);
        } else if (p instanceof CompositeFactPattern) {
            fixConstraints((CompositeFactPattern) p);
        }
    }
1342709
private boolean match(SyntacticChunk chunk, Element element, int baseTokenIndex, Sentence sentence) {
		boolean match;
		boolean negated;
		// Sees if the mask must or not match.
		// Negated is optional, so it can be null, true or false.
		// If null, consider as false.
		 if (element.isNegated() == null) {
			match = false;
			negated = false;
		} else {
			match = element.isNegated().booleanValue();
			negated = element.isNegated().booleanValue();
		}
		for (Mask mask : element.getMask()) {
			// If the token must match the mask.
			if (!negated) {
				// If not negated, match starts as false and just one match is needed to make it true.
				if (mask.getLexemeMask() != null && mask.getLexemeMask().equalsIgnoreCase(chunk.toPlainText())) {
					match = true;
				} else if (mask.getTagMask() != null && chunk.getMorphologicalTag() != null) {
					match = match | (chunk.getMorphologicalTag().matchExact(mask.getTagMask(), false) && chunk.getSyntacticTag().match(mask.getTagMask()));
				} else if (mask.getPrimitiveMask() != null /*&& chunk.getTokens().size() > 0*/ && matchLemma(chunk.getChildChunks().get(0).getMainToken(), mask.getPrimitiveMask())) {
					match = true;
				} else if (mask.getTagReference() != null && chunk.getMorphologicalTag() != null) {
					TagMask t = RuleUtils.createTagMaskFromReferenceSyntatic(mask.getTagReference(), sentence, baseTokenIndex);
					match = match | (chunk.getMorphologicalTag().match(t, false) && (t.getSyntacticFunction() == null || chunk.getSyntacticTag().match(t)));
				} else if (mask.getOutOfBounds() != null
				     && chunk.getTokens().get(0).getLexeme().equals(OUT_OF_BOUNDS)) {
                    match = true;
                }
			} else { // The token must NOT match the mask.
				// If negated, match starts as true and just one match is needed to make it false.
				if ( mask.getLexemeMask() != null && mask.getLexemeMask().equalsIgnoreCase(chunk.toPlainText())) {
					match = false;
				} else if (mask.getTagMask() != null) {
					match = match & !(chunk.getMorphologicalTag().matchExact(mask.getTagMask(), false) && (mask.getTagMask().getSyntacticFunction() == null || chunk.getSyntacticTag().match(mask.getTagMask())));
				} else if (mask.getPrimitiveMask() != null /*&& chunk.getTokens().size() > 0*/ && matchLemma(chunk.getChildChunks().get(0).getMainToken(), mask.getPrimitiveMask())) {
                    match = false;
              }  else if (mask.getTagReference() != null) {
					TagMask t = RuleUtils.createTagMaskFromReferenceSyntatic(mask.getTagReference(), sentence, baseTokenIndex);
					match = match & !(chunk.getMorphologicalTag().match(t,false) && (t.getSyntacticFunction() == null || chunk.getSyntacticTag().match(t)));
			  } else if (mask.getOutOfBounds() != null && 
			        sentence.getSyntacticChunks().indexOf(chunk) == sentence.getSyntacticChunks().size() - 1) {
                    match = false;
              }
			}
		}
		return match;
	}
107189
def _is_not_considered_falsey(value, ignore_types=()):
    '''
    
    '''
    return isinstance(value, bool) or type(value) in ignore_types or value
640102
private boolean isRealSentence(AnalyzedSentence sentence) {
    AnalyzedTokenReadings[] tokens = sentence.getTokensWithoutWhitespace();
    if (tokens.length > 0) {
      AnalyzedTokenReadings lastToken = tokens[tokens.length - 1];
      return lastToken.hasPosTag("PKT") && StringUtils.equalsAny(lastToken.getToken(), ".", "?", "!");
    }
    return false;
  }
882125
def check_arg_compatibility(args: argparse.Namespace):
    

    if args.lhuc is not None:
        # Actually this check is a bit too strict
        check_condition(args.encoder != C.CONVOLUTION_TYPE or args.decoder != C.CONVOLUTION_TYPE,
                        "LHUC is not supported for convolutional models yet.")
        check_condition(args.decoder != C.TRANSFORMER_TYPE or C.LHUC_STATE_INIT not in args.lhuc,
                        "The %s options only applies to RNN models" % C.LHUC_STATE_INIT)

    if args.decoder_only:
        check_condition(args.decoder != C.TRANSFORMER_TYPE and args.decoder != C.CONVOLUTION_TYPE,
                        "Decoder pre-training currently supports RNN decoders only.")
500830
def _assert_type_bounds_are_not_conflicting(current_type_bound, previous_type_bound,
                                            location, match_query):
    """"""
    if all((current_type_bound is not None,
            previous_type_bound is not None,
            current_type_bound != previous_type_bound)):
        raise AssertionError(
            u'Conflicting type bounds calculated at location {}: {} vs {} '
            u'for query {}'.format(location, previous_type_bound, current_type_bound, match_query))
1048617
def tokenize(self, sentence,
                 normalized=True,
                 is_feature=False,
                 is_surface=False,
                 return_list=False,
                 func_normalizer=normalize_text):
        # type: (text_type, bool, bool, bool, bool, Callable[[str], str])->Union[List[str], TokenizedSenetence]
        
        if six.PY2 and isinstance(sentence, str):
            sentence = sentence.decode(self.string_encoding)
        else:
            pass

        # decide normalization function depending on dictType
        if func_normalizer is None and self._dictType == 'neologd' and is_neologdn_valid:
            normalized_sentence = neologdn.normalize(sentence)
        elif func_normalizer is None and self._dictType == 'neologd' and is_neologdn_valid == False:
            raise Exception("You could not call neologd dictionary bacause you do NOT install the package neologdn.")
        elif func_normalizer == normalize_text:
            normalized_sentence = normalize_text(sentence, dictionary_mode=self._dictType)
        elif func_normalizer is None:
            normalized_sentence = sentence
        else:
            normalized_sentence = func_normalizer(sentence)

        # don't delete this variable. The variable "encoded_text" protects sentence from deleting
        if six.PY2:
            encoded_text = normalized_sentence.encode(self.string_encoding)
        else:
            encoded_text = normalized_sentence

        if six.PY2:
            tokenized_objects = []
            node = self.mecabObj.parseToNode(encoded_text)
            node = node.next
            while node.next is not None:
                word_surface = node.surface.decode(self.string_encoding)

                tuple_pos, word_stem = self.__feature_parser(node.feature.decode(self.string_encoding), word_surface)

                tokenized_obj = TokenizedResult(
                    node_obj=node,
                    tuple_pos=tuple_pos,
                    word_stem=word_stem,
                    word_surface=word_surface,
                    is_feature=is_feature,
                    is_surface=is_surface
                )
                tokenized_objects.append(tokenized_obj)
                node = node.next

            tokenized_sentence = TokenizedSenetence(
                sentence=sentence,
                tokenized_objects=tokenized_objects)
        else:
            parsed_result = self.mecabObj.parse(encoded_text)
            tokenized_objects = self.__postprocess_analyzed_result(
                string_mecab_parsed_result=parsed_result,
                is_feature=is_feature,
                is_surface=is_surface
            )
            tokenized_sentence = TokenizedSenetence(
                sentence=sentence,
                tokenized_objects=tokenized_objects
            )  # type: TokenizedSenetence

        if return_list:
            return tokenized_sentence.convert_list_object()
        else:
            return tokenized_sentence
317093
private boolean isAlleleCorrect(String allele, boolean acceptAmbiguousBases) {
        if (StringUtils.isNotEmpty(allele)) {
            for (char base : allele.toCharArray()) {
                if (!isValidBase(base, acceptAmbiguousBases)) {
                    return false;
                }
            }
        }
        return true;
    }