Helper function detects a certain clause in tag tokens list.
    Returns its value.
515117
def detect_clause(parser, clause_name, tokens, as_filter_expr=True):
    
    if clause_name in tokens:
        t_index = tokens.index(clause_name)
        clause_value = tokens[t_index + 1]
        if as_filter_expr:
            clause_value = parser.compile_filter(clause_value)
        del tokens[t_index:t_index + 2]
    else:
        clause_value = None
    return clause_value
904060
function checkAtruler(i) {
  const start = i;
  let l;

  if (i >= tokensLength) return 0;

  if (l = checkAtkeyword(i)) i += l;
  else return 0;

  if (l = checkTsets(i)) i += l;

  if (l = checkAtrulers(i)) i += l;
  else return 0;

  return i - start;
}
9310
def _token_list_matches_tree(self, token_list):
        
        try:
            parsed_selector = cssselect.parse(
                ''.join(token.as_css() for token in token_list))[0]

            return bool(
                self.tree.xpath(
                    self.xpath_translator.selector_to_xpath(parsed_selector)))
        except:
            # On error, assume the selector matches the tree
            return True
1082567
def _scan_for_tokens(contents):
    """"""
    # Regexes are in priority order. Changing the order may alter the
    # behavior of the lexer
    scanner = re.Scanner([
        # Things inside quotes
        (r"(?<![^\s\(])([\"\'])(?:(?=(\\?))\2.)*?\1(?![^\s\)])",
         lambda s, t: (TokenType.QuotedLiteral, t)),
        # Numbers on their own
        (r"(?<![^\s\(])-?[0-9]+(?![^\s\)\(])", lambda s, t: (TokenType.Number,
                                                             t)),
        # Left Paren
        (r"\(", lambda s, t: (TokenType.LeftParen, t)),
        # Right Paren
        (r"\)", lambda s, t: (TokenType.RightParen, t)),
        # Either a valid function name or variable name.
        (r"(?<![^\s\(])[a-zA-z_][a-zA-Z0-9_]*(?![^\s\)\(])",
         lambda s, t: (TokenType.Word, t)),
        # Variable dereference.
        (r"(?<![^\s\(])\${[a-zA-z_][a-zA-Z0-9_]*}(?![^\s\)])",
         lambda s, t: (TokenType.Deref, t)),
        # Newline
        (r"\n", lambda s, t: (TokenType.Newline, t)),
        # Whitespace
        (r"\s+", lambda s, t: (TokenType.Whitespace, t)),
        # The beginning of a double-quoted string, terminating at end of line
        (r"(?<![^\s\(\\])[\"]([^\"]|\\[\"])*$",
         lambda s, t: (TokenType.BeginDoubleQuotedLiteral, t)),
        # The end of a double-quoted string
        (r"[^\s]*(?<!\\)[\"](?![^\s\)])",
         lambda s, t: (TokenType.EndDoubleQuotedLiteral, t)),
        # The beginning of a single-quoted string, terminating at end of line
        (r"(?<![^\s\(\\])[\']([^\']|\\[\'])*$",
         lambda s, t: (TokenType.BeginSingleQuotedLiteral, t)),
        # The end of a single-quoted string
        (r"[^\s]*(?<!\\)[\'](?![^\s\)])",
         lambda s, t: (TokenType.EndSingleQuotedLiteral, t)),
        # Begin-RST Comment Block
        (r"#.rst:$", lambda s, t: (TokenType.BeginRSTComment, t)),
        # Begin Inline RST
        (r"#\[=*\[.rst:$", lambda s, t: (TokenType.BeginInlineRST, t)),
        # End Inline RST
        (r"#\]=*\]$", lambda s, t: (TokenType.EndInlineRST, t)),
        # Comment
        (r"#", lambda s, t: (TokenType.Comment, t)),
        # Catch-all for literals which are compound statements.
        (r"([^\s\(\)]+|[^\s\(]*[^\)]|[^\(][^\s\)]*)",
         lambda s, t: (TokenType.UnquotedLiteral, t))
    ])

    tokens_return = []

    lines = contents.splitlines(True)
    lineno = 0
    for line in lines:
        lineno += 1
        col = 1

        tokens, remaining = scanner.scan(line)
        if remaining != "":
            msg = "Unknown tokens found on line {0}: {1}".format(lineno,
                                                                 remaining)
            raise RuntimeError(msg)

        for token_type, token_contents in tokens:
            tokens_return.append(Token(type=token_type,
                                       content=token_contents,
                                       line=lineno,
                                       col=col))

            col += len(token_contents)

    return tokens_return
254061
function parseArgumentList() {
                    var token, expr, args = [];

                    while (true) {
                        expr = parseExpression();
                        if (typeof expr === 'undefined') {
                            break;
                        }
                        args.push(expr);
                        token = lexer.peek();
                        if (!matchOp(token, ',')) {
                            break;
                        }
                        lexer.next();
                    }

                    return args;
                }
978296
public void checkTags(Doc doc, Tag[] tags, boolean areInlineTags) {
        if (tags == null) {
            return;
        }
        Taglet taglet;
        for (int i = 0; i < tags.length; i++) {
            String name = tags[i].name();
            if (name.length() > 0 && name.charAt(0) == '@') {
                name = name.substring(1, name.length());
            }
            if (! (standardTags.contains(name) || customTags.containsKey(name))) {
                if (standardTagsLowercase.contains(StringUtils.toLowerCase(name))) {
                    message.warning(tags[i].position(), "doclet.UnknownTagLowercase", tags[i].name());
                    continue;
                } else {
                    message.warning(tags[i].position(), "doclet.UnknownTag", tags[i].name());
                    continue;
                }
            }
            //Check if this tag is being used in the wrong location.
            if ((taglet = customTags.get(name)) != null) {
                if (areInlineTags && ! taglet.isInlineTag()) {
                    printTagMisuseWarn(taglet, tags[i], "inline");
                }
                if ((doc instanceof RootDoc) && ! taglet.inOverview()) {
                    printTagMisuseWarn(taglet, tags[i], "overview");
                } else if ((doc instanceof PackageDoc) && ! taglet.inPackage()) {
                    printTagMisuseWarn(taglet, tags[i], "package");
                } else if ((doc instanceof ClassDoc) && ! taglet.inType()) {
                    printTagMisuseWarn(taglet, tags[i], "class");
                } else if ((doc instanceof ConstructorDoc) && ! taglet.inConstructor()) {
                    printTagMisuseWarn(taglet, tags[i], "constructor");
                } else if ((doc instanceof FieldDoc) && ! taglet.inField()) {
                    printTagMisuseWarn(taglet, tags[i], "field");
                } else if ((doc instanceof MethodDoc) && ! taglet.inMethod()) {
                    printTagMisuseWarn(taglet, tags[i], "method");
                }
            }
        }
    }
70024
private boolean match(Token token, Element element, int baseTokenIndex, Sentence sentence) {
		boolean match;
		boolean negated;
		// Sees if the mask must or not match.
		// Negated is optional, so it can be null, true or false.
		// If null, consider as false.
		if (element.isNegated() == null) {
			match = false;
			negated = false;
		} else {
			match = element.isNegated().booleanValue();
			negated = element.isNegated().booleanValue();
		}
		for (Mask mask : element.getMask()) {
			// If the token must match the mask.
			if (!negated) {
				// If not negated, match starts as false and just one match is needed to make it true.
				if (mask.getLexemeMask() != null && mask.getLexemeMask().equalsIgnoreCase(token.getLexeme())) {
					match = true;
				} else if (mask.getPrimitiveMask() != null && matchLemma(token, mask.getPrimitiveMask())) {
					match = true;
				} else if (mask.getTagMask() != null && token.getMorphologicalTag() != null) {
					match = match | token.getMorphologicalTag().matchExact(mask.getTagMask(), false);
				} else if (mask.getTagReference() != null && token.getMorphologicalTag() != null) {
					match = match | token.getMorphologicalTag().match(RuleUtils.createTagMaskFromReference(mask.getTagReference(), sentence, baseTokenIndex), false);
				} else if (mask.getOutOfBounds() != null && (baseTokenIndex == 0 || baseTokenIndex == sentence.getTokens().size() -1)) {
					match = false;
				}
			} else { // The token must NOT match the mask.
				// If negated, match starts as true and just one match is needed to make it false.
				if (mask.getLexemeMask() != null && mask.getLexemeMask().equalsIgnoreCase(token.getLexeme())) {
					match = false;
				} else if (mask.getPrimitiveMask() != null && matchLemma(token, mask.getPrimitiveMask())) {
                  match = false;
                } else if (mask.getTagMask() != null && token!=null && token.getMorphologicalTag() != null) {
					match = match & !token.getMorphologicalTag().matchExact(mask.getTagMask(),false);
				} else if (mask.getTagReference() != null && token!=null && token.getMorphologicalTag() != null) {
					match = match & !token.getMorphologicalTag().match(RuleUtils.createTagMaskFromReference(mask.getTagReference(), sentence, baseTokenIndex), false);
				} else if (mask.getOutOfBounds() != null && (baseTokenIndex == 0 || baseTokenIndex == sentence.getTokens().size() -1)) {
					match = false;
				}
			}
		}
		return match;
	}
468450
def get_tag_value(resource, tag, utf_8=False):
        """"""

        tags = {k.lower(): v for k, v in resource.get('tags', {}).items()}
        value = tags.get(tag, False)

        if value is not False:
            if utf_8:
                value = value.encode('utf8').decode('utf8')
        return value
747469
def tag(self, sentence, tokenize=True):
        
        #: Do not process empty strings (Issue #3)
        if sentence.strip() == "":
            return []
        #: Do not process strings consisting of a single punctuation mark (Issue #4)
        elif sentence.strip() in PUNCTUATION:
            if self.include_punc:
                _sym = sentence.strip()
                if _sym in tuple('.?!'):
                    _tag = "."
                else:
                    _tag = _sym
                return [(_sym, _tag)]
            else:
                return []
        if tokenize:
            _tokenized = " ".join(self.tokenizer.tokenize(sentence))
            sentence = _tokenized
        # Sentence is tokenized before it is passed on to pattern.de.tag
        # (i.e. it is either submitted tokenized or if )
        _tagged = pattern_tag(sentence, tokenize=False,
                              encoding=self.encoding,
                              tagset=self.tagset)
        if self.include_punc:
            return _tagged
        else:
            _tagged = [
                (word, t) for word, t in _tagged if not PUNCTUATION_REGEX.match(
                    unicode(t))]
            return _tagged
799130
def get_tags(self, tagtype):
        '''  '''
        return [t for t in self.__tags if t.tagtype == tagtype]
958151
def parse_expression(clause):
    
    if isinstance(clause, Expression):
        return clause
    elif hasattr(clause, "getName") and clause.getName() != "field":
        if clause.getName() == "nested":
            return AttributeSelection.from_statement(clause)
        elif clause.getName() == "function":
            return SelectFunction.from_statement(clause)
        else:
            return Value(resolve(clause[0]))
    else:
        return Field(clause[0])
988719
def _find_any_text(self, tag):
        
        text = ''
        if tag is not None:
            text = six.text_type(tag)
            text = re.sub(r'(<[^>]+>)', '', text)
            text = re.sub(r'\s', ' ', text)
            text = text.strip()
        return text
125300
function getValueQuery(tagInfo) {
        var query;
        if (tagInfo.token.string === "=") {
            return "";
        }
        // Remove quotation marks in query.
        query = tagInfo.token.string.substr(1, tagInfo.offset - 1);

        // Get the last option to use as a query to support multiple options.
        return query.split(/\s+/).slice(-1)[0];
    }
659075
def tag_clauses(self):
        """"""
        if not self.is_tagged(CLAUSE_ANNOTATION):
            self.tag_clause_annotations()

        def from_sentence(words):
            """Function that extracts clauses from a signle sentence."""
            clauses = defaultdict(list)
            start = words[0][START]
            end = words[0][END]
            clause = words[0][CLAUSE_IDX]
            for word in words:
                if word[CLAUSE_IDX] != clause:
                    clauses[clause].append((start, end))
                    start, clause = word[START], word[CLAUSE_IDX]
                end = word[END]
            clauses[clause].append((start, words[-1][END]))
            clauses = [(key, {START: [s for s, e in clause], END: [e for s, e in clause]}) for key, clause in clauses.items()]
            return [v for k, v in sorted(clauses)]

        clauses = []
        sentences = self.divide()
        for sentence in sentences:
            clauses.extend(from_sentence(sentence))

        self[CLAUSES] = clauses
        return self
593699
private boolean find(Clause clause, char[] chars, int[] cursor) {
    int start = cursor[0];
    int end = cursor[1];
    while (end - start >= clause.minlen)
      if (matchClause(clause, chars, cursor))
        return true;
      else
        cursor[0] = ++start;
    return false;
  }
517610
private void checkForTokens( String content, int startOffset, int endOffset )
  {
    while( startOffset <= endOffset )
    {
      // skip the delimiters to find the start of a new token
      while( isDelimiter( content.substring( startOffset, startOffset + 1 ) ) )
      {
        if( isError( startOffset ) )
        {
          setCharacterAttributes( startOffset, 1, _error, true );
        }
        else if( isWarning( startOffset ) )
        {
          setCharacterAttributes( startOffset, 1, _warning, true );
        }

        if( startOffset < endOffset )
        {
          startOffset++;
        }
        else
        {
          return;
        }
      }
      // Extract and process the entire token
      if( isQuoteDelimiter( content.substring( startOffset, startOffset + 1 ) ) )
      {
        startOffset = getQuoteToken( content, startOffset, endOffset );
      }
      else if( isCharQuoteDelimiter( content.substring( startOffset, startOffset + 1 ) ) )
      {
        startOffset = getQuoteToken( content, startOffset, endOffset );
      }
      else
      {
        startOffset = getOtherToken( content, startOffset, endOffset );
      }
    }
  }
1312021
def tag(self, tokens):
        """"""
        # Lazy load model first time we tag
        if not self.classes:
            self.load(self.model)
        prev, prev2 = self.START
        tags = []
        for i, token in enumerate(tokens):
            tag = self.tagdict.get(token)
            if not tag:
                features = self._get_features(i, tokens, prev, prev2)
                tag = self.perceptron.predict(features)
            tags.append((token, tag))
            prev2 = prev
            prev = tag
        return tags