sentences: an array of annotated sentences A sentence is an array of annotated tokens A token is an object with (token, tag, testTag, ruleList)
170791
function Corpus(data, typeOfCorpus, SentenceClass) {
  this.wordCount = 0;
  this.sentences = [];
  if (data) {
    // For other types of corpora add a case here and supply a parsing method
    switch (typeOfCorpus) {
      case BROWN:
        this.parseBrownCorpus(data, SentenceClass);
        break;
      case JSON:
        // Assume it is a JSON object of a corpus
        var that = this;
        data.sentences.forEach(function(s) {
          var taggedSentence = new SentenceClass(s.taggedWords);
          that.sentences.push(taggedSentence);
          that.wordCount += s.taggedWords.length;
        });
        break;
    }
  }
}
99196
def parse_sentence(obj: dict) -> BioCSentence:
    """"""
    sentence = BioCSentence()
    sentence.offset = obj['offset']
    sentence.infons = obj['infons']
    sentence.text = obj['text']
    for annotation in obj['annotations']:
        sentence.add_annotation(parse_annotation(annotation))
    for relation in obj['relations']:
        sentence.add_relation(parse_relation(relation))
    return sentence
1317262
function validateSentence(parsed) {
  const sentences = parsed.sentences();
  return [
    !isSingleSentence(sentences) ? 'Must be one and only one sentence.' : null,
    !isNotTooShort(parsed) ? 'Must be at least two words long.' : null,
    !isNotPunctuated(sentences) ? 'Must not have any end punctuation.' : null,
  ].filter(Boolean);
}
501607
def _tokenize(sentence):
    ''''''

    _tokens = nltk.word_tokenize(sentence)
    tokens = [stemmer.stem(tk) for tk in _tokens]
    return tokens
63015
def main(argv=None):
    ''''''

    t = CrfTokenizer()
    print t.tokenize("This is a sentence.")
    print t.tokenize("Buy???This...Now!!!")
    print t.tokenize("The <bold>only</bold> source.")
    print t.tokenize("The<bold>only</bold>source.")
    print t.tokenize("Big&gt;little.")
    print t.tokenize("Big & little.")
    print t.tokenize("blond&curly.")
    print t.tokenize("&brokenHtml")
    t.setGroupPunctuation(True)
    t.setRecognizeHtmlTags(True)
    t.setRecognizeHtmlEntities(True)
    print t.tokenize("Buy???This...Now!!!")
    print t.tokenize("The <bold>only</bold> source.")
    print t.tokenize("The<bold>only</bold>source.")
    print t.tokenize("Big&gt;little.")
    print t.tokenize("Big & little.")
    print t.tokenize("blond&curly.")
    print t.tokenize("&brokenHtml")
    t.setSkipHtmlTags(True)
    t.setSkipHtmlEntities(True)
    print t.tokenize("Buy???This...Now!!!")
    print t.tokenize("The <bold>only</bold> source.")
    print t.tokenize("The<bold>only</bold>source.")
    print t.tokenize("Big&gt;little.")
    print t.tokenize("Big & little.")
    print t.tokenize("blond&curly.")
    print t.tokenize("&brokenHtml")
    t.setTokenPrefix("X:")
    print t.tokenize("Tokenize with prefixes.")
    t.setTokenPrefix(None)
    print t.tokenize("No more  prefixes.")
    t.setRecognizePunctuation(False)
    print t.tokenize("This is a sentence.")
    print t.tokenize("Buy???This...Now!!!")
    print t.tokenize("The <bold>only</bold> source.")
    print t.tokenize("The<bold>only</bold>source.")
    print t.tokenize("Big&gt;little.")
    print t.tokenize("Big & little.")
    print t.tokenize("blond&curly.")
    print t.tokenize("&brokenHtml")
    print t.tokenize("A line break goes here\n\t \rand a new line starts")
    t.setRecognizeLinebreaks(True)
    print t.tokenize("A line break goes here\n\r \rand a new line starts")
428195
public static SpanFeatureAnnotation annotate(AnnotatedSentence sentence,
      FeatureVectorGenerator<StringContext> featureGenerator) {
    int numTerminals = sentence.size();
    Tensor[] featureVectors = new Tensor[numTerminals * (numTerminals + 1) / 2];
    for (int i = 0; i < numTerminals; i++) {
      for (int j = i; j < numTerminals; j++) {
        int spanIndex = getSpanIndex(i, j, numTerminals);
        featureVectors[spanIndex] = featureGenerator.apply(
            new StringContext(i, j, sentence));
      }
    }
    return new SpanFeatureAnnotation(featureVectors, numTerminals);
  }
1352996
private function ANNOTATION() {
        $text = '';
        do {
            $text .= $this->ch;
            $this->consume();
        } while ($this->isLETTER());

        $tokenType = null;
        switch ($text) {
            case '@var':
                $tokenType = Token::T_VAR;
                break;
            case '@get':
            case '@post':
            case '@request':
            case '@session':
            case '@config':
            case '@service':
            case '@currentEvent':
                $tokenType = Token::T_PROPERTY_INJECTOR;
                break;
            case '@keepDefault':
            case '@queued':
            case '@queue':
                $tokenType = Token::T_ACTION;
                break;
            default:
                $tokenType = Token::T_UNKNOWN_ANNOTATION;
        }
        return new Token($tokenType, $text);
    }
747469
def tag(self, sentence, tokenize=True):
        
        #: Do not process empty strings (Issue #3)
        if sentence.strip() == "":
            return []
        #: Do not process strings consisting of a single punctuation mark (Issue #4)
        elif sentence.strip() in PUNCTUATION:
            if self.include_punc:
                _sym = sentence.strip()
                if _sym in tuple('.?!'):
                    _tag = "."
                else:
                    _tag = _sym
                return [(_sym, _tag)]
            else:
                return []
        if tokenize:
            _tokenized = " ".join(self.tokenizer.tokenize(sentence))
            sentence = _tokenized
        # Sentence is tokenized before it is passed on to pattern.de.tag
        # (i.e. it is either submitted tokenized or if )
        _tagged = pattern_tag(sentence, tokenize=False,
                              encoding=self.encoding,
                              tagset=self.tagset)
        if self.include_punc:
            return _tagged
        else:
            _tagged = [
                (word, t) for word, t in _tagged if not PUNCTUATION_REGEX.match(
                    unicode(t))]
            return _tagged
159071
def sentence_ends(self):
        """"""
        if not self.is_tagged(SENTENCES):
            self.tokenize_sentences()
        return self.ends(SENTENCES)
23146
private void addEvents(String sentence) {
    String[] parts = sentence.split(" ");
    String outcome = NameFinderME.OTHER;
    List toks = new ArrayList();
    List outcomes = new ArrayList();
    for (int pi = 0, pl = parts.length; pi < pl; pi++) {
      if (parts[pi].equals("<START>")) {
        outcome = NameFinderME.START;
      }
      else if (parts[pi].equals("<END>")) {
        outcome = NameFinderME.OTHER;
      }
      else { //regular token
        toks.add(parts[pi]);
        outcomes.add(outcome);
        if (outcome.equals(NameFinderME.START)) {
          outcome = NameFinderME.CONTINUE;
        }
      }
    }
    events = new Event[toks.size()];
    for (int ti = 0, tl = toks.size(); ti < tl; ti++) {
      events[ti] = new Event((String) outcomes.get(ti), cg.getContext(ti, toks, outcomes, prevTags));
    }
    for (int ti=0,tl=toks.size();ti<tl;ti++) {
      prevTags.put(toks.get(ti),outcomes.get(ti));
    }
  }
1290082
def tokenize(self, sentence,
                 normalize=True,
                 is_feature=False,
                 is_surface=False,
                 return_list=False,
                 func_normalizer=text_preprocess.normalize_text):
        # type: (text_type, bool, bool, bool, bool, Callable[[str],str]) -> Union[List[str], TokenizedSenetence]
        
        assert isinstance(normalize, bool)
        assert isinstance(sentence, text_type)
        normalized_sentence = func_normalizer(sentence)
        if six.PY2:
            normalized_sentence = normalized_sentence.encode('utf-8')

        result = self.__list_tags(self.kytea.getTags(normalized_sentence))

        token_objects = [
            self.__extract_morphological_information(
                kytea_tags_tuple=kytea_tags,
                is_feature=is_feature
            )
            for kytea_tags in result]

        if return_list:
            tokenized_objects = TokenizedSenetence(
                sentence=sentence,
                tokenized_objects=token_objects
            )
            return tokenized_objects.convert_list_object()
        else:
            tokenized_objects = TokenizedSenetence(
                sentence=sentence,
                tokenized_objects=token_objects)

            return tokenized_objects
625519
def tag(self, tokens):
        """"""
        if not self._loaded_model:
            self.load(self.model)
        tags = [None] * len(tokens)
        norm = self._normalize(tokens)
        length = len(norm)
        # A set of allowed indexes for matches to start or end at
        delims = [0] + [i for span in [m.span() for m in self.delimiters.finditer(norm)] for i in span] + [length]
        # Token indices
        token_at_index = []
        for i, t in enumerate(tokens):
            token_at_index.extend([i] * (len(self.lexicon[t].normalized) + 1))
        start_i = 0
        end_i = 1
        matches = {}
        next_start = end_i
        # TODO: This could be a little more efficient by skipping indexes forward to next delim points.
        while True:
            current = norm[start_i:end_i]
            if self._dawg.has_keys_with_prefix(current):
                # print('%s:%s:%s' % (start_i, end_i, current))
                # If the current span is in the dawg, and isn't followed by an alphanumeric character
                if current in self._dawg and start_i in delims and end_i in delims:
                    # print(current)
                    # Subsequent longer matches with same start_i will overwrite values in matches dict
                    matches[start_i] = (start_i, end_i, current)
                    # We can skip forward to after this match next time we increment start_i
                    next_start = end_i
                # Increment end_i provided we aren't already at the end of the input
                if end_i < length:
                    end_i += 1
                    continue
            # Increment start_i provided we aren't already at the end of the input
            start_i = next_start
            if start_i >= length - 1:
                break
            end_i = start_i + 1
            next_start = end_i
        # Apply matches as tags to the relevant tokens
        for start_i, end_i, current in matches.values():
            start_token = token_at_index[start_i]
            end_token = token_at_index[end_i]
            # Possible for match to start in 'I' token from prev match. Merge matches by not overwriting to 'B'.
            if not tags[start_token] == 'I-%s' % self.entity:
                tags[start_token] = 'B-%s' % self.entity
            tags[start_token+1:end_token+1] = ['I-%s' % self.entity] * (end_token - start_token)
        tokentags = list(zip(tokens, tags))
        return tokentags
724623
function Tokenizer(str, tags) {
            this.str = str;
            this.index = 0;
            this.len = str.length;
            this.lineno = 0;
            this.colno = 0;

            this.in_code = false;

            tags = tags || {};
            this.tags = {
                BLOCK_START: tags.blockStart || BLOCK_START,
                BLOCK_END: tags.blockEnd || BLOCK_END,
                VARIABLE_START: tags.variableStart || VARIABLE_START,
                VARIABLE_END: tags.variableEnd || VARIABLE_END,
                COMMENT_START: tags.commentStart || COMMENT_START,
                COMMENT_END: tags.commentEnd || COMMENT_END
            };
        }
878490
def tag(ctx, model, cs, corpus, output):
    """"""
    click.echo('chemdataextractor.dict.tag')
    tagger = CsDictCemTagger(model=model) if cs else CiDictCemTagger(model=model)
    for line in corpus:
        sentence = []
        goldsentence = []
        for t in line.split():
            token, tag = t.rsplit(u'/', 1)
            goldsentence.append((token, tag))
            sentence.append(token)
        if sentence:
            tokentags = tagger.tag(sentence)
            for i, tokentag in enumerate(tokentags):
                goldtokentag = goldsentence[i]
                if goldtokentag[1] not in {u'B-CM', u'I-CM'} and tokentag[1] in {u'B-CM', u'I-CM'}:
                    print(line)
                    print(tokentag[0])

            output.write(u' '.join(u'/'.join(tokentag) for tokentag in tagger.tag(sentence)))
            output.write(u'\n')
        else:
            output.write(u'\n')
415064
public Token findVerb(Sentence sentence) {
    List<SyntacticChunk> syntChunks = sentence.getSyntacticChunks();

    for (int i = 0; i < syntChunks.size(); i++) {
      String tag = syntChunks.get(i).getTag();

      if (tag.equals("P") || tag.equals("MV") || tag.equals("PMV")
          || tag.equals("AUX") || tag.equals("PAUX"))
        return syntChunks.get(i).getTokens().get(0);
    }

    return null;
  }
358615
static List<GoogleToken> getGoogleTokens(String sentence, boolean addStartToken, Tokenizer wordTokenizer) {
    List<GoogleToken> result = new ArrayList<>();
    if (addStartToken) {
      result.add(new GoogleToken(LanguageModel.GOOGLE_SENTENCE_START, 0, 0));
    }
    List<String> tokens = wordTokenizer.tokenize(sentence);
    int startPos = 0;
    for (String token : tokens) {
      if (!StringTools.isWhitespace(token)) {
        result.add(new GoogleToken(token, startPos, startPos+token.length()));
      }
      startPos += token.length();
    }
    return result;
  }
1004047
function arrayDeclaration () {
    var elementFns = [];
    if (peekToken().text != ']') {
      do {
        elementFns.push(expression());
      } while (expect(','));
    }
    consume(']');
    return function(self, locals){
      var array = [];
      for ( var i = 0; i < elementFns.length; i++) {
        array.push(elementFns[i](self, locals));
      }
      return array;
    };
  }