Various metadata about what preceded a word can (only) be drawn from the previous word.
9998
def populate_previous_punctuation(previous_word, current_word)
      if !previous_word || previous_word.ends_sentence?
        current_word.begins_sentence = true
      else
        current_word.punctuation_before = previous_word.punctuation_after
      end
    end
1195746
def pre(*content, sep='\n'):
    
    return _md(_join(*content, sep=sep), symbols=MD_SYMBOLS[3])
1162442
function(word)
		{
			var c = word[0];
			for (var i = 0; i < 3; i++)
				word[i] = word[i+1];
			word[3] = c;
			
			return word;
		}
1201184
function _createSpotInEnteredMagicWords(spot, chars) {
  const firstChar = chars.substring(0, 1)
  const remainingChars = chars.substring(1)
  const nextSpot = (spot[firstChar] = spot[firstChar] || {})
  if (remainingChars) {
    return _createSpotInEnteredMagicWords(nextSpot, remainingChars)
  } else {
    return nextSpot
  }
}
1244684
def _commonWordStart(self, words):
        
        if not words:
            return ''

        length = 0
        firstWord = words[0]
        otherWords = words[1:]
        for index, char in enumerate(firstWord):
            if not all([word[index] == char for word in otherWords]):
                break
            length = index + 1

        return firstWord[:length]
379852
function _getWishIdsInEnteredMagicWords(word) {
  const startingCharWishesObj = _climbDownChain(
    _enteredMagicWords,
    word.split(''),
  )
  if (startingCharWishesObj) {
    return _getPropFromPosterity(startingCharWishesObj, 'wishes', true)
  } else {
    return []
  }
}
193850
function constainRestrictedWords(text) {
	var i,
		restictedWords = ['today', 'tomorrow', 'yesterday', 'tonight'];

	text = text.toLowerCase();
	i = restictedWords.length;
	while (i--) {
		if (text.indexOf(restictedWords[i]) > -1) {
			return true;
		}
	}
	return false;
}
514925
def targetwords(self, index, targetwords, alignment):
        """"""
        return [ targetwords[x] for x in alignment[index] ]
1133249
def extend_trailing_preposition(max_words,words)
    while words.length < max_words && words[-1] && words[-1][PREPOSITION_REGEX]
      words << model.next_word(words[-1],randomizer)
    end
    words
  end
792267
def surround_previous_word(input_str):
    '''
    
    '''
    start = None
    end = None
    for i, char in enumerate(reversed(input_str)):
        if start is None:
            if char in '{}()[]<>?|':
                return input_str
            elif char != ' ':
                start = i
        else:
            if char in '{}()[]<>?| ':
                end = i
                break
    if start is None:
        return input_str
    if end is None:
        end = len(input_str)
    new_str = ''
    for i, char in enumerate(reversed(input_str)):
        if char == ' ' and i + 1 == start:
            continue
        if i == start:
            new_str += ') '
        elif i == end:
            new_str += '('
        new_str += char
    if end == len(input_str):
        new_str += '('
    return new_str[::-1]
29371
def _get_features(self, i, word, context, prev, prev2):
        '''
        '''
        def add(name, *args):
            features[' '.join((name,) + tuple(args))] += 1

        i += len(self.START)
        features = defaultdict(int)
        # It's useful to have a constant feature, which acts sort of like a prior
        add('bias')
        add('i suffix', word[-3:])
        add('i pref1', word[0])
        add('i-1 tag', prev)
        add('i-2 tag', prev2)
        add('i tag+i-2 tag', prev, prev2)
        add('i word', context[i])
        add('i-1 tag+i word', prev, context[i])
        add('i-1 word', context[i-1])
        add('i-1 suffix', context[i-1][-3:])
        add('i-2 word', context[i-2])
        add('i+1 word', context[i+1])
        add('i+1 suffix', context[i+1][-3:])
        add('i+2 word', context[i+2])
        return features
103262
private void appendLiteral(int word)
  {
    // when we have a zero sequence of the maximum lenght (that is,
    // 00.00000.1111111111111111111111111 = 0x01FFFFFF), it could happen
    // that we try to append a zero literal because the result of the given operation must be an
    // empty set. Whitout the following test, we would have increased the
    // counter of the zero sequence, thus obtaining 0x02000000 that
    // represents a sequence with the first bit set!
    if (lastWordIndex == 0 && word == ConciseSetUtils.ALL_ZEROS_LITERAL && words[0] == 0x01FFFFFF) {
      return;
    }

    // first addition
    if (lastWordIndex < 0) {
      words[lastWordIndex = 0] = word;
      return;
    }

    final int lastWord = words[lastWordIndex];
    if (word == ConciseSetUtils.ALL_ZEROS_LITERAL) {
      if (lastWord == ConciseSetUtils.ALL_ZEROS_LITERAL) {
        words[lastWordIndex] = 1;
      } else if (isZeroSequence(lastWord)) {
        words[lastWordIndex]++;
      } else if (!simulateWAH && containsOnlyOneBit(getLiteralBits(lastWord))) {
        words[lastWordIndex] = 1 | ((1 + Integer.numberOfTrailingZeros(lastWord)) << 25);
      } else {
        words[++lastWordIndex] = word;
      }
    } else if (word == ConciseSetUtils.ALL_ONES_LITERAL) {
      if (lastWord == ConciseSetUtils.ALL_ONES_LITERAL) {
        words[lastWordIndex] = ConciseSetUtils.SEQUENCE_BIT | 1;
      } else if (isOneSequence(lastWord)) {
        words[lastWordIndex]++;
      } else if (!simulateWAH && containsOnlyOneBit(~lastWord)) {
        words[lastWordIndex] = ConciseSetUtils.SEQUENCE_BIT | 1 | ((1 + Integer.numberOfTrailingZeros(~lastWord))
                                                                   << 25);
      } else {
        words[++lastWordIndex] = word;
      }
    } else {
      words[++lastWordIndex] = word;
    }
  }
1054512
def word_starts(self):
        """"""
        if not self.is_tagged(WORDS):
            self.tokenize_words()
        return self.starts(WORDS)
703184
function sharedPrefix(current, old, searchLength) {
    for (var i = 0; i < searchLength; i++) {
      if (diffBasic(current[i], old[i])) {
        return i;
      }
    }
    return searchLength;
  }
826862
def positions(self, word):
        
        right = len(word) - self.right
        return [i for i in self.hd.positions(word) if self.left <= i <= right]
1300435
def _get_features(self, i, context, prev, prev2):
        """"""
        w = self.lexicon[context[i]]
        features = [
            'bias',
            'w:shape=%s' % w.shape,
            'w:lower=%s' % w.lower,
            'p1:tag=%s' % prev,
            'p2:tag=%s' % prev2,
            'p1:tag+w:lower=%s+%s' % (prev, w.lower),
            'p1:tag+p2:tag=%s+%s' % (prev, prev2),
        ]
        if w.like_number:
            features.append('w:like_number')
        elif w.is_punct:
            features.append('w:is_punct')
        elif w.like_url:
            features.append('w:like_url')
        else:
            features.extend([
                'w:suffix2=%s' % w.lower[-2:],
                'w:suffix3=%s' % w.lower[-3:],
                'w:suffix4=%s' % w.lower[-4:],
                'w:suffix5=%s' % w.lower[-5:],
                'w:prefix1=%s' % w.lower[:1],
                'w:prefix2=%s' % w.lower[:2],
                'w:prefix3=%s' % w.lower[:3],
            ])
            if w.is_alpha:
                features.append('w:is_alpha')
            elif w.is_hyphenated:
                features.append('w:is_hyphenated')
            if w.is_upper:
                features.append('w:is_upper')
            elif w.is_lower:
                features.append('w:is_lower')
            elif w.is_title:
                features.append('w:is_title')
        if self.clusters and w.cluster:
            features.extend([
                'w:cluster4=%s' % w.cluster[:4],
                'w:cluster6=%s' % w.cluster[:6],
                'w:cluster10=%s' % w.cluster[:10],
                'w:cluster20=%s' % w.cluster[:20],
            ])
        # Add features for previous tokens if present
        if i > 0:
            p1 = self.lexicon[context[i-1]]
            features.extend([
                'p1:lower=%s' % p1.lower,
                'p1:shape=%s' % p1.shape,
            ])
            if not (p1.like_number or p1.is_punct or p1.like_url):
                features.append('p1:suffix3=%s' % p1.lower[-3:])
            if self.clusters and p1.cluster:
                features.extend([
                    'p1:cluster4=%s' % p1.cluster[:4],
                    'p1:cluster6=%s' % p1.cluster[:6],
                    'p1:cluster10=%s' % p1.cluster[:10],
                    'p1:cluster20=%s' % p1.cluster[:20],
                ])
            if i > 1:
                p2 = self.lexicon[context[i-2]]
                features.extend([
                    'p2:lower=%s' % p2.lower,
                    'p2:shape=%s' % p2.shape,
                ])
                if self.clusters and p2.cluster:
                    features.extend([
                        'p2:cluster4=%s' % p2.cluster[:4],
                        'p2:cluster6=%s' % p2.cluster[:6],
                        'p2:cluster10=%s' % p2.cluster[:10],
                        'p2:cluster20=%s' % p2.cluster[:20],
                    ])
        # Add features for next tokens if present
        end = len(context) - 1
        if i < end:
            n1 = self.lexicon[context[i+1]]
            features.extend([
                'n1:lower=%s' % n1.lower,
                'n1:shape=%s' % n1.shape
            ])
            if not (n1.like_number or n1.is_punct or n1.like_url):
                features.append('n1:suffix3=%s' % n1.lower[-3:])
            if self.clusters and n1.cluster:
                features.extend([
                    'n1:cluster4=%s' % n1.cluster[:4],
                    'n1:cluster6=%s' % n1.cluster[:6],
                    'n1:cluster10=%s' % n1.cluster[:10],
                    'n1:cluster20=%s' % n1.cluster[:20],
                ])
            if i < end - 1:
                n2 = self.lexicon[context[i+2]]
                features.extend([
                    'n2:lower=%s' % n2.lower,
                    'n2:shape=%s' % n2.shape
                ])
                if self.clusters and n2.cluster:
                    features.extend([
                        'n2:cluster4=%s' % n2.cluster[:4],
                        'n2:cluster6=%s' % n2.cluster[:6],
                        'n2:cluster10=%s' % n2.cluster[:10],
                        'n2:cluster20=%s' % n2.cluster[:20],
                    ])
        # Add position features
        if i == 0:
            features.append('-firsttoken-')
        elif i == 1:
            features.append('-secondtoken-')
        elif i == end - 1:
            features.append('-secondlasttoken-')
        elif i == end:
            features.append('-lasttoken-')
        return features
1050104
function(word, labels) {
    let links = this.root.links
    let text = word.text
    // If the word has tags, it is a parameter.
    if (word.tags.length > 0) {
      links = this.root.paramLinks
      text = word.tags[0]
    }
    if (links.has(text)) {
      // That word already exists.
      // Use it, and complement it with the word's tags.
      let node = links.get(text).target
      labels.forEach(label => {
        if (!node.labelTags.has(label)) {
          node.labelTags.set(label, [])
        }
        let tags = node.labelTags.get(label)
        word.tags.forEach(tag => {
          if (tags.indexOf(tag) === -1) {
            tags.push(tag)
          }
        })
      })
      return node
    } else {
      links.set(text, new Link(this.brain,
        new Node(this.brain, word, labels, this.root)))
      return links.get(text).target
    }
  }