Computes the Word Error Rate, defined as the edit distance between the
        two provided sentences after tokenizing to words.
        Arguments:
            s1 (string): space-separated sentence
            s2 (string): space-separated sentence
359923
def wer(self, s1, s2):
        

        # build mapping of words to integers
        b = set(s1.split() + s2.split())
        word2char = dict(zip(b, range(len(b))))

        # map the words to a char array (Levenshtein packages only accepts
        # strings)
        w1 = [chr(word2char[w]) for w in s1.split()]
        w2 = [chr(word2char[w]) for w in s2.split()]

        return Lev.distance(''.join(w1), ''.join(w2))
246182
def _infer_spaces(s):
    
    s = s.lower()

    # Find the best match for the i first characters, assuming cost has
    # been built for the i-1 first characters.
    # Returns a pair (match_cost, match_length).
    def best_match(i):
        candidates = enumerate(reversed(cost[max(0, i - MAXWORD):i]))
        return min((c + WORDCOST.get(s[i-k-1: i], 9e999), k + 1)
                    for k, c in candidates)

    # Build the cost array.
    cost = [0]
    for i in range(1, len(s) + 1):
        c, k = best_match(i)
        cost.append(c)

    # Backtrack to recover the minimal-cost string.
    out = []
    i = len(s)
    while i > 0:
        c, k = best_match(i)
        assert c == cost[i]
        out.append(s[i-k:i])
        i -= k

    return u" ".join(reversed(out))
1262453
public static int getWorstCaseEditDistance(
      int sourceLength, int targetLength, int changeCost, int openGapCost, int continueGapCost) {

    int maxLen = Math.max(sourceLength, targetLength);
    int minLen = Math.min(sourceLength, targetLength);

    // Compute maximum cost of changing one string into another.  If the
    // lengths differ, you'll need maxLen - minLen insertions or deletions.
    int totChangeCost =
        scriptCost(openGapCost, continueGapCost, maxLen - minLen) + minLen * changeCost;

    // Another possibility is to just delete the entire source and insert the
    // target, and not do any changes.
    int blowAwayCost =
        scriptCost(openGapCost, continueGapCost, sourceLength)
            + scriptCost(openGapCost, continueGapCost, targetLength);

    return Math.min(totChangeCost, blowAwayCost);
  }
1126870
def _levenshtein_distance(sentence1, sentence2):
    '''
    
    '''
    first = any2utf8(sentence1).decode('utf-8', 'ignore')
    second = any2utf8(sentence2).decode('utf-8', 'ignore')
    sentence1_len, sentence2_len = len(first), len(second)
    maxlen = max(sentence1_len, sentence2_len)
    if sentence1_len > sentence2_len:
        first, second = second, first

    distances = range(len(first) + 1)
    for index2, char2 in enumerate(second):
        new_distances = [index2 + 1]
        for index1, char1 in enumerate(first):
            if char1 == char2:
                new_distances.append(distances[index1])
            else:
                new_distances.append(1 + min((distances[index1],
                                             distances[index1 + 1],
                                             new_distances[-1])))
        distances = new_distances
    levenshtein = distances[-1]
    d = float((maxlen - levenshtein)/maxlen)
    # smoothing
    s = (sigmoid(d * 6) - 0.5) * 2
    # print("smoothing[%s| %s]: %s -> %s" % (sentence1, sentence2, d, s))
    return s
1369861
private void addErrorCorrection(StringBuilder adjustedString, int codewordSize, int dataBlocks, int eccBlocks) {

        int x, poly, startWeight;

        /* Split into codewords and calculate Reed-Solomon error correction codes */
        switch (codewordSize) {
            case 6:
                x = 32;
                poly = 0x43;
                startWeight = 0x20;
                break;
            case 8:
                x = 128;
                poly = 0x12d;
                startWeight = 0x80;
                break;
            case 10:
                x = 512;
                poly = 0x409;
                startWeight = 0x200;
                break;
            case 12:
                x = 2048;
                poly = 0x1069;
                startWeight = 0x800;
                break;
            default:
                throw new OkapiException("Unrecognized codeword size: " + codewordSize);
        }

        ReedSolomon rs = new ReedSolomon();
        int[] data = new int[dataBlocks + 3];
        int[] ecc = new int[eccBlocks + 3];

        for (int i = 0; i < dataBlocks; i++) {
            for (int weight = 0; weight < codewordSize; weight++) {
                if (adjustedString.charAt((i * codewordSize) + weight) == '1') {
                    data[i] += (x >> weight);
                }
            }
        }

        rs.init_gf(poly);
        rs.init_code(eccBlocks, 1);
        rs.encode(dataBlocks, data);

        for (int i = 0; i < eccBlocks; i++) {
            ecc[i] = rs.getResult(i);
        }

        for (int i = (eccBlocks - 1); i >= 0; i--) {
            for (int weight = startWeight; weight > 0; weight = weight >> 1) {
                if ((ecc[i] & weight) != 0) {
                    adjustedString.append('1');
                } else {
                    adjustedString.append('0');
                }
            }
        }
    }
922079
public int compareTo(Error s) {
    if (getStart() < s.getStart()) {
      return -1;
    } else if (getStart() == s.getStart()) {
      if (getEnd() > s.getEnd()) {
        return -1;
      } else if (getEnd() < s.getEnd()) {
        return 1;
      } else {
        return 0;
      }
    } else {
      return 1;
    }
  }
573822
def calculate_mean_edit_distance_and_loss(iterator, dropout, reuse):
    
    '''
    # Obtain the next batch of data
    (batch_x, batch_seq_len), batch_y = iterator.get_next()

    # Calculate the logits of the batch
    logits, _ = create_model(batch_x, batch_seq_len, dropout, reuse=reuse)

    # Compute the CTC loss using TensorFlow's `ctc_loss`
    total_loss = tf.nn.ctc_loss(labels=batch_y, inputs=logits, sequence_length=batch_seq_len)

    # Calculate the average loss across the batch
    avg_loss = tf.reduce_mean(total_loss)

    # Finally we return the average loss
    return avg_loss
1135073
def token_distance(t1, t2, initial_match_penalization):
    """"""
    if isinstance(t1, NameInitial) or isinstance(t2, NameInitial):
        if t1.token == t2.token:
            return 0
        if t1 == t2:
            return initial_match_penalization
        return 1.0
    return _normalized_edit_dist(t1.token, t2.token)
1196953
def stn(s, length, encoding, errors):
    
    s = s.encode(encoding, errors)
    return s[:length] + (length - len(s)) * NUL
158120
def p_error_raise(p):
    
    q = make_number(1, lineno=p.lineno(2))
    r = make_binary(p.lineno(1), 'MINUS',
                    make_typecast(TYPE.ubyte, p[2], p.lineno(1)), q,
                    lambda x, y: x - y)
    p[0] = make_sentence('ERROR', r)
199921
def edit_distance(seq1, seq2, action_function=lowest_cost_action, test=operator.eq):
    
    m = len(seq1)
    n = len(seq2)
    # Special, easy cases:
    if seq1 == seq2:
        return 0, n
    if m == 0:
        return n, 0
    if n == 0:
        return m, 0
    v0 = [0] * (n + 1)     # The two 'error' columns
    v1 = [0] * (n + 1)
    m0 = [0] * (n + 1)     # The two 'match' columns
    m1 = [0] * (n + 1)
    for i in range(1, n + 1):
        v0[i] = i
    for i in range(1, m + 1):
        v1[0] = i
        for j in range(1, n + 1):
            cost = 0 if test(seq1[i - 1], seq2[j - 1]) else 1
            # The costs
            ins_cost = v1[j - 1] + 1
            del_cost = v0[j] + 1
            sub_cost = v0[j - 1] + cost
            # Match counts
            ins_match = m1[j - 1]
            del_match = m0[j]
            sub_match = m0[j - 1] + int(not cost)

            action = action_function(ins_cost, del_cost, sub_cost, ins_match,
                                     del_match, sub_match, cost)

            if action in [EQUAL, REPLACE]:
                v1[j] = sub_cost
                m1[j] = sub_match
            elif action == INSERT:
                v1[j] = ins_cost
                m1[j] = ins_match
            elif action == DELETE:
                v1[j] = del_cost
                m1[j] = del_match
            else:
                raise Exception('Invalid dynamic programming option returned!')
                # Copy the columns over
        for i in range(0, n + 1):
            v0[i] = v1[i]
            m0[i] = m1[i]
    return v1[n], m1[n]
1154167
def _word_badness(word):
    
    if word.endswith('e'):
        return len(word) - 2
    elif word.endswith('ess'):
        return len(word) - 10
    elif word.endswith('ss'):
        return len(word) - 4
    else:
        return len(word)
161925
def _compute_distance_fast(self):
        """"""
        d, m = edit_distance(self.seq1, self.seq2,
                             action_function=self.action_function,
                             test=self.test)
        if self.dist:
            assert d == self.dist
        if self._matches:
            assert m == self._matches
        self.dist = d
        self._matches = m
299129
def word_matches(s1, s2, n=3):
    
    return __matches(s1, s2, word_ngrams, n=n)
1186234
private double compare_(String s1, String s2) {
    // before we begin, note the length of the strings
    int shortlen = Math.min(s1.length(), s2.length());
    int longlen = Math.max(s1.length(), s2.length());
    
    int removed = 0; // total length of common substrings
    while (true) {
      // first, we identify the longest common substring
      int longest = 0;
      int longesti = 0;
      int longestj = 0;
    
      int[][] matrix = new int[s1.length()][s2.length()];
      for (int i = 0; i < s1.length(); i++) {
        for (int j = 0; j < s2.length(); j++) {
          if (s1.charAt(i) == s2.charAt(j)) {
            if (i == 0 || j == 0)
              matrix[i][j] = 1;
            else
              matrix[i][j] = matrix[i - 1][j - 1] + 1;

            if (matrix[i][j] > longest) {
              longest = matrix[i][j];
              longesti = i;
              longestj = j;
            }
          } else
            matrix[i][j] = 0;
        }
      }

      longesti++; // this solves an off-by-one problem
      longestj++; // this solves an off-by-one problem

      // at this point we know the length of the longest common
      // substring, and also its location, since it ends at indexes
      // longesti and longestj.

      if (longest < minlen)
        break; // all remaining common substrings are too short, so we stop

      // now we slice away the common substrings
      s1 = s1.substring(0, longesti - longest) + s1.substring(longesti);
      s2 = s2.substring(0, longestj - longest) + s2.substring(longestj);
      removed += longest;
    }

    return formula.compute(removed, shortlen, longlen);
  }
1348324
def partial_token_sort_ratio(s1, s2, force_ascii=True, full_process=True):
    
    return _token_sort(s1, s2, partial=True, force_ascii=force_ascii, full_process=full_process)
104346
def ed(s1, s2):
    '''
    '''
    m, n = len(s1), len(s2)
    if m < n:
        m, n = n, m         # ensure n <= m, to use O(min(n,m)) space
        s1, s2 = s2, s1
    d = list(range(n+1))
    for i in range(m):
        p = i
        d[0] = i+1
        for j in range(n):
            t = 0 if s1[i] == s2[j] else 1
            p, d[j+1] = d[j+1], min(p+t, d[j]+1, d[j+1]+1)
    return d[n]