Construct a suffix array for a given generic token array.
170995
public int[] buildSuffixArray(T[] tokens) {
        final int length = tokens.length;
        /*
         * Allocate slightly more space, some suffix construction strategies need it and
         * we don't want to waste space for multiple symbol mappings.
         */
        input = new int[length + SuffixArrays.MAX_EXTRA_TRAILING_SPACE];

        //System.out.println("Assigning token ids ...");

        /*
         * We associate every token to an id, all `equal´ tokens to the same id.
         * The suffix array is built using only the the ids.
         */
        tokIDs = new TreeMap<>(comparator);

        for (int i = 0; i < length; i++) {
            tokIDs.putIfAbsent(tokens[i], i);
            input[i] = tokIDs.get(tokens[i]);
        }

        //System.out.println("Token ids assigned.");

        return delegate.buildSuffixArray(input, 0, length);
    }
208480
def strip_suffix(string, suffix):
    """"""
    if string.endswith(suffix):
        return string[:-(len(suffix))]
    return string
1032763
private static int[][] translate(Map<String, Integer> symbol, Map<String, Integer> suffix, String[] o) {
        int[][] seq = new int[o.length][2];
        
        for (int i = 0; i < o.length; i++) {
            Integer index = symbol.get(o[i]);
            if (index != null) {
                seq[i][0] = index;
            } else {
                seq[i][0] = 0;
            }
            
            index = null;
            if (o[i].length() > 2) {
                index = suffix.get(o[i].substring(o[i].length() - 2));
            }            
            if (index != null) {
                seq[i][1] = index;
            } else {
                seq[i][1] = -1;
            }
        }

        return seq;
    }
200385
ArrayTagSet addAll(Tag[] ts, int tsLength) {
    if (tsLength == 0) {
      return this;
    } else if (length == 0) {
      Arrays.sort(ts, 0, tsLength, TAG_COMPARATOR);
      int len = dedup(ts, 0, ts, 0, tsLength);
      return new ArrayTagSet(toStringArray(ts, len));
    } else {
      String[] newTags = new String[(length + tsLength) * 2];
      Arrays.sort(ts, 0, tsLength, TAG_COMPARATOR);
      int newLength = merge(newTags, tags, length, ts, tsLength);
      return new ArrayTagSet(newTags, newLength);
    }
  }
520217
public static <T> SuffixData createWithLCP(T[] input, ISuffixArrayBuilder builder, Comparator<? super T> comparator) {
        final GenericArrayAdapter adapter = new GenericArrayAdapter(builder, comparator);
        final int[] sa = adapter.buildSuffixArray(input);
        final int[] lcp = computeLCP(adapter.input, 0, input.length, sa);
        return new SuffixData(sa, lcp);
    }
316829
def _suffix_rules(token, tag="NN"):
    
    if isinstance(token, (list, tuple)):
        token, tag = token
    if token.endswith("ing"):
        tag = "VBG"
    if token.endswith("ly"):
        tag = "RB"
    if token.endswith("s") and not token.endswith(("is", "ous", "ss")):
        tag = "NNS"
    if token.endswith(("able", "al", "ful", "ible", "ient", "ish", "ive", "less", "tic", "ous")) or "-" in token:
        tag = "JJ"
    if token.endswith("ed"):
        tag = "VBN"
    if token.endswith(("ate", "ify", "ise", "ize")):
        tag = "VBP"
    return [token, tag]
141175
@SuppressWarnings("unchecked")
  public <E extends T> T[] concat(T[] oldArray, E newElement) {
    T[] arr = (T[]) Array.newInstance(type, oldArray.length + 1);
    System.arraycopy(oldArray, 0, arr, 0, oldArray.length);
    arr[oldArray.length] = newElement;
    return  arr;
  }
643765
def get_array_indices(self):
        """"""

        for token in self.tokens:
            if isinstance(token, SquareBrackets):
                # Use [1:-1] index to discard the square brackets
                yield token.tokens[1:-1]
542668
private static String[] tokenizePathAsArray(String path) {
        String root = null;
        if (FileUtil.isAbsolutePath(path)) {
            String[] s = FileUtil.dissect(path);
            root = s[0];
            path = s[1];
        }
        char sep = File.separatorChar;
        int start = 0;
        int len = path.length();
        int count = 0;
        for (int pos = 0; pos < len; pos++) {
            if (path.charAt(pos) == sep) {
                if (pos != start) {
                    count++;
                }
                start = pos + 1;
            }
        }
        if (len != start) {
            count++;
        }
        String[] l = new String[count + ((root == null) ? 0 : 1)];

        if (root != null) {
            l[0] = root;
            count = 1;
        } else {
            count = 0;
        }
        start = 0;
        for (int pos = 0; pos < len; pos++) {
            if (path.charAt(pos) == sep) {
                if (pos != start) {
                    String tok = path.substring(start, pos);
                    l[count++] = tok;
                }
                start = pos + 1;
            }
        }
        if (len != start) {
            String tok = path.substring(start);
            l[count/*++*/] = tok;
        }
        return l;
    }
590919
public static SuffixData createWithLCP(int[] input, int start, int length,
                                           ISuffixArrayBuilder builder) {
        final int[] sa = builder.buildSuffixArray(input, start, length);
        final int[] lcp = computeLCP(input, start, length, sa);
        return new SuffixData(sa, lcp);
    }
534186
def get_suffixes(arr):
    
    arr = tuple(arr)
    return [arr]
    return (arr[i:] for i in range(len(arr)))
1131355
public List<String> extractSuffix(int length, int size, boolean extend)
    {
        TFDictionary suffixTreeSet = new TFDictionary();
        for (String key : tfDictionary.keySet())
        {
            if (key.length() > length)
            {
                suffixTreeSet.add(key.substring(key.length() - length, key.length()));
                if (extend)
                {
                    for (int l = 1; l < length; ++l)
                    {
                        suffixTreeSet.add(key.substring(key.length() - l, key.length()));
                    }
                }
            }
        }

        if (extend)
        {
            size *= length;
        }

        return extract(suffixTreeSet, size);
    }
820038
def compile_suffix_regex(entries):
    
    expression = "|".join([piece + "$" for piece in entries if piece.strip()])
    return re.compile(expression)
157699
def _segment_with_tokens(text, tokens):
        """"""
        list_form = []
        text_ptr = 0
        for token in tokens:
            inter_token_string = []
            while not text[text_ptr:].startswith(token):
                inter_token_string.append(text[text_ptr])
                text_ptr += 1
                if text_ptr >= len(text):
                    raise ValueError("Tokenization produced tokens that do not belong in string!")
            text_ptr += len(token)
            if inter_token_string:
                list_form.append(''.join(inter_token_string))
            list_form.append(token)
        if text_ptr < len(text):
            list_form.append(text[text_ptr:])
        return list_form
1099778
private String[] parseStringArray(final String list) {
		StringTokenizer tokenizer = new StringTokenizer(list, ",", false);
		int length = tokenizer.countTokens();
		String[] arr = new String[length];

		for (int i = 0; tokenizer.hasMoreElements(); i++) {
			arr[i] = cleanSpaces(tokenizer.nextToken());
		}

		return arr;
	}
1329160
def tokenlist(item, sep, suppress=True):
    """"""
    if suppress:
        sep = sep.suppress()
    return item + ZeroOrMore(sep + item) + Optional(sep)
1135117
def suffixes(self):
        """"""
        name = self.name
        if name.endswith('.'):
            return []
        name = name.lstrip('.')
        return ['.' + suffix for suffix in name.split('.')[1:]]