MAX_TERM_COUNT = 10000       # There are 39,000 terms in the GO!
769684
def parse(filename, MAX_TERM_COUNT=1000):
    
    with open(filename, "r") as f:

        termId = None
        name = None
        desc = None
        parents = []

        termCount = 0
        for l in f.readlines():
            if l.startswith("id:"):
                termId = l.strip()[4:]
            if l.startswith("name:"):
                name = l.strip()[6:]
            elif l.startswith("def:"):
                desc = l.strip()[5:]
            elif l.startswith("is_a:"):
                pid = l.strip()[6:].split(" ", 1)[0]
                parents.append(pid)
            if len(l) == 1:     # newline
                # save
                if termId is not None and name is not None:
                    terms[termId] = {'name': name, 'desc': desc,
                                     'parents': parents[:], 'children': []}
                    termId = None
                    name = None
                    parents = []
                    termCount += 1
                    if MAX_TERM_COUNT is not None and \
                       termCount > MAX_TERM_COUNT:
                        break

    count = 0
    for tid, tdict in terms.items():
        count += 1      # purely for display
        for p in tdict['parents']:
            if p in terms.keys():
                terms[p]['children'].append(tid)

    # Get unique term IDs for Tag Groups.
    tagGroups = set()
    for tid, tdict in terms.items():
        # Only create Tags for GO:terms that are 'leafs' of the tree
        if len(tdict['children']) == 0:
            for p in tdict['parents']:
                tagGroups.add(p)

    return tagGroups, terms
818065
def f2p(phrase, max_word_size=15, cutoff=3):
    

    results = f2p_list(phrase, max_word_size, cutoff)
    return ' '.join(i[0][0] for i in results)
1049760
function countWords (text, ParseTree, isWord) {
  isWord = isWord || {};
  function countWord (word) {
    if (word)
      isWord[word.toLowerCase()] = true;
  }
  function countWordsAtNodes (nodes) {
    if (nodes)
      nodes.forEach (countWordsAtNode);
  }
  function countWordsAtNode (node) {
    if (typeof(node) === 'object') {
      if (node.functag)
	countWord (ParseTree.funcChar + node.functag);
      switch (node.type) {
      case 'lookup':
	countWord (ParseTree.varChar + node.varname);
        break
      case 'assign':
	countWord (ParseTree.varChar + node.varname);
        countWordsAtNodes (node.value);
        countWordsAtNodes (node.local);
        break
      case 'alt':
        node.opts.forEach (countWordsAtNodes);
        break
      case 'rep':
        countWordsAtNodes (node.unit);
        break
      case 'func':
	countWord (ParseTree.funcChar + node.funcname);
	switch (node.funcname) {
	case 'eval':
          countWordsAtNodes (node.value);
          countWordsAtNodes (node.args);
	  break
	case 'strictquote':
	case 'quote':
	case 'unquote':
        default:
          countWordsAtNodes (node.args);
          break
	}
        break
      case 'cond':
        if (ParseTree.isTraceryExpr (node))
	  countWord (ParseTree.traceryChar + ParseTree.traceryVarName(node) + ParseTree.traceryChar);
        countWordsAtNodes (node.test);
        countWordsAtNodes (node.t);
        countWordsAtNodes (node.f);
        break
      case 'root':
      case 'alt_sampled':
        countWordsAtNodes (node.rhs);
        break
      case 'rep_sampled':
        node.reps.forEach (countWordsAtNodes);
        break
      case 'sym':
	countWord (ParseTree.symChar + ParseTree.leftBraceChar + (node.user || ParseTree.defaultUser) + '/' + node.name + ParseTree.rightBraceChar);
        countWordsAtNodes (node.rhs);
        countWordsAtNodes (node.bind);
        break
      default:
	break
      }
    } else if (typeof(node) === 'string')
      node
      .replace(/[^a-zA-Z0-9_]/g,' ')  // these are the characters we keep
      .replace(/\s+/g,' ').replace(/^ /,'').replace(/ $/,'')  // collapse all runs of space & remove start/end space
      .split(' ')
      .forEach (countWord);
  }
  
  var parsed = ParseTree.parseRhs (text);
  countWordsAtNodes (parsed);

  return isWord;
}
1141720
function updateWordCount() {
        var wordCount = document.getElementsByClassName('entry-word-count')[0],
            editorValue = editor.getValue();

        if (editorValue.length) {
            wordCount.innerHTML = editorValue.match(/\S+/g).length + ' words';
        }
    }
821503
void buildPT_T() {
    int numTags = tagIndex.size();
    m_TT = new double[numTags][numTags];
    m_T = new double[numTags];
    double[] tmp = new double[numTags];
    for (IntTaggedWord word : words) {
      double tot = 0.0;
      for (int t = 0; t < numTags; t++) {
        IntTaggedWord iTW = new IntTaggedWord(word.word, t);
        tmp[t] = seenCounter.getCount(iTW);
        tot += tmp[t];
      }
      if (tot < 10) {
        continue;
      }
      for (int t = 0; t < numTags; t++) {
        for (int t2 = 0; t2 < numTags; t2++) {
          if (tmp[t2] > 0.0) {
            double c = tmp[t] / tot;
            m_T[t] += c;
            m_TT[t2][t] += c;
          }
        }
      }
    }
  }
178491
def log_term_frequency(document, term)
      count = document.term_count(term)
      if count > 0
        1 + log(count)
      else
        0
      end
    end
313208
def fetch(self):
        

        entries = []
        for activity in self.activities['entries']:
            entries.append(
                    [element for element in [activity['title'],
                        activity['content'][0]['value']]])

        return entries[0:self.max_entries]
209010
static function snippet($text, $keywords, $limit, $max_length, $around = array('<b>', '</b>'))
	{
		$result = array();

		$keywords = self::normalize($keywords);
		list($pre, $post) = $around;
		$prev = array();
		$after = false;

		foreach(self::normalize_hash($text) as $word_orig => $word_norm)
		{
			$is_kw = false;
			if(in_array($word_norm, $keywords))
			{
				$prev[] = $pre.$word_orig.$post;
				$is_kw = true;
			}
			else
				$prev[] = $word_orig;

			if($is_kw)
			{
				if($after)
				{
					$result = array_merge($result, $prev);
					$prev = array();
				}
				else
				{
					$result = array_merge($result, count($prev) > $limit ? array('&#133') : array(), array_slice($prev, -$limit-1));
					$prev = array();
				}

				$after = true;
			}
			else
			{
				if($after && count($prev) >= $limit)
				{
					$after = false;
					$result = array_merge($result, $prev);
					$prev = array();
				}
			}
		}

		if($after)
			$result = array_merge($result, array('&#133'), $prev);

		return strip_text(join(' ', $result).'&#133;', $max_length);
	}
1327563
def enumerate_query_by_limit(q, limit=1000):
    
    for offset in count(0, limit):
        r = q.offset(offset).limit(limit).all()

        for row in r:
            yield row

        if len(r) < limit:
            break
197286
def p_term_list(self, p):
        ''''''
        if p[1] is None:
            p[0] = []
        elif len(p) == 4:
            p[1].append(p[3])
            p[0] = p[1]
        elif len(p) == 2:
            p[0] = [p[1]]
548609
def train_phrases(paths, out='data/bigram_model.phrases', tokenizer=word_tokenize, **kwargs):
    
    n = 0
    for path in paths:
        print('Counting lines for {0}...'.format(path))
        n += sum(1 for line in open(path, 'r'))
    print('Processing {0} lines...'.format(n))

    # Change to use less memory. Default is 40m.
    kwargs = {
        'max_vocab_size': 40000000,
        'threshold': 8.
    }.update(kwargs)

    print('Training bigrams...')
    bigram = Phrases(_phrase_doc_stream(paths, n, tokenizer=word_tokenize), **kwargs)

    print('Saving...')
    bigram.save(out)
1045264
def get_tokens(max_value):
  
  vocab = [str(i) for i in range(max_value)]
  vocab = set(vocab)
  vocab.update(CodeOp.LITERALS)
  vocab.update(CodeOp.KEYWORDS)
  vocab |= set("".join(vocab))
  return sorted(vocab)
428958
def convert_examples_to_features(examples, tokenizer, max_seq_length,
                                 doc_stride, max_query_length, is_training):
    """"""

    unique_id = 1000000000

    features = []
    for (example_index, example) in enumerate(examples):
        query_tokens = tokenizer.tokenize(example.question_text)

        if len(query_tokens) > max_query_length:
            query_tokens = query_tokens[0:max_query_length]

        tok_to_orig_index = []
        orig_to_tok_index = []
        all_doc_tokens = []
        for (i, token) in enumerate(example.doc_tokens):
            orig_to_tok_index.append(len(all_doc_tokens))
            sub_tokens = tokenizer.tokenize(token)
            for sub_token in sub_tokens:
                tok_to_orig_index.append(i)
                all_doc_tokens.append(sub_token)

        tok_start_position = None
        tok_end_position = None
        if is_training and example.is_impossible:
            tok_start_position = -1
            tok_end_position = -1
        if is_training and not example.is_impossible:
            tok_start_position = orig_to_tok_index[example.start_position]
            if example.end_position < len(example.doc_tokens) - 1:
                tok_end_position = orig_to_tok_index[example.end_position + 1] - 1
            else:
                tok_end_position = len(all_doc_tokens) - 1
            (tok_start_position, tok_end_position) = _improve_answer_span(
                all_doc_tokens, tok_start_position, tok_end_position, tokenizer,
                example.orig_answer_text)

        # The -3 accounts for [CLS], [SEP] and [SEP]
        max_tokens_for_doc = max_seq_length - len(query_tokens) - 3

        # We can have documents that are longer than the maximum sequence length.
        # To deal with this we do a sliding window approach, where we take chunks
        # of the up to our max length with a stride of `doc_stride`.
        _DocSpan = collections.namedtuple(  # pylint: disable=invalid-name
            "DocSpan", ["start", "length"])
        doc_spans = []
        start_offset = 0
        while start_offset < len(all_doc_tokens):
            length = len(all_doc_tokens) - start_offset
            if length > max_tokens_for_doc:
                length = max_tokens_for_doc
            doc_spans.append(_DocSpan(start=start_offset, length=length))
            if start_offset + length == len(all_doc_tokens):
                break
            start_offset += min(length, doc_stride)

        for (doc_span_index, doc_span) in enumerate(doc_spans):
            tokens = []
            token_to_orig_map = {}
            token_is_max_context = {}
            segment_ids = []
            tokens.append("[CLS]")
            segment_ids.append(0)
            for token in query_tokens:
                tokens.append(token)
                segment_ids.append(0)
            tokens.append("[SEP]")
            segment_ids.append(0)

            for i in range(doc_span.length):
                split_token_index = doc_span.start + i
                token_to_orig_map[len(tokens)] = tok_to_orig_index[split_token_index]

                is_max_context = _check_is_max_context(doc_spans, doc_span_index,
                                                       split_token_index)
                token_is_max_context[len(tokens)] = is_max_context
                tokens.append(all_doc_tokens[split_token_index])
                segment_ids.append(1)
            tokens.append("[SEP]")
            segment_ids.append(1)

            input_ids = tokenizer.convert_tokens_to_ids(tokens)

            # The mask has 1 for real tokens and 0 for padding tokens. Only real
            # tokens are attended to.
            input_mask = [1] * len(input_ids)

            # Zero-pad up to the sequence length.
            while len(input_ids) < max_seq_length:
                input_ids.append(0)
                input_mask.append(0)
                segment_ids.append(0)

            assert len(input_ids) == max_seq_length
            assert len(input_mask) == max_seq_length
            assert len(segment_ids) == max_seq_length

            start_position = None
            end_position = None
            if is_training and not example.is_impossible:
                # For training, if our document chunk does not contain an annotation
                # we throw it out, since there is nothing to predict.
                doc_start = doc_span.start
                doc_end = doc_span.start + doc_span.length - 1
                out_of_span = False
                if not (tok_start_position >= doc_start and
                        tok_end_position <= doc_end):
                    out_of_span = True
                if out_of_span:
                    start_position = 0
                    end_position = 0
                else:
                    doc_offset = len(query_tokens) + 2
                    start_position = tok_start_position - doc_start + doc_offset
                    end_position = tok_end_position - doc_start + doc_offset
            if is_training and example.is_impossible:
                start_position = 0
                end_position = 0
            if example_index < 20:
                logger.info("*** Example ***")
                logger.info("unique_id: %s" % (unique_id))
                logger.info("example_index: %s" % (example_index))
                logger.info("doc_span_index: %s" % (doc_span_index))
                logger.info("tokens: %s" % " ".join(tokens))
                logger.info("token_to_orig_map: %s" % " ".join([
                    "%d:%d" % (x, y) for (x, y) in token_to_orig_map.items()]))
                logger.info("token_is_max_context: %s" % " ".join([
                    "%d:%s" % (x, y) for (x, y) in token_is_max_context.items()
                ]))
                logger.info("input_ids: %s" % " ".join([str(x) for x in input_ids]))
                logger.info(
                    "input_mask: %s" % " ".join([str(x) for x in input_mask]))
                logger.info(
                    "segment_ids: %s" % " ".join([str(x) for x in segment_ids]))
                if is_training and example.is_impossible:
                    logger.info("impossible example")
                if is_training and not example.is_impossible:
                    answer_text = " ".join(tokens[start_position:(end_position + 1)])
                    logger.info("start_position: %d" % (start_position))
                    logger.info("end_position: %d" % (end_position))
                    logger.info(
                        "answer: %s" % (answer_text))

            features.append(
                InputFeatures(
                    unique_id=unique_id,
                    example_index=example_index,
                    doc_span_index=doc_span_index,
                    tokens=tokens,
                    token_to_orig_map=token_to_orig_map,
                    token_is_max_context=token_is_max_context,
                    input_ids=input_ids,
                    input_mask=input_mask,
                    segment_ids=segment_ids,
                    start_position=start_position,
                    end_position=end_position,
                    is_impossible=example.is_impossible))
            unique_id += 1

    return features
534088
def term_count_buckets(self):

        

        buckets = {}
        for term, count in self.term_counts().items():
            if count in buckets: buckets[count].append(term)
            else: buckets[count] = [term]

        return buckets
410815
function TermDataStage(term_info, term_accs){

    // Ready logging.
    var logger = new bbop.logger();
    logger.DEBUG = true;
    function ll(str){ logger.kvetch('JSM02: ' + str); }
    ll('');
    ll('TermDataStage start...');

    // The number of requests that we will make.
    var request_count = 0;
    var run_funs = [];

    // Now, cycle though all of the posible pairs of terms while setting
    // and unsetting the query filter on the manager.
    ll(' Gathering batch functions for term data...');
    // Different.
    var mixed_pairs = [];
    for(var v_i = 0; v_i < term_accs.length; v_i++){
	for(var h_i = 0; h_i < v_i; h_i++){

	    var v = term_accs[v_i];
	    var h = term_accs[h_i];
	    
	    mixed_pairs.push([v, h]);
	}
    }

    var filter_strs = _get_filters(filter_manager);
    console.log('pass filter state: ', filter_strs);

    us.each(mixed_pairs, function(pair){

	var v = pair[0];
	var h = pair[1];
	ll(' Setup search for: ' + v + ' and ' + h);

	// Code here will be ignored by JSHint, as we are defining
	// functions in a loop, which apparently is usually
	// naughty.
	/* jshint ignore:start */
	run_funs.push(function(){
	    
	    // Manager settings.
	    var go = _new_manager();
	    go.set_personality('bioentity');
	    go.add_query_filter('document_category', 'bioentity');
	    go.set_results_count(0); // we don't need any actual rows returned
	    go.set_facet_limit(0); // don't need any actual facets returned
	    //go.debug(false);
	    
	    // Stack on the filters from the filter box.
	    us.each(filter_strs, function(fas){
		go.add_query_filter_as_string(fas, []);
	    });

	    // Set the next query.
	    go.add_query_filter(closure_type, v);
	    go.add_query_filter(closure_type, h);
	    
	    return go.search();
	});
	/* jshint ignore:end */
	
	request_count++;
    });

    // Reflexive.
    us.each(term_accs, function(r){

	ll(' Setup reflexive search for: ' + r);

	// Code here will be ignored by JSHint, as we are defining
	// functions in a loop, which apparently is usually naughty.
	/* jshint ignore:start */
	run_funs.push(function(){

	    // Manager settings.
	    var go = _new_manager();
	    go.set_personality('bioentity');
	    go.add_query_filter('document_category', 'bioentity');
	    go.set_results_count(0); // we don't need any actual rows returned
	    go.set_facet_limit(0); // don't need any actual facets returned
	    //go.debug(false);
	    
	    // Stack on the filters from the filter box.
	    us.each(filter_strs, function(fas){
		go.add_query_filter_as_string(fas, []);
	    });

    	    // Set the next query.
    	    go.add_query_filter(closure_type, r);
	    
	    return go.search();
	});
	/* jshint ignore:end */
	
	request_count++;
    });

    // Now that we know how many requests we will make, initialize the
    // progress bar.
    jQuery("#progress-text").empty();
    jQuery("#progress-text").append('<b><span id="progress-count">0</span> of '+
				    request_count + '</b>');
    //jQuery("#progress-bar").empty();
    jQuery("#progress-bar").progressbar({ max: request_count });
    jQuery("#progress-widget").show();

    // Actually serially fetch the data.
    var seen_links = {};
    var max_count = 0;
    var requests_done = 0;
    // Fetch the data and grab the number we want.
    var accumulator_fun = function(resp){	

	// Update the bar.
	requests_done++;
	jQuery('#progress-bar').progressbar({ value: requests_done });
	jQuery("#progress-count").empty();
	jQuery("#progress-count").append(requests_done);

	// Count is easy.
	var count = resp.total_documents();
	if( count > max_count ){
	    max_count = count;
	}

	// Now let's try and fiqure out which terms we were looking
	// at...
	var fqs = resp.query_filters();
	//console.log(resp);
	var fprops = fqs[closure_type];

	var axes = [];
	us.each(fprops, function(fval, fkey){
	    axes.push(fkey);
	});	
	var axis1 = axes[0];
	var axis2 = axes[1] || axis1; // the reflexive case

	ll(' Data from: (' + axis1 + ', ' + axis2 + '); ' + count);

	// Structure for the links we've seen, level 1.
	if( typeof(seen_links[axis1]) === 'undefined' ){
	    seen_links[axis1] = {}; }
	if( typeof(seen_links[axis2]) === 'undefined' ){
	    seen_links[axis2] = {}; }
	// Mark the links we've seen, level 2.
	if( typeof(seen_links[axis2][axis1]) === 'undefined' ){
	    seen_links[axis2][axis1] = count; }
	if( typeof(seen_links[axis1][axis2]) === 'undefined' ){
	    seen_links[axis1][axis2] = count; }
    };

    // The final function is the data renderer.
    var final_fun = function(){
	ll('Starting final in TermDataStage...');

	// We're done, so hide the progress bar again and show the
	// order selector.
	jQuery('#progress-widget').hide();
	jQuery("#order-selector").show();

	// We want this to be fully defined at the end of the accumulator.
	var data = {
    	    graph: seen_links,
    	    nodes: [],
    	    links: []
	};

	// Now we map out term information into the same format as the
	// example.
	us.each(term_accs, function(acc){
	    var node = {
		'id': acc,
		'name': term_info[acc]['name'],
		'source': term_info[acc]['source'],
		'index': term_info[acc]['index']
	    };
	    data.nodes.push(node);
	});

	// Next, we put our link information into the same format as
	// the example.
	var already_done_links = {};
	us.each(seen_links, function(sub_data, sub){
	    us.each(sub_data, function(link_count, obj){

		// Only add links that we haven't yet.
		var li1 = sub + '_' + obj;
		var li2 = obj + '_' + sub;
		if( typeof(already_done_links[li1]) === 'undefined' ){
		    
		    // Add it to our done list. Either way
		    // we'll catch it.
		    already_done_links[li1] = true;
		    already_done_links[li2] = true;
		    
		    // Push the new link data.
		    var link = {
			'source': term_info[sub]['index'],
			'target': term_info[obj]['index'],
			'value': link_count
		    };
		    data.links.push(link);
		}
	    });
	});

	ll(' Seen links: ' + bbop.dump(seen_links));
	ll(' Data: ' + bbop.dump(data));
	ll(' Max count: ' + max_count);
	// //ll(h + ', ' + v + ': ' + count);
	// ll('accumulate: ' + axis1 + ', ' + axis2 + ': ' + count);

	ll('Completed TermDataStage!');
	RenderStage(data, max_count);
	//PlotStage(data, max_count);
    };

    // Create and run coordinating manager.
    var manager = _new_manager();
    manager.run_promise_functions(
	run_funs,
	accumulator_fun,
	final_fun,
	function(err){
	    alert(err.toString());
	});
}
537879
private Term getMaxTerm(int i) {
        Term maxTerm = terms[i];
        if (maxTerm == null) {
            return null;
        }
        Term term = maxTerm;
        while ((term = term.next()) != null) {
            maxTerm = term;
        }
        return maxTerm;
    }
701304
def remove_terms_used_in_less_than_num_docs(self, threshold):
        '''
        
        '''
        term_counts = self._X.astype(bool).astype(int).sum(axis=0).A[0]
        terms_to_remove = np.where(term_counts < threshold)[0]
        return self.remove_terms_by_indices(terms_to_remove)