Run keyword extraction on given PDF file for given taxonomy.
773607
def extract(filepath, taxonomy, output_mode, output_limit,
            spires, match_mode, detect_author_keywords, extract_acronyms,
            rebuild_cache, only_core_tags, no_cache):
    """"""
    if not filepath or not taxonomy:
        print("No PDF file or taxonomy given!", file=sys.stderr)
        sys.exit(0)

    click.echo(
        ">>> Going extract keywords from {0} as '{1}'...".format(
            filepath, output_mode
        )
    )
    if not os.path.isfile(filepath):
        click.echo(
            "Path to non-existing file\n",
        )
        sys.exit(1)

    result = get_keywords_from_local_file(
        local_file=filepath,
        taxonomy_name=taxonomy,
        output_mode=output_mode,
        output_limit=output_limit,
        spires=spires,
        match_mode=match_mode,
        no_cache=no_cache,
        with_author_keywords=detect_author_keywords,
        rebuild_cache=rebuild_cache,
        only_core_tags=only_core_tags,
        extract_acronyms=extract_acronyms
    )
    click.echo(result)
917137
def _AnalyzeKeywords(self, keywords):
    """"""
    start_time = rdfvalue.RDFDatetime.Now() - rdfvalue.Duration("180d")
    filtered_keywords = []

    for k in keywords:
      if k.startswith(self.START_TIME_PREFIX):
        try:
          start_time = rdfvalue.RDFDatetime.FromHumanReadable(
              k[self.START_TIME_PREFIX_LEN:])
        except ValueError:
          pass
      else:
        filtered_keywords.append(k)

    if not filtered_keywords:
      filtered_keywords.append(".")

    return start_time, filtered_keywords
1127451
def multi_keyword(tokens, keyword_parts):
    """"""
    tokens = iter(tokens)
    matched_tokens = []
    limit = len(keyword_parts)

    for idx in six.moves.range(limit):
        try:
            token = next(tokens)
        except StopIteration:
            return

        if (not token or token.name != "symbol" or
                token.value.lower() != keyword_parts[idx]):
            return

        matched_tokens.append(token)

    return TokenMatch(None, token.value, matched_tokens)
891626
def get_vocab(args):
    """"""
    counter = nlp.data.Counter()
    start = time.time()
    for filename in args.files:
        print('Starting processing of {} after {:.1f} seconds.'.format(
            filename,
            time.time() - start))
        with open(filename, 'r') as f:
            tokens = itertools.chain.from_iterable((l.split() for l in f))
            counter.update(tokens)

    if args.max_word_length:
        counter = {
            w: c
            for w, c in counter.items() if len(w) < args.max_word_length
        }

    total_time = time.time() - start
    print('Finished after {:.1f} seconds.'.format(total_time))
    num_words = sum(counter.values())
    print('Got {} words. Processed {:.1f} per second.'.format(
        num_words, num_words / total_time))

    start = time.time()
    print('Starting creation of vocabulary.')
    vocab = nlp.Vocab(counter, max_size=args.max_size, min_freq=args.min_freq,
                      unknown_token=None, padding_token=None, bos_token=None,
                      eos_token=None)
    with open(args.vocab_output, 'w') as f:
        f.write(vocab.to_json())
    print('Finished creation of vocabulary after {:.1f} seconds.'.format(
        time.time() - start))

    print('Writing word counts.')
    start = time.time()
    idx_to_counts = [counter[t] for t in vocab.idx_to_token]
    with open(args.counts_output, 'w') as f:
        json.dump(idx_to_counts, f)
    print('Finished writing word counts after {:.1f} seconds..'.format(
        time.time() - start))
1124425
public void processDocument(BufferedReader document) throws IOException {
        Queue<String> prevWords = new ArrayDeque<String>(windowSize);
        Queue<String> nextWords = new ArrayDeque<String>(windowSize);

        Iterator<String> documentTokens = 
            IteratorFactory.tokenizeOrdered(document);

        String focusWord = null;

        // prefetch the first windowSize words 
        for (int i = 0; i < windowSize && documentTokens.hasNext(); ++i)
            nextWords.offer(documentTokens.next());
        
        while (!nextWords.isEmpty()) {
            focusWord = nextWords.remove();

            // shift over the window to the next word
            if (documentTokens.hasNext()) {
                String windowEdge = documentTokens.next(); 
                nextWords.offer(windowEdge);
            }    

            // If we are filtering the semantic vectors, check whether this word
            // should have its semantics calculated.  In addition, if there is a
            // filter and it would have excluded the word, do not keep its
            // semantics around
            boolean calculateSemantics =
                semanticFilter.isEmpty() || semanticFilter.contains(focusWord)
                && !focusWord.equals(IteratorFactory.EMPTY_TOKEN);
            
            if (calculateSemantics) {
                IntegerVector focusMeaning = getSemanticVector(focusWord);

                // Sum up the index vector for all the surrounding words.  If
                // permutations are enabled, permute the index vector based on
                // its relative position to the focus word.
                int permutations = -(prevWords.size());        
                for (String word : prevWords) {
                    // Skip the addition of any words that are excluded from the
                    // filter set.  Note that by doing the exclusion here, we
                    // ensure that the token stream maintains its existing
                    // ordering, which is necessary when permutations are taken
                    // into account.
                    if (word.equals(IteratorFactory.EMPTY_TOKEN)) {
                        ++permutations;
                        continue;
                    }
                    
                    TernaryVector iv = wordToIndexVector.get(word);
                    if (usePermutations) {
                        iv = permutationFunc.permute(iv, permutations);
                        ++permutations;
                    }
                    
                    add(focusMeaning, iv);
                }
            
                // Repeat for the words in the forward window.
                permutations = 1;
                for (String word : nextWords) {
                    // Skip the addition of any words that are excluded from the
                    // filter set.  Note that by doing the exclusion here, we
                    // ensure that the token stream maintains its existing
                    // ordering, which is necessary when permutations are taken
                    // into account.
                    if (word.equals(IteratorFactory.EMPTY_TOKEN)) {
                        ++permutations;
                        continue;
                    }
                
                    TernaryVector iv = wordToIndexVector.get(word);
                    if (usePermutations) {
                        iv = permutationFunc.permute(iv, permutations);
                        ++permutations;
                    }

                    add(focusMeaning, iv);
                }
            }

            // Last put this focus word in the prev words and shift off the
            // front of the previous word window if it now contains more words
            // than the maximum window size
            prevWords.offer(focusWord);
            if (prevWords.size() > windowSize) {
                prevWords.remove();
            }
        }    

        document.close();
    }
831467
def save_keywords(filename, xml):
    """"""
    tmp_dir = os.path.dirname(filename)
    if not os.path.isdir(tmp_dir):
        os.mkdir(tmp_dir)

    file_desc = open(filename, "w")
    file_desc.write(xml)
    file_desc.close()
371595
def main(lang="en", output_dir=None, n_iter=25):
    
    nlp = spacy.blank(lang)
    # add the tagger to the pipeline
    # nlp.create_pipe works for built-ins that are registered with spaCy
    tagger = nlp.create_pipe("tagger")
    # Add the tags. This needs to be done before you start training.
    for tag, values in TAG_MAP.items():
        tagger.add_label(tag, values)
    nlp.add_pipe(tagger)

    optimizer = nlp.begin_training()
    for i in range(n_iter):
        random.shuffle(TRAIN_DATA)
        losses = {}
        # batch up the examples using spaCy's minibatch
        batches = minibatch(TRAIN_DATA, size=compounding(4.0, 32.0, 1.001))
        for batch in batches:
            texts, annotations = zip(*batch)
            nlp.update(texts, annotations, sgd=optimizer, losses=losses)
        print("Losses", losses)

    # test the trained model
    test_text = "I like blue eggs"
    doc = nlp(test_text)
    print("Tags", [(t.text, t.tag_, t.pos_) for t in doc])

    # save model to output directory
    if output_dir is not None:
        output_dir = Path(output_dir)
        if not output_dir.exists():
            output_dir.mkdir()
        nlp.to_disk(output_dir)
        print("Saved model to", output_dir)

        # test the save model
        print("Loading from", output_dir)
        nlp2 = spacy.load(output_dir)
        doc = nlp2(test_text)
        print("Tags", [(t.text, t.tag_, t.pos_) for t in doc])
778603
def tag_analysis(self):
        """"""
        if not self.is_tagged(WORDS):
            self.tokenize_words()
        sentences = self.divide(WORDS, SENTENCES)
        for sentence in sentences:
            texts = [word[TEXT] for word in sentence]
            all_analysis = vabamorf.analyze(texts, **self.__kwargs)
            for word, analysis in zip(sentence, all_analysis):
                word[ANALYSIS] = analysis[ANALYSIS]
                word[TEXT] = analysis[TEXT]
        return self
587146
def get_vocabulary(preprocess_output_dir, name):
  
  vocab_file = os.path.join(preprocess_output_dir, CATEGORICAL_ANALYSIS % name)
  if not file_io.file_exists(vocab_file):
    raise ValueError('File %s not found in %s' %
                     (CATEGORICAL_ANALYSIS % name, preprocess_output_dir))

  labels = python_portable_string(
      file_io.read_file_to_string(vocab_file)).split('\n')
  label_values = [x for x in labels if x]  # remove empty lines

  return label_values
437487
def start_indexing(self):
        
        for filepath in self.filepaths:
            with open(filepath) as fp:
                blob = fp.read()
                self.words.extend(self.tokenize(blob))
1154859
def keyword(self, text):
        """"""
        cls = self.KEYWORDS[text]
        self.push_token(cls(text, self.lineno, self.offset))
846286
def text_extract(path, password=None):
    """"""
    pdf = Info(path, password).pdf

    return [pdf.getPage(i).extractText() for i in range(pdf.getNumPages())]
629253
def _read_words(filename):
  """"""
  with tf.gfile.GFile(filename, "r") as f:
    if sys.version_info[0] >= 3:
      return f.read().replace("\n", " %s " % EOS).split()
    else:
      return f.read().decode("utf-8").replace("\n", " %s " % EOS).split()
1347419
function run(keywordsToRun, keywordsDefined, passedArguments) {
        var vars = helpers.createLocalEnv(passedArguments);
        keywordsToRun = helpers.createKeywords(keywordsToRun);

        return {
            then: function(callback) {
                runKeywords(keywordsToRun, keywordsDefined, vars, callback);
            }
        };
    }
974776
def extract_files(files):
    """"""
    expanded_files = []
    legal_extensions = [".md", ".txt", ".rtf", ".html", ".tex", ".markdown"]

    for f in files:
        # If it's a directory, recursively walk through it and find the files.
        if os.path.isdir(f):
            for dir_, _, filenames in os.walk(f):
                for filename in filenames:
                    fn, file_extension = os.path.splitext(filename)
                    if file_extension in legal_extensions:
                        joined_file = os.path.join(dir_, filename)
                        expanded_files.append(joined_file)

        # Otherwise add the file directly.
        else:
            expanded_files.append(f)

    return expanded_files
1052045
def from_file(cls, path, encoding, dialect, fields, converters, field_index):
        """"""

        return cls(open(path, 'r', encoding=encoding), dialect, fields, converters, field_index)
1252597
def contains_extractor(document):
    """"""
    tokens = _get_document_tokens(document)
    features = dict((u'contains({0})'.format(w), True) for w in tokens)
    return features