Read input document, and output words.
173162
def words(ctx, input, output):
    """"""
    log.info('chemdataextractor.read.elements')
    log.info('Reading %s' % input.name)
    doc = Document.from_file(input)
    for element in doc.elements:
        if isinstance(element, Text):
            for sentence in element.sentences:
                output.write(u' '.join(sentence.raw_tokens))
                output.write(u'\n')
244316
def read_files(filenames, with_name=False):
    """"""
    text = [read_file(filename) for filename in filenames]
    if with_name:
        return dict(zip(filenames, text))
    return text
766595
def read_input_data(filename):
    """"""
    logging.info('Opening file %s for reading input', filename)
    input_file = open(filename, 'r')
    data = []
    labels = []
    for line in input_file:
        tokens = line.split(',', 1)
        labels.append(tokens[0].strip())
        data.append(tokens[1].strip())
    return labels, data
81296
def read(self):
        """"""

        line_method = self._header_line
        section = None
        while 1:
            token = self.tok.get(True, True)
            if token.is_eol_or_eof():
                break
            if token.is_comment():
                u = token.value.upper()
                if u == 'HEADER':
                    line_method = self._header_line
                elif u == 'QUESTION' or u == 'ZONE':
                    line_method = self._question_line
                    section = self.message.question
                elif u == 'ANSWER' or u == 'PREREQ':
                    line_method = self._rr_line
                    section = self.message.answer
                elif u == 'AUTHORITY' or u == 'UPDATE':
                    line_method = self._rr_line
                    section = self.message.authority
                elif u == 'ADDITIONAL':
                    line_method = self._rr_line
                    section = self.message.additional
                self.tok.get_eol()
                continue
            self.tok.unget(token)
            line_method(section)
1102719
def process_input(self, paragraph):
        """"""
        try:
            lang, body = paragraph.split(None, 1)
        except ValueError:
            lang, body = paragraph, None

        if not lang.strip().startswith('%'):
            lang = 'scala'
            body = paragraph.strip()

        else:
            lang = lang.strip()[1:]

        if lang == 'md':
            self.build_markdown(lang, body)
        else:
            self.build_code(lang, body)
189926
def read(self, file):
        """"""
        content = self._read_content(file)
        self._validate(content)
        self._parse(content)

        return self
1298130
def readlines(self):
        
        continuation = False
        while True:
            yield self.readline(continuation)
            continuation = True
603150
public void processDocument(BufferedReader document) throws IOException {
        Queue<String> prevWords = new ArrayDeque<String>(windowSize);
        Queue<String> nextWords = new ArrayDeque<String>(windowSize);

        Iterator<String> documentTokens = 
            IteratorFactory.tokenizeOrdered(document);

        String focusWord = null;

        // Prefetch the first windowSize words.  As soon as a word enters the
        // nextWords buffer increase its occurrence count.
        for (int i = 0; i < windowSize && documentTokens.hasNext(); ++i) 
            nextWords.offer(documentTokens.next());        
        
        while (!nextWords.isEmpty()) {
            
            focusWord = nextWords.remove();

            // shift over the window to the next word
            if (documentTokens.hasNext()) {
                String windowEdge = documentTokens.next(); 
                nextWords.offer(windowEdge);
            }    
                
            // Don't bother calculating the semantics for empty tokens
            // (i.e. words that were filtered out)
            if (!focusWord.equals(IteratorFactory.EMPTY_TOKEN)) {
                SemanticVector focusMeaning = getSemanticVector(focusWord);

                // Sum up the index vector for all the surrounding words.  If
                // permutations are enabled, permute the index vector based on
                // its relative position to the focus word.
                int permutations = -(prevWords.size());        
                for (String word : prevWords) {
                    // Skip the addition of any words that are excluded from the
                    // filter set.  Note that by doing the exclusion here, we
                    // ensure that the token stream maintains its existing
                    // ordering, which is necessary when permutations are taken
                    // into account.
                    if (word.equals(IteratorFactory.EMPTY_TOKEN)) {
                        ++permutations;
                        continue;
                    }
                    
                    TernaryVector iv = wordToIndexVector.get(word);
                    if (usePermutations) {
                        iv = permutationFunc.permute(iv, permutations);
                        ++permutations;
                    }
                            
                    updateSemantics(focusMeaning, word, iv);
                }
                
                // Repeat for the words in the forward window.
                permutations = 1;
                for (String word : nextWords) {
                    // Skip the addition of any words that are excluded from the
                    // filter set.  Note that by doing the exclusion here, we
                    // ensure that the token stream maintains its existing
                    // ordering, which is necessary when permutations are taken
                    // into account.
                    if (word.equals(IteratorFactory.EMPTY_TOKEN)) {
                        ++permutations;
                        continue;
                    }
                    
                    TernaryVector iv = wordToIndexVector.get(word);
                    if (usePermutations) {
                        iv = permutationFunc.permute(iv, permutations);
                        ++permutations;
                    }
                    
                    updateSemantics(focusMeaning, word, iv);
                }
            }

            // Last put this focus word in the prev words and shift off the
            // front of the previous word window if it now contains more words
            // than the maximum window size
            prevWords.offer(focusWord);

            // Increment the frequency count for the word now that it has been
            // seen and processed.
            Integer count = wordToOccurrences.get(focusWord);
            wordToOccurrences.put(focusWord, (count == null) ? 1 : count + 1);

            if (prevWords.size() > windowSize) {
                prevWords.remove();
            }
        }    

        document.close();
    }
93831
def tag(ctx, input, output):
    """"""
    log.info('chemdataextractor.pos.tag')
    log.info('Reading %s' % input.name)
    doc = Document.from_file(input)
    for element in doc.elements:
        if isinstance(element, Text):
            for sentence in element.sentences:
                output.write(u' '.join(u'/'.join([token, tag]) for token, tag in sentence.pos_tagged_tokens))
                output.write(u'\n')
752568
def word_texts(self):
        """"""
        if not self.is_tagged(WORDS):
            self.tokenize_words()
        return [word[TEXT] for word in self[WORDS]]
959545
private void scanText() {

        if (isLetter(mInput)) {
            mOutput.append(mInput);
            mPos++;

        } else {
            mType = TokenType.TEXT;
            mFinnished = true;
        }
    }
1054398
def load_word_file(filename):
    """"""
    words_file = resource_filename(__name__, "words/%s" % filename)
    handle = open(words_file, 'r')
    words = handle.readlines()
    handle.close()
    return words
865662
def _readSentence(self):
        
        reply_word, words = self.protocol.readSentence()
        words = dict(parseWord(word) for word in words)
        return reply_word, words
491060
def read(tex):
    
    if isinstance(tex, str):
        tex = tex
    else:
        tex = ''.join(itertools.chain(*tex))
    buf, children = Buffer(tokenize(tex)), []
    while buf.hasNext():
        content = read_tex(buf)
        if content is not None:
            children.append(content)
    return TexEnv('[tex]', children), tex
166479
def translate(self):
        """"""
        visitor = CheckTranslator(self.document,
                                  contents=self.contents,
                                  filename=self.filename,
                                  ignore=self.ignore)
        self.document.walkabout(visitor)
        self.checkers += visitor.checkers
935567
def read(*args):
    """"""
    return io.open(os.path.join(HERE, *args), encoding="utf-8").read()
953377
def _read(self, filename):
        """"""
        data = {}
        parsers = [
            FirstDataParser(), CoordinateParser(), EnergyGradParser(),
            SkipApproxHessian(), HessianParser(), MassParser(),
        ]
        with open(filename) as f:
            while True:
                line = f.readline()
                if line == "":
                    break
                # at each line, a parsers checks if it has to process a piece of
                # file. If that happens, the parser gets control over the file
                # and reads as many lines as it needs to collect data for some
                # attributes.
                for parser in parsers:
                    if parser.test(line, data):
                        parser.read(line, f, data)
                        break
        self.__dict__.update(data)