there is exactly one document per line
12219
public void addDocument(Document doc) throws IOException {
		// needs to differenciate SimpleDocuments from MultiField ones
		// each class has its own way of serializing
		String serial = doc.getStringSerialization();
		raw_file_buffer.write(serial);
	}
478996
def multi_cell(self, w, h, txt='', border=0, align='J', fill=0, split_only=False):
        ""
        txt = self.normalize_text(txt)
        ret = [] # if split_only = True, returns splited text cells
        cw=self.current_font['cw']
        if(w==0):
            w=self.w-self.r_margin-self.x
        wmax=(w-2*self.c_margin)*1000.0/self.font_size
        s=txt.replace("\r",'')
        nb=len(s)
        if(nb>0 and s[nb-1]=="\n"):
            nb-=1
        b=0
        if(border):
            if(border==1):
                border='LTRB'
                b='LRT'
                b2='LR'
            else:
                b2=''
                if('L' in border):
                    b2+='L'
                if('R' in border):
                    b2+='R'
                if ('T' in border):
                    b=b2+'T'
                else:
                    b=b2
        sep=-1
        i=0
        j=0
        l=0
        ns=0
        nl=1
        while(i<nb):
            #Get next character
            c=s[i]
            if(c=="\n"):
                #Explicit line break
                if(self.ws>0):
                    self.ws=0
                    if not split_only:
                        self._out('0 Tw')
                if not split_only:
                    self.cell(w,h,substr(s,j,i-j),b,2,align,fill)
                else:
                    ret.append(substr(s,j,i-j))
                i+=1
                sep=-1
                j=i
                l=0
                ns=0
                nl+=1
                if(border and nl==2):
                    b=b2
                continue
            if(c==' '):
                sep=i
                ls=l
                ns+=1
            if self.unifontsubset:
                l += self.get_string_width(c) / self.font_size*1000.0
            else:
                l += cw.get(c,0)
            if(l>wmax):
                #Automatic line break
                if(sep==-1):
                    if(i==j):
                        i+=1
                    if(self.ws>0):
                        self.ws=0
                        if not split_only:
                            self._out('0 Tw')
                    if not split_only:
                        self.cell(w,h,substr(s,j,i-j),b,2,align,fill)
                    else:
                        ret.append(substr(s,j,i-j))
                else:
                    if(align=='J'):
                        if ns>1:
                            self.ws=(wmax-ls)/1000.0*self.font_size/(ns-1)
                        else:
                            self.ws=0
                        if not split_only:
                            self._out(sprintf('%.3f Tw',self.ws*self.k))
                    if not split_only:
                        self.cell(w,h,substr(s,j,sep-j),b,2,align,fill)
                    else:
                        ret.append(substr(s,j,sep-j))
                    i=sep+1
                sep=-1
                j=i
                l=0
                ns=0
                nl+=1
                if(border and nl==2):
                    b=b2
            else:
                i+=1
        #Last chunk
        if(self.ws>0):
            self.ws=0
            if not split_only:
                self._out('0 Tw')
        if(border and 'B' in border):
            b+='B'
        if not split_only:
            self.cell(w,h,substr(s,j,i-j),b,2,align,fill)
            self.x=self.l_margin
        else:
            ret.append(substr(s,j,i-j))
        return ret
1280257
def lex_document(self, cli, document):
        
        # Cache of already lexed lines.
        cache = {}

        # Pygments generators that are currently lexing.
        line_generators = {}  # Map lexer generator to the line number.

        def get_syntax_sync():
            " The Syntax synchronisation objcet that we currently use. "
            if self.sync_from_start(cli):
                return SyncFromStart()
            else:
                return self.syntax_sync

        def find_closest_generator(i):
            " Return a generator close to line 'i', or None if none was fonud. "
            for generator, lineno in line_generators.items():
                if lineno < i and i - lineno < self.REUSE_GENERATOR_MAX_DISTANCE:
                    return generator

        def create_line_generator(start_lineno, column=0):
            """
            Create a generator that yields the lexed lines.
            Each iteration it yields a (line_number, [(token, text), ...]) tuple.
            """
            def get_tokens():
                text = '\n'.join(document.lines[start_lineno:])[column:]

                # We call `get_tokens_unprocessed`, because `get_tokens` will
                # still replace \r\n and \r by \n.  (We don't want that,
                # Pygments should return exactly the same amount of text, as we
                # have given as input.)
                for _, t, v in self.pygments_lexer.get_tokens_unprocessed(text):
                    yield t, v

            return enumerate(split_lines(get_tokens()), start_lineno)

        def get_generator(i):
            """
            Find an already started generator that is close, or create a new one.
            """
            # Find closest line generator.
            generator = find_closest_generator(i)
            if generator:
                return generator

            # No generator found. Determine starting point for the syntax
            # synchronisation first.

            # Go at least x lines back. (Make scrolling upwards more
            # efficient.)
            i = max(0, i - self.MIN_LINES_BACKWARDS)

            if i == 0:
                row = 0
                column = 0
            else:
                row, column = get_syntax_sync().get_sync_start_position(document, i)

            # Find generator close to this point, or otherwise create a new one.
            generator = find_closest_generator(i)
            if generator:
                return generator
            else:
                generator = create_line_generator(row, column)

            # If the column is not 0, ignore the first line. (Which is
            # incomplete. This happens when the synchronisation algorithm tells
            # us to start parsing in the middle of a line.)
            if column:
                next(generator)
                row += 1

            line_generators[generator] = row
            return generator

        def get_line(i):
            " Return the tokens for a given line number. "
            try:
                return cache[i]
            except KeyError:
                generator = get_generator(i)

                # Exhaust the generator, until we find the requested line.
                for num, line in generator:
                    cache[num] = line
                    if num == i:
                        line_generators[generator] = i

                        # Remove the next item from the cache.
                        # (It could happen that it's already there, because of
                        # another generator that started filling these lines,
                        # but we want to synchronise these lines with the
                        # current lexer's state.)
                        if num + 1 in cache:
                            del cache[num + 1]

                        return cache[num]
            return []

        return get_line
505789
@Override
  public void printProbsDocument(List<IN> document) {

    Pair<int[][][], int[]> p = documentToDataAndLabels(document);
    int[][][] data = p.first();

    // FactorTable[] factorTables =
    // CRFLogConditionalObjectiveFunction.getCalibratedCliqueTree(weights, data,
    // labelIndices, classIndex.size());
    CRFCliqueTree cliqueTree = CRFCliqueTree.getCalibratedCliqueTree(weights, data, labelIndices, classIndex.size(),
        classIndex, flags.backgroundSymbol);

    // for (int i = 0; i < factorTables.length; i++) {
    for (int i = 0; i < cliqueTree.length(); i++) {
      IN wi = document.get(i);
      System.out.print(wi.get(CoreAnnotations.TextAnnotation.class) + '\t');
      for (Iterator<String> iter = classIndex.iterator(); iter.hasNext();) {
        String label = iter.next();
        int index = classIndex.indexOf(label);
        // double prob = Math.pow(Math.E, factorTables[i].logProbEnd(index));
        double prob = cliqueTree.prob(i, index);
        System.out.print(label + '=' + prob);
        if (iter.hasNext()) {
          System.out.print("\t");
        } else {
          System.out.print("\n");
        }
      }
    }
  }
757993
private ArrayList<TextPiece> combineLines(ArrayList<TextPiece> wordsOfAPage) {
	ArrayList<TextPiece> linesOfAPage = new ArrayList<TextPiece>();

	/*
	 * loops over all words
	 */
	TextPiece line = null;
	for (int i = 0; i < wordsOfAPage.size(); i++) {
	    TextPiece prevWord = (i == 0) ? null : wordsOfAPage.get(i - 1);
	    TextPiece currentWord = wordsOfAPage.get(i);
	    TextPiece nextWord = (i == wordsOfAPage.size() - 1) ? null
		    : wordsOfAPage.get(i + 1);

	    float gapToPrevWord = (prevWord == null) ? Integer.MAX_VALUE
		    : currentWord.getX() - prevWord.getX()
			    - prevWord.getWidth();
	    float gapToNextWord = (nextWord == null) ? Integer.MAX_VALUE
		    : nextWord.getX() - currentWord.getX()
			    - currentWord.getWidth();

	    /*
	     * Decides whether this word is the beginning of a new line
	     */
	    boolean isNewLine;
	    if (prevWord != null
		    && (Math.abs(currentWord.getY() - prevWord.getY()) <= Math
			    .max(prevWord.getYScale(), prevWord.getFontSize()))
		    && (currentWord.getX() > prevWord.getX())
		    && ((gapToPrevWord < m_docInfo.getAverageCharWidth() * 1.5) || (gapToPrevWord < 5.0f))) {
		isNewLine = false;
	    } else
		isNewLine = true;

	    boolean isSpaceOrEmpty = currentWord.getText() == " "
		    || currentWord.getText() == "";
	    if (isNewLine || line == null) {
		if (isSpaceOrEmpty) {
		} // ignore empty new line
		else {
		    /*
		     * Creates a new line and add it to collection
		     */
		    line = new TextPiece();
		    linesOfAPage.add(line);
		    /*
		     * Gets info of this line
		     */
		    line.setX(currentWord.getX());
		    line.setY(currentWord.getY());
		    line.setEndX(currentWord.getX() + currentWord.getWidth());
		    line.setEndY(currentWord.getY()
			    + Math.max(currentWord.getYScale(),
				    currentWord.getFontSize()));
		    line.setXScale(currentWord.getXScale());
		    line.setWidth(currentWord.getWidth());
		    line.setHeight(Math.max(currentWord.getFontSize(),
			    currentWord.getYScale()));
		    line.setText(currentWord.getText());
		    line.setFontSize(currentWord.getFontSize());
		}
	    } else {
		/*
		 * This word is still in the same line: combines it line will
		 * not be null here, otherwise it's a bug
		 */
		if (line.getText().length() == 1
			&& currentWord.getXScale() > prevWord.getXScale()
			&& currentWord.getX() > prevWord.getX()
			&& currentWord.getY() > prevWord.getY()
			&& prevWord.getY() + prevWord.getXScale() > currentWord
				.getY()) {
		    line.setSuperScriptBeginning(true);
		}
		if (currentWord.getX() - line.getEndX() > 1.5) { // try to fix
								 // the space
								 // problem
								 // inherited
								 // from PDFBOX
		    line.setText(line.getText() + " " + currentWord.getText());
		} else
		    line.setText(line.getText() + currentWord.getText());
		line.setWidth(line.getWidth() + currentWord.getWidth());

		/*
		 * Updates the end-x
		 */
		line.setEndX(currentWord.getX() + currentWord.getWidth());

		if (currentWord.getXScale() > line.getXScale())
		    line.setXScale(currentWord.getXScale());

		line.setY(currentWord.getY());
		line.setEndY(currentWord.getY()
			+ Math.max(currentWord.getYScale(),
				currentWord.getFontSize()));
	    }
	}

	return linesOfAPage;
    }
1236705
def __isListOfTexts(self, docs):
        
        return isinstance(docs, list) and \
               all(isinstance(d, (basestring, Text)) for d in docs)
677878
function (line, lineNum) {
            if (line.trim() === '') {
                return;
            }


            function logAnyDelimiterType(line) {
                blocks.forEach(function (blockDef, blockIdx) {
                    if (blockDef.start.test(line) === true) {
                        fileStartDelimiters.push([blockIdx, lineNum, line.trim()]);
                    }
                    if (blockDef.end.test(line) === true) {
                        fileEndDelimiters.push([blockIdx, lineNum, line.trim()]);
                    }
                });
            }

            logAnyDelimiterType(line);


            var startCount = fileStartDelimiters.length;
            var endCount = fileEndDelimiters.length;

            /**
             * Checks that amount of start/end blocks are equal.
             *
             * @param blockDef
             * @param blockIdx
             */
            var checkBlocksParity = function (blockDef, blockIdx) {
                var block = blockStats[blockIdx];

                /**
                 * 'if' for start block check
                 */
                if (blockDef.start.test(line) === true) {

                    block.lastStartLine = lineNum;

                    if (block.startCount > block.endCount) {
                        generateMessage({
                            pattern: line.trim(),
                            lineNum: block.lastStartLine,
                            caseNum: 1
                        });
                    }

                    block.startCount++;
                }


                /**
                 * 'if' for end block check
                 */
                if (blockDef.end.test(line) === true) {

                    block.lastEndLine = lineNum;

                    if (block.endCount >= block.startCount) {
                        generateMessage({
                            pattern: line.trim(),
                            lineNum: block.lastEndLine,
                            caseNum: 2
                        });
                    }

                    block.endCount++;
                }
            };


            /**
             * Checks if any two (or more) pairs of start/end
             * blocks are intersecting.
             *
             * @param blockDef
             * @param blockIdx
             */
            var checkBlocksIntersection = function (blockDef, blockIdx) {
                if (blockDef.start.test(line)) {
                    blocksStack.push([blockIdx, lineNum, line.trim()]);
                }

                if (blockDef.end.test(line) && typeof last(blocksStack) !== "undefined") {
                    if (last(blocksStack)[0] === blockIdx) {
                        blocksStack.pop();
                    } else {
                        generateMessage({
                            start: last(blocksStack)[2],
                            pattern: line.trim(),
                            lineNum: last(blocksStack)[1],
                            endLineNum: lineNum,
                            caseNum: 3
                        });
                    }
                } else if (startCount - endCount > 1) {
                    generateMessage({
                        start: last(blocksStack)[2],
                        pattern: line.trim(),
                        lineNum: last(blocksStack)[1],
                        endLineNum: lineNum,
                        caseNum: 3
                    });
                }
            };

            if (options.parityCheck === true) {
                blocks.forEach(checkBlocksParity);
            }

            if (options.intersectionCheck === true) {
                blocks.forEach(checkBlocksIntersection);
            }
        }
231162
def next(self):
        
        in_doc, lines, doc_tag = None, [], None

        for i, line in enumerate(self.stream):
            self.linepos += 1
            #print(i, line)

            if line.startswith("---"):
                # Include only lines in the form:
                #  "--- !tag"
                #  "---"
                # Other lines are spurious.
                in_doc = False
                l = line[3:].strip().lstrip()

                if l.startswith("!"):
                    # "--- !tag"
                    doc_tag = l
                    in_doc = True
                elif not l:
                    #  "---"
                    in_doc = True
                    doc_tag = None

                if in_doc:
                    lineno = self.linepos

            if in_doc:
                lines.append(line)

            if in_doc and line.startswith("..."):
                return YamlDoc(text="".join(lines), lineno=lineno, tag=doc_tag)

        raise StopIteration("Cannot find next YAML document in %s" % self.filename)
282143
def join_next_line(self, separator=' '):
        
        if not self.document.on_last_line:
            self.cursor_position += self.document.get_end_of_line_position()
            self.delete()

            # Remove spaces.
            self.text = (self.document.text_before_cursor + separator +
                         self.document.text_after_cursor.lstrip(' '))
757701
function (data) {
            this.source = data.source;

            var linesCount = 0;
            var coveredLinesCount = 0;

            for (var i in data) {
                if (i == "source") {
                    continue;
                }
                linesCount++;
                if (data[i] > 0) {
                    coveredLinesCount++;
                }
            }
            this.linesCount = linesCount;
            this.coveredLinesCount = coveredLinesCount;
        }
954433
private boolean twoPhaseCurrentDocMatches() throws IOException {
    if (docId != spans1.spans.docID()) {
      reset();
      docId = spans1.spans.docID();
    }
    if (docId == NO_MORE_DOCS) {
      return false;
    } else {
      int spans2DocId = spans2.spans.docID();
      if (spans2DocId < docId) {
        spans2DocId = spans2.spans.advance(docId);
      }
      if (docId != spans2DocId) {
        return spans1.spans.nextStartPosition() != NO_MORE_POSITIONS;
      } else {
        return goToNextStartPosition();
      }
    }
  }
1304060
def straight_line_show(title, length=100, linestyle="=", pad=0):
        
        print(StrTemplate.straight_line(
            title=title, length=length, linestyle=linestyle, pad=pad))
945516
function () {

                $scope.currentFiles.forEach(function (file) {

                    if (file.selected) {
                        ScribeAPI.log({
                            path : file.path
                        }, function (data) {
                            if (!Array.isArray($scope.lines)) {
                                $scope.lines = [];
                            }
                            $scope.lines = $scope.lines.concat(data);
                        });
                    }

                });
            }
105941
private boolean goToNextDoc() throws IOException {
    if (docId == NO_MORE_DOCS) {
      return true;
    } else {
      int spans1DocId = spans1.spans.nextDoc();
      int spans2DocId = spans2.spans.docID();
      docId = Math.max(spans1DocId, spans2DocId);
      while (spans1DocId != spans2DocId && docId != NO_MORE_DOCS) {
        if (spans1DocId < spans2DocId) {
          spans1DocId = spans1.spans.advance(spans2DocId);
          docId = spans1DocId;
        } else {
          spans2DocId = spans2.spans.advance(spans1DocId);
          docId = spans2DocId;
        }
      }
      if (docId != NO_MORE_DOCS && !goToNextStartPosition()) {
        reset();
        return false;
      } else {
        return true;
      }
    }
  }
436668
function docBlock($lines) {
    if (!is_array($lines)) {
      $lines = array($lines);
    }

    $lines = array_merge(
      array("/**"),
      array_map(function ($line) {
        if (empty($line)) {
          return ' *';
        }
        return " * $line";
      }, $lines),
      array(" */")
    );

    return $lines;
  }
104673
def insert_one(self, document, **kwargs):
        
        self._arctic_lib.check_quota()
        return self._collection.insert_one(document, **kwargs)
1181084
def unindent(self):
        

        cursor = self.textCursor()
        if not cursor.hasSelection():
            cursor.movePosition(QTextCursor.StartOfBlock)
            line = foundations.strings.to_string(self.document().findBlockByNumber(cursor.blockNumber()).text())
            indent_marker = re.match(r"({0})".format(self.__indent_marker), line)
            if indent_marker:
                foundations.common.repeat(cursor.deleteChar, len(indent_marker.group(1)))
        else:
            block = self.document().findBlock(cursor.selectionStart())
            while True:
                block_cursor = self.textCursor()
                block_cursor.setPosition(block.position())
                indent_marker = re.match(r"({0})".format(self.__indent_marker), block.text())
                if indent_marker:
                    foundations.common.repeat(block_cursor.deleteChar, len(indent_marker.group(1)))
                if block.contains(cursor.selectionEnd()):
                    break
                block = block.next()
        return True