Outer parsing method: Processing code blocks first

@param cursor     text cursor
@param paragraphs current paragraphs
@return is code block found
161274
private boolean handleCodeBlock(TextCursor cursor, ArrayList<MDSection> paragraphs) {
        if (mode != MODE_ONLY_LINKS) {
            int blockStart = findCodeBlockStart(cursor);
            if (blockStart >= 0) {
                int blockEnd = findCodeBlockEnd(cursor, blockStart);
                if (blockEnd >= 0) {
                    // Adding Text Block if there are some elements before code block
                    if (cursor.currentOffset < blockStart) {
                        handleTextBlock(cursor, blockStart, paragraphs);
                    }

                    String codeContent = cursor.text.substring(cursor.currentOffset + 3, blockEnd - 3).trim();

                    // TODO: Better removing of empty leading and tailing lines
                    // Required to remove only ONE line
                    if (codeContent.startsWith("\n")) {
                        codeContent = codeContent.substring(1);
                    }
                    if (codeContent.endsWith("\n")) {
                        codeContent = codeContent.substring(0, codeContent.length() - 1);
                    }

                    cursor.currentOffset = blockEnd;
                    paragraphs.add(new MDSection(new MDCode(codeContent)));
                    return true;
                }
            }
        }

        // Adding remaining text blocks
        if (cursor.currentOffset < cursor.text.length()) {
            handleTextBlock(cursor, cursor.text.length(), paragraphs);
        }
        return false;
    }
537741
public void process(JCas jcas) throws AnalysisEngineProcessException {
		Integer offset = 0; // a cursor of sorts to keep up with the position in the document text
		
		// grab the document text
		String docText = jcas.getDocumentText();
		// get [sentence-tokens[word-tokens]] from the MaxentTagger
		TokenizerFactory<Word> fac = PTBTokenizerFactory.newTokenizerFactory();  
		fac.setOptions("ptb3Escaping=false,untokenizable=noneKeep");
		List<List<HasWord>> tokenArray = MaxentTagger.tokenizeText(new StringReader(docText), fac);
		
		// iterate over sentences in this document
		for(List<HasWord> sentenceToken : tokenArray) {
			List<TaggedWord> taggedSentence = mt.tagSentence(sentenceToken);
			ListIterator<TaggedWord> twit = taggedSentence.listIterator();
			
			// create a sentence object. gets added to index or discarded depending on configuration
			Sentence sentence = new Sentence(jcas);
			sentence.setBegin(offset);
			
			Integer wordCount = 0;
			// iterate over words in this sentence
			for(HasWord wordToken : sentenceToken) {
				Token t = new Token(jcas);
				TaggedWord tw = twit.next();
				
				// if pos is supposed to be added, iterate through the tagged tokens and set pos
				if(annotate_partofspeech) {
					t.setPos(tw.tag());
				}
				
				String thisWord = wordToken.word();
				
				if(docText.indexOf(thisWord, offset) < 0) {
					Logger.printDetail(component, "A previously tagged token wasn't found in the document text: \"" + thisWord + "\". " +
							"This may be due to unpredictable punctuation tokenization; hence this token isn't tagged.");
					continue; // jump to next token: discards token
				} else {
					offset = docText.indexOf(thisWord, offset); // set cursor to the starting position of token in docText
					t.setBegin(offset);
					++wordCount;
				}
				
				offset += thisWord.length(); // move cursor behind the word
				t.setEnd(offset);
				
				// add tokens to indexes.
				if(annotate_tokens) {
					t.addToIndexes();
				}
			}
			
			// if flag is set, also tag sentences
			if(annotate_sentences) {
				if(wordCount == 0)
					sentence.setEnd(offset);
				else
					sentence.setEnd(offset-1);
				sentence.addToIndexes();
			}
		}
		
		// TODO: DEBUG
		FSIterator fsi = jcas.getAnnotationIndex(Sentence.type).iterator();
		while(fsi.hasNext()) {
			Sentence s = (Sentence) fsi.next();
			if(s.getBegin() < 0 || s.getEnd() < 0) {
				System.err.println("Sentence: " + s.getBegin() + ":" + s.getEnd() + " = " + s.getCoveredText());
				System.err.println("wrong index in text: " + jcas.getDocumentText());
				System.exit(-1);
			}
		}
		FSIterator fsi2 = jcas.getAnnotationIndex(Token.type).iterator();
		while(fsi2.hasNext()) {
			Token t = (Token) fsi2.next();
			if(t.getBegin() < 0 || t.getEnd() < 0) {
				System.err.println("In text: " + jcas.getDocumentText());
				System.err.println("Token: " + t.getBegin() + ":" + t.getEnd());
				System.exit(-1);
			}
		}
	}
1212607
function(node) {
			//skip nodes with no name
			if (!node.name) {
				return true;
			}
	
			//skip formatting elements
			if (node.name === "b" || node.name === "i" || node.name === "em") { //$NON-NLS-0$ //$NON-NLS-1$ //$NON-NLS-2$
				return true;
			}
	
			//skip paragraphs and other blocks of formatted text
			if (node.name === "tt" || node.name === "code" || node.name === "blockquote") { //$NON-NLS-0$ //$NON-NLS-1$ //$NON-NLS-2$ //$NON-NLS-3$
				return true;
			}
			//include the element if we have no reason to skip it
			return false;
		}
1021553
def parse_text_block(text_indent = nil, from = nil)
      empty_lines = 0
      first_line = true
      embedded = nil
      case from
      when :from_tag
        first_line = true
      when :from_embedded
        embedded = true
      end

      close_bracket = false
      until @lines.empty?
        if @lines.first =~ /\A\s*>?\s*\Z/
          next_line
          @stacks.last << [:newline]
          empty_lines += 1 if text_indent
        else
          indent = get_indent(@lines.first)
          break if indent <= @indents.last
          if @lines.first =~ /\A\s*>/
            indent += 1 #$1.size if $1
            close_bracket = true
          else
            close_bracket = false
          end

          if empty_lines > 0
            @stacks.last << [:slim, :interpolate, "\n" * empty_lines]
            empty_lines = 0
          end

          next_line

          # The text block lines must be at least indented
          # as deep as the first line.
          if text_indent && indent < text_indent
            # special case for a leading '>' being back 1 char
            unless first_line && close_bracket && (text_indent - indent == 1)
              @line.lstrip!
              syntax_error!('Unexpected text indentation')
            end
          end

          @line.slice!(0, text_indent || indent)
          unless embedded
            @line = $' if @line =~ /\A>/
            # a code comment
            if @line =~ /(\A|[^\\])#([^{]|\Z)/
              @line = $` + $1
            end
          end
          @stacks.last << [:newline] if !first_line && !embedded
          @stacks.last << [:slim, :interpolate, (text_indent ? "\n" : '') + @line] << [:newline]

          # The indentation of first line of the text block
          # determines the text base indentation.
          text_indent ||= indent

          first_line = false
        end
      end
    end
103259
def _do_code_blocks(self, text):
        """"""
        code_block_re = re.compile(r'''
            (?:\n\n|\A\n?)
            (               # $1 = the code block -- one or more lines, starting with a space/tab
              (?:
                (?:[ ]{%d} | \t)  # Lines must start with a tab or a tab-width of spaces
                .*\n+
              )+
            )
            ((?=^[ ]{0,%d}\S)|\Z)   # Lookahead for non-space at line-start, or end of doc
            ''' % (self.tab_width, self.tab_width),
            re.M | re.X)
        return code_block_re.sub(self._code_block_sub, text)
1311716
public boolean process(Context context) throws NestingException {
        Source source = context.getSource();

        int count = 0;
        while (source.hasNext() &&
                (strong || context.hasNextAdjustedForTerminator()) &&
                (max < 0 || count < max)) {
            int offset = source.getOffset();
            boolean parsed = false;

            if ((source.nextMayBeConstant() || hasCrazyCode) && !context.checkBadTag(offset)) {
                boolean suspicious = false;

                for (ProcCode code : cachedCodes) {
                    if (code.suspicious(context)) {
                        suspicious = true;
                        if (code.process(context)) {
                            parsed = true;
                            break;
                        }
                    }
                }

                if (suspicious && !parsed && !hasCheck) {
                    context.addBadTag(offset);
                }
            }

            if (!parsed) {
                if (strong) {
                    // If scope is strong and has not a code from scope then stop the scope processing
                    break;
                } else if (ignoreText) {
                    source.incOffset();
                } else {
                    try {
                        context.getTarget().append(source.next());
                    } catch (IOException e) {
                        // Nothing! Because StringBuilder doesn't catch IOException
                    }
                }
            } else {
                count++;
            }
        }

        return min < 0 || count >= min;
    }
173619
function(block, parent) {
    if (!block.nodes.length) return false;

    // line count
    var lines = block.nodes[block.nodes.length - 1].line - block.nodes[0].line + 1;
    if (lines === 1) return false;

    // word count of Text node values
    var words = 0;
    // number of Code nodes that are in their own lines
    var codesWithOwnLine = 0;
    // if the previous node was the first in its line
    var prevStartLine = false;

    for (var i = 0; i < block.nodes.length; i++) {
      var node = block.nodes[i];
      var prev = block.nodes[i - 1] || parent || {line: -1};
      var next = block.nodes[i];

      if (node.type === 'Text') {
        words += (node.val.match(/\w+(\s+|$)/g) || []).length;
      } else if (node.type === 'Code' && node.buffer && !node.block) {
        if ((node.line > prev.line || prev.type === 'Text' && prev.val === '\n') && prevStartLine) {
          codesWithOwnLine++;
        }
      } else {
        // Technically Tags can also be interpolated, but determine whether to
        // use multiple dot blocks or one single dot block is way too
        // complicated. KISS.
        return false;
      }
      prevStartLine = node.line > prev.line || prev.type === 'Text' && prev.val === '\n';
    }

    return words > 0 && codesWithOwnLine / lines < 0.35;
  }
1346962
def parseBlock(self, contextStack, currentColumnIndex, text):
        
        startColumnIndex = currentColumnIndex
        countOfNotMatchedSymbols = 0
        highlightedSegments = []
        textTypeMap = []
        ruleTryMatchResult = None
        while currentColumnIndex < len(text):
            textToMatchObject = TextToMatchObject(currentColumnIndex,
                                                   text,
                                                   self.parser.deliminatorSet,
                                                   contextStack.currentData())
            for rule in self.rules:
                ruleTryMatchResult = rule.tryMatch(textToMatchObject)
                if ruleTryMatchResult is not None:  # if something matched
                    _logger.debug('\tmatched rule %s at %d',
                                  rule.shortId(),
                                  currentColumnIndex)
                    if countOfNotMatchedSymbols > 0:
                        highlightedSegments.append((countOfNotMatchedSymbols, self.format))
                        textTypeMap += [self.textType for i in range(countOfNotMatchedSymbols)]
                        countOfNotMatchedSymbols = 0

                    if ruleTryMatchResult.rule.context is not None:
                        newContextStack = ruleTryMatchResult.rule.context.getNextContextStack(contextStack,
                                                                                              ruleTryMatchResult.data)
                    else:
                        newContextStack = contextStack

                    format = ruleTryMatchResult.rule.format if ruleTryMatchResult.rule.attribute else newContextStack.currentContext().format
                    textType = ruleTryMatchResult.rule.textType or newContextStack.currentContext().textType

                    highlightedSegments.append((ruleTryMatchResult.length,
                                                format))
                    textTypeMap += textType * ruleTryMatchResult.length

                    currentColumnIndex += ruleTryMatchResult.length

                    if newContextStack != contextStack:
                        lineContinue = isinstance(ruleTryMatchResult.rule, LineContinue)

                        return currentColumnIndex - startColumnIndex, newContextStack, highlightedSegments, textTypeMap, lineContinue

                    break  # for loop
            else:  # no matched rules
                if self.fallthroughContext is not None:
                    newContextStack = self.fallthroughContext.getNextContextStack(contextStack)
                    if newContextStack != contextStack:
                        if countOfNotMatchedSymbols > 0:
                            highlightedSegments.append((countOfNotMatchedSymbols, self.format))
                            textTypeMap += [self.textType for i in range(countOfNotMatchedSymbols)]
                        return (currentColumnIndex - startColumnIndex, newContextStack, highlightedSegments, textTypeMap, False)

                currentColumnIndex += 1
                countOfNotMatchedSymbols += 1

        if countOfNotMatchedSymbols > 0:
            highlightedSegments.append((countOfNotMatchedSymbols, self.format))
            textTypeMap += [self.textType for i in range(countOfNotMatchedSymbols)]

        lineContinue = ruleTryMatchResult is not None and \
                       isinstance(ruleTryMatchResult.rule, LineContinue)

        return currentColumnIndex - startColumnIndex, contextStack, highlightedSegments, textTypeMap, lineContinue
1275251
function parseElementBlock(stream, state) {
        if(stream.eat("<")) {
            if(stream.match("?")) {
                pushContext(state, TAG_INSTRUCTION);
                state.tokenize = parseProcessingInstructionStartTag;
                return STYLE_INSTRUCTION;
            } else if(stream.match("!--")) {
                // new context: comment
                pushContext(state, TAG_COMMENT);
                return chain(stream, state, inBlock(STYLE_COMMENT, "-->",
                    state.context == null ? parseDocument : parseElementBlock));
            } else if(stream.match("![CDATA[")) {
                // new context: CDATA section
                pushContext(state, TAG_CDATA);
                return chain(stream, state, inBlock(STYLE_TEXT, "]]>",
                    state.context == null ? parseDocument : parseElementBlock));
            } else if(stream.eatSpace() || stream.eol() ) {
                stream.skipToEnd();
                return STYLE_ERROR;
            } else {
                // element
                state.tokenize = parseElementTagName;
                return STYLE_ELEMENT_NAME;
            }
        } else if(stream.eat("&")) {
            stream.eatWhile(/[^;]/);
            stream.eat(";");
            return STYLE_ENTITIES;
        } else {
            // new context: text
            pushContext(state, TAG_TEXT);
            state.tokenize = parseText;
            return null;
        }
        
        state.tokenize = state.context == null ? parseDocument : parseElementBlock;
        stream.skipToEnd();
        return null;
    }
1010683
private String[] segment(final String builtText) {

    // these are fine because they do not affect offsets
    String line = builtText.trim();
    line = RuleBasedTokenizer.doubleSpaces.matcher(line).replaceAll(" ");

    if (this.isHardParagraph) {
      // convert every (spurious) paragraph in newlines and keep them
      line = paragraph.matcher(line).replaceAll("\n$1");
    } else {
      // end of sentence markers, paragraph mark and beginning of link
      line = endPunctLinkPara.matcher(line).replaceAll("$1\n$2$3");
      line = conventionalPara.matcher(line).replaceAll("$1\n$2$3");
      line = endInsideQuotesPara.matcher(line).replaceAll("$1\n$3$4");
      line = multiDotsParaStarters.matcher(line).replaceAll("$1\n$2$3");
      // remove spurious paragraphs
      line = alphaNumParaLowerNum.matcher(line).replaceAll("$1 $3");
      line = spuriousParagraph.matcher(line).replaceAll(" $2");
    }
    // non-period end of sentence markers (?!) followed by sentence starters.
    line = noPeriodSpaceEnd.matcher(line).replaceAll("$1\n$2");
    // multi-dots followed by sentence starters
    line = multiDotsSpaceStarters.matcher(line).replaceAll("$1\n$2");
    // end of sentence inside quotes or brackets
    line = endInsideQuotesSpace.matcher(line).replaceAll("$1\n$2");
    // end of sentence marker, sentence starter punctuation and upper case.
    line = punctSpaceUpper.matcher(line).replaceAll("$1\n$2");
    // end of sentence markers, maybe space or paragraph mark and beginning of
    // link
    line = endPunctLinkSpace.matcher(line).replaceAll("$1\n$2");

    // special case of multi-punctuation
    line = punctSpaceMultiPunct.matcher(line).replaceAll("$1\n$2");

    // non breaker segments everything else with some exceptions
    final String[] lines = line.split("\n");
    final String[] sentences = this.nonBreaker.segmenterExceptions(lines);
    return sentences;
  }
1308489
def __select_text_under_cursor_blocks(self, cursor):
        

        start_block = self.document().findBlock(cursor.selectionStart()).firstLineNumber()
        end_block = self.document().findBlock(cursor.selectionEnd()).firstLineNumber()
        cursor.setPosition(self.document().findBlockByLineNumber(start_block).position())
        cursor.movePosition(QTextCursor.StartOfLine, QTextCursor.MoveAnchor)
        cursor.movePosition(QTextCursor.Down, QTextCursor.KeepAnchor, end_block - start_block)
        cursor.movePosition(QTextCursor.EndOfLine, QTextCursor.KeepAnchor)
663276
public static Parser<Void> blockComment(Parser<Void> begin, Parser<Void> end, Parser<?> commented) {
    return Parsers.sequence(begin, end.not().next(commented).skipMany(), end);
  }
404939
def parse_block(self, block, lineno, indent):
        
        tree = self.driver.parse_tokens(self.wrap_toks(block, lineno, indent))
        tree.future_features = frozenset()
        return tree
31298
def pre_process_code_block(block):
        
        if 'indent' in block and block['indent']:
            indent = r'^' + block['indent']
            block['content'] = re.sub(indent, '', block['icontent'],
                                      flags=re.MULTILINE)
900248
public List<List<Token>> process(final String text) {
        final List<List<Token>> paragraph = new ArrayList<>();
        currentSentence = new ArrayList<>();
        final Tokens tokens = splitText(text);

        while (tokens.hasNext()) {
            final Token t = tokens.next();
            final String trimmedWord = t.text.trim();

            // skip spaces
            if (trimmedWord.isEmpty()) continue;

            if (((mode == WITH_PUNCTUATION) || (mode == WITHOUT_PUNCTUATION && isLetterOrDigit(initChar(t.text))))) {
                boolean canBreakSentence = true;
                if (t.text.contains("'")) {
                    wordContainsApostrophe(t);
                }
                else if (".".equals(trimmedWord)) {
                    canBreakSentence = wordIsFullStop(t);
                }
                else if (":".equals(trimmedWord)) {
                    wordIsColon(tokens, t);
                }
                else currentSentence.add(t);

                // handling the end of a sentence
                if (canBreakSentence && equalss(trimmedWord, ".", ";", "?", "!")) {
                    paragraph.add(currentSentence);
                    currentSentence = new ArrayList<>();
                }
            }
        }

        if (!currentSentence.isEmpty()) paragraph.add(currentSentence);
        return paragraph;
    }
816508
function blockComment(editor) {
        editor = editor || EditorManager.getFocusedEditor();
        if (!editor) {
            return;
        }

        var edits = [],
            lineCommentSels = [];
        _.each(editor.getSelections(), function (sel) {
            var mode = editor.getModeForRange(sel.start, sel.end),
                edit = {edit: [], selection: [sel]}; // default edit in case we don't have a mode for this selection
            if (mode) {
                var language = editor.document.getLanguage().getLanguageForMode(mode.name || mode);

                if (language.hasBlockCommentSyntax()) {
                    // getLineCommentPrefixes always return an array, and will be empty if no line comment syntax is defined
                    edit = _getBlockCommentPrefixSuffixEdit(editor, language.getBlockCommentPrefix(), language.getBlockCommentSuffix(),
                                                            language.getLineCommentPrefixes(), sel);
                    if (!edit) {
                        // This is only null if the block comment code found that the selection is within a line-commented line.
                        // Add this to the list of line-comment selections we need to handle. Since edit is null, we'll skip
                        // pushing anything onto the edit list for this selection.
                        lineCommentSels.push(sel);
                    }
                }
            }
            if (edit) {
                edits.push(edit);
            }
        });

        // Handle any line-comment edits. It's okay if these are out-of-order with the other edits, since
        // they shouldn't overlap, and `doMultipleEdits()` will take care of sorting the edits so the
        // selections can be tracked appropriately.
        edits.push.apply(edits, _getLineCommentEdits(editor, lineCommentSels, "block"));

        editor.setSelections(editor.document.doMultipleEdits(edits));
    }
661082
function parse(code) {
  code = code + '\n';
  var lines = code.split(/\r\n|\r|\n/);

  var header;
  var sections = [];

  var comments = [];
  var inComment = false, section;
  lines.forEach(function(line) {
    // comment block
    if (/^\/\*/.test(line)) {
      inComment = true;
    }
    if (/\*\/$/.test(line)) {
      inComment = false;
      line = line.replace(/\*\/$/, '');
      comments.push(cleanLine(line));
      return;
    }

    if (/^\/\//.test(line)) {
      comments.push(cleanLine(line));
      return;
    }

    if (inComment) {
      comments.push(cleanLine(line));
      return;
    }

    // it is not comment now
    if (comments.length) {
      section = parseSection(comments);
      if (section) {
        sections.push(section);
      } else if (!header && !sections.length) {
        header = comments.join('\n');
      }
      // reset storage
      comments = [];
    }
  });
  return {header: header, sections: sections};
}