In-book search - returns a highlighted version of the HTML.
355541
def in_book_search_highlighted_results(request):
    """"""
    results = {}

    args = request.matchdict
    ident_hash = args['ident_hash']

    page_ident_hash = args['page_ident_hash']
    try:
        page_uuid, _ = split_ident_hash(page_ident_hash)
    except IdentHashShortId as e:
        page_uuid = get_uuid(e.id)
    except IdentHashMissingVersion as e:
        page_uuid = e.id
    args['page_uuid'] = page_uuid

    args['search_term'] = request.params.get('q', '')

    query_type = request.params.get('query_type', '')
    combiner = ''
    if query_type:
        if query_type.lower() == 'or':
            combiner = '_or'

    # Get version from URL params
    id, version = split_ident_hash(ident_hash)
    args['uuid'] = id
    args['version'] = version

    with db_connect() as db_connection:
        with db_connection.cursor() as cursor:
            cursor.execute(SQL['get-collated-state'], args)
            res = cursor.fetchall()
            if res and res[0][0]:
                statement = SQL['get-in-collated-book-search-full-page']
            else:
                statement = SQL['get-in-book-search-full-page']
            cursor.execute(statement.format(combiner=combiner), args)
            res = cursor.fetchall()

            results['results'] = {'query': [],
                                  'total': len(res),
                                  'items': []}
            results['results']['query'] = {
                'search_term': args['search_term'],
                'collection_id': ident_hash,
            }
            for uuid, version, title, headline, rank in res:
                results['results']['items'].append({
                    'rank': '{}'.format(rank),
                    'id': '{}'.format(page_ident_hash),
                    'title': '{}'.format(title),
                    'html': '{}'.format(headline),
                })

    resp = request.response
    resp.status = '200 OK'
    resp.content_type = 'application/json'
    resp.body = json.dumps(results)

    return resp
926697
def highlightAllMatches(self):
        
        # Remove any previous highlighting.
        self.qteWidget.SCISetStylingEx(0, 0, self.styleOrig)
        SCI = self.qteWidget

        # Compile a list of spans that contain the specified string.
        self.compileMatchList()

        # Return if the substring does not exist in the text.
        if len(self.matchList) == 0:
            return

        # ------------------------------------------------------------
        # Make a copy of the style bits of the document, overwrite
        # those parts containing a substring, and then write them
        # back all at once. This is much faster than calling the
        # styling methods repeatedly.
        # ------------------------------------------------------------

        # Make a copy of the document style bits and determine the
        # cursor position in the document.
        style = bytearray(self.styleOrig)
        pos = SCI.getCursorPosition()
        cur = SCI.positionFromLineIndex(*pos)

        # Style all matches.
        self.selMatchIdx = 0
        for start, stop in self.matchList:
            if start < cur:
                self.selMatchIdx += 1
            style[start:stop] = bytes(b'\x1e') * (stop - start)

        if self.selMatchIdx >= len(self.matchList):
            self.selMatchIdx = 0

        # Style the first match after the current cursor position
        # differently to indicate that it is the currently
        # selected one.
        start, stop = self.matchList[self.selMatchIdx]
        style[start:stop] = bytes(b'\x1f') * (stop - start)

        # Place the cursor at the start of the currently selected match.
        line, col = SCI.lineIndexFromPosition(start)
        SCI.setCursorPosition(line, col)

        # Apply the modified style array to the document.
        self.qteWidget.SCISetStylingEx(0, 0, style)
820798
function highlight(text, lang, options){

    function makeSafe(str){
      return str
        .replace(/\r\n|\n\r|\r/g, '\n')
        .replace(/&/g, '&amp;')
        .replace(/</g, '&lt;');
    }

    function normalize(text){
      text = text
        // cut first empty lines
        .replace(/^(?:\s*[\n]+)+?([ \t]*)/, '$1')
        .trimRight();

      // fix empty strings
      text = text.replace(/\n[ \t]+\n/g, '\n\n');

      // normalize text offset
      var minOffset = 1000;
      var lines = text.split(/\n+/);
      var startLine = Number(text.match(/^function/) != null); // fix for function.toString()

      for (var i = startLine; i < lines.length; i++)
      {
        var m = lines[i].match(/^\s*/);
        if (m[0].length < minOffset)
          minOffset = m[0].length;
        if (minOffset == 0)
          break;
      }

      if (minOffset > 0)
        text = text.replace(new RegExp('(^|\\n) {' + minOffset + '}', 'g'), '$1');

      return text;
    }

    function defaultWrapper(line, idx){
      return (
        '<div class="line ' + (idx % 2 ? 'odd' : 'even') + lineClass + '">' +
          '<span class="lineContent">' +
            (!options.noLineNumber
              ? '<input class="lineNumber" value="' + lead(idx + 1, numberWidth) + '" type="none" unselectable="on" readonly="readonly" tabindex="-1" />' +
                '<span class="over"></span>'
              : ''
            ) +
            line + '\r\n' +
          '</span>' +
        '</div>'
      );
    }

    //  MAIN PART

    if (!options)
      options = {};

    // prepare text
    var rangeStart = -1;
    var rangeEnd = -1;
    var rangeName = '';

    if (options.range)
    {
      var left = makeSafe(text.substr(0, options.range[0]));
      var range = makeSafe(text.substring(options.range[0], options.range[1]));
      var right = makeSafe(text.substr(options.range[1]));

      rangeStart = left.length;
      rangeEnd = rangeStart + range.length;
      rangeName = options.range[2] || 'range';

      text = left + range + right;
    }
    else
    {
      text = makeSafe(text);
    }

    if (!options.keepFormat)
      text = normalize(text || '');

    var parser = LANG_PARSER[lang] || LANG_PARSER.text;
    var html = parser(text, rangeStart, rangeEnd, rangeName);

    var lines = html.split('\n');
    var numberWidth = String(lines.length).length;
    var lineClass = (options.noLineNumber ? '' : ' hasLineNumber');
    var offsetRx = new RegExp('^(<span class="' + rangeName + '">)?([ \\t]+)');

    lines = lines
      .map(function(line){
        return line.replace(offsetRx, function(m, rangeSpan, spaces){
          return (rangeSpan || '') + repeat('\xA0', spaces.replace(/\t/g, '  ').length);
        });
      })
      .map(options.wrapper || defaultWrapper);

    return options.lines ? lines : lines.join('');
  }
815036
def xpath_book(request, uuid, version, return_json=True):
    

    xpath_string = request.params.get('q')
    results = execute_xpath(xpath_string, 'xpath', uuid, version)
    if return_json:
        return results
    else:
        return xpath_book_html(request, results)
194176
def html(self):
    '''
    
    '''

    if not getattr(self, '_html', False):
      query_params = {
        'prop': 'revisions',
        'rvprop': 'content',
        'rvlimit': 1,
        'rvparse': '',
        'titles': self.title
      }

      request = _wiki_request(query_params)
      self._html = request['query']['pages'][self.pageid]['revisions'][0]['*']

    return self._html
949386
def highlight_matches(self):
        """"""
        if self.is_code_editor and self.highlight_button.isChecked():
            text = self.search_text.currentText()
            words = self.words_button.isChecked()
            regexp = self.re_button.isChecked()
            self.editor.highlight_found_results(text, words=words,
                                                regexp=regexp)
1231568
def _get_markup(self, text, is_markdown, **kwargs):
        
        if is_markdown:
            return markdown.to_html(
                text,
                config=kwargs,
                search_path=self.search_path)

        return html_entry.process(
            text,
            config=kwargs,
            search_path=self.search_path)
980331
def highlight_block(self, text):
        """"""
        text = to_text_string(text)
        previous_state = tbh.get_state(self.currentBlock().previous())
        
        if previous_state == self.COMMENT:
            self.setFormat(0, len(text), self.formats["comment"])
        else:
            previous_state = self.NORMAL
            self.setFormat(0, len(text), self.formats["normal"])
        
        tbh.set_state(self.currentBlock(), previous_state)
        match = self.PROG.search(text)        

        match_count = 0
        n_characters = len(text)
        # There should never be more matches than characters in the text.
        while match and match_count < n_characters:
            match_dict = match.groupdict()
            for key, value in list(match_dict.items()):
                if value:
                    start, end = match.span(key)
                    if previous_state == self.COMMENT:
                        if key == "multiline_comment_end":
                            tbh.set_state(self.currentBlock(), self.NORMAL)
                            self.setFormat(end, len(text),
                                           self.formats["normal"])
                        else:
                            tbh.set_state(self.currentBlock(), self.COMMENT)
                            self.setFormat(0, len(text),
                                           self.formats["comment"])
                    else:
                        if key == "multiline_comment_start":
                            tbh.set_state(self.currentBlock(), self.COMMENT)
                            self.setFormat(start, len(text),
                                           self.formats["comment"])
                        else:
                            tbh.set_state(self.currentBlock(), self.NORMAL)
                            try:
                                self.setFormat(start, end-start,
                                               self.formats[key])
                            except KeyError:
                                # happens with unmatched end-of-comment;
                                # see issue 1462
                                pass
            
            match = self.PROG.search(text, match.end())
            match_count += 1
        
        self.highlight_spaces(text)
1055218
def get_highlights(self):
        '''
        '''
        # NOTE: all Instapaper API methods use POST except this one!
        path = '/'.join([self.RESOURCE, str(self.object_id), 'highlights'])
        response = self.client.request(path, method='GET', api_version='1.1')
        items = response['data']
        highlights = []
        for item in items:
            if item.get('type') == 'error':
                raise Exception(item.get('message'))
            elif item.get('type') == 'highlight':
                highlights.append(Highlight(self, **item))
        return highlights
1025539
function highlightKeywords() {
      var q = url('?q');
      if (q !== null) {
        var keywords = q.split("%20");
        keywords.forEach(function (keyword) {
          if (keyword !== "") {
            $('.data-searchable *').mark(keyword);
            $('article *').mark(keyword);
          }
        });
      }
    }
488128
def _parse_title_url(html_chunk):
    
    title = html_chunk.find("div", {"class": "comment"})

    if not title:
        return _parse_alt_title(html_chunk), None

    title = title[0].find("h2")
    if not title:
        return _parse_alt_title(html_chunk), None

    # look for the url of the book if present
    url = None
    url_tag = title[0].find("a")
    if url_tag:
        url = url_tag[0].params.get("href", None)
        title = url_tag

    return title[0].getContent(), normalize_url(BASE_URL, url)
346172
def highlight_block(self, text):
        """"""
        text = to_text_string(text)
        inside_comment = tbh.get_state(self.currentBlock().previous()) == self.INSIDE_COMMENT
        self.setFormat(0, len(text),
                       self.formats["comment" if inside_comment else "normal"])
        
        match = self.PROG.search(text)
        index = 0
        while match:
            for key, value in list(match.groupdict().items()):
                if value:
                    start, end = match.span(key)
                    index += end-start
                    if key == "comment_start":
                        inside_comment = True
                        self.setFormat(start, len(text)-start,
                                       self.formats["comment"])
                    elif key == "comment_end":
                        inside_comment = False
                        self.setFormat(start, end-start,
                                       self.formats["comment"])
                    elif inside_comment:
                        self.setFormat(start, end-start,
                                       self.formats["comment"])
                    elif key == "define":
                        self.setFormat(start, end-start,
                                       self.formats["number"])
                    else:
                        self.setFormat(start, end-start, self.formats[key])
                    
            match = self.PROG.search(text, match.end())
        
        self.highlight_spaces(text)
        
        last_state = self.INSIDE_COMMENT if inside_comment else self.NORMAL
        tbh.set_state(self.currentBlock(), last_state)
555417
function highlight() {
    $('pre code').each(function (i, block) {
      hljs.highlightBlock(block);
    });
    $('pre code[highlight-lines]').each(function (i, block) {
      if (block.innerHTML === "") return;
      var lines = block.innerHTML.split('\n');

      queryString = block.getAttribute('highlight-lines');
      if (!queryString) return;

      var ranges = queryString.split(',');
      for (var j = 0, range; range = ranges[j++];) {
        var found = range.match(/^(\d+)\-(\d+)?$/);
        if (found) {
          // consider region as `{startlinenumber}-{endlinenumber}`, in which {endlinenumber} is optional
          var start = +found[1];
          var end = +found[2];
          if (isNaN(end) || end > lines.length) {
            end = lines.length;
          }
        } else {
          // consider region as a sigine line number
          if (isNaN(range)) continue;
          var start = +range;
          var end = start;
        }
        if (start <= 0 || end <= 0 || start > end || start > lines.length) {
          // skip current region if invalid
          continue;
        }
        lines[start - 1] = '<span class="line-highlight">' + lines[start - 1];
        lines[end - 1] = lines[end - 1] + '</span>';
      }

      block.innerHTML = lines.join('\n');
    });
  }
1343886
def _find_title(html_file):
    """"""
    # TODO Is it necessary to read files like this?
    with html_file.open() as f:
        for line in f:
            if '<title>' in line:
                # + 7 to skip len('<title>')
                return line[line.index('<title>') + 7:line.index('</title>')]

    return '(Unknown)'
291607
function() {
					// Do not apply if nothing is found.
					if ( this._.cursors.length < 1 )
						return;

					// Remove the previous highlight if there's one.
					if ( this._.highlightRange )
						this.removeHighlight();

					// Apply the highlight.
					var range = this.toDomRange(),
						bookmark = range.createBookmark();
					highlightStyle.applyToRange( range, editor );
					range.moveToBookmark( bookmark );
					this._.highlightRange = range;

					// Scroll the editor to the highlighted area.
					var element = range.startContainer;
					if ( element.type != CKEDITOR.NODE_ELEMENT )
						element = element.getParent();
					element.scrollIntoView();

					// Update the character cursors.
					this.updateFromDomRange( range );
				}
1069178
def _search(self):
        
        text = "Search: {}".format(self.search_string)
        footerwid = urwid.AttrMap(urwid.Text(text), 'footer')
        self.top.footer = footerwid
        search_items = []
        for grp in self.items_org:
            done = False
            for idx, item in enumerate(grp):
                if isinstance(item, urwid.Columns):
                    for col_idx, col in enumerate(item.contents):
                        if isinstance(col[0], urwid.decoration.AttrMap):
                            grp[idx][col_idx].set_label(splittext(col[0].base_widget.label,
                                                                  self.search_string,
                                                                  ''))
                            if self.search_string.lower() in col[0].base_widget.label.lower():
                                grp[idx][col_idx].set_label(splittext(col[0].base_widget.label,
                                                                      self.search_string,
                                                                      'search'))
                                done = True
                elif isinstance(item, urwid.Text):
                    grp[idx].set_text(splittext(item.text, self.search_string, ''))
                    if self.search_string.lower() in item.text.lower():
                        grp[idx].set_text(splittext(item.text, self.search_string, 'search'))
                        done = True
            if done is True:
                search_items.extend(grp)
        self.items = search_items
        self.top.body = urwid.ListBox(self.items)
        if self.items:
            self.top.body.focus_position = 2 if self.compact is False else 0
            # Trick urwid into redisplaying the cursor
            self.top.keypress(self.tui.get_cols_rows(), "")
            self.no_matches = False
        else:
            self.no_matches = True
            footerwid = urwid.AttrMap(urwid.Text(text + "  No Matches"), 'footer')
            self.top.footer = footerwid
1326301
def search_for_book_in_misc(citation, kbs):
    
    citation_year = year_from_citation(citation)
    for citation_element in citation:
        LOGGER.debug(u"Searching for book title in: %s", citation_element['misc_txt'])
        for title in kbs['books']:
            startIndex = find_substring_ignore_special_chars(citation_element['misc_txt'], title)
            if startIndex != -1:
                line = kbs['books'][title.upper()]
                book_year = line[2].strip(';')
                book_authors = line[0]
                book_found = False
                if citation_year == book_year:
                    # For now consider the citation as valid, we are using
                    # an exact search, we don't need to check the authors
                    # However, the code below will be useful if we decide
                    # to introduce fuzzy matching.
                    book_found = True

                    for author in get_possible_author_names(citation):
                        if find_substring_ignore_special_chars(book_authors, author) != -1:
                            book_found = True

                    for author in re.findall('[a-zA-Z]{4,}', book_authors):
                        if find_substring_ignore_special_chars(citation_element['misc_txt'], author) != -1:
                            book_found = True

                    if book_found:
                        LOGGER.debug(u"Book found: %s", title)
                        book_element = {'type': 'BOOK',
                                        'misc_txt': '',
                                        'authors': book_authors,
                                        'title': line[1],
                                        'year': book_year}
                        citation.append(book_element)
                        citation_element['misc_txt'] = cut_substring_with_special_chars(citation_element['misc_txt'], title, startIndex)
                        # Remove year from misc txt
                        citation_element['misc_txt'] = remove_year(citation_element['misc_txt'], book_year)
                        return True

        LOGGER.debug("Book not found!")

    return False