Run each line through log2code and group by matched pattern.
514284
def run(self):
        """"""
        if ProfileCollection and isinstance(self.mloginfo.logfile,
                                            ProfileCollection):
            print("\n    not available for system.profile collections\n")
            return

        codelines = defaultdict(lambda: 0)
        non_matches = 0

        # get log file information
        logfile = self.mloginfo.logfile
        if logfile.start and logfile.end and not self.mloginfo.args['verbose']:
            progress_start = self.mloginfo._datetime_to_epoch(logfile.start)
            progress_total = (self.mloginfo._datetime_to_epoch(logfile.end) -
                              progress_start)
        else:
            self.mloginfo.progress_bar_enabled = False

        for i, logevent in enumerate(self.mloginfo.logfile):
            cl, _ = self.log2code(logevent.line_str)

            # update progress bar every 1000 lines
            if self.mloginfo.progress_bar_enabled and (i % 1000 == 0):
                if logevent.datetime:
                    progress_curr = self.mloginfo._datetime_to_epoch(logevent
                                                                     .datetime)
                    (self.mloginfo
                     .update_progress(float(progress_curr - progress_start) /
                                      progress_total))

            if cl:
                codelines[cl.pattern] += 1
            else:
                if logevent.operation:
                    # skip operations (command, insert, update, delete,
                    # query, getmore)
                    continue
                if not logevent.thread:
                    # skip the lines that don't have a thread name
                    # (usually map/reduce or assertions)
                    continue
                if len(logevent.split_tokens) - logevent.datetime_nextpos <= 1:
                    # skip empty log messages (after thread name)
                    continue
                if ("warning: log line attempted" in logevent.line_str and
                        "over max size" in logevent.line_str):
                    # skip lines that are too long
                    continue

                # everything else is a real non-match
                non_matches += 1
                if self.mloginfo.args['verbose']:
                    print("couldn't match:" + logevent)

        # clear progress bar again
        if self.mloginfo.progress_bar_enabled:
            self.mloginfo.update_progress(1.0)

        if self.mloginfo.args['verbose']:
            print('')

        for cl in sorted(codelines, key=lambda x: codelines[x], reverse=True):
            print("%8i  %s" % (codelines[cl], " ... ".join(cl)))

        print('')
        if non_matches > 0:
            print("distinct couldn't match %i lines" % non_matches)
            if not self.mloginfo.args['verbose']:
                print("to show non-matched lines, run with --verbose.")
635040
async def _process_lines(self, pattern: Optional[str] = None) -> None:
        """"""
        if pattern is not None:
            cmp = re.compile(pattern)

        _LOGGER.debug("Start working with pattern '%s'.", pattern)

        # read lines
        while self.is_running:
            try:
                line = await self._input.readline()
                if not line:
                    break
                line = line.decode()
            except Exception:  # pylint: disable=broad-except
                break

            match = True if pattern is None else cmp.search(line)
            if match:
                _LOGGER.debug("Process: %s", line)
                await self._que.put(line)

        try:
            await self._loop.run_in_executor(None, self._proc.wait)
        finally:
            await self._que.put(None)
            _LOGGER.debug("Close read ffmpeg output.")
425004
def run(self, arguments=None):
        """"""
        LogFileTool.run(self, arguments)

        for i, self.logfile in enumerate(self.args['logfile']):
            if i > 0:
                print("\n ------------------------------------------\n")

            if self.logfile.datetime_format == 'ctime-pre2.4':
                # no milliseconds when datetime format doesn't support it
                start_time = (self.logfile.start.strftime("%Y %b %d %H:%M:%S")
                              if self.logfile.start else "unknown")
                end_time = (self.logfile.end.strftime("%Y %b %d %H:%M:%S")
                            if self.logfile.start else "unknown")
            else:
                # include milliseconds
                start_time = (self.logfile.start.strftime("%Y %b %d "
                                                          "%H:%M:%S.%f")[:-3]
                              if self.logfile.start else "unknown")
                end_time = (self.logfile.end.strftime("%Y %b %d "
                                                      "%H:%M:%S.%f")[:-3]
                            if self.logfile.start else "unknown")

            print("     source: %s" % self.logfile.name)
            print("       host: %s"
                  % (self.logfile.hostname + ':' + str(self.logfile.port)
                     if self.logfile.hostname else "unknown"))
            print("      start: %s" % (start_time))
            print("        end: %s" % (end_time))

            # TODO: add timezone if iso8601 format
            print("date format: %s" % self.logfile.datetime_format)
            print("     length: %s" % len(self.logfile))
            print("     binary: %s" % (self.logfile.binary or "unknown"))

            version = (' -> '.join(self.logfile.versions) or "unknown")

            # if version is unknown, go by date
            if version == 'unknown':
                if self.logfile.datetime_format == 'ctime-pre2.4':
                    version = '< 2.4 (no milliseconds)'
                elif self.logfile.datetime_format == 'ctime':
                    version = '>= 2.4.x ctime (milliseconds present)'
                elif (self.logfile.datetime_format == "iso8601-utc" or
                      self.logfile.datetime_format == "iso8601-local"):
                    if self.logfile.has_level:
                        version = '>= 3.0 (iso8601 format, level, component)'
                    else:
                        version = '= 2.6.x (iso8601 format)'

            print("    version: %s" % version)
            print("    storage: %s"
                  % (self.logfile.storage_engine or 'unknown'))

            # now run all sections
            for section in self.sections:
                if section.active:
                    print("\n%s" % section.name.upper())
                    section.run()
68434
def run(self, *args):
        """"""

        params = self.parser.parse_args(args)

        code = self.unify(params.matching, params.sources,
                          params.fast_matching, params.no_strict,
                          params.interactive, params.recovery)

        return code
1186808
def parse(self, lines):
        """"""
        pattern = re.compile(r"""^(?P<path>.+?)
                                 :(?P<msg>.+)
                                 :(?P<line_nr>\d+?)
                                 :(?P<col>\d+?)$""", re.VERBOSE)
        return self._parse_by_pattern(lines, pattern)
456207
def logevents(self):
        """"""
        for key in self.groups:
            for logevent in self.groups[key]:
                yield logevent
528043
def add_line(self, logevent):
        """"""
        key = None
        self.empty = False
        self.groups.setdefault(key, list()).append(logevent)
882846
def scan(pattern, string, *args, **kwargs):
    
    return compile(pattern, *args, **kwargs).scan(string)
972340
function postprocess(value, logs, options) {
  var tokens = []
  var start = 0
  var match
  var content
  var info
  var parameters
  var end
  var markdown

  EXPRESSION_LOG.lastIndex = 0
  match = EXPRESSION_LOG.exec(value)

  while (match) {
    end = EXPRESSION_LOG.lastIndex

    content = value.slice(start, end - match[0].length)

    parameters = match[2].split(/\s*,\s*/)

    info = parameters[0]
    info = unquote(info.trim())
    info = logs[info]

    if (info && info.value) {
      tokens = tokens.concat(script(content, options), log(info))
    } else {
      parameters = parameters.slice(1).join(', ')

      tokens = tokens.concat(
        script(content + match[1] + parameters + match[3], options)
      )
    }

    start = end
    match = EXPRESSION_LOG.exec(value)
  }

  if (start < value.length) {
    tokens = tokens.concat(script(value.slice(start), options))
  }

  markdown = []

  tokens.forEach(each)

  markdown = markdown.filter(filter)

  return markdown

  function each(token) {
    var prev
    var lang

    if (token.type === 'markdown') {
      markdown = markdown.concat(parse(token.value))
    } else {
      prev = markdown[markdown.length - 1]
      lang = 'lang' in token ? token.lang : token.type

      if (prev && prev.type === 'code' && prev.lang === lang) {
        prev.value += token.value ? '\n' + token.value : ''
      } else {
        markdown.push({
          type: 'code',
          lang: lang,
          value: token.value
        })
      }
    }
  }

  function filter(token) {
    if ('value' in token) {
      token.value = trimTrailingLines(token.value)

      return token.value !== ''
    }

    return true
  }
}
549016
def find_lines(self, line):
        """"""
        for other_line in self.lines:
            if other_line.match(line):
                yield other_line
1202224
def _starts_with_regex(line, regex):
    """"""
    if not regex.startswith('^'):
        regex = '^' + regex
    reg = re.compile(regex)
    return reg.match(line)
401364
def parse(line):
    
    values = None
    matches = re.search(regex, line)
    if matches:

        # Standard values
        values = {
            'date_time': matches.group(DATE_TIME),
            'log_level': matches.group(LOG_LEVEL),
            'process_id': matches.group(PROCESS_ID),
            'thread_name': matches.group(THREAD_NAME),
            'logger_name': matches.group(LOGGER_NAME),
            'log_message': matches.group(LOG_MESSAGE)
        }

        # Optional transaction tracking information
        if matches.group(TRANSACTION):
            values["transaction"] = {
                "app": matches.group(TRANSACTION_APP),
                "id": matches.group(TRANSACTION_ID),
                "span": matches.group(TRANSACTION_SPAN),
                "exported": matches.group(TRANSACTION_EXPORTED)
            }

    return values
659354
def accept_line(self, logevent):
        """"""
        if self.regex_mode:
            return bool(re.search(self.field, logevent.line_str))
        else:
            return getattr(logevent, self.field) is not None
653036
def search(lines, pattern):
    
    p = pattern.replace("*", ".*")
    test = re.compile(p)
    result = []
    for l in lines:
        if test.search(l):
            result.append(l)
    return result
1288672
def lines_once(cls, code, **kwargs):
        
        if kwargs:
            g = cls.with_id_processor()
            g._append_context(kwargs)
        else:
            g = cls()
        g.lines(code)
        return g.code
1076888
def grep_log(self, expr, filename='system.log', from_mark=None):
        
        matchings = []
        pattern = re.compile(expr)
        with open(os.path.join(self.get_path(), 'logs', filename)) as f:
            if from_mark:
                f.seek(from_mark)
            for line in f:
                m = pattern.search(line)
                if m:
                    matchings.append((line, m))
        return matchings
123195
def parse_lines(log_parsers, fileinp):
    """"""
    while 1:
        logentry = fileinp.readline()
        if not logentry:
            break
        elif not logentry.rstrip():
            continue  # skip newlines

        processed = False
        for lp in log_parsers:
            if lp.grok(logentry):
                processed = True
        if not processed:
            # error: none of the logparsers worked on the line
            logger = logging.getLogger('logparser')
            logger.warning(
                #'Could not parse line %s, in file %s >>>%s<<<',
                #fileinp.lineno(), fileinp.filename(), line.rstrip())
                'Could not parse line >>>%s<<<', logentry.rstrip())
            print('Could not parse line >>>%s<<<' % logentry.rstrip())