Parse dmesg boot log
11599
public function getDevs()
    {

        // Time?
        if (!empty($this->settings['timer'])) {
            $t = new Timer('Hardware Devices');
        }

        $hw = new Hwpci(null, '/usr/share/misc/pci_vendors');
        $hw->work('dragonfly');

        return $hw->result();
    }
670353
private String fedoraLogEntry(DigitalObject object, String pid) {
        String message = fedoraMessageTemplate.replace("[[PID]]", pid);
        return message.replace("[[OID]]", object.getId());
    }
519241
def _get_ldap_msg(e):
        """"""
        msg = e
        if hasattr(e, 'message'):
            msg = e.message
            if 'desc' in e.message:
                msg = e.message['desc']
            elif hasattr(e, 'args'):
                msg = e.args[0]['desc']
        return msg
466527
def parse_sysctl(text):
    '''  '''
    lines = text.splitlines()
    results = {}
    for line in lines:
        key, _, value = line.decode('ascii').partition(': ')

        if key == 'hw.memsize':
            value = int(value)

        elif key == 'vm.swapusage':
            values = value.split()[2::3]            # every third token
            su_unit = values[0][-1].lower()         # get unit, 'M'
            PAGESIZE = 1024
            if su_unit == 'm':
                PAGESIZE = 1024 * 1024

            value = [ (float(val[:-1]) * PAGESIZE) for val in values ]

        results[key] = value
    return results
175642
def get_deps(cfg=None,deps=[]):
    
    if not cfg is None:
        if not 'deps' in cfg:
            cfg['deps']=deps
        else:
            deps=cfg['deps']
    if not len(deps)==0:
        for dep in deps:
            if not dep in cfg:
                runbashcmd(f'conda install {dep}',test=cfg['test'])
                cfg[dep]=dep
    logging.info(f"{len(deps)} deps installed.")
    return cfg
407964
def logs(context, format):
    """"""
    fritz = context.obj
    fritz.login()

    messages = fritz.get_logs()
    if format == "plain":
        for msg in messages:
            merged = "{} {} {}".format(msg.date, msg.time, msg.message.encode("UTF-8"))
            click.echo(merged)

    if format == "json":
        entries = [msg._asdict() for msg in messages]
        click.echo(json.dumps({
            "entries": entries,
        }))
773195
def CLI(ctx, config, list_lookups):
    """"""
    loglevel = 'info'
    verbosity = getattr(logging, loglevel.upper(), 'INFO')
    #verbosity = logging.DEBUG
    ctx.obj = {
        'verbosity': verbosity,
        'logfile': None,
        'config': {'lookups': {}},
    }
    #logfmt = '%(levelname)-8s | %(message)s'
    logfmt = '%(message)s'
    logging.basicConfig(format=logfmt, level=verbosity)
    debug('CLI >  CWD="{0}"'.format(os.getcwd()))
    with open(CONFIG_FILE, 'ab+') as phonedb:
        phonedb.seek(0)
        rawdata = phonedb.read()
        if rawdata:
            data = json.loads(rawdata, object_pairs_hook=OrderedDict)
            debug(data)
            ctx.obj['config'] = data
            if 'lookups' not in ctx.obj['config']:
                ctx.obj['config']['lookups'] = {}

    if 'sms_gateways' not in ctx.obj['config']:
        """
        gwsorted = sorted(ctx.obj['config']['sms_gateways'].items())
        gwordered = OrderedDict()
        for (k, v) in gwsorted:
            gwordered[k] = v
        ctx.obj['config']['sms_gateways'] = gwordered
        """
        info('loading SMS gateways...')
        sms_gateways = {}
        with open('carrier-gateways', 'rb') as gateways:
            gwdata = gateways.read()
            import re
            re_carrier = r'(?P<carrier>\w+[\w\&\(\)\-\'\s]*[\w\)\(]+)\s+'
            re_gateway = r'10digitphonenumber(?P<gateway>\@\w[\w\.]*\w)\s+'
            gwre = re.compile(re_carrier + re_gateway)
            gws = gwre.findall(gwdata)
            for gw in gwre.finditer(gwdata):
                gwinfo = gw.groupdict()
                debug(gw.groupdict())
                sms_gateways[gwinfo['carrier']] = gwinfo['gateway']
            num_lines = len([l for l in gwdata.split('\n') if len(l)])
            debug('lines={0} ({1} / 2)'.format(num_lines/2, num_lines))
            debug('matches={0}'.format(len(data)))
        ctx.obj['config']['sms_gateways'] = sms_gateways
        with open(CONFIG_FILE, 'wb') as cfg:
            cfg.write(json.dumps(ctx.obj['config'], indent=2))
        info('SMS gateways written to config')

    if list_lookups:
        sms_updated = False
        from fuzzywuzzy import fuzz
        from fuzzywuzzy import process
        lookups = ctx.obj['config']['lookups']
        info('cached lookups:')
        for number, pinfo in lookups.iteritems():
            owner = pinfo['comment']
            common = pinfo['body']['national_format']
            carrier = pinfo['body']['carrier']['name']
            pntype = pinfo['body']['carrier']['type']
            info('  {0}: {1}'.format(owner, common))
            info('    - carrier: {0}'.format(carrier))
            info('    - type: {0}'.format(pntype))
            debug('    - unformatted: {0}'.format(number))
            if 'sms' not in lookups[number]:
                carriers = ctx.obj['config']['sms_gateways'].keys()
                fuzzy = process.extract(carrier, carriers)
                fuzz_max = 0
                result = []
                gateway_key = None
                for f in fuzzy:
                    if f[1] >= fuzz_max:
                        fuzz_max = f[1]
                        result.append(f[0])
                debug(result)
                if len(result) > 1:
                    if len(sorted(result)[0]) <= len(carrier):
                        gateway_key = sorted(result)[0]
                    else:
                        gateway_key = result[0]
                else:
                    gateway_key = result[0]
                gw_suffix = ctx.obj['config']['sms_gateways'][gateway_key]
                ctx.obj['config']['lookups'][number]['sms'] = '{0}{1}'.format(
                    number, gw_suffix)
                sms_updated = True
            info('    - SMS gateway: {0}'.format(
                ctx.obj['config']['lookups'][number]['sms']))
        if sms_updated:
            with open(CONFIG_FILE, 'wb') as cfg:
                cfg.write(json.dumps(ctx.obj['config'], indent=2))
        ctx.exit()
743036
def ParseCFlags(self, start_line=0):
    """"""
    modname = None                      # name of current module
    modlist = []
    flag = None
    for line_num in range(start_line, len(self.output)):  # collect flags
      line = self.output[line_num].rstrip()
      if not line:                      # blank lines terminate flags
        if flag:                        # save last flag
          modlist.append(flag)
          flag = None
        continue

      mobj = self.module_c_re.match(line)
      if mobj:                          # start of a new module
        modname = mobj.group(1)
        logging.debug('Module: %s' % line)
        if flag:
          modlist.append(flag)
        self.module_list.append(modname)
        self.modules.setdefault(modname, [])
        modlist = self.modules[modname]
        flag = None
        continue

      mobj = self.flag_c_re.match(line)
      if mobj:                          # start of a new flag
        if flag:                        # save last flag
          modlist.append(flag)
        logging.debug('Flag: %s' % line)
        flag = Flag(mobj.group(1),  mobj.group(2))
        continue

      # append to flag help.  type and default are part of the main text
      if flag:
        flag.help += ' ' + line.strip()
      else:
        logging.info('Extra: %s' % line)
    if flag:
      modlist.append(flag)
633547
def _parse_fmadm_faulty(output):
    '''
    
    '''
    def _merge_data(summary, fault):
        result = {}
        uuid = summary['event-id']
        del summary['event-id']

        result[uuid] = OrderedDict()
        result[uuid]['summary'] = summary
        result[uuid]['fault'] = fault
        return result

    result = {}
    summary = []
    summary_data = {}
    fault_data = {}
    data_key = None

    for line in output.split("\n"):
        # we hit a divider
        if line.startswith('-'):
            if summary and summary_data and fault_data:
                # we have data, store it and reset
                result.update(_merge_data(summary_data, fault_data))

                summary = []
                summary_data = {}
                fault_data = {}
                continue
            else:
                # we don't have all data, colelct more
                continue

        # if we do not have the header, store it
        if not summary:
            summary.append(line)
            continue

        # if we have the header but no data, store the data and parse it
        if summary and not summary_data:
            summary.append(line)
            summary_data = _parse_fmdump("\n".join(summary))[0]
            continue

        # if we have a header and data, assume the other lines are details
        if summary and summary_data:
            # if line starts with a whitespace and we already have a key, append
            if line.startswith(' ') and data_key:
                fault_data[data_key] = "{0}\n{1}".format(
                    fault_data[data_key],
                    line.strip()
                )
            # we have a key : value line, parse it
            elif ':' in line:
                line = line.split(':')
                data_key = line[0].strip()
                fault_data[data_key] = ":".join(line[1:]).strip()
                # note: for some reason Chassis_id is lobbed ofter Platform, fix that here
                if data_key == 'Platform':
                    fault_data['Chassis_id'] = fault_data[data_key][fault_data[data_key].index('Chassis_id'):].split(':')[-1].strip()
                    fault_data[data_key] = fault_data[data_key][0:fault_data[data_key].index('Chassis_id')].strip()

    # we have data, store it and reset
    result.update(_merge_data(summary_data, fault_data))

    return result
154452
def parse_log_line(line):
    
    matches = re.search(r'^\[([^\]]+)\] ([^:]+: .*)', line)
    error = re.search(r'Traceback', line)
    if error:
        return {'line': line, 'step': 'error'}
    if not matches:
        return {'line': line, 'step': None}

    tstamp = matches.group(1)
    msg = matches.group(2)

    if not msg.find('Timing: ') >= 0:
        return {'line': line, 'step': None}

    when = datetime.strptime(tstamp, '%Y-%m-%dT%H:%MZ').replace(
        tzinfo=pytz.timezone('UTC'))
    step = msg.split(":")[-1].strip()
    return {'line': line, 'step': step, 'when': when}
1223358
def parse_logs(self, f):
        """"""
        data = {}
        for l in f.splitlines():
            # catch empty lines
            if len(l) == 0:
                continue
            s = l.split("\t")
            data_ = []
            # catch lines with descriptive content: "Of pairs used:"
            for i in s[1:]:
                if len(i) == 0:
                    continue
                try:
                    i.replace('(', '')
                    i.replace(')', '')
                    i.replace(',', '')

                    data_.append(float(i))
                except ValueError:
                    data_.append(i)
            if len(data_) == 0:
                continue
            if s[0].startswith('short range'):
                s[0] = 'short range'
            elif s[0].startswith('same fragment'):
                s[0] = 'same fragment'
            s[0] = s[0].capitalize()
            data[s[0]] = data_
        return data
479059
def log(msg, level=0):
    '''
    
    '''

    red = '\033[91m'
    endc = '\033[0m'

    # configure the logging module
    cfg = {
        'version': 1,
        'disable_existing_loggers': False,
        'formatters': {
            'stdout': {
                'format': '[%(levelname)s]: %(asctime)s - %(message)s',
                'datefmt': '%x %X'
            },
            'stderr': {
                'format': red + '[%(levelname)s]: %(asctime)s - %(message)s' + endc,
                'datefmt': '%x %X'
            }
        },
        'handlers': {
            'stdout': {
                'class': 'logging.StreamHandler',
                'level': 'DEBUG',
                'formatter': 'stdout'
            },
            'stderr': {
                'class': 'logging.StreamHandler',
                'level': 'ERROR',
                'formatter': 'stderr'
            }
        },
        'loggers': {
            'info': {
                'handlers': ['stdout'],
                'level': 'INFO',
                'propagate': True
            },
            'error': {
                'handlers': ['stderr'],
                'level': 'ERROR',
                'propagate': False
            }
        }
    }

    dictConfig(cfg)

    lg = 'info' if level == 0 else 'error'
    lvl = 20 if level == 0 else 40

    logger = logging.getLogger(lg)
    logger.log(lvl, msg)
293868
def _parse(self, msg_dict):
        '''
        
        '''
        error_present = False
        # log.debug('Matching the message:')
        # log.debug(msg_dict)
        for message in self.compiled_messages:
            # log.debug('Matching using:')
            # log.debug(message)
            match_on = message['match_on']
            if match_on not in msg_dict:
                # log.debug('%s is not a valid key in the partially parsed dict', match_on)
                continue
            if message['tag'] != msg_dict[match_on]:
                continue
            if '__python_fun__' in message:
                return {
                    'model': message['model'],
                    'error': message['error'],
                    '__python_fun__': message['__python_fun__']
                }
            error_present = True
            match = message['line'].search(msg_dict['message'])
            if not match:
                continue
            positions = message.get('positions', {})
            values = message.get('values')
            ret = {
                'model': message['model'],
                'mapping': message['mapping'],
                'replace': message['replace'],
                'error': message['error']
            }
            for key in values.keys():
                # Check if the value needs to be replaced
                if key in message['replace']:
                    result = napalm_logs.utils.cast(match.group(positions.get(key)), message['replace'][key])
                else:
                    result = match.group(positions.get(key))
                ret[key] = result
            return ret
        if error_present is True:
            log.info('Configured regex did not match for os: %s tag %s', self._name, msg_dict.get('tag', ''))
        else:
            log.info('Syslog message not configured for os: %s tag %s', self._name, msg_dict.get('tag', ''))
1251458
def parse!
      begin
        parse
      rescue StandardError => e
        raise e if @options[:trace] || e.is_a?(SystemExit)
        $stderr.puts "#{e.class}: " + e.message.to_s
        exit 1
      end
      exit 0
    end
284435
def load_parsers(self):
        """"""
        for parser in sorted(logdissect.parsers.__all__):
            self.parse_modules[parser] = \
                __import__('logdissect.parsers.' + parser, globals(), \
                locals(), [logdissect]).ParseModule()
878197
def parse_message(message, nodata=False):
    
    header = read_machine_header(message)
    h_len = __get_machine_header_length(header)
    meta_raw = message[h_len:h_len + header['meta_len']]
    meta = __parse_meta(meta_raw, header)
    data_start = h_len + header['meta_len']
    data = b''
    if not nodata:
        data = __decompress(
            meta,
            message[data_start:data_start + header['data_len']]
        )
    return header, meta, data
1283741
def _parse_fmadm_config(output):
    '''
    
    '''
    result = []
    output = output.split("\n")

    # extract header
    header = [field for field in output[0].lower().split(" ") if field]
    del output[0]

    # parse entries
    for entry in output:
        entry = [item for item in entry.split(" ") if item]
        entry = entry[0:3] + [" ".join(entry[3:])]

        # prepare component
        component = OrderedDict()
        for field in header:
            component[field] = entry[header.index(field)]

        result.append(component)

    # keying
    keyed_result = OrderedDict()
    for component in result:
        keyed_result[component['module']] = component
        del keyed_result[component['module']]['module']

    result = keyed_result

    return result