Parse raw log lines and convert it to a dictionary with extracted features.
11484
def _extract_line_features(self):
        
        for line in self.line_iterator:
            m = re.search(GENERIC_IPV4_REGEX, line)

            data = {
                'raw': line
            }

            if m:
                data['ip'] = m.group('ip')

            self.parsed_lines.append(data)
686986
def parse_item(lines):
    
    info = {}
    for line in lines:
        key, value = line.split(': ', 1)
        if key in LIST_KEYS:
            info.setdefault(key, []).append(value)
        else:
            assert key not in info
            info[key] = value
    
    if 'Subtag' in info or 'Tag' in info:
        yield info
31224
def parse(self, line):
        
        tree = list(self.parser.raw_parse(line))[0]
        tree = tree[0]
        return tree
909067
def _convert_to_dict(stdout):
    

    lines = stdout.split("\n")
    lines = list(filter(None, lines))
    info_dict, j = _get_dict(lines, 0, 0, 0)
    return info_dict
591094
def parse_line(self, line, lineno):
        """"""
        match = self.RE_TINDERBOXPRINT.match(line) if line else None
        if match:
            line = match.group('line')

            for regexp_item in self.TINDERBOX_REGEXP_TUPLE:
                match = regexp_item['re'].match(line)
                if match:
                    artifact = match.groupdict()
                    # handle duplicate fields
                    for to_field, from_field in regexp_item['duplicates_fields'].items():
                        # if to_field not present or None copy form from_field
                        if to_field not in artifact or artifact[to_field] is None:
                            artifact[to_field] = artifact[from_field]
                    artifact.update(regexp_item['base_dict'])
                    self.artifact.append(artifact)
                    return

            # default case: consider it html content
            # try to detect title/value splitting on <br/>
            artifact = {"content_type": "raw_html", }
            if "<br/>" in line:
                title, value = line.split("<br/>", 1)
                artifact["title"] = title
                artifact["value"] = value
            # or similar long lines if they contain a url
            elif "href" in line and "title" in line:
                def parse_url_line(line_data):
                    class TpLineParser(HTMLParser):

                        def handle_starttag(self, tag, attrs):
                            d = dict(attrs)
                            artifact["url"] = d['href']
                            artifact["title"] = d['title']

                        def handle_data(self, data):
                            artifact["value"] = data

                    p = TpLineParser()
                    p.feed(line_data)
                    p.close()

                # strip ^M returns on windows lines otherwise
                # handle_data will yield no data 'value'
                parse_url_line(line.replace('\r', ''))
            else:
                artifact["value"] = line
            self.artifact.append(artifact)
1090004
def _from_dict(cls, _dict):
        """"""
        args = {}
        xtra = _dict.copy()
        if 'log_messages' in _dict:
            args['log_messages'] = [
                LogMessage._from_dict(x) for x in (_dict.get('log_messages'))
            ]
            del xtra['log_messages']
        else:
            raise ValueError(
                'Required property \'log_messages\' not present in OutputData JSON'
            )
        if 'text' in _dict:
            args['text'] = _dict.get('text')
            del xtra['text']
        else:
            raise ValueError(
                'Required property \'text\' not present in OutputData JSON')
        if 'generic' in _dict:
            args['generic'] = [
                DialogRuntimeResponseGeneric._from_dict(x)
                for x in (_dict.get('generic'))
            ]
            del xtra['generic']
        if 'nodes_visited' in _dict:
            args['nodes_visited'] = _dict.get('nodes_visited')
            del xtra['nodes_visited']
        if 'nodes_visited_details' in _dict:
            args['nodes_visited_details'] = [
                DialogNodeVisitedDetails._from_dict(x)
                for x in (_dict.get('nodes_visited_details'))
            ]
            del xtra['nodes_visited_details']
        args.update(xtra)
        return cls(**args)
586755
def parse_text_to_dict(self, txt):
        
        op = {}
        print('TODO - import NLP, split into verbs / nouns')
        op['nouns'] = txt
        op['verbs'] = txt
        
        return op
678964
def parse (cls, line, lineno, log, cmddict=None):
    
    start  = line.find('%')
    pos    = SeqPos(line, lineno, start + 1, len(line))
    result = None

    if start >= 0:
      result = cls(line[start:], pos)

    return result
1201999
def _report_line_to_dict(cls, line):
        ''''''
        data = line.split('\t')
        if len(data) != len(report.columns):
            return None

        d = dict(zip(report.columns, data))
        for key in report.int_columns:
            try:
                d[key] = int(d[key])
            except:
                assert d[key] == '.'

        for key in report.float_columns:
            try:
                d[key] = float(d[key])
            except:
                assert d[key] == '.'

        d['flag'] = flag.Flag(int(d['flag']))
        return d
589358
def parse_lines(lines: [str], units: Units, use_na: bool = True) -> [dict]:  # type: ignore
    
    parsed_lines = []
    prob = ''
    while lines:
        raw_line = lines[0].strip()
        line = core.sanitize_line(raw_line)
        # Remove prob from the beginning of a line
        if line.startswith('PROB'):
            # Add standalone prob to next line
            if len(line) == 6:
                prob = line
                line = ''
            # Add to current line
            elif len(line) > 6:
                prob = line[:6]
                line = line[6:].strip()
        if line:
            parsed_line = (parse_na_line if use_na else parse_in_line)(line, units)
            for key in ('start_time', 'end_time'):
                parsed_line[key] = core.make_timestamp(parsed_line[key])  # type: ignore
            parsed_line['probability'] = core.make_number(prob[4:])  # type: ignore
            parsed_line['raw'] = raw_line
            parsed_line['sanitized'] = prob + ' ' + line if prob else line
            prob = ''
            parsed_lines.append(parsed_line)
        lines.pop(0)
    return parsed_lines
590573
def parse(logfile, time_format):
  ""

  records = [server.record_from_txt(line, only_elapsed=True,
    time_format=time_format) for line in sys.stdin.readlines()]

  # TODO: make this code better.
  def output(summary):
    width = max([len(p[0]) for p in summary]) + 3
    print '\n'.join([
      "%s%s%s" % (p[0], ' ' * (width - len(p[0])),
        colored(minutes_to_txt(p[1]), 'red')) for p in summary])

  output(server.summarize(records))
467275
def _to_dict(self):
        """"""
        _dict = {}
        if hasattr(self, 'features') and self.features is not None:
            _dict['features'] = self.features
        if hasattr(self,
                   'text_characters') and self.text_characters is not None:
            _dict['text_characters'] = self.text_characters
        if hasattr(self, 'text_units') and self.text_units is not None:
            _dict['text_units'] = self.text_units
        return _dict
1168686
def _parse_dict(features, new_names):
        """"""
        feature_collection = OrderedDict()
        for feature_type, feature_names in features.items():
            try:
                feature_type = FeatureType(feature_type)
            except ValueError:
                ValueError('Failed to parse {}, keys of the dictionary have to be instances '
                           'of {}'.format(features, FeatureType.__name__))

            feature_collection[feature_type] = feature_collection.get(feature_type, OrderedDict())

            if feature_names is ...:
                feature_collection[feature_type] = ...

            if feature_type.has_dict() and feature_collection[feature_type] is not ...:
                feature_collection[feature_type].update(FeatureParser._parse_feature_names(feature_names, new_names))

        return feature_collection
1035969
def to_dict(self):
        
        return {'logs_folder': self.logs_folder,
                'log_file': self.log_file,
                'log_console': self.log_console,
                'log_name': self.log_name,
                'log_filename': self.log_filename,
                'keep_osm_tags': self.keep_osm_tags
                }
328787
def normalized_log_entries(raw_entries):
    ''''''
    entry_start = r'([0-9:, \-]+) - .* - (\w+) - (.*)$'
    entry = None
    # process start/end here - avoid parsing log entries twice
    for line in raw_entries:
        m = re.match(entry_start, line)
        if m:
            # this is the start of a new entry
            # spit out the one previously built up (if any)
            if entry is not None:
                yield entry
            (log_time, log_level, log_text) = m.groups()
            # convert time
            log_timestamp = _timestamp_from_string(log_time)
            # join level and first line of message
            msg = '[{}] {}'.format(log_level, log_text)
            entry = {
                'timestamp': log_timestamp,
                'message': msg,
            }
        else:
            # additional line(s) for entry (i.e. stack trace)
            entry['message'] = entry['message'] + line
    if entry is not None:
        # return the final entry
        yield entry
453005
def from_string(cls, line, ignore_bad_cookies=False,
                    ignore_bad_attributes=True):
        ""
        cookie_dict = parse_one_response(
            line, ignore_bad_cookies=ignore_bad_cookies,
            ignore_bad_attributes=ignore_bad_attributes)
        if not cookie_dict:
            return None
        return cls.from_dict(
            cookie_dict, ignore_bad_attributes=ignore_bad_attributes)
973980
def parse_metadata(lines):
    
    meta = defaultdict(list)
    for line in lines:
        line = line.rstrip()
        if line.startswith("!"):
            if "_table_begin" in line or "_table_end" in line:
                continue
            key, value = __parse_entry(line)
            meta[key].append(value)

    return dict(meta)