获取收藏夹日志

        :return: 收藏夹日志中的操作，返回生成器
        :rtype: CollectActivity.Iterable
344680
def logs(self):
        
        import time
        from datetime import datetime
        from .answer import Answer
        from .question import Question
        from .acttype import CollectActType

        self._make_soup()
        gotten_feed_num = 20
        offset = 0
        data = {
            'start': 0,
            '_xsrf': self.xsrf
        }
        api_url = self.url + 'log'
        while gotten_feed_num == 20:
            data['offset'] = offset
            res = self._session.post(url=api_url, data=data)
            gotten_feed_num = res.json()['msg'][0]
            soup = BeautifulSoup(res.json()['msg'][1])
            offset += gotten_feed_num
            zm_items = soup.find_all('div', class_='zm-item')

            for zm_item in zm_items:
                act_time = datetime.strptime(zm_item.find('time').text, "%Y-%m-%d %H:%M:%S")
                if zm_item.find('ins'):
                    link = zm_item.find('ins').a
                    act_type = CollectActType.INSERT_ANSWER
                elif zm_item.find('del'):
                    link = zm_item.find('del').a
                    act_type = CollectActType.DELETE_ANSWER
                else:
                    continue
                try:
                    answer_url = Zhihu_URL + link['href']
                    question_url = re_a2q.match(answer_url).group(1)
                    question = Question(question_url, link.text)
                    answer = Answer(
                        answer_url, question, session=self._session)
                    yield CollectActivity(
                        act_type, act_time, self.owner, self, answer)
                except AttributeError:
                    act_type = CollectActType.CREATE_COLLECTION
                    yield CollectActivity(
                        act_type, act_time, self.owner, self)
            data['start'] = zm_items[-1]['id'][8:]
            time.sleep(0.5)
747447
def logs(self):
        """"""
        if self._resources is None:
            self.__init()
        if "logs" in self._resources:
            url = self._url + "/logs"
            return _logs.Log(url=url,
                             securityHandler=self._securityHandler,
                             proxy_url=self._proxy_url,
                             proxy_port=self._proxy_port,
                             initialize=True)
        else:
            return None
1324237
def _get_log(self):
        

        result = yield from self.manager.query("GET", "containers/{}/logs".format(self._cid), params={"stderr": 1, "stdout": 1})
        return result
611201
def log(self):
        
        logserv = self.system.request_service('LogStoreService')
        return logserv.lastlog(html=False)
515333
def get_log_report(log: Union[logging.Logger,
                              logging.PlaceHolder]) -> Dict[str, Any]:
    
    if isinstance(log, logging.Logger):
        # suppress invalid error for Logger.manager:
        # noinspection PyUnresolvedReferences
        return {
            '(object)': str(log),
            'level': log.level,
            'disabled': log.disabled,
            'propagate': log.propagate,
            'parent': str(log.parent),
            'manager': str(log.manager),
            'handlers': [get_handler_report(h) for h in log.handlers],
        }
    elif isinstance(log, logging.PlaceHolder):
        return {
            "(object)": str(log),
        }
    else:
        raise ValueError("Unknown object type: {!r}".format(log))
1212392
def logs(self):
        """"""
        if not self.parent.loaded: self.parent.load()
        logs = self.parent.p.logs_dir.flat_directories
        logs.sort(key=lambda x: x.mod_time)
        return logs
737139
def publish(self):
        

        try:
             for collection in self.settings.get("scheduler").get("collections"):
                yield self.publish_for_collection(collection)
        except Exception as ex:
            self.logger.error(ex)
638578
def dump_to_log(self):
        
        if self._event_dict:
            logger.info('Events:')
            for event_str, count_int in sorted(self._event_dict.items()):
                logger.info('  {}: {}'.format(event_str, count_int))
        else:
            logger.info('No Events')
1297445
def Logs(c_type=None, grep_list=None):
    
    def get_logs(logs, log_entries):
        try:
            for log in logs:
                if str(container.name) in log_entries:
                    log_entries[str(container.name)].append(log)
                else:
                    log_entries[str(container.name)] = [log]
        except Exception as e:  # pragma: no cover
            logger.error('Unable to get logs for ' +
                         str(container.name) +
                         ' because: ' + str(e))
        return log_entries

    status = (True, None)
    log_entries = {}
    d_client = docker.from_env()
    containers = d_client.containers.list(all=True,
                                          filters={'label': 'vent'})
    logger.debug('containers found: ' + str(containers))
    comp_c = containers
    if c_type:
        try:
            comp_c = [c for c in containers
                      if (c_type
                          in c.attrs['Config']['Labels']['vent.groups'])]
        except Exception as e:  # pragma: no cover
            logger.error('Unable to limit containers by: ' +
                         str(c_type) + ' because: ' +
                         str(e))

    if grep_list:
        for expression in grep_list:
            for container in comp_c:
                try:
                    # 'logs' stores each line containing the expression
                    logs = [log for log in container.logs().split('\n')
                            if expression in log]
                    log_entries = get_logs(logs, log_entries)
                except Exception as e:  # pragma: no cover
                    logger.info('Unable to get logs for ' +
                                str(container) +
                                ' because: ' + str(e))
    else:
        for container in comp_c:
            try:
                logs = container.logs().split('\n')
                log_entries = get_logs(logs, log_entries)
            except Exception as e:  # pragma: no cover
                logger.info('Unabled to get logs for ' +
                            str(container) +
                            ' because: ' + str(e))

    status = (True, log_entries)
    return status
599482
def parse_logs(log_list, date, machine_name, log_type):
    
    output = []
    count = fail = skip = updated = 0

    # Check things are setup correctly
    try:
        machine = Machine.objects.get(name=machine_name)
    except Machine.DoesNotExist:
        return "ERROR: Couldn't find machine named: %s" % machine_name

    if log_type == "alogger":
        parser = AloggerParser()
    else:
        parser = get_parser(log_type)

    # Process each line
    for line_no, line in enumerate(log_list):
        try:
            data = parser.line_to_dict(line)
        except ValueError:
            output.append("%d: Error reading line" % line_no)
            continue

        # if parser returns None, nothing to do, continue to next line
        if data is None:
            skip += 1
            continue

        # check for required fields
        required_fields = [
            'user', 'project', 'jobid', 'jobname',
            'cpu_usage', 'cores',
            'act_wall_time', 'est_wall_time',
            'mem', 'vmem', 'list_pmem', 'list_mem', 'list_pvmem',
            'ctime', 'qtime', 'etime', 'start',
        ]

        for field in required_fields:
            if field not in data:
                output.append(
                    "line %d: %s field not given." % (line_no, field))
                fail = fail + 1
                continue

        # Process user --> account
        try:
            account = Account.objects.get(
                username=data['user'],
                date_deleted__isnull=True)

        except Account.DoesNotExist:
            # Couldn't find user account - Assign to user None
            output.append(
                "line %d: Couldn't find user account for username=%s."
                % (line_no, data['user']))
            fail += 1
            continue

        except Account.MultipleObjectsReturned:
            output.append(
                "line %d: Username %s has multiple active accounts."
                % (line_no, data['user']))
            fail += 1
            continue

        # Process project
        if data['project'] is None:
            output.append(
                "line %d: Project was not supplied." % (line_no))
            fail += 1
            continue

        try:
            project = Project.objects.get(pid=data['project'])

        except Project.DoesNotExist:
            output.append(
                "line %d: Couldn't find specified project %s"
                % (line_no, data['project']))
            fail += 1
            continue

        # memory calculations
        if machine.mem_per_core:
            avail_mem_per_core = machine.mem_per_core * 1024
            avail_mem_for_job = avail_mem_per_core * data['cores']

            if data['list_pmem'] * data['cores'] > data['list_mem']:
                memory_used_per_core = data['list_pmem']
                memory_used_for_job = data['list_pmem'] * data['cores']
            else:
                memory_used_per_core = data['list_mem'] / data['cores']
                memory_used_for_job = data['list_mem']

            if memory_used_for_job > avail_mem_for_job:
                data['cpu_usage'] = ceil(
                    memory_used_per_core / avail_mem_per_core
                    * data['act_wall_time']
                    * data['cores'])

        # apply scaling factor to cpu_usage
        data['cpu_usage'] = data['cpu_usage'] * machine.scaling_factor

        # Everything is good so add entry
        queue, created = Queue.objects.get_or_create(name=data['queue'])

        try:
            cpujob, created = CPUJob.objects.get_or_create(jobid=data['jobid'])
            cpujob.account = account
            cpujob.username = data['user']
            cpujob.project = project
            cpujob.machine = machine
            cpujob.date = date
            cpujob.queue = queue
            cpujob.cpu_usage = data['cpu_usage']
            cpujob.est_wall_time = data['est_wall_time']
            cpujob.act_wall_time = data['act_wall_time']
            cpujob.mem = data['mem']
            cpujob.vmem = data['vmem']
            cpujob.ctime = data['ctime']
            cpujob.qtime = data['qtime']
            cpujob.etime = data['etime']
            cpujob.start = data['start']
            cpujob.cores = data['cores']
            cpujob.exit_status = data['exit_status']
            cpujob.jobname = data['jobname']
            cpujob.list_mem = data['list_mem']
            cpujob.list_vmem = data['list_vmem']
            cpujob.list_pmem = data['list_pmem']
            cpujob.list_pvmem = data['list_pvmem']
            cpujob.save()

        except Exception as e:
            output.append(
                "line %d: Failed to insert a line  - %s" % (line_no, e))
            fail += 1
            continue

        if created:
            count += 1
        else:
            updated += 1

    summary = (
        'Inserted : %i\nUpdated  : %i\nFailed   : %i\nSkiped   : %i'
        % (count, updated, fail, skip)
    )

    logger.debug('Inserted : %i' % count)
    logger.debug('Updated  : %i' % updated)
    logger.debug('Failed   : %i' % fail)
    logger.debug('Skiped   : %i' % skip)

    return summary, output
759315
def get_log_entries(self, start_date=None, end_date=None, filters=list(),
                        limit=None):
        '''
        
        '''
        query = text_helper.format_block('''
        SELECT "localhost",
               logs.message,
               logs.level,
               logs.category,
               logs.log_name,
               logs.file_path,
               logs.line_num,
               logs.timestamp
        FROM logs
        WHERE 1
        ''')
        query = self._add_timestamp_condition_sql(query, start_date, end_date)

        def transform_filter(filter):
            level = filter.get('level', None)
            category = filter.get('category', None)
            name = filter.get('name', None)
            if level is None:
                raise AttributeError("level is mandatory parameter.")
            resp = "(logs.level <= %d" % (int(level), )
            if category is not None:
                resp += " AND logs.category == '%s'" % (category, )
            if name is not None:
                resp += " AND logs.log_name == '%s'" % (name, )
            resp += ')'
            return resp

        filter_strings = map(transform_filter, filters)
        if filter_strings:
            query += " AND (%s)\n" % (' OR '.join(filter_strings), )
        query += " ORDER BY logs.timestamp, rowid"
        if limit:
            query += " LIMIT %s" % (limit, )

        d = self._db.runQuery(query)
        d.addCallback(self._decode, entry_type='log')
        return d
281656
def log_to_api(self):
        
        if self.entries:
            try:
                headers = {'Content-Type': 'application/json'}
                self.session.post('/v2/logs/app', headers=headers, json=self.entries)
                # self.entries = []  # clear entries
            except Exception:
                # best effort on api logging
                pass
859104
def get_logger(self):
        
        if Global.LOGGER:
            Global.LOGGER.debug('configuring a logger')
        if self._logger_instance is not None:
            return self._logger_instance

        self._logger_instance = logging.getLogger("flowsLogger")
        self._logger_instance.setLevel(logging.DEBUG)

        log_format = '%(asctime)s - [%(levelname)s]|%(thread)d\t%(message)s'
        log_date_format = '%Y-%m-%d %H:%M:%S'
        formatter = logging.Formatter(log_format, log_date_format)

        new_log_stream_handler = logging.StreamHandler()
        new_log_stream_handler.setFormatter(formatter)
        new_log_stream_handler.setLevel(logging.INFO)

        self._logger_instance.addHandler(new_log_stream_handler)

        return self._logger_instance
968552
def log(self, msg, level="info"):
        
        if not self.config.get("log_file"):
            # If level was given as a str then convert to actual level
            level = LOG_LEVELS.get(level, level)
            syslog(level, u"{}".format(msg))
        else:
            # Binary mode so fs encoding setting is not an issue
            with open(self.config["log_file"], "ab") as f:
                log_time = time.strftime("%Y-%m-%d %H:%M:%S")
                # nice formating of data structures using pretty print
                if isinstance(msg, (dict, list, set, tuple)):
                    msg = pformat(msg)
                    # if multiline then start the data output on a fresh line
                    # to aid readability.
                    if "\n" in msg:
                        msg = u"\n" + msg
                out = u"{} {} {}\n".format(log_time, level.upper(), msg)
                try:
                    # Encode unicode strings to bytes
                    f.write(out.encode("utf-8"))
                except (AttributeError, UnicodeDecodeError):
                    # Write any byte strings straight to log
                    f.write(out)
493912
def process_log(self, **log_item):
        
        logger.debug("Processing log item: %s", log_item)
        self.log_batch.append(log_item)
        if len(self.log_batch) >= self.log_batch_size:
            self._post_log_batch()
771977
def collect_output(self, out_file=None):
        
        if self.logger.isEnabledFor(logging.INFO):
            self.logger.info(
                "collecting output %s", " to %s" % out_file if out_file else ''
            )
            self.logger.info("self.output %s", self.output)
        return collect_output(self.output, out_file)
23248
def create_log(self):
        """"""
        return EventLog(
            self.networkapi_url,
            self.user,
            self.password,
            self.user_ldap)