Lambda Entrypoint - Log Subscriber

    Format log events and relay to sentry (direct or sqs)
503744
def process_log_event(event, context):
    
    init()
    # Grab the actual error log payload
    serialized = event['awslogs'].pop('data')
    data = json.loads(zlib.decompress(
        base64.b64decode(serialized), 16 + zlib.MAX_WBITS))
    msg = get_sentry_message(config, data)
    if msg is None:
        return
    if config['sentry_dsn']:
        # Deliver directly to sentry
        send_sentry_message(config['sentry_dsn'], msg)
    elif config['sentry_sqs']:
        # Delivery indirectly via sqs
        sqs.send_message(
            QueueUrl=config['sentry_sqs'])
488169
def invoke_lambda_function( self,
                                function_name,
                                payload,
                                invocation_type='Event',
                                log_type='Tail',
                                client_context=None,
                                qualifier=None
                            ):
        
        return self.lambda_client.invoke(
            FunctionName=function_name,
            InvocationType=invocation_type,
            LogType=log_type,
            Payload=payload
        )
574897
def setup_model_event_subscribers(config, model_cls, schema):
    
    events_map = get_events_map()
    model_events = schema.get('_event_handlers', {})
    event_kwargs = {'model': model_cls}

    for event_tag, subscribers in model_events.items():
        type_, action = event_tag.split('_')
        event_objects = events_map[type_][action]

        if not isinstance(event_objects, list):
            event_objects = [event_objects]

        for sub_name in subscribers:
            sub_func = resolve_to_callable(sub_name)
            config.subscribe_to_events(
                sub_func, event_objects, **event_kwargs)
392315
def get_function(session_factory, name, role, sns_topic, log_groups,
                 subject="Lambda Error", pattern="Traceback"):
    

    # Lazy import to avoid runtime dependency
    from c7n.mu import (
        LambdaFunction, PythonPackageArchive, CloudWatchLogSubscription)

    config = dict(
        name=name,
        handler='logsub.process_log_event',
        runtime='python2.7',
        memory_size=512,
        timeout=15,
        role=role,
        description='Custodian Ops Error Notify',
        events=[
            CloudWatchLogSubscription(
                session_factory, log_groups, pattern)])

    archive = PythonPackageArchive()
    archive.add_py_file(__file__)
    archive.add_contents(
        'config.json', json.dumps({
            'topic': sns_topic,
            'subject': subject
        }))
    archive.close()

    return LambdaFunction(config, archive)
951839
function triggerRunnerEvents() {
    var logs = [];
    var _stdoutWrite = process.stdout.write;
    process.stdout.write = function() {
        logs.push(arguments[0]);
    };
    reporter.jasmineStarted();
    for (var i=0; i<env._suites.length; i++) {
        var s = env._suites[i];
        triggerSuiteEvents(s);
    }
    reporter.jasmineDone();
    process.stdout.write = _stdoutWrite;
    return logs;
}
1232219
function _observeSubscriptionSyncEvents(args) {
  var rtm = args.rtm;
  var filterFn = args.filterFn;

  rtm.on('subscription-created', function(subscription) {
    if (subscription.allowRead !== true) return;

    filterFn(subscription).then(function(ok) {
      if (ok) {
        rtm.subscribe({channel: subscription.channelId});
      }
    });
  });

  rtm.on('subscription-deleted', function(subscription) {
    rtm.unsubscribe({channel: subscription.channelId});
  });

  rtm.on('subscription-updated', function(subscription) {
    if (subscription.allowRead === false) {
      return rtm.unsubscribe({channel: subscription.channelId});
    }
    filterFn(subscription).then(function(ok) {
      if (ok) {
        rtm.subscribe({channel: subscription.channelId});
      }
    });
  });
}
586547
def submit(self, func, *args, **kwargs):
        
        self.op_sequence += 1
        self.sqs.send_message(
            QueueUrl=self.map_queue,
            MessageBody=utils.dumps({'args': args, 'kwargs': kwargs}),
            MessageAttributes={
                'sequence_id': {
                    'StringValue': str(self.op_sequence),
                    'DataType': 'Number'},
                'op': {
                    'StringValue': named(func),
                    'DataType': 'String',
                },
                'ser': {
                    'StringValue': 'json',
                    'DataType': 'String'}}
        )

        self.futures[self.op_sequence] = f = SQSFuture(
            self.op_sequence)
        return f
699832
function log( func, msg ){
  var logline = '['+func+'] '+msg;

  if ( logInfo ) {
    if ( logInfoStr == '' ) {
      console.log( logline );      
    } else if ( logInfoStr.indexOf( func ) >= 0 ) { 
      console.log( logline );
    }
  }

  // send log to event broker
  if ( loggerEvents ) { 
    var logBroker = getEventBroker('log');
    logBroker.cleanupQueue( 1000 );
    logBroker.queueEvent( { text: logline, channel:'log' } );
  }
  
}
1234519
def cli(env):
    """"""
    mgr = SoftLayer.EventLogManager(env.client)

    event_log_types = mgr.get_event_log_types()

    table = formatting.Table(COLUMNS)

    for event_log_type in event_log_types:
        table.add_row([event_log_type])

    env.fout(table)
1332041
def message_to_event(direction, message):
    
    server = message.server

    # change numerics into nice names
    if message.verb in numerics:
        message.verb = numerics[message.verb]
    verb = message.verb.lower()

    # modify public/private verbs
    if verb == 'privmsg':
        if server.is_channel(message.params[0]):
            verb = 'pubmsg'
    if verb == 'notice':
        verb = 'privnotice'
        if server.is_channel(message.params[0]):
            verb = 'pubnotice'
    elif verb == 'mode':
        verb = 'umode'
        if server.is_channel(message.params[0]):
            verb = 'cmode'

    # this is the same as ircreactor does
    info = message.__dict__
    info['direction'] = direction
    info['verb'] = verb

    if 'time' in info['tags']:
        info['server_time'] = time.strptime(info['tags']['time'],
                                            '%Y-%m-%dT%H:%M:%S.%fZ')

    infos = [[verb, info], ]

    # handle shitty ctcp
    if verb in ('privmsg', 'pubmsg', 'privnotice', 'pubnotice'):
        infos = ctcp_unpack_message(info)

    # work on each info object separately
    i = -1
    while i < (len(infos) - 1):
        i += 1
        name = infos[i][NAME_ATTR]

        # standard message attributes
        for attr, param_map in _verb_param_map.items():
            # escaping
            escaped = False
            if attr.startswith('escaped_'):
                attr = attr.lstrip('escaped_')
                escaped = True

            for param_number, verbs in param_map.items():
                if len(infos[i][INFO_ATTR]['params']) > param_number and name in verbs:
                    value = infos[i][INFO_ATTR]['params'][param_number]
                    if escaped:
                        value = escape(value)
                    infos[i][INFO_ATTR][attr] = value

        # custom processing
        if name == 'welcome':
            # for servers where a low nicklen makes them silently truncate our nick
            server.nick = server.istring(infos[i][INFO_ATTR]['nick'])

        # custom message attributes
        if name == 'ctcp':
            if infos[i][INFO_ATTR]['ctcp_verb'] == 'action':
                info = dict(infos[i][INFO_ATTR])
                info['message'] = info['ctcp_text']
                if server.is_channel(info['target']):
                    name = 'pubaction'
                    info['channel'] = info['target']
                else:
                    name = 'privaction'
                infos.append([name, info])

        if name == 'umode' and len(infos[i][INFO_ATTR]['params']) > 1:
            modestring = infos[i][INFO_ATTR]['params'][1:]
            modes = parse_modes(modestring)

            infos[i][INFO_ATTR]['modestring'] = ' '.join(modestring).strip()
            infos[i][INFO_ATTR]['modes'] = modes

        if name == 'cmode' and len(infos[i][INFO_ATTR]['params']) > 1:
            modestring = infos[i][INFO_ATTR]['params'][1:]
            chanmodes = server.features.get('chanmodes')
            prefixes = list(server.features.get('prefix').keys())
            modes = parse_modes(modestring, chanmodes, prefixes)

            infos[i][INFO_ATTR]['modestring'] = ' '.join(modestring).strip()
            infos[i][INFO_ATTR]['modes'] = modes

        if name == 'cmodeis':
            if len(infos[i][INFO_ATTR]['params']) > 2:
                modestring = infos[i][INFO_ATTR]['params'][2:]
                chanmodes = server.features.get('chanmodes')
                modes = parse_modes(modestring, chanmodes)

                infos[i][INFO_ATTR]['modestring'] = ' '.join(modestring).strip()
                infos[i][INFO_ATTR]['modes'] = modes
            else:
                infos[i][INFO_ATTR]['modestring'] = ''
                infos[i][INFO_ATTR]['modes'] = []

        if name == 'namreply':
            channel_name = infos[i][INFO_ATTR]['params'][2]
            server.info.create_channel(channel_name)
            channel = server.info.channels.get(channel_name)

            nice_names = []
            channel_prefixes = {}
            if len(infos[i][INFO_ATTR]['params']) > 3:
                raw_names = infos[i][INFO_ATTR]['params'][3].split(' ')
            else:
                raw_names = []

            for name in raw_names:
                # InspIRCd sends us an empty last param because they are cool
                if not len(name):
                    continue

                prefixes = ''
                while name[0] in server.features.available['prefix'].values():
                    prefixes += name[0]
                    name = name[1:]

                nick = NickMask(name).nick

                server.info.create_user(nick)
                nice_names.append(name)
                server.info.create_user(name)
                user = server.info.users.get(nick)
                channel_prefixes[user] = prefixes
                channel.add_user(nick, prefixes=prefixes)

            infos[i][INFO_ATTR]['users'] = ','.join(nice_names)
            infos[i][INFO_ATTR]['prefixes'] = channel_prefixes

        # source / target mapping
        for attr in ('source', 'target', 'channel'):
            if attr in infos[i][INFO_ATTR] and infos[i][INFO_ATTR][attr]:
                source = infos[i][INFO_ATTR][attr]
                if server.is_channel(source):
                    server.info.create_channel(source)
                    infos[i][INFO_ATTR][attr] = server.info.channels.get(source)
                elif '.' in source and server.is_server(source):
                    server.info.create_server(source)
                    infos[i][INFO_ATTR][attr] = server.info.servers.get(source)
                elif server.is_nick(source):
                    server.info.create_user(source)
                    infos[i][INFO_ATTR][attr] = server.info.users.get(NickMask(source).nick)
                else:  # we assume this is a user with messed up characters
                    server.info.create_user(source)
                    infos[i][INFO_ATTR][attr] = server.info.users.get(NickMask(source).nick)

        if 'channels' in infos[i][INFO_ATTR] and infos[i][INFO_ATTR]['channels']:
            channels = []
            for chan in infos[i][INFO_ATTR]['channels'].split(','):
                server.info.create_channel(chan)
                channels.append(server.info.channels.get(chan))
            infos[i][INFO_ATTR]['channels'] = channels

        if 'users' in infos[i][INFO_ATTR] and infos[i][INFO_ATTR]['users']:
            users = []
            for user in infos[i][INFO_ATTR]['users'].split(','):
                server.info.create_user(user)
                users.append(server.info.users.get(NickMask(user).nick))
            infos[i][INFO_ATTR]['users'] = users

        # custom from_to attribute for ease in bots
        verb = infos[i][INFO_ATTR]['verb']
        dir = infos[i][INFO_ATTR]['direction']
        source = infos[i][INFO_ATTR].get('source')
        target = infos[i][INFO_ATTR].get('target')

        if verb in ['pubmsg', 'pubnotice', 'pubaction']:
            infos[i][INFO_ATTR]['from_to'] = target
        elif verb in ['privmsg', 'privnotice', 'privaction']:
            if dir == 'out':
                infos[i][INFO_ATTR]['from_to'] = target
            elif dir == 'in':
                if 'echo-message' in server.capabilities.enabled:
                    infos[i][INFO_ATTR]['from_to'] = target
                else:
                    infos[i][INFO_ATTR]['from_to'] = source
        if 'from_to' in infos[i][INFO_ATTR] and infos[i][INFO_ATTR]['from_to'] is None:
            del infos[i][INFO_ATTR]['from_to']

        # convenience function so unnecessary messages can get ignored easily
        infos[i][INFO_ATTR]['will_be_echod'] = False
        if verb in ['pubmsg', 'pubnotice', 'privmsg', 'privnotice']:
            if dir == 'out' and 'echo-message' in server.capabilities.enabled:
                infos[i][INFO_ATTR]['will_be_echod'] = True

        if 'from_to' in infos[i][INFO_ATTR] and infos[i][INFO_ATTR]['from_to'].is_server:
            del infos[i][INFO_ATTR]['from_to']

    return infos
651000
def _process_event_queue(self):
    
    try:
      event = self._event_queue.get(timeout=0.05)
    except queue.Empty:
      return

    try:
      subscription, is_initial_event, files = (event['subscription'],
                                               event['is_fresh_instance'],
                                               event['files'] if PY3 else [f.decode('utf-8') for f in event['files']])
    except (KeyError, UnicodeDecodeError) as e:
      self._logger.warn('%r raised by invalid watchman event: %s', e, event)
      return

    self._logger.debug('processing {} files for subscription {} (first_event={})'
                       .format(len(files), subscription, is_initial_event))

    # The first watchman event is a listing of all files - ignore it.
    if not is_initial_event:
      if subscription == self._fs_event_service.PANTS_PID_SUBSCRIPTION_NAME:
        self._maybe_invalidate_scheduler_pidfile()
      else:
        self._handle_batch_event(files)

    if not self._watchman_is_running.is_set():
      self._watchman_is_running.set()

    self._event_queue.task_done()
1301487
def _EntryToEvent(entry, handlers, transformers):
  """"""
  event = rdf_events.AuditEvent(
      timestamp=entry.timestamp,
      user=entry.username,
      action=handlers[entry.router_method_name])

  for fn in transformers:
    fn(entry, event)

  return event
4661
def logs(awsclient, function_name, start_dt, end_dt=None, tail=False):
    
    log.debug('Getting cloudwatch logs for: %s', function_name)
    log_group_name = '/aws/lambda/%s' % function_name

    current_date = None
    start_ts = datetime_to_timestamp(start_dt)
    if end_dt:
        end_ts = datetime_to_timestamp(end_dt)
    else:
        end_ts = None

    # tail mode
    # we assume that logs can arrive late but not out of order
    # so we hold the timestamp of the last logentry and start the next iteration
    # from there
    while True:
        logentries = filter_log_events(awsclient, log_group_name,
                                       start_ts=start_ts, end_ts=end_ts)
        if logentries:
            for e in logentries:
                actual_date, actual_time = decode_format_timestamp(e['timestamp'])
                if current_date != actual_date:
                    # print the date only when it changed
                    current_date = actual_date
                    log.info(current_date)
                log.info('%s  %s' % (actual_time, e['message'].strip()))
        if tail:
            if logentries:
                start_ts = logentries[-1]['timestamp'] + 1
            time.sleep(2)
            continue
        break
770579
function handleEntryPoints(opts, io, nextHandler) {
  var entries = opts.entries
    , bundlerOpts

  bundlerOpts = opts.bundler

  if(!bundlerOpts || !bundlerOpts.legacy) {
    return nextHandler
  }

  if(!opts.entries || !opts.bundler) {
    return nextHandler
  }

  return handle

  function handle(server, req, resp, parsed) {
    if(!(parsed.pathname in entries) && !('browserify' in parsed.query)) {
      return nextHandler(server, req, resp, parsed)
    }

    var entryPath = entries[parsed.pathname]
      , args = bundlerOpts.flags.slice()
      , bundler
      , output

    if(entryPath) {
      args.unshift(entryPath)
    }

    args.unshift(bundlerOpts.command)
    parsed.loggedPathname = ansicolors.magenta(
        parsed.pathname + ' -> ' + args.map(toLocal).join(' ')
    )
    args.shift()

    bundler = spawn(bundlerOpts.command, args)
    bundler.stderr.pipe(accumError(io.error, resp))
    resp.setHeader('content-type', 'text/javascript')
    bundler.stdout.pipe(resp)
  }

  function toLocal(file) {
    return file.replace(opts.cwd, '.')
  }
}
656297
def _connect_sentry(message, result):
    '''
    
    '''
    pillar_data = __salt__['pillar.raw']()
    grains = __salt__['grains.items']()
    raven_config = pillar_data['raven']
    hide_pillar = raven_config.get('hide_pillar')
    sentry_data = {
        'result': result,
        'pillar': 'HIDDEN' if hide_pillar else pillar_data,
        'grains': grains
    }
    data = {
        'platform': 'python',
        'culprit': message,
        'level': 'error'
    }
    tags = {}
    if 'tags' in raven_config:
        for tag in raven_config['tags']:
            tags[tag] = grains[tag]

    if _ret_is_not_error(result):
        data['level'] = 'info'

    if raven_config.get('report_errors_only') and data['level'] != 'error':
        return

    if raven_config.get('dsn'):
        client = Client(raven_config.get('dsn'), transport=HTTPTransport)
    else:
        try:
            servers = []
            for server in raven_config['servers']:
                servers.append(server + '/api/store/')
            client = Client(
                servers=servers,
                public_key=raven_config['public_key'],
                secret_key=raven_config['secret_key'],
                project=raven_config['project'],
                transport=HTTPTransport
            )
        except KeyError as missing_key:
            log.error(
                'Sentry returner needs key \'%s\' in pillar',
                missing_key
            )
            return

    try:
        msgid = client.capture(
            'raven.events.Message',
            message=message,
            data=data,
            extra=sentry_data,
            tags=tags
        )
        log.info('Message id %s written to sentry', msgid)
    except Exception as exc:
        log.error('Can\'t send message to sentry: %s', exc, exc_info=True)
881287
def setup_logging(self):
        """"""
        date_format = '%Y-%m-%dT%H:%M:%S'
        log_format = '%(asctime)s %(levelname)s: %(message)s'

        if self.opts.verbose:
            lvl = logging.DEBUG
        else:
            lvl = logging.INFO
            # Requests is a bit chatty
            logging.getLogger('requests').setLevel('WARNING')

        self.logger.setLevel(lvl)

        stdout = logging.StreamHandler(sys.stdout)
        stdout.setLevel(lvl)
        formatter = logging.Formatter(log_format, date_format)
        stdout.setFormatter(formatter)
        self.logger.addHandler(stdout)

        # Decided not to use stderr
        # stderr = logging.StreamHandler(sys.stderr)
        # stderr.setLevel(logging.ERROR)  # Error and above go to both stdout & stderr
        # formatter = logging.Formatter(log_format, date_format)
        # stderr.setFormatter(formatter)
        # self.logger.addHandler(stderr)

        log = self.opts.log or self.config['crony'].get('log_file')
        if log:
            logfile = logging.FileHandler(log)
            logfile.setLevel(lvl)
            formatter = logging.Formatter(log_format, date_format)
            logfile.setFormatter(formatter)
            self.logger.addHandler(logfile)

        if self.sentry_client:
            sentry = SentryHandler(self.sentry_client)
            sentry.setLevel(logging.ERROR)
            self.logger.addHandler(sentry)

        self.logger.debug('Logging setup complete.')
901585
function loggerEmitter(opts) {

  if (!(this instanceof loggerEmitter)) return new loggerEmitter(opts); // new is optional
  events.EventEmitter.call(this);
  opts = opts || {};
  var logger = this;

  logger.noConsole   = opts.noConsole || false;
  logger.noErrors    = opts.noErrors  || false;
  logger.log         = log;
  logger.log.logger  = logger;
  logger.log.inspect = inspect;

  function log(x) {
    var msg;
    if (util.isError(x)) {
      msg = x.stack || x.message;
      if (!logger.noErrors) {
        logger.emit('error', x); // throws if no listener
      }
      else {
        logger.emit('log', msg);
      }
    }
    else {
      msg = util.format.apply(logger, arguments);
      logger.emit('log', msg);
    }

    if (!logger.noConsole) {
      console.log(msg);
    }

    return msg;
  }

  function inspect() { return 'logger-emitter'; }
}