runBatch - gets and processes logs for a range of blocks
765164
async function runBatch(opts, context) {
  const fromBlock = opts.fromBlock
  const toBlock = opts.toBlock
  let lastLogBlock = undefined

  console.log(
    'Looking for logs from block ' + fromBlock + ' to ' + (toBlock || 'Latest')
  )

  const eventTopics = Object.keys(context.signatureToRules)
  const logs = await web3.eth.getPastLogs({
    fromBlock: web3.utils.toHex(fromBlock), // Hex required for infura
    toBlock: toBlock ? web3.utils.toHex(toBlock) : "latest",  // Hex required for infura
    topics: [eventTopics]
  })

  if (logs.length > 0) {
    console.log('' + logs.length + ' logs found')
  }

  for (const log of logs) {
    const contractVersion = context.addressToVersion[log.address]
    if (contractVersion == undefined) {
      continue // Skip - Not a trusted contract
    }
    const contractName = contractVersion.contractName
    const rule = context.signatureToRules[log.topics[0]][contractName]
    if (rule == undefined) {
      continue // Skip - No handler defined
    }
    lastLogBlock = log.blockNumber
    // Process it
    await handleLog(log, rule, contractVersion, context)
  }
  return lastLogBlock
}
726859
def _parse_run_control(block):
    """"""
    _, max_iterations, strain_ratio, _, _ = parse_fixed_width(
        2 * [(5, int)] + [(10, float)] + 2 * [(5, int)], block)

    return propagation.EquivalentLinearCalculation(
        strain_ratio, max_iterations, tolerance=10.)
266314
def processing_block_list():
    """"""
    pb_list = ProcessingBlockList()
    return dict(active=pb_list.active,
                completed=pb_list.completed,
                aborted=pb_list.aborted)
81684
def dump_logs(job_queue='run_reach_queue', job_status='RUNNING'):
    """"""
    jobs = get_jobs(job_queue, job_status)
    for job in jobs:
        get_job_log(job, write_file=True)
397713
private void logBlock(int blockIndex, int startIndex, int blockLength)
   {
      if (m_log != null)
      {
         m_log.println("Block Index: " + blockIndex);
         m_log.println("Length: " + blockLength + " (" + Integer.toHexString(blockLength) + ")");
         m_log.println();
         m_log.println(FastTrackUtility.hexdump(m_buffer, startIndex, blockLength, true, 16, ""));
         m_log.flush();
      }
   }
509708
def run_and_print_log(workflow, highlight=None):
    
    from noodles.run.threading.sqlite3 import run_parallel
    from noodles import serial

    import io
    import logging

    log = io.StringIO()
    log_handler = logging.StreamHandler(log)
    formatter = logging.Formatter('%(asctime)s - %(message)s')
    log_handler.setFormatter(formatter)

    logger = logging.getLogger('noodles')
    logger.setLevel(logging.INFO)
    logger.handlers = [log_handler]

    result = run_parallel(
        workflow, n_threads=4, registry=serial.base, db_file='tutorial.db',
        always_cache=True, echo_log=False)
    display_text(log.getvalue(), highlight or [], split_at=40)
    return result
142189
def get_log(self, offset, count=10, callback=None):
        '''
        
        '''
        params = {'offset': offset, 'count': count}
        return self.execute_command('getLog', params, callback=callback)
521862
def run(self):
        """"""
        grouping = Grouping(group_by=lambda x: (x.namespace, x.operation,
                                                x.pattern))
        logfile = self.mloginfo.logfile

        if logfile.start and logfile.end:
            progress_start = self.mloginfo._datetime_to_epoch(logfile.start)
            progress_total = (self.mloginfo._datetime_to_epoch(logfile.end) -
                              progress_start)
        else:
            self.mloginfo.progress_bar_enabled = False

        for i, le in enumerate(logfile):
            # update progress bar every 1000 lines
            if self.mloginfo.progress_bar_enabled and (i % 1000 == 0):
                if le.datetime:
                    progress_curr = self.mloginfo._datetime_to_epoch(le
                                                                     .datetime)
                    if progress_total:
                        (self.mloginfo
                         .update_progress(float(progress_curr -
                                                progress_start) /
                                          progress_total))

            if (le.operation in ['query', 'getmore', 'update', 'remove'] or
                    le.command in ['count', 'findandmodify',
                                   'geonear', 'find']):
                lt = LogTuple(namespace=le.namespace, operation=op_or_cmd(le),
                              pattern=le.pattern, duration=le.duration)
                grouping.add(lt)

        grouping.sort_by_size()

        # clear progress bar again
        if self.mloginfo.progress_bar_enabled:
            self.mloginfo.update_progress(1.0)

        # no queries in the log file
        if len(grouping) < 1:
            print('no queries found.')
            return

        titles = ['namespace', 'operation', 'pattern', 'count', 'min (ms)',
                  'max (ms)', 'mean (ms)', '95%-ile (ms)', 'sum (ms)']
        table_rows = []

        for g in grouping:
            # calculate statistics for this group
            namespace, op, pattern = g

            group_events = [le.duration for le in grouping[g]
                            if le.duration is not None]

            stats = OrderedDict()
            stats['namespace'] = namespace
            stats['operation'] = op
            stats['pattern'] = pattern
            stats['count'] = len(group_events)
            stats['min'] = min(group_events) if group_events else '-'
            stats['max'] = max(group_events) if group_events else '-'
            stats['mean'] = 0
            if np:
                stats['95%'] = (np.percentile(group_events, 95)
                                if group_events else '-')
            else:
                stats['95%'] = 'n/a'
            stats['sum'] = sum(group_events) if group_events else '-'
            stats['mean'] = (stats['sum'] / stats['count']
                             if group_events else '-')

            if self.mloginfo.args['verbose']:
                stats['example'] = grouping[g][0]
                titles.append('example')

            table_rows.append(stats)

        # sort order depending on field names
        reverse = True
        if self.mloginfo.args['sort'] in ['namespace', 'pattern']:
            reverse = False

        table_rows = sorted(table_rows,
                            key=itemgetter(self.mloginfo.args['sort']),
                            reverse=reverse)
        print_table(table_rows, titles, uppercase_headers=False)
        print('')
210915
def get_processing_block_event():
    """"""
    event = DB.rpoplpush('processing_block_events',
                         'processing_block_event_history')
    if event:
        event = json.loads(event.decode('utf-8'))
    return event
369037
def get(block_id):
    """"""
    _url = get_root_url()
    try:
        block = DB.get_block_details([block_id]).__next__()
        response = block

        response['links'] = {
            'self': '{}'.format(request.url),
            'list': '{}/processing-blocks'.format(_url),
            'home': '{}'.format(_url)
        }
        return block
    except IndexError as error:
        response = dict(message='Unable to GET Processing Block',
                        id='{}'.format(block_id),
                        error=error.__str__())
        response['links'] = {
            'list': '{}/processing-blocks'.format(_url),
            'home': '{}'.format(_url)
        }
        return response, HTTPStatus.NOT_FOUND
874660
def process_queue
      loop do
        sleep 1 until processing?
        buffer = @queue.shift(BATCH_SIZE)
        if buffer.size > 0
          rpc_client.notification('/containers/log_batch', [buffer])
          sleep 0.01
        else
          sleep 1
        end
      end
    end
1240709
def logs(ctx, services, num, follow):
    """"""
    logger.debug("running command %s (%s)", ctx.command.name, ctx.params,
                 extra={"command": ctx.command.name, "params": ctx.params})

    home = ctx.obj["HOME"]
    services_path = os.path.join(home, SERVICES)

    tail_threads = []
    for service in services:
        logpath = os.path.join(services_path, service, LOGS_DIR, STDOUTLOG)
        if os.path.exists(logpath):
            logger.debug("tailing %s", logpath)
            # TODO: Print log lines from multiple services sorted by timestamp
            t = threading.Thread(target=Tailer, kwargs={"name": service,
                                                        "nlines": num,
                                                        "filepath": logpath,
                                                        "follow": follow})
            t.daemon = True
            t.start()
            tail_threads.append(t)

    if tail_threads:
        while tail_threads[0].isAlive():
            tail_threads[0].join(0.1)
248134
def _take_batch(self, min_records):
        ''''''

        if not self._buffer:
            return []

        enough_messages = len(self._buffer) >= min_records
        enough_time = time.time() - self.time_last_batch_sent >= self.batch_delay_seconds
        ready = enough_messages or enough_time

        if not ready:
            return []

        result = list(self._buffer)
        self._buffer.clear()
        return result
1154567
def iterateBlocksFrom(block):
        
        count = 0
        while block.isValid() and count < MAX_SEARCH_OFFSET_LINES:
            yield block
            block = block.next()
            count += 1
1138847
def atlasdb_get_zonefiles_by_block( from_block, to_block, offset, count, name=None, con=None, path=None ):
    
    ret = None

    if count > 100:
        return {'error' : 'Count must be less than 100'}

    with AtlasDBOpen(con=con, path=path) as dbcon:

        sql = 'SELECT name,zonefile_hash,txid,block_height,inv_index FROM zonefiles WHERE block_height >= ? AND block_height <= ?'
        args = (from_block, to_block)

        if name:
            sql += ' AND name = ?'
            args += (name,)

        sql += 'ORDER BY inv_index LIMIT ? OFFSET ?;'
        args += (count, offset)

        cur = dbcon.cursor()
        res = atlasdb_query_execute( cur, sql, args )

        ret = []

        for zfinfo in res:
            ret.append({
                'name' : zfinfo['name'],
                'zonefile_hash' : zfinfo['zonefile_hash'],
                'block_height' : zfinfo['block_height'],
                'txid' : zfinfo['txid'],
                'inv_index': zfinfo['inv_index'],
            })

    return ret
328312
def _iter_withprogress(iterable, progress, log):
    """"""
    before_all = time.time()
    before = before_all
    n = 0
    for i, o in enumerate(iterable):
        yield o
        n = i+1
        if n % progress == 0:
            after = time.time()
            log('%s rows in %.2fs; batch in %.2fs (%d rows/s)'
                % (n, after-before_all, after-before, progress/(after-before)))
            before = after
    after_all = time.time()
    log('%s rows in %.2fs (%d rows/s)'
        % (n, after_all-before_all, n/(after_all-before_all)))
774183
def loadBatch(self, records):
        
        try:
            curr_batch = records[:self.batchSize()]
            next_batch = records[self.batchSize():]
            
            curr_records = list(curr_batch)
            if self._preloadColumns:
                for record in curr_records:
                    record.recordValues(self._preloadColumns)
            
            if len(curr_records) == self.batchSize():
                self.loadedRecords[object, object].emit(curr_records,
                                                        next_batch)
            else:
                self.loadedRecords[object].emit(curr_records)
        
        except ConnectionLostError:
            self.connectionLost.emit()
        
        except Interruption:
            pass