Returns the total execution time and task quantity by worker
176454
def getWorkerInfo(dataTask):
    """"""
    workertime = []
    workertasks = []
    for fichier, vals in dataTask.items():
        if hasattr(vals, 'values'):
            #workers_names.append(fichier)
            # Data from worker
            totaltime = sum([a['executionTime'] for a in vals.values()])
            totaltasks = sum([1 for a in vals.values()])
            workertime.append(totaltime)
            workertasks.append(totaltasks)
    return workertime, workertasks
1083165
def get_max_hours(context):
    """"""
    progress = context['project_progress']
    return max([0] + [max(p['worked'], p['assigned']) for p in progress])
782484
def check_for_completion
      return unless all_work_units_complete?
      set_next_status
      outs = gather_outputs_from_work_units
      return queue_for_workers([outs]) if merging?
      if complete?
        update_attributes(:outputs => outs, :time => time_taken)
        CloudCrowd.log "Job ##{id} (#{action}) #{display_status}." unless ENV['RACK_ENV'] == 'test'
        CloudCrowd.defer { fire_callback } if callback_url
      end
      self
    end
33145
def extract_worker_exc(*arg, **kw):
    """"""
    _self = arg[0]
    if not isinstance(_self, StrategyBase):
        # Run for StrategyBase instance only
        return
    # Iterate over workers to get their task and queue
    for _worker_prc, _main_q, _rslt_q in _self._workers:
        _task = _worker_prc._task
        if _task.action == 'setup':
            # Ignore setup
            continue
        # Do till queue is empty for the worker
        while True:
            try:
                _exc = _rslt_q.get(block=False, interceptor=True)
                RESULT[_task.name].add(_exc)
            except Empty:
                break
686336
def worker(task_queue, result_queue, timer, timeout=Conf.TIMEOUT):
    
    name = current_process().name
    logger.info(_('{} ready for work at {}').format(name, current_process().pid))
    task_count = 0
    if timeout is None:
        timeout = -1
    # Start reading the task queue
    for task in iter(task_queue.get, 'STOP'):
        result = None
        timer.value = -1  # Idle
        task_count += 1
        # Get the function from the task
        logger.info(_('{} processing [{}]').format(name, task['name']))
        f = task['func']
        # if it's not an instance try to get it from the string
        if not callable(task['func']):
            try:
                module, func = f.rsplit('.', 1)
                m = importlib.import_module(module)
                f = getattr(m, func)
            except (ValueError, ImportError, AttributeError) as e:
                result = (e, False)
                if error_reporter:
                    error_reporter.report()
        # We're still going
        if not result:
            db.close_old_connections()
            timer_value = task['kwargs'].pop('timeout', timeout)
            # signal execution
            pre_execute.send(sender="django_q", func=f, task=task)
            # execute the payload
            timer.value = timer_value  # Busy
            try:
                res = f(*task['args'], **task['kwargs'])
                result = (res, True)
            except Exception as e:
                result = ('{} : {}'.format(e, traceback.format_exc()), False)
                if error_reporter:
                    error_reporter.report()
        with timer.get_lock():
            # Process result
            task['result'] = result[0]
            task['success'] = result[1]
            task['stopped'] = timezone.now()
            result_queue.put(task)
            timer.value = -1  # Idle
            # Recycle
            if task_count == Conf.RECYCLE:
                timer.value = -2  # Recycled
                break
    logger.info(_('{} stopped doing work').format(name))
1191468
def count(self, task):
        
        return len([c for c in task.corrections if c["event"]["@class"] == self.event_class])
137209
def num_workers(self):
        
        size = ctypes.c_int()
        check_call(_LIB.MXKVStoreGetGroupSize(self.handle, ctypes.byref(size)))
        return size.value
159298
public int getActiveCount() {
        final ReentrantLock mainLock = this.mainLock;
        mainLock.lock();
        try {
            int n = 0;
            for (Worker w : workers)
                if (w.isLocked())
                    ++n;
            return n;
        } finally {
            mainLock.unlock();
        }
    }
559948
def inspect_workers(self):
        
        workers = tuple(self.workers.values())
        expired = tuple(w for w in workers if not w.is_alive())

        for worker in expired:
            self.workers.pop(worker.pid)

        return ((w.pid, w.exitcode) for w in expired if w.exitcode != 0)
382339
private void collectWorkerInfo(List<WorkerInfo> workerInfoList) {
    initVariables();
    for (WorkerInfo workerInfo : workerInfoList) {
      long usedBytes = workerInfo.getUsedBytes();
      long capacityBytes = workerInfo.getCapacityBytes();
      mSumCapacityBytes += capacityBytes;
      mSumUsedBytes += usedBytes;

      String workerName = workerInfo.getAddress().getHost();

      Map<String, Long> totalBytesOnTiers = workerInfo.getCapacityBytesOnTiers();
      for (Map.Entry<String, Long> totalBytesTier : totalBytesOnTiers.entrySet()) {
        String tier = totalBytesTier.getKey();
        long value = totalBytesTier.getValue();
        mSumCapacityBytesOnTierMap.put(tier,
            value + mSumCapacityBytesOnTierMap.getOrDefault(tier, 0L));

        Map<String, String> map = mCapacityTierInfoMap.getOrDefault(tier, new HashMap<>());
        map.put(workerName, FormatUtils.getSizeFromBytes(value));
        mCapacityTierInfoMap.put(tier, map);
      }

      Map<String, Long> usedBytesOnTiers = workerInfo.getUsedBytesOnTiers();
      for (Map.Entry<String, Long> usedBytesTier: usedBytesOnTiers.entrySet()) {
        String tier = usedBytesTier.getKey();
        long value = usedBytesTier.getValue();
        mSumUsedBytesOnTierMap.put(tier,
            value + mSumUsedBytesOnTierMap.getOrDefault(tier, 0L));

        Map<String, String> map = mUsedTierInfoMap.getOrDefault(tier, new HashMap<>());
        map.put(workerName, FormatUtils.getSizeFromBytes(value));
        mUsedTierInfoMap.put(tier, map);
      }
    }
  }
377198
def get_project_totals(entries, date_headers, hour_type=None, overtime=False,
                       total_column=False, by='user'):
    
    totals = [0 for date in date_headers]
    rows = []
    for thing, thing_entries in groupby(entries, lambda x: x[by]):
        name, thing_id, date_dict = date_totals(thing_entries, by)
        dates = []
        for index, day in enumerate(date_headers):
            if isinstance(day, datetime.datetime):
                day = day.date()
            if hour_type:
                total = date_dict.get(day, {}).get(hour_type, 0)
                dates.append(total)
            else:
                billable = date_dict.get(day, {}).get('billable', 0)
                nonbillable = date_dict.get(day, {}).get('non_billable', 0)
                total = billable + nonbillable
                dates.append({
                    'day': day,
                    'billable': billable,
                    'nonbillable': nonbillable,
                    'total': total
                })
            totals[index] += total
        if total_column:
            dates.append(sum(dates))
        if overtime:
            dates.append(find_overtime(dates))
        dates = [date or '' for date in dates]
        rows.append((name, thing_id, dates))
    if total_column:
        totals.append(sum(totals))
    totals = [t or '' for t in totals]
    yield (rows, totals)
1091296
def progress(self, *restrictions, display=True):
        
        todo = self._jobs_to_do(restrictions)
        total = len(todo)
        remaining = len(todo - self.target)
        if display:
            print('%-20s' % self.__class__.__name__,
                  'Completed %d of %d (%2.1f%%)   %s' % (
                      total - remaining, total, 100 - 100 * remaining / (total+1e-12),
                      datetime.datetime.strftime(datetime.datetime.now(), '%Y-%m-%d %H:%M:%S')), flush=True)
        return remaining, total
789437
def view_task_durations(token, dstore):
    
    task = token.split(':')[1]  # called as task_duration:task_name
    array = dstore['task_info/' + task]['duration']
    return '\n'.join(map(str, array))
223155
def get_proc_title(self):
        
        has_worker = bool(getattr(self, 'worker', None))

        title_parts = [self.prog_name.replace('.py', ('#%s' % self.worker.id) if has_worker else '')]

        status = 'init'
        if has_worker and self.worker.status:
            status = self.worker.status
            if self.worker.end_forced:
                status += ' - ending'
        title_parts.append('[%s]' % status)

        if has_worker and self.worker.queues:
            title_parts.append('queues=%s' % ','.join(self.worker.queues))

        if has_worker and self.worker.status:
            # add infos about the main loop
            title_parts.append('loop=%s/%s' % (self.worker.num_loops, self.worker.max_loops))

            # and about the number of jobs to run
            title_parts.append('waiting=%s' % self.worker.count_waiting_jobs())

            # and about the number of delayed jobs
            title_parts.append('delayed=%s' % self.worker.count_delayed_jobs())

            # and about elapsed time
            if self.worker.start_date:
                duraiton_message = 'duration=%s'
                duration_args = (timedelta(seconds=int(round(self.worker.elapsed.total_seconds()))), )
                if self.worker.max_duration:
                    duraiton_message += '/%s'
                    duration_args += (self.worker.max_duration, )
                title_parts.append(duraiton_message % duration_args)

        return ' '.join(title_parts)
1079703
def start_worker(self, row):
        ''''''
        def on_worker_started(worker, fs_id):
            pass

        def on_worker_received(worker, fs_id, received, received_total):
            GLib.idle_add(do_worker_received, fs_id, received, received_total)

        def do_worker_received(fs_id, received, received_total):
            self.download_speed_add(received)
            row = None
            if fs_id in self.workers:
                row = self.workers[fs_id][1]
            else:
                row = self.get_row_by_fsid(fs_id)
            if not row:
                return

            row[CURRSIZE_COL] = received_total
            curr_size = util.get_human_size(row[CURRSIZE_COL], False)[0]
            total_size = util.get_human_size(row[SIZE_COL])[0]
            row[PERCENT_COL] = int(row[CURRSIZE_COL] / row[SIZE_COL] * 100)
            row[HUMANSIZE_COL] = '{0} / {1}'.format(curr_size, total_size)
            self.update_task_db(row)

        def on_worker_downloaded(worker, fs_id):
            GLib.idle_add(do_worker_downloaded, fs_id)

        def do_worker_downloaded(fs_id):
            row = None
            if fs_id in self.workers:
                row = self.workers[fs_id][1]
            else:
                row = self.get_row_by_fsid(fs_id)
            if not row:
                return
            row[CURRSIZE_COL] = row[SIZE_COL]
            row[STATE_COL] = State.FINISHED
            row[PERCENT_COL] = 100
            total_size = util.get_human_size(row[SIZE_COL])[0]
            row[HUMANSIZE_COL] = '{0} / {1}'.format(total_size, total_size)
            row[STATENAME_COL] = StateNames[State.FINISHED]
            self.update_task_db(row)
            self.check_commit(force=True)
            self.workers.pop(row[FSID_COL], None)
            self.app.toast(_('{0} downloaded'.format(row[NAME_COL])))
            self.launch_app(fs_id)
            self.scan_tasks()

        def on_worker_network_error(worker, fs_id):
            GLib.idle_add(do_worker_network_error, fs_id)

        def do_worker_network_error(fs_id):
            row = self.workers.get(fs_id, None)
            if row:
                row = row[1]
            else:
                row = self.get_row_by_fsid(fs_id)
                if not row:
                    return
            row[STATE_COL] = State.ERROR
            row[STATENAME_COL] = StateNames[State.ERROR]
            self.update_task_db(row)
            self.remove_worker(row[FSID_COL], stop=False)
            if self.app.profile['retries-each']:
                GLib.timeout_add(self.app.profile['retries-each'] * 60000,
                                 self.restart_task, row)
            else:
                self.app.toast(_('Error occurs will downloading {0}').format(
                               row[NAME_COL]))
            self.scan_tasks()

        def do_worker_disk_error(fs_id, tmp_filepath):
            # do not retry on disk-error
            self.app.toast(_('Disk Error: failed to read/write {0}').format(
                           tmp_filepath))

        def on_worker_disk_error(worker, fs_id, tmp_filepath):
            GLib.idle_add(do_worker_disk_error, fs_id, tmp_filepath)

        if not row or row[FSID_COL] in self.workers:
            return
        row[STATE_COL] = State.DOWNLOADING
        row[STATENAME_COL] = StateNames[State.DOWNLOADING]
        worker = Downloader(self, row)
        self.workers[row[FSID_COL]] = (worker, row)
        worker.connect('started', on_worker_started)
        worker.connect('received', on_worker_received)
        worker.connect('downloaded', on_worker_downloaded)
        worker.connect('network-error', on_worker_network_error)
        worker.connect('disk-error', on_worker_disk_error)
        worker.start()
491016
def get(self):
        ''''''
        if self.bucket.get() < 1:
            return None
        now = time.time()
        self.mutex.acquire()
        try:
            task = self.priority_queue.get_nowait()
            self.bucket.desc()
        except Queue.Empty:
            self.mutex.release()
            return None
        task.exetime = now + self.processing_timeout
        self.processing.put(task)
        self.mutex.release()
        return task.taskid
876652
def results(self):
        """"""
        tt = None
        for i, tf in enumerate(self[:-1]):
            tt = self[i+1]
            q = Queue(tf.qsize)
            tf.set_out(q, tt.workers)
            tt.set_in(q, tf.workers)

        if tt is None: # we have only one pool
            tt = self[0]
        q = Queue(tt.qsize)
        err_q = Queue()
        tt.set_out(q, 1)

        for t in self:
            t.set_err(err_q)
            t._start()

        for item in iterqueue(q, tt.workers):
            yield item

        errors = list(iterqueue(err_q, sum(t.workers for t in self)))

        for t in self:
            t._join()

        if len(errors) > 0:
            task_name, ex = errors[0]
            if len(errors) == 1:
                msg = 'The task "%s" raised %s' % (task_name, repr(ex),)
            else:
                msg = '%d tasks raised an exeption. First error reported on task "%s": %s' % (len(errors), task_name, repr(ex))

            raise TaskException(msg)