Print an update about the current number of jobs running
503967
def print_update(self, stream=sys.stdout, job_stats=None):
        
        if job_stats is None:
            job_stats = JobStatusVector()
            job_det_list = []
            job_det_list += self._scatter_link.jobs.values()

            for job_dets in job_det_list:
                if job_dets.status == JobStatus.no_job:
                    continue
                job_stats[job_dets.status] += 1

        stream.write("Status :\n  Total  : %i\n  Unknown: %i\n" %
                     (job_stats.n_total, job_stats[JobStatus.unknown]))
        stream.write("  Not Ready: %i\n  Ready: %i\n" %
                     (job_stats[JobStatus.not_ready], job_stats[JobStatus.ready]))
        stream.write("  Pending: %i\n  Running: %i\n" %
                     (job_stats[JobStatus.pending], job_stats[JobStatus.running]))
        stream.write("  Done: %i\n  Failed: %i\n" %
                     (job_stats[JobStatus.done], job_stats[JobStatus.failed]))
182197
def tail(self, stream='stdout', num_lines=10):
        
        target = self._map_string_to_file(stream)
        if not target:  # no current temp file
            last_run = self.backend.get_latest_run_log(self.parent_job.job_id,
                                                       self.name)
            if not last_run:
                return None
            return self._tail_string(last_run['tasks'][self.name][stream],
                                     num_lines)
        else:
            return self._tail_temp_file(target, num_lines)
401543
def status(self):
        """"""
        # If there is no script it is either ready or a lost duplicate #
        if not self.script_path.exists:
            if self.name in     jobs.names: return "DUPLICATE"
            if self.name not in jobs.names: return "READY"
        # It is submitted already #
        if self.name in jobs.names:
            if jobs[self.name]['type'] == 'queued':  return "QUEUED"
            if jobs[self.name]['type'] == 'running': return "RUNNING"
        # So the script exists for sure but it is not in the queue #
        if not self.kwargs['out_file'].exists: return "ABORTED"
        # Let's look in log file #
        if 'CANCELED'          in self.log_tail: return "CANCELLED"
        if 'slurmstepd: error' in self.log_tail: return "CANCELLED"
        # It all looks good #
        if 'SLURM: end at'     in self.log_tail: return "FINISHED"
        # At this point we have no idea #
        return "INTERUPTED"
179878
def _show_status_for_work(self, work):
    
    work_count = len(work.work)
    work_completed = {}
    work_completed_count = 0
    for v in itervalues(work.work):
      if v['is_completed']:
        work_completed_count += 1
        worker_id = v['claimed_worker_id']
        if worker_id not in work_completed:
          work_completed[worker_id] = {
              'completed_count': 0,
              'last_update': 0.0,
          }
        work_completed[worker_id]['completed_count'] += 1
        work_completed[worker_id]['last_update'] = max(
            work_completed[worker_id]['last_update'],
            v['claimed_worker_start_time'])
    print('Completed {0}/{1} work'.format(work_completed_count,
                                          work_count))
    for k in sorted(iterkeys(work_completed)):
      last_update_time = time.strftime(
          '%Y-%m-%d %H:%M:%S',
          time.localtime(work_completed[k]['last_update']))
      print('Worker {0}: completed {1}   last claimed work at {2}'.format(
          k, work_completed[k]['completed_count'], last_update_time))
1048610
public static void printAllJobs(List<JobExecutionInfo> jobExecutionInfos, int resultsLimit) {
        if (jobExecutionInfos == null) {
            System.err.println("No jobs found.");
            System.exit(1);
        }

        List<String> labels = Arrays.asList("Job Name", "State", "Last Run Started", "Last Run Completed",
                "Schedule", "Last Run Records Processed", "Last Run Records Failed");
        List<String> flags = Arrays.asList("-", "-", "-", "-", "-", "", "");
        List<List<String>> data = new ArrayList<>();
        for (JobExecutionInfo jobInfo : jobExecutionInfos) {
            List<String> entry = new ArrayList<>();
            entry.add(jobInfo.getJobName());
            entry.add(jobInfo.getState().toString());
            entry.add(dateTimeFormatter.print(jobInfo.getStartTime()));
            entry.add(dateTimeFormatter.print(jobInfo.getEndTime()));

            entry.add(extractJobSchedule(jobInfo));

            // Add metrics
            MetricArray metrics = jobInfo.getMetrics();
            Double recordsProcessed = null;
            Double recordsFailed = null;
            try {
                for (Metric metric : metrics) {
                    if (metric.getName().equals(MetricNames.ExtractorMetrics.RECORDS_READ_METER)) {
                        recordsProcessed = Double.parseDouble(metric.getValue());
                    } else if (metric.getName().equals(MetricNames.ExtractorMetrics.RECORDS_FAILED_METER)) {
                        recordsFailed = Double.parseDouble(metric.getValue());
                    }
                }

                if (recordsProcessed != null && recordsFailed != null) {
                    entry.add(recordsProcessed.toString());
                    entry.add(recordsFailed.toString());
                }
            } catch (NumberFormatException ex) {
                System.err.println("Failed to process metrics");
            }
            if (recordsProcessed == null || recordsFailed == null) {
                entry.add("-");
                entry.add("-");
            }

            data.add(entry);
        }
        new CliTablePrinter.Builder()
                .labels(labels)
                .data(data)
                .flags(flags)
                .delimiterWidth(2)
                .build()
                .printTable();

        if (jobExecutionInfos.size() == resultsLimit) {
            System.out.println("\nWARNING: There may be more jobs (# of results is equal to the limit)");
        }
    }
269642
def notify(self, new_jobs_count):
        

        if not self.use_notify():
            return

        # Not really useful to send more than 100 notifs (to be configured)
        count = min(new_jobs_count, 100)

        notify_key = redis_key("notify", self)

        context.connections.redis.lpush(notify_key, *([1] * count))
        context.connections.redis.expire(notify_key, max(1, int(context.get_current_config()["max_latency"] * 2)))
1321776
def pprint(self, num=10):
        
        def takeAndPrint(time, rdd):
            taken = rdd.take(num + 1)
            print("-------------------------------------------")
            print("Time: %s" % time)
            print("-------------------------------------------")
            for record in taken[:num]:
                print(record)
            if len(taken) > num:
                print("...")
            print("")

        self.foreachRDD(takeAndPrint)
396829
def _print(self, force_flush=False):
        
        self._stream_flush()
        next_perc = self._calc_percent()
        if self.update_interval:
            do_update = time.time() - self.last_time >= self.update_interval
        elif force_flush:
            do_update = True
        else:
            do_update = next_perc > self.last_progress

        if do_update and self.active:
            self.last_progress = next_perc
            self._cache_percent_indicator(self.last_progress)
            if self.track:
                self._cached_output += ' Time elapsed: ' + \
                                       self._get_time(self._elapsed())
                self._cache_eta()
            if self.item_id:
                self._cache_item_id()
            self._stream_out('\r%s' % self._cached_output)
            self._stream_flush()
            self._cached_output = ''
81684
def dump_logs(job_queue='run_reach_queue', job_status='RUNNING'):
    """"""
    jobs = get_jobs(job_queue, job_status)
    for job in jobs:
        get_job_log(job, write_file=True)
855901
def set_status(db, job_id, status):
    
    assert status in (
        'created', 'submitted', 'executing', 'complete', 'aborted', 'failed'
    ), status
    if status in ('created', 'complete', 'failed', 'aborted'):
        is_running = 0
    else:  # 'executing'
        is_running = 1
    if job_id < 0:
        rows = db('SELECT id FROM job ORDER BY id DESC LIMIT ?x', -job_id)
        if not rows:
            return 0
        job_id = rows[-1].id
    cursor = db('UPDATE job SET status=?x, is_running=?x WHERE id=?x',
                status, is_running, job_id)
    return cursor.rowcount
369884
def job_count_enabled(self):
		
		enabled = 0
		for job_desc in self._jobs.values():
			if job_desc['enabled']:
				enabled += 1
		return enabled
854507
def _PrintExtractionStatusUpdateLinear(self, processing_status):
    
    for worker_status in processing_status.workers_status:
      status_line = (
          '{0:s} (PID: {1:d}) - events produced: {2:d} - file: {3:s} '
          '- running: {4!s}\n').format(
              worker_status.identifier, worker_status.pid,
              worker_status.number_of_produced_events,
              worker_status.display_name,
              worker_status.status not in definitions.ERROR_STATUS_INDICATORS)
      self._output_writer.Write(status_line)
885971
def report_on_jobs(self):
        
        hasChildren = []
        readyToRun = []
        zombies = []
        hasLogFile = []
        hasServices = []
        services = []
        properties = set()

        for job in self.jobsToReport:
            if job.logJobStoreFileID is not None:
                hasLogFile.append(job)

            childNumber = reduce(lambda x, y: x + y, map(len, job.stack) + [0])
            if childNumber > 0:  # Total number of successors > 0
                hasChildren.append(job)
                properties.add("HAS_CHILDREN")
            elif job.command is not None:
                # Job has no children and a command to run. Indicates job could be run.
                readyToRun.append(job)
                properties.add("READY_TO_RUN")
            else:
                # Job has no successors and no command, so is a zombie job.
                zombies.append(job)
                properties.add("IS_ZOMBIE")
            if job.services:
                hasServices.append(job)
                properties.add("HAS_SERVICES")
            if job.startJobStoreID or job.terminateJobStoreID or job.errorJobStoreID:
                # These attributes are only set in service jobs
                services.append(job)
                properties.add("IS_SERVICE")

        jobStats = {'hasChildren': hasChildren,
                    'readyToRun': readyToRun,
                    'zombies': zombies,
                    'hasServices': hasServices,
                    'services': services,
                    'hasLogFile': hasLogFile,
                    'properties': properties,
                    'childNumber': childNumber}
        return jobStats
17916
def _PrintAnalysisStatusUpdateLinear(self, processing_status):
    
    for worker_status in processing_status.workers_status:
      status_line = (
          '{0:s} (PID: {1:d}) - events consumed: {2:d} - running: '
          '{3!s}\n').format(
              worker_status.identifier, worker_status.pid,
              worker_status.number_of_consumed_events,
              worker_status.status not in definitions.ERROR_STATUS_INDICATORS)
      self._output_writer.Write(status_line)
151905
def poll(self):
        
        task = self.task

        # total job count
        n_jobs = len(self.submission_data)

        # store the number of consecutive polling failures and get the maximum number of polls
        n_poll_fails = 0
        if task.walltime == NO_FLOAT:
            max_polls = sys.maxsize
        else:
            max_polls = int(math.ceil((task.walltime * 3600.) / (task.poll_interval * 60.)))

        # update variable attributes for polling
        self.poll_data.n_finished_min = task.acceptance * (1 if task.acceptance > 1 else n_jobs)
        self.poll_data.n_failed_max = task.tolerance * (1 if task.tolerance > 1 else n_jobs)

        # track finished and failed jobs in dicts holding status data
        finished_jobs = OrderedDict()
        failed_jobs = OrderedDict()

        # start the poll loop
        for i in six.moves.range(max_polls):
            # sleep after the first iteration
            if i > 0:
                time.sleep(task.poll_interval * 60)

            # determine the currently active jobs, i.e., the jobs whose states should be checked
            active_jobs = OrderedDict()
            for job_num, data in six.iteritems(self.submission_data.jobs):
                if job_num in finished_jobs or job_num in failed_jobs:
                    continue
                if self.skip_jobs.get(job_num):
                    finished_jobs[job_num] = self.status_data_cls.job_data(
                        status=self.job_manager.FINISHED, code=0)
                else:
                    data = data.copy()
                    if not data["job_id"]:
                        data["job_id"] = self.status_data_cls.dummy_job_id
                    active_jobs[job_num] = data
            self.poll_data.n_active = len(active_jobs)

            # query job states
            job_ids = [data["job_id"] for data in six.itervalues(active_jobs)]  # noqa: F812
            _states, errors = self.job_manager.query_batch(job_ids)
            if errors:
                print("{} error(s) occured during job status query of task {}:".format(
                    len(errors), task.task_id))
                tmpl = "    {}"
                for i, err in enumerate(errors):
                    print(tmpl.format(err))
                    if i + 1 >= self.show_errors:
                        remaining = len(errors) - self.show_errors
                        if remaining > 0:
                            print("    ... and {} more".format(remaining))
                        break

                n_poll_fails += 1
                if task.poll_fails > 0 and n_poll_fails > task.poll_fails:
                    raise Exception("poll_fails exceeded")
                else:
                    continue
            else:
                n_poll_fails = 0

            # states stores job_id's as keys, so replace them by using job_num's
            states = OrderedDict()
            for job_num, data in six.iteritems(active_jobs):
                states[job_num] = self.status_data_cls.job_data(**_states[data["job_id"]])

            # store jobs per status and take further actions depending on the status
            pending_jobs = OrderedDict()
            running_jobs = OrderedDict()
            newly_failed_jobs = OrderedDict()
            retry_jobs = OrderedDict()
            for job_num, data in six.iteritems(states):
                if data["status"] == self.job_manager.PENDING:
                    pending_jobs[job_num] = data
                    task.forward_dashboard_event(self.dashboard, data, "status.pending", job_num)

                elif data["status"] == self.job_manager.RUNNING:
                    running_jobs[job_num] = data
                    task.forward_dashboard_event(self.dashboard, data, "status.running", job_num)

                elif data["status"] == self.job_manager.FINISHED:
                    finished_jobs[job_num] = data
                    self.poll_data.n_active -= 1
                    task.forward_dashboard_event(self.dashboard, data, "status.finished", job_num)

                elif data["status"] in (self.job_manager.FAILED, self.job_manager.RETRY):
                    newly_failed_jobs[job_num] = data
                    self.poll_data.n_active -= 1

                    # retry or ultimately failed?
                    if self.job_retries[job_num] < task.retries:
                        self.job_retries[job_num] += 1
                        self.submission_data.attempts[job_num] += 1
                        data["status"] = self.job_manager.RETRY
                        retry_jobs[job_num] = self.submission_data.jobs[job_num]["branches"]
                        task.forward_dashboard_event(self.dashboard, data, "status.retry", job_num)
                    else:
                        failed_jobs[job_num] = data
                        task.forward_dashboard_event(self.dashboard, data, "status.failed", job_num)

                else:
                    raise Exception("unknown job status '{}'".format(data["status"]))

            # gather some counts
            n_pending = len(pending_jobs)
            n_running = len(running_jobs)
            n_finished = len(finished_jobs)
            n_retry = len(retry_jobs)
            n_failed = len(failed_jobs)
            n_unsubmitted = len(self.submission_data.unsubmitted_jobs)

            # log the status line
            counts = (n_pending, n_running, n_finished, n_retry, n_failed)
            if task.parallel_jobs > 0:
                counts = (n_unsubmitted,) + counts
            status_line = self.job_manager.status_line(counts, last_counts=True, sum_counts=n_jobs,
                color=True, align=task.align_status_line)
            task.publish_message(status_line)
            self.last_status_counts = counts

            # inform the scheduler about the progress
            task.publish_progress(100. * n_finished / n_jobs)

            # log newly failed jobs
            if newly_failed_jobs:
                print("{} failed job(s) in task {}:".format(len(newly_failed_jobs), task.task_id))
                tmpl = "    job: {}, branches: {}, id: {job_id}, status: {status}, code: {code}, " \
                    "error: {error}"
                for i, (job_num, data) in enumerate(six.iteritems(newly_failed_jobs)):
                    branches = self.submission_data.jobs[job_num]["branches"]
                    print(tmpl.format(job_num, ",".join(str(b) for b in branches), **data))
                    if i + 1 >= self.show_errors:
                        remaining = len(newly_failed_jobs) - self.show_errors
                        if remaining > 0:
                            print("    ... and {} more".format(remaining))
                        break

            # infer the overall status
            reached_end = n_jobs == n_finished + n_failed
            finished = n_finished >= self.poll_data.n_finished_min
            failed = n_failed > self.poll_data.n_failed_max
            unreachable = n_jobs - n_failed < self.poll_data.n_finished_min
            if finished:
                # write status output
                if "status" in self._outputs:
                    status_data = self.status_data_cls()
                    status_data.jobs.update(finished_jobs)
                    status_data.jobs.update(states)
                    self._outputs["status"].dump(status_data, formatter="json", indent=4)
                break
            elif failed:
                failed_nums = [job_num for job_num in failed_jobs if job_num not in retry_jobs]
                raise Exception("tolerance exceeded for jobs {}".format(failed_nums))
            elif unreachable:
                err = None
                if reached_end:
                    err = "acceptance of {} not reached, total jobs: {}, failed jobs: {}"
                elif task.check_unreachable_acceptance:
                    err = "acceptance of {} unreachable, total jobs: {}, failed jobs: {}"
                if err:
                    raise Exception(err.format(self.poll_data.n_finished_min, n_jobs, n_failed))

            # configurable poll callback
            task.poll_callback(self.poll_data)

            # trigger automatic resubmission and submission of unsubmitted jobs
            self.submit(retry_jobs)

            # break when no polling is desired
            # we can get to this point when there was already a submission and the no_poll
            # parameter was set so that only failed jobs are resubmitted once
            if task.no_poll:
                break
        else:
            # walltime exceeded
            raise Exception("walltime exceeded")
1085512
def poll_for_server_running(job_id):
    
    sys.stdout.write('Waiting for server in {0} to initialize ...'.format(job_id))
    sys.stdout.flush()
    desc = dxpy.describe(job_id)
    # Keep checking until the server has begun or it has failed.
    while(SERVER_READY_TAG not in desc['tags'] and desc['state'] != 'failed'):
        time.sleep(SLEEP_PERIOD)
        sys.stdout.write('.')
        sys.stdout.flush()
        desc = dxpy.describe(job_id)

    # If the server job failed, provide friendly advice.
    if desc['state'] == 'failed':
        msg = RED('Error:') + ' Server failed to run.\n'
        msg += 'You may want to check the job logs by running:'
        msg += BOLD('dx watch {0}'.format(job_id))
        err_exit(msg)
980725
public void display() {
    System.out.printf("Job id=%s, name=%s\n", getJobID(), getJobName());
    System.out.printf("Configuration file: %s\n", getJobID());
    System.out.printf("Tracking URL: %s\n", getTrackingURL());

    System.out.printf("Completion: map: %f reduce %f\n",
        100.0 * this.mapProgress, 100.0 * this.reduceProgress);

    System.out.println("Job total maps = " + totalMaps);
    System.out.println("Job completed maps = " + completedMaps);
    System.out.println("Map percentage complete = " + mapProgress);
    System.out.println("Job total reduces = " + totalReduces);
    System.out.println("Job completed reduces = " + completedReduces);
    System.out.println("Reduce percentage complete = " + reduceProgress);
    System.out.flush();
  }