Helper for job-type specific statistics-based properties.
3580
def _job_statistics(self):
        """"""
        statistics = self._properties.get("statistics", {})
        return statistics.get(self._JOB_TYPE, {})
6493
def collection_for_job(cls, job):
        
        return cls.collection(job_model_repr=job.get_model_repr(), identifier=getattr(job, '_cached_identifier', job.identifier.hget()))
246069
function (jobName, customParams, callback) {
      [jobName, customParams, callback] = doArgs(arguments, ['string', ['object', {}], 'function']);

      this.job_info(jobName, function (error, data) {
        if (error) {
          callback(error, data);
          return;
        }

        const lastResultUrl = data.lastBuild.url;

        doRequest({
          urlPattern: [lastResultUrl + API, jobName]
        }, customParams, callback);
      });
    }
946325
private static List<? extends Tag<?>> addAdditionalMetadataTags(Properties jobProps, List<? extends Tag<?>> inputTags) {
    List<Tag<?>> metadataTags = Lists.newArrayList(inputTags);
    String jobId;

    // generate job id if not already set
    if (jobProps.containsKey(ConfigurationKeys.JOB_ID_KEY)) {
      jobId = jobProps.getProperty(ConfigurationKeys.JOB_ID_KEY);
    } else {
      jobId = JobLauncherUtils.newJobId(JobState.getJobNameFromProps(jobProps));
      jobProps.put(ConfigurationKeys.JOB_ID_KEY, jobId);
    }

    String jobExecutionId = Long.toString(Id.Job.parse(jobId).getSequence());

    // only inject flow tags if a flow name is defined
    if (jobProps.containsKey(ConfigurationKeys.FLOW_NAME_KEY)) {
      metadataTags.add(new Tag<>(TimingEvent.FlowEventConstants.FLOW_GROUP_FIELD,
          jobProps.getProperty(ConfigurationKeys.FLOW_GROUP_KEY, "")));
      metadataTags.add(
          new Tag<>(TimingEvent.FlowEventConstants.FLOW_NAME_FIELD, jobProps.getProperty(ConfigurationKeys.FLOW_NAME_KEY)));

      // use job execution id if flow execution id is not present
      metadataTags.add(new Tag<>(TimingEvent.FlowEventConstants.FLOW_EXECUTION_ID_FIELD,
          jobProps.getProperty(ConfigurationKeys.FLOW_EXECUTION_ID_KEY, jobExecutionId)));
    }

    metadataTags.add(new Tag<>(TimingEvent.FlowEventConstants.JOB_GROUP_FIELD,
        jobProps.getProperty(ConfigurationKeys.JOB_GROUP_KEY, "")));
    metadataTags.add(new Tag<>(TimingEvent.FlowEventConstants.JOB_NAME_FIELD,
        jobProps.getProperty(ConfigurationKeys.JOB_NAME_KEY, "")));
    metadataTags.add(new Tag<>(TimingEvent.FlowEventConstants.JOB_EXECUTION_ID_FIELD, jobExecutionId));

    LOGGER.debug("GobblinHelixJobLauncher.addAdditionalMetadataTags: metadataTags {}", metadataTags);

    return metadataTags;
  }
61594
def build_joblist(jobgraph):
    """"""
    jobset = set()
    for job in jobgraph:
        jobset = populate_jobset(job, jobset, depth=1)
    return list(jobset)
707413
public void updateMetrics() {
    Counters counters = getCounters();
    for (Counters.Group group : counters) {
      jobMetrics.setTag("group", group.getDisplayName());
      for (Counters.Counter counter : group) {
        jobMetrics.setTag("counter", counter.getDisplayName());
        jobMetrics.setMetric("value", (float) counter.getCounter());
        jobMetrics.update();
      }
    }
  }
171985
def job_data(cls, job_id=dummy_job_id, branches=None, **kwargs):
        
        return dict(job_id=job_id, branches=branches or [])
1232673
def job_ids(log_stream):
    """"""
    id_rows = [line for line in log_stream if 'scancel' in line]
    jobs = [id_row.strip()[-7:-1] for id_row in id_rows]
    return jobs
1162477
public void printJobExecutionStatistics() {
    /*
     * Print Job Counters
     */
    System.out.println("JOB COUNTERS *********************************************");
    int size = this._job.size();
    java.util.Iterator<Map.Entry<Enum, String>> kv = this._job.entrySet().iterator();
    for (int i = 0; i < size; i++)
    {
      Map.Entry<Enum, String> entry = (Map.Entry<Enum, String>) kv.next();
      Enum key = entry.getKey();
      String value = entry.getValue();
      System.out.println("Key:<" + key.name() + ">, value:<"+ value +">"); 
    }
    /*
     * 
     */
    System.out.println("MAP COUNTERS *********************************************");
    int size1 = this._mapTaskList.size();
    for (int i = 0; i < size1; i++)
    {
      System.out.println("MAP TASK *********************************************");
      this._mapTaskList.get(i).printKeys();
    }
    /*
     * 
     */
    System.out.println("REDUCE COUNTERS *********************************************");
    int size2 = this._mapTaskList.size();
    for (int i = 0; i < size2; i++)
    {
      System.out.println("REDUCE TASK *********************************************");
      this._reduceTaskList.get(i).printKeys();
    }
  }
68514
def aggregate_stats(stats_files):
  """"""
  all_stats = {}
  for fname in stats_files:
    with tf.gfile.Open(fname) as f:
      stats = json.loads(f.read())
      for k, v in stats.iteritems():
        if k not in all_stats:
          if isinstance(v, list):
            all_stats[k] = []
          else:
            all_stats[k] = 0

        if isinstance(v, list):
          all_stats[k].extend(v)
        else:
          all_stats[k] += v

  stats = all_stats
  ref_coverage = float(stats["total_found_refs"]) / stats["total_original_refs"]
  len_bounds = [0, 2, 10, 100, 1000, 5000, 10000, 20000, 50000, 100000, 1000000]
  len_counts, len_bounds = np.histogram(stats["ref_lengths"], len_bounds)
  len_dist = len_counts.astype(np.float32) / len_counts.sum()
  wiki_coverage = (float(stats["num_wikis_written"]) /
                   stats["total_original_wikis"])
  wikis_skipped_no_ref = (float(stats["wikis_skipped_no_refs"]) /
                          stats["total_original_wikis"])
  wikis_skipped_no_lead = (float(stats["wikis_skipped_short_lead"]) /
                           stats["total_original_wikis"])
  wiki_ref_coverage = [
      float(found) / orig for found, orig
      in zip(stats["wiki_found_refs"], stats["wiki_original_refs"]) if found
  ]
  coverage_bounds = np.arange(21).astype(np.float32) / 20
  coverage_counts, coverage_bounds = np.histogram(wiki_ref_coverage,
                                                  coverage_bounds)
  coverage_dist = coverage_counts.astype(np.float32) / coverage_counts.sum()

  agg_stats = dict(
      total_original_wikis=stats["total_original_wikis"],
      total_original_refs=stats["total_original_refs"],
      wiki_coverage=wiki_coverage,
      wikis_skipped_no_ref=wikis_skipped_no_ref,
      wikis_skipped_no_lead=wikis_skipped_no_lead,
      overall_ref_coverage=ref_coverage,
      per_wiki_ref_coverage_dist=list((coverage_dist * 100).astype(int)),
      per_wiki_ref_coverage_bounds=list((coverage_bounds * 100).astype(int)),
      ref_len_dist=list((len_dist * 100).astype(int)),
      ref_len_bounds=list(len_bounds),
  )
  return agg_stats
1267221
def job_stats_enhanced(job_id):
    
    stats_dict = {}
    with os.popen('sacct --noheader --format JobId,Elapsed,TotalCPU,Partition,NTasks,AveRSS,State,ExitCode,start,end -P -j ' + str(job_id)) as f:
        try:
            line = f.readline()
            if line in ["SLURM accounting storage is disabled",
                        "slurm_load_job error: Invalid job id specified"]:
                raise
            cols = line.split('|')
            stats_dict['job_id'] = cols[0]
            stats_dict['wallclock'] = get_timedelta(cols[1])
            stats_dict['cpu'] = get_timedelta(cols[2])
            stats_dict['queue'] = cols[3]
            stats_dict['status'] = cols[6]
            stats_dict['exit_code'] = cols[7].split(':')[0]
            stats_dict['start'] = cols[8]
            stats_dict['end'] = cols[9]

            steps = []
            for line in f:
                step = {}
                cols = line.split('|')
                step_val = cols[0].split('.')[1]
                step['step'] = step_val
                step['wallclock'] = get_timedelta(cols[1])
                step['cpu'] = get_timedelta(cols[2])
                step['ntasks'] = cols[4]
                step['status'] = cols[6]
                step['exit_code'] = cols[7].split(':')[0]
                step['start'] = cols[8]
                step['end'] = cols[9]
                steps.append(step)
            stats_dict['steps'] = steps
        except:
            with os.popen('squeue -h -j %s' % str(job_id)) as f:
                try:
                    for line in f:
                        new_line = re.sub(' +', ' ', line.strip())
                        job_id = int(new_line.split(' ')[0])
                        state = new_line.split(' ')[4]
                        stats_dict['job_id'] = str(job_id)
                        stats_dict['status'] = state
                except:
                    print('SLURM: Error reading job stats')
                    stats_dict['status'] = 'UNKNOWN'
    with os.popen('squeue --format %%S -h -j ' + str(job_id)) as f:
        try:
            line = f.readline()
            if len(line) > 0:
                stats_dict['start_time'] = line
            else:
                stats_dict['start_time'] = ""
        except:
            print('SLURM: Error getting start time')
    return stats_dict
904935
public static JobMetrics get(String jobName, String jobId) {
    return get(new JobState(jobName, jobId));
  }
1040243
def job_stats_enhanced(job_id):
    
    stats_dict = {}
    with os.popen('qstat -xf ' + str(job_id)) as f:
        try:
            line = f.readline().strip()
            while line:
                if line.startswith('Job Id:'):
                    stats_dict['job_id'] = line.split(':')[1].split('.')[0].strip()
                elif line.startswith('resources_used.walltime'):
                    stats_dict['wallclock'] = get_timedelta(line.split('=')[1])
                elif line.startswith('resources_used.cput'):
                    stats_dict['cpu'] = get_timedelta(line.split('=')[1])
                elif line.startswith('queue'):
                    stats_dict['queue'] = line.split('=')[1].strip()
                elif line.startswith('job_state'):
                    stats_dict['status'] = line.split('=')[1].strip()
                elif line.startswith('Exit_status'):
                    stats_dict['exit_code'] = line.split('=')[1].strip()
                elif line.startswith('Exit_status'):
                    stats_dict['exit_code'] = line.split('=')[1].strip()
                elif line.startswith('stime'):
                    stats_dict['start'] = line.split('=')[1].strip()
                line = f.readline().strip()
            if stats_dict['status'] == 'F' and 'exit_code' not in stats_dict:
                stats_dict['status'] = 'CA'
            elif stats_dict['status'] == 'F' and stats_dict['exit_code'] == '0':
                stats_dict['status'] = 'PBS_F'
        except Exception as e:
            with os.popen('qstat -xaw ' + str(job_id)) as f:
                try:
                    output = f.readlines()
                    for line in output:
                        if str(job_id) in line:
                            cols = line.split()
                            stats_dict['job_id'] = cols[0].split('.')[0]
                            stats_dict['queue'] = cols[2]
                            stats_dict['status'] = cols[9]
                            stats_dict['wallclock'] = get_timedelta(cols[10])
                            return stats_dict
                except Exception as e:
                    print(e)
                    print('PBS: Error reading job stats')
                    stats_dict['status'] = 'UNKNOWN'
    return stats_dict
1312606
def _build_job_dict(self):
        """"""
        if self.args['dry_run']:
            status = JobStatus.unknown
        else:
            status = JobStatus.not_ready

        base_config = self.scatter_link.args

        for jobkey, job_config in sorted(self._job_configs.items()):
            full_job_config = base_config.copy()
            full_job_config.update(job_config)
            ScatterGather._make_scatter_logfile_name(jobkey,
                                                     self.linkname, full_job_config)
            logfile = job_config.get('logfile')
            self._scatter_link._register_job(key=jobkey,
                                             job_config=full_job_config,
                                             logfile=logfile,
                                             status=status)
899674
public function stats($jobType = null) {
		if (!Configure::read('Queue.isStatisticEnabled')) {
			throw new NotFoundException('Not enabled');
		}

		$stats = $this->QueuedJobs->getFullStats($jobType);

		$jobTypes = $this->QueuedJobs->find()->where()->find('list', ['keyField' => 'job_type', 'valueField' => 'job_type'])->distinct('job_type')->toArray();
		$this->set(compact('stats', 'jobTypes'));
	}
715107
def resultsFor( self, ps ):
        """"""
        k = self._parametersAsIndex(ps)
        if k in self._results.keys():
            # filter out pending job ids, which can be anything except dicts
            return [ res for res in self._results[k] if isinstance(res, dict) ]
        else:
            return []
453626
def job(func_or_queue, connection=None, *args, **kwargs):
    """"""
    decorated_func = _job(func_or_queue, connection=connection, *args, **kwargs)

    if callable(func_or_queue):
        @wraps(decorated_func)
        def wrapper(*args, **kwargs):
            with log.fields(uuid=uuid.uuid4(),
                            job_name=decorated_func.__name__):
                return decorated_func(*args, **kwargs)
        return wrapper
    else:
        def decorator(func):
            @decorated_func
            @wraps(func)
            def wrapper(*args, **kwargs):
                with log.fields(uuid=uuid.uuid4(),
                                job_name=func.__name__):
                    return func(*args, **kwargs)
            return wrapper
        return decorator