Get all queued and running jobs for a results file.
765552
def getJobIDsForResultsFile(self, resultsFile):
        
        jobIDs = []
        for line in self._runParasol(['-extended', 'list', 'jobs'])[1]:
            fields = line.strip().split()
            if len(fields) == 0 or fields[-1] != resultsFile:
                continue
            jobID = fields[0]
            jobIDs.append(int(jobID))
        return set(jobIDs)
408851
def _get_tasks_by_queue(self):
        """"""
        task_map = {}
        _checker = None

        # Ask the persistence engine for an Async to use for checking if the
        # context is complete.
        if self._persistence_engine:
            _checker = self._persistence_engine.context_completion_checker

        for async in self._tasks:
            queue = async.get_queue()
            if _checker:
                async.update_options(_context_checker=_checker)

            task = async.to_task()
            task_map.setdefault(queue, []).append(task)

        return task_map
96363
def run_all
      results = runner.run([options[:spec_dir]])

      self.last_failed_paths = results.keys
      self.last_run_failed   = !results.empty?

      throw :task_has_failed if last_run_failed
    end
789894
def list(self, allJobs=False):
        

        jobs = self.server.call('get', '/job')

        return [Job(job['id'], self.server) for job in jobs if allJobs or self._job_owned(job)]
779834
def clean_jobs(self, link, job_dict=None, clean_all=False):
        
        failed = False
        if job_dict is None:
            job_dict = link.jobs

        for job_details in job_dict.values():
            # clean failed jobs
            if job_details.status == JobStatus.failed or clean_all:
                # clean_job(job_details.logfile, job_details.outfiles, self._dry_run)
                clean_job(job_details.logfile, {}, self._dry_run)
                job_details.status = JobStatus.ready
        if failed:
            return JobStatus.failed
        return JobStatus.done
581553
def get(self, *jids):
        ''''''
        if jids:
            return [
                Job(self.client, **j) for j in
                json.loads(self.client('multiget', *jids))]
        return []
1048610
public static void printAllJobs(List<JobExecutionInfo> jobExecutionInfos, int resultsLimit) {
        if (jobExecutionInfos == null) {
            System.err.println("No jobs found.");
            System.exit(1);
        }

        List<String> labels = Arrays.asList("Job Name", "State", "Last Run Started", "Last Run Completed",
                "Schedule", "Last Run Records Processed", "Last Run Records Failed");
        List<String> flags = Arrays.asList("-", "-", "-", "-", "-", "", "");
        List<List<String>> data = new ArrayList<>();
        for (JobExecutionInfo jobInfo : jobExecutionInfos) {
            List<String> entry = new ArrayList<>();
            entry.add(jobInfo.getJobName());
            entry.add(jobInfo.getState().toString());
            entry.add(dateTimeFormatter.print(jobInfo.getStartTime()));
            entry.add(dateTimeFormatter.print(jobInfo.getEndTime()));

            entry.add(extractJobSchedule(jobInfo));

            // Add metrics
            MetricArray metrics = jobInfo.getMetrics();
            Double recordsProcessed = null;
            Double recordsFailed = null;
            try {
                for (Metric metric : metrics) {
                    if (metric.getName().equals(MetricNames.ExtractorMetrics.RECORDS_READ_METER)) {
                        recordsProcessed = Double.parseDouble(metric.getValue());
                    } else if (metric.getName().equals(MetricNames.ExtractorMetrics.RECORDS_FAILED_METER)) {
                        recordsFailed = Double.parseDouble(metric.getValue());
                    }
                }

                if (recordsProcessed != null && recordsFailed != null) {
                    entry.add(recordsProcessed.toString());
                    entry.add(recordsFailed.toString());
                }
            } catch (NumberFormatException ex) {
                System.err.println("Failed to process metrics");
            }
            if (recordsProcessed == null || recordsFailed == null) {
                entry.add("-");
                entry.add("-");
            }

            data.add(entry);
        }
        new CliTablePrinter.Builder()
                .labels(labels)
                .data(data)
                .flags(flags)
                .delimiterWidth(2)
                .build()
                .printTable();

        if (jobExecutionInfos.size() == resultsLimit) {
            System.out.println("\nWARNING: There may be more jobs (# of results is equal to the limit)");
        }
    }
105263
def Jobs():
    
    jobs = [0, 0, 0, 0]

    # get running jobs
    try:
        d_client = docker.from_env()
        c = d_client.containers.list(all=False,
                                     filters={'label': 'vent-plugin'})
        files = []
        for container in c:
            jobs[1] += 1
            if 'file' in container.attrs['Config']['Labels']:
                if container.attrs['Config']['Labels']['file'] not in files:
                    files.append(container.attrs['Config']['Labels']['file'])
        jobs[0] = len(files)
    except Exception as e:  # pragma: no cover
        logger.error('Could not get running jobs ' + str(e))

    # get finished jobs
    try:
        d_client = docker.from_env()
        c = d_client.containers.list(all=True,
                                     filters={'label': 'vent-plugin',
                                              'status': 'exited'})

        file_names = []
        tool_names = []
        finished_jobs = []
        path_dirs = PathDirs()
        manifest = join(path_dirs.meta_dir, 'status.json')

        if exists(manifest):
            file_status = 'a'
        else:
            file_status = 'w'

        # get a list of past jobs' file names if status.json exists
        if file_status == 'a':
            with open(manifest, 'r') as infile:
                for line in infile:
                    finished_jobs.append(json.loads(line))

            # get a list of file names so we can check against each container
            file_names = [d['FileName'] for d in finished_jobs]

            # multiple tools can run on 1 file. Use a tuple to status check
            tool_names = [(d['FileName'], d['VentPlugin'])
                          for d in finished_jobs]

        for container in c:
            jobs[3] += 1

            if 'file' in container.attrs['Config']['Labels']:
                # make sure the file name and the tool tup exists because
                # multiple tools can run on 1 file.
                if (container.attrs['Config']['Labels']['file'],
                    container.attrs['Config']['Labels']['vent.name']) not in \
                        tool_names:
                    # TODO figure out a nicer way of getting desired values
                    # from containers.attrs.
                    new_file = {}
                    new_file['FileName'] = \
                        container.attrs['Config']['Labels']['file']
                    new_file['VentPlugin'] = \
                        container.attrs['Config']['Labels']['vent.name']
                    new_file['StartedAt'] = \
                        container.attrs['State']['StartedAt']
                    new_file['FinishedAt'] = \
                        container.attrs['State']['FinishedAt']
                    new_file['ID'] = \
                        container.attrs['Id'][:12]

                    # create/append a json file with all wanted information
                    with open(manifest, file_status) as outfile:
                        json.dump(new_file, outfile)
                        outfile.write('\n')

                # delete any containers with 'vent-plugin' in the groups
                if 'vent-plugin' in container.attrs['Config']['Labels']:
                    container.remove()

        # add extra one to account for file that just finished if the file was
        # just created since file_names is processed near the beginning
        if file_status == 'w' and len(file_names) == 1:
            jobs[2] = len(set(file_names)) + 1
        else:
            jobs[2] = len(set(file_names))

        jobs[3] = jobs[3] - jobs[1]

    except Exception as e:  # pragma: no cover
        logger.error('Could not get finished jobs ' + str(e))

    return tuple(jobs)
722929
def _get_results(FetcherClass, services, kwargs, num_results=None, fast=0, verbose=False, timeout=None):
    
    results = []

    if not num_results or fast:
        num_results = len(services)

    with futures.ThreadPoolExecutor(max_workers=len(services)) as executor:
        fetches = {}
        for service in services[:num_results]:
            tail = [x for x in services if x is not service]
            random.shuffle(tail)
            srv = FetcherClass(services=[service] + tail, verbose=verbose, timeout=timeout)
            fetches[executor.submit(srv.action, **kwargs)] = srv

        if fast == 1:
            raise NotImplementedError
            # ths code is a work in progress. futures.FIRST_COMPLETED works differently than I thought...
            to_iterate, still_going = futures.wait(fetches, return_when=futures.FIRST_COMPLETED)
            for x in still_going:
                try:
                    x.result(timeout=1.001)
                except futures._base.TimeoutError:
                    pass

        elif fast > 1:
            raise Exception("fast level greater than 1 not yet implemented")
        else:
            to_iterate = futures.as_completed(fetches)

        for future in to_iterate:
            service = fetches[future]
            results.append([service, future.result()])

    return results
171045
async def main():
    redis = await create_pool(RedisSettings())

    job = await redis.enqueue_job('the_task')

    # get the job's id
    print(job.job_id)
    

    # get information about the job, will include results if the job has finished, but
    # doesn't await the job's result
    debug(await job.info())
    """
    >   docs/examples/job_results.py:23 main
    JobDef(
        function='the_task',
        args=(),
        kwargs={},
        job_try=None,
        enqueue_time=datetime.datetime(2019, 4, 23, 13, 58, 56, 781000),
        score=1556027936781
    ) (JobDef)
    """

    # get the Job's status
    print(await job.status())
    """
    >  JobStatus.queued
    """

    # poll redis for the job result, if the job raised an exception,
    # it will be raised here
    # (You'll need the worker running at the same time to get a result here)
    print(await job.result(timeout=5))
    """
    >  42
    """
52002
def join(self):
        
        self._jobs.join()
        try:
            return self._results, self._errors
        finally:
            self._clear()
1100822
def status_search(self, status):
        """"""
        json = self._fetch_json()
        jobs = json['response']
        for job in jobs:
            job_info = jobs[job]
            if job_info['status'].lower() == status.lower():
                yield self._build_results(jobs, job)
1080418
def results
      return self.get_jobs.map do |task|
        next if task == nil
        next if task.server == nil
        {
          :job_id => task.job_id,
          :name   => task.info[0],
          :host   => task.server.name,
          :status => task.status,
          :error  => task.error,
          :total  => self.get_total_execution_time(task.status, task.time_start, task.time_end),
          :start  => task.time_start,
          :end    => task.time_end,
          :worker => task
        }
      end
    end
120328
def resumable(self):
        ''''''
        # First, find the jids of all the jobs registered to this client.
        # Then, get the corresponding job objects
        jids = self.client.workers[self.client.worker_name]['jobs']
        jobs = self.client.jobs.get(*jids)

        # We'll filter out all the jobs that aren't in any of the queues
        # we're working on.
        queue_names = set([queue.name for queue in self.queues])
        return [job for job in jobs if job.queue_name in queue_names]
212350
def get_jobs_from_queue(self, queue: str, max_jobs: int) -> List[Job]:
        """"""
        jobs_json_string = self._run_script(
            self._get_jobs_from_queue,
            self._to_namespaced(queue),
            self._to_namespaced(RUNNING_JOBS_KEY.format(self._id)),
            JobStatus.RUNNING.value,
            max_jobs
        )

        jobs = json.loads(jobs_json_string.decode())
        jobs = [Job.deserialize(job) for job in jobs]

        return jobs
1043666
def getIssuedBatchJobIDs(self):
        
        issuedJobs = set()
        for resultsFile in itervalues(self.resultsFiles):
            issuedJobs.update(self.getJobIDsForResultsFile(resultsFile))

        return list(issuedJobs)
379705
def read_results(self):
        """"""
        
        if self.submited:
            # The job has been submitted. Check the state.
            if not self.job_ready() :
                if self.block :
                    while not self.job_ready() :
                        time.sleep(self.job_check_time)
                else :
                    raise CalcNotReadyError
                    

            # Assume the calc finished. Copy the files back.
            subprocess.call(self.copy_in_cmd % {
                'ldir': self.wdir,
                'rdir': os.path.join(self.parameters['rdir'],os.path.split(self.directory)[-1]),
                'user': self.parameters['user'],
                'host': self.parameters['host']
            }, shell=True)
                
        
        fn=os.path.join(self.directory,'pw.out')
        # Read the pan-ultimate line of the output file
        try: 
            ln=open(fn).readlines()[-2]
            if ln.find('JOB DONE.')>-1 :
                # Job is done we can read the output
                r=read_quantumespresso_textoutput(fn)
                self.submited=False
                self.jobid=None
            else :
                # Job not ready.
                raise CalcNotReadyError
        except (IOError, IndexError) :
            # Job not ready.
            raise CalcNotReadyError
        
        # All is fine - really read the results
        self.calc.read_results(self)