method iterated thru all documents in all job collections and builds a tree of known system state
10940
def _build_tree_by_level(self, time_qualifier, collection_name, since):
        
        invalid_tree_records = dict()
        invalid_tq_records = dict()

        try:
            job_records = self.job_dao.get_all(collection_name, since)
            for job_record in job_records:
                tree = self.get_tree(job_record.process_name)
                if tree is None:
                    utils.increment_family_property(job_record.process_name, invalid_tree_records)
                    continue

                job_time_qualifier = context.process_context[job_record.process_name].time_qualifier
                if time_qualifier != job_time_qualifier:
                    utils.increment_family_property(job_record.process_name, invalid_tq_records)
                    continue

                tree.update_node(job_record)

        except LookupError:
            self.logger.warning('No job records in {0}.'.format(collection_name))

        for name, counter in invalid_tree_records.items():
            self.logger.warning('Skipping {0} job records for {1} since no tree is handling it.'
                                .format(counter, name))

        for name, counter in invalid_tq_records.items():
            self.logger.warning('Skipping {0} job records for {1} since the process has different time qualifier.'
                                .format(counter, name))
142736
public final void reloadAll() {
        final List<String> list = new ArrayList<>();
        for (final RootDocument mongo : repositoryManager
                .getRootDocumentRepository().findAll()) {
            list.add(mongo.getDbName());
        }
        reset();
        for (final String dbname : list) {
            loadDocument(dbname);
        }
    }
625745
def get_collection_tree(collection_ident, cursor):
    """"""
    cursor.execute('''
    WITH RECURSIVE t(node, parent, document, path) AS (
        SELECT tr.nodeid, tr.parent_id, tr.documentid, ARRAY[tr.nodeid]
        FROM trees tr
        WHERE tr.documentid = %s and tr.is_collated = 'False'
    UNION ALL
        SELECT c.nodeid, c.parent_id, c.documentid, path || ARRAY[c.nodeid]
        FROM trees c JOIN t ON c.parent_id = t.node
        WHERE NOT c.nodeid = ANY(t.path)
    )
    SELECT t.document, m.portal_type
    FROM t JOIN modules m
    ON t.document = m.module_ident''', [collection_ident])
    for i in cursor.fetchall():
        yield i
1352441
def _prepare_job(request, candidates):
    
    temp_dir = tempfile.mkdtemp()
    inifiles = []
    arch = request.FILES.get('archive')
    if arch is None:
        # move each file to a new temp dir, using the upload file names,
        # not the temporary ones
        for each_file in request.FILES.values():
            new_path = os.path.join(temp_dir, each_file.name)
            shutil.move(each_file.temporary_file_path(), new_path)
            if each_file.name in candidates:
                inifiles.append(new_path)
        return inifiles
    # else extract the files from the archive into temp_dir
    return readinput.extract_from_zip(arch, candidates)
938531
def bulk_launch(self, jobs=None, filter=None, all=False): # pylint: disable=redefined-builtin
        
        json = None
        if jobs is not None:
            schema = JobSchema(exclude=('id', 'status', 'package_name', 'config_name', 'device_name', 'result_id', 'user_id', 'created', 'updated', 'automatic'))
            jobs_json = self.service.encode(schema, jobs, many=True)
            json = {self.RESOURCE: jobs_json}

        schema = JobSchema()
        resp = self.service.post(self.base,
                                 params={'bulk': 'launch', 'filter': filter, 'all': all}, json=json)
        return self.service.decode(schema, resp, many=True)
466192
function(job, done) {
                    Async.whilst(
                        function() { return !job.properties().isDone; },
                        function(iterationDone) {
                            job.fetch(function(err, job) {
                                if (err) {
                                    callback(err);
                                }
                                else {
                                    // If the user asked for verbose output,
                                    // then write out the status of the search
                                    var properties = job.properties();
                                    if (isVerbose) {
                                        var progress    = (properties.doneProgress * 100.0) + "%";
                                        var scanned     = properties.scanCount;
                                        var matched     = properties.eventCount;
                                        var results     = properties.resultCount;
                                        var stats = "-- " +
                                            progress + " done | " +
                                            scanned  + " scanned | " +
                                            matched  + " matched | " +
                                            results  + " results";
                                        print("\r" + stats + "                                          ");
                                    }
                                    
                                    Async.sleep(1000, iterationDone);
                                }
                            });
                        },
                        function(err) {
                            if (isVerbose) {
                                print("\r");
                            }
                            done(err, job);
                        }
                    );
                }
535395
def run(self):
        """"""
        config = self.state.document.settings.env.config

        # Get all processes:
        processes = get_processes(config.autoprocess_process_dir, config.autoprocess_source_base_url)
        process_nodes = []

        for process in sorted(processes, key=itemgetter('name')):
            process_nodes.extend(self.make_process_node(process))

        return process_nodes
820252
def job_status(self, project, job_id):
        
        all_jobs = self.list_jobs(project)
        for state in constants.JOB_STATES:
            job_ids = [job['id'] for job in all_jobs[state]]
            if job_id in job_ids:
                return state
        return ''
435198
def set_all_revisions_to_in_process(self, ids):
        

        predicate = {
            "_id" : {
                "$in" : [ ObjectId(id) for id in ids ]
            }
        }

        set = {"$set": { "inProcess": True }}

        yield self.revisions.collection.update(predicate, set, multi=True)
1312606
def _build_job_dict(self):
        """"""
        if self.args['dry_run']:
            status = JobStatus.unknown
        else:
            status = JobStatus.not_ready

        base_config = self.scatter_link.args

        for jobkey, job_config in sorted(self._job_configs.items()):
            full_job_config = base_config.copy()
            full_job_config.update(job_config)
            ScatterGather._make_scatter_logfile_name(jobkey,
                                                     self.linkname, full_job_config)
            logfile = job_config.get('logfile')
            self._scatter_link._register_job(key=jobkey,
                                             job_config=full_job_config,
                                             logfile=logfile,
                                             status=status)
736623
def _makeJobSuccessorReadyToRun(self, jobGraph, jobNode):
        """"""
        successorJobStoreID = jobNode.jobStoreID
        #Build map from successor to predecessors.
        if successorJobStoreID not in self.toilState.successorJobStoreIDToPredecessorJobs:
            self.toilState.successorJobStoreIDToPredecessorJobs[successorJobStoreID] = []
        self.toilState.successorJobStoreIDToPredecessorJobs[successorJobStoreID].append(jobGraph)

        if jobNode.predecessorNumber > 1:
            return self._checkSuccessorReadyToRunMultiplePredecessors(jobGraph, jobNode, successorJobStoreID)
        else:
            return True
876121
def paths_from_buildroot_iter(self):
    """"""
    for f in self:
      yield os.path.join(self.rel_root, f)
1262883
def list(self, job_ids, print_array_jobs = False, print_dependencies = False, long = False, print_times = False, status=Status, names=None, ids_only=False):
    """"""
    # configuration for jobs
    fields = ("job-id", "grid-id", "queue", "status", "job-name")
    lengths = (6, 17, 11, 12, 16)
    dependency_length = 0

    if print_dependencies:
      fields += ("dependencies",)
      lengths += (25,)
      dependency_length = lengths[-1]

    if long:
      fields += ("submitted command",)
      lengths += (43,)

    format = "{:^%d}  " * len(lengths)
    format = format % lengths

    # if ids_only:
    #   self.lock()
    #   for job in self.get_jobs():
    #     print(job.unique, end=" ")
    #   self.unlock()
    #   return

    array_format = "{0:^%d}  {1:>%d}  {2:^%d}  {3:^%d}" % lengths[:4]
    delimiter = format.format(*['='*k for k in lengths])
    array_delimiter = array_format.format(*["-"*k for k in lengths[:4]])
    header = [fields[k].center(lengths[k]) for k in range(len(lengths))]

    # print header
    if not ids_only:
      print('  '.join(header))
      print(delimiter)

    self.lock()
    for job in self.get_jobs(job_ids):
      job.refresh()
      if job.status in status and (names is None or job.name in names):
        if ids_only:
          print(job.unique, end=" ")
        else:
          print(job.format(format, dependency_length))
        if print_times:
          print(times(job))

        if (not ids_only) and print_array_jobs and job.array:
          print(array_delimiter)
          for array_job in job.array:
            if array_job.status in status:
              print(array_job.format(array_format))
              if print_times:
                print(times(array_job))
          print(array_delimiter)

    self.unlock()
1253540
def _createJobStateFile(self):
        
        jobStateFile = os.path.join(self.localTempDir, '.jobState')
        jobState = {'jobPID': os.getpid(),
                    'jobName': self.jobName,
                    'jobDir': self.localTempDir,
                    'deferredFunctions': []}
        with open(jobStateFile + '.tmp', 'wb') as fH:
            dill.dump(jobState, fH)
        os.rename(jobStateFile + '.tmp', jobStateFile)
        return jobStateFile
889757
def atrm(*args):
    '''
    
    '''

    if not args:
        return {'jobs': {'removed': [], 'tag': None}}

    if args[0] == 'all':
        if len(args) > 1:
            opts = list(list(map(str, [j['job'] for j in atq(args[1])['jobs']])))
            ret = {'jobs': {'removed': opts, 'tag': args[1]}}
        else:
            opts = list(list(map(str, [j['job'] for j in atq()['jobs']])))
            ret = {'jobs': {'removed': opts, 'tag': None}}
    else:
        opts = list(list(map(str, [i['job'] for i in atq()['jobs']
            if i['job'] in args])))
        ret = {'jobs': {'removed': opts, 'tag': None}}

    # call atrm for each job in ret['jobs']['removed']
    for job in ret['jobs']['removed']:
        res_job = __salt__['cmd.run_all']('atrm {job}'.format(
            job=job
        ))
        if res_job['retcode'] > 0:
            if 'failed' not in ret['jobs']:
                ret['jobs']['failed'] = {}
            ret['jobs']['failed'][job] = res_job['stderr']

    # remove failed from list
    if 'failed' in ret['jobs']:
        for job in ret['jobs']['failed']:
            ret['jobs']['removed'].remove(job)

    return ret
597756
def _serialiseJob(self, jobStore, jobsToJobGraphs, rootJobGraph):
        
        # Pickle the job so that its run method can be run at a later time.
        # Drop out the children/followOns/predecessors/services - which are
        # all recorded within the jobStore and do not need to be stored within
        # the job
        self._children, self._followOns, self._services = [], [], []
        self._directPredecessors, self._promiseJobStore = set(), None
        # The pickled job is "run" as the command of the job, see worker
        # for the mechanism which unpickles the job and executes the Job.run
        # method.
        with jobStore.writeFileStream(rootJobGraph.jobStoreID) as (fileHandle, fileStoreID):
            pickle.dump(self, fileHandle, pickle.HIGHEST_PROTOCOL)
        # Note that getUserScript() may have been overridden. This is intended. If we used
        # self.userModule directly, we'd be getting a reference to job.py if the job was
        # specified as a function (as opposed to a class) since that is where FunctionWrappingJob
        #  is defined. What we really want is the module that was loaded as __main__,
        # and FunctionWrappingJob overrides getUserScript() to give us just that. Only then can
        # filter_main() in _unpickle( ) do its job of resolving any user-defined type or function.
        userScript = self.getUserScript().globalize()
        jobsToJobGraphs[self].command = ' '.join(('_toil', fileStoreID) + userScript.toCommand())
        #Update the status of the jobGraph on disk
        jobStore.update(jobsToJobGraphs[self])
1017636
function(partition) {
                     var self = this;
                     var results = {
                         'id': [],
                         'partition': [],
                         'nameUUID': [],
                         'status': []
                     };
                     self.data.partition.map(function(val, i) { // for each partition
                         if (val.match(partition)) { // if the job is on @partition
                             for (var key in self.data) { // keep all the {'key':'value'} corresponding
                                 results[key].push(self.data[key][i]);
                             }
                         }
                     });
                     return results;
                 }