Store job info, and publish an update

    Parameters
    ----------
    to_deposit : dict
        The job info
514177
def _deposit_payload(to_deposit):
    
    pubsub = to_deposit['pubsub']
    id = to_deposit['id']

    with r_client.pipeline() as pipe:
        pipe.set(id, json.dumps(to_deposit), ex=REDIS_KEY_TIMEOUT)
        pipe.publish(pubsub, json.dumps({"update": [id]}))
        pipe.execute()
790838
def get_job_info():
    

    jobid = get_job_id()

    if jobid == '':
        return None

    info = get_qstat_info('-ft {0}'.format(jobid), 'Job Id:')

    # Select the dict for this job (there should only be one entry in any case)
    info = info['Job Id: {}'.format(jobid)]

    # Add the jobid to the dict and then return
    info['Job_ID'] = jobid

    return info
1237844
def job_success(self, job, queue, job_result):
        

        # display what was done
        obj = job.get_object()
        message = '[%s|%s] %s [%s]' % (queue.name.hget(),
                                       obj.pk.get(),
                                       job_result,
                                       threading.current_thread().name)
        self.log(message)

        # default stuff: update job and queue statuses, and do logging
        super(FullNameWorker, self).job_success(job, queue, job_result)

        # update the queue's dates
        queue_fields_to_update = {
            'last_job_date': str(datetime.utcnow())
        }
        if not queue.first_job_date.hget():
            queue_fields_to_update['first_job_date'] = queue_fields_to_update['last_job_date']
        queue.hmset(**queue_fields_to_update)

        # update the jobs counter on the queue
        queue.jobs_counter.hincrby(1)

        # save a ref to the job to display at the end of this example script
        self.jobs.append(int(job.pk.get()))
449655
def process_update_job_from_file(file_path, report_email = nil, postback_url = nil, options = {})
      data = options
      data['file'] = file_path
      process_job(:update, data, report_email, postback_url, 'file')
    end
954423
def get_job_output(self, job_id):
        
        log.debug("GetJobOutput %r", job_id)
        path = "/%s/jobs/%s/live/out" % (self.account, job_id)
        res, content = self._request(path, "GET")
        if res["status"] != "200":
            raise errors.MantaAPIError(res, content)
        keys = content.splitlines(False)
        return keys
830848
def job(self):
        
        if self._submitter and hasattr(self._submitter, '_job_access'):
            return self._submitter._job_access()
        return None
272124
def store_job(opts, load, event=None, mminion=None):
    '''
    
    '''
    # Generate EndTime
    endtime = salt.utils.jid.jid_to_time(salt.utils.jid.gen_jid(opts))
    # If the return data is invalid, just ignore it
    if any(key not in load for key in ('return', 'jid', 'id')):
        return False
    if not salt.utils.verify.valid_id(opts, load['id']):
        return False
    if mminion is None:
        mminion = salt.minion.MasterMinion(opts, states=False, rend=False)

    job_cache = opts['master_job_cache']
    if load['jid'] == 'req':
        # The minion is returning a standalone job, request a jobid
        load['arg'] = load.get('arg', load.get('fun_args', []))
        load['tgt_type'] = 'glob'
        load['tgt'] = load['id']

        prep_fstr = '{0}.prep_jid'.format(opts['master_job_cache'])
        try:
            load['jid'] = mminion.returners[prep_fstr](nocache=load.get('nocache', False))
        except KeyError:
            emsg = "Returner '{0}' does not support function prep_jid".format(job_cache)
            log.error(emsg)
            raise KeyError(emsg)

        # save the load, since we don't have it
        saveload_fstr = '{0}.save_load'.format(job_cache)
        try:
            mminion.returners[saveload_fstr](load['jid'], load)
        except KeyError:
            emsg = "Returner '{0}' does not support function save_load".format(job_cache)
            log.error(emsg)
            raise KeyError(emsg)
    elif salt.utils.jid.is_jid(load['jid']):
        # Store the jid
        jidstore_fstr = '{0}.prep_jid'.format(job_cache)
        try:
            mminion.returners[jidstore_fstr](False, passed_jid=load['jid'])
        except KeyError:
            emsg = "Returner '{0}' does not support function prep_jid".format(job_cache)
            log.error(emsg)
            raise KeyError(emsg)

    if event:
        # If the return data is invalid, just ignore it
        log.info('Got return from %s for job %s', load['id'], load['jid'])
        event.fire_event(load,
                         salt.utils.event.tagify([load['jid'], 'ret', load['id']], 'job'))
        event.fire_ret_load(load)

    # if you have a job_cache, or an ext_job_cache, don't write to
    # the regular master cache
    if not opts['job_cache'] or opts.get('ext_job_cache'):
        return

    # do not cache job results if explicitly requested
    if load.get('jid') == 'nocache':
        log.debug('Ignoring job return with jid for caching %s from %s',
                  load['jid'], load['id'])
        return

    # otherwise, write to the master cache
    savefstr = '{0}.save_load'.format(job_cache)
    getfstr = '{0}.get_load'.format(job_cache)
    fstr = '{0}.returner'.format(job_cache)
    updateetfstr = '{0}.update_endtime'.format(job_cache)
    if 'fun' not in load and load.get('return', {}):
        ret_ = load.get('return', {})
        if 'fun' in ret_:
            load.update({'fun': ret_['fun']})
        if 'user' in ret_:
            load.update({'user': ret_['user']})

    # Try to reach returner methods
    try:
        savefstr_func = mminion.returners[savefstr]
        getfstr_func = mminion.returners[getfstr]
        fstr_func = mminion.returners[fstr]
    except KeyError as error:
        emsg = "Returner '{0}' does not support function {1}".format(job_cache, error)
        log.error(emsg)
        raise KeyError(emsg)

    if job_cache != 'local_cache':
        try:
            mminion.returners[savefstr](load['jid'], load)
        except KeyError as e:
            log.error("Load does not contain 'jid': %s", e)

    mminion.returners[fstr](load)

    if (opts.get('job_cache_store_endtime')
            and updateetfstr in mminion.returners):
        mminion.returners[updateetfstr](load['jid'], endtime)
77682
def store(self, database, validate=True, role=None):
        
        if validate:
            self.validate()
        self._id, self._rev = database.save(self.to_primitive(role=role))
        return self
267677
def _update_log_record(self, record):
    
    if not hasattr(record, 'hostname'):
        record.hostname = '-'
    if not hasattr(record, 'job_id'):
        record.job_id = self.job_id
594485
def update_job(self, job_details):
        
        other = self.get_details(job_details.jobname,
                                 job_details.jobkey)
        other.timestamp = job_details.timestamp
        other.status = job_details.status
        other.update_table_row(self._table, other.dbkey - 1)
        return other
999729
def _process_state_embryo(self, job_record):
        
        uow, is_duplicate = self.insert_and_publish_uow(job_record, 0, 0)
        try:
            target_state = self._compute_next_job_state(job_record)
            self.update_job(job_record, uow, target_state)
        except ValueError:
            # do no processing for the future timeperiods
            pass
303068
def add_job_to_db(self, key, job):
        """"""
        job_msg = self.registry.deep_encode(job)
        prov = prov_key(job_msg)

        def set_link(duplicate_id):
            self.cur.execute(
                'update "jobs" set "link" = ?, "status" = ? where "id" = ?',
                (duplicate_id, Status.DUPLICATE, key))

        with self.lock:
            self.cur.execute(
                'select * from "jobs" where "prov" = ? '
                'and (("result" is not null) or '
                '("session" = ? and "link" is null))',
                (prov, self.session))
            rec = self.cur.fetchone()
            rec = JobEntry(*rec) if rec is not None else None

            self.cur.execute(
                'update "jobs" set "prov" = ?, "version" = ?, "function" = ?, '
                '"arguments" = ?, "status" = ? where "id" = ?',
                (prov, job_msg['data']['hints'].get('version'),
                 json.dumps(job_msg['data']['function']),
                 json.dumps(job_msg['data']['arguments']),
                 Status.WAITING,
                 key))

            if not rec:
                # no duplicate found, go on
                return 'initialized', None

            set_link(rec.id)

            if rec.result is not None and rec.status == Status.WORKFLOW:
                # the found duplicate returned a workflow
                if rec.link is not None:
                    # link is set, so result is fully realized
                    self.cur.execute(
                        'select * from "jobs" where "id" = ?',
                        (rec.link,))
                    rec = self.cur.fetchone()
                    assert rec is not None, "database integrity violation"
                    rec = JobEntry(*rec)

                else:
                    # link is not set, the result is still waited upon
                    assert rec.session == self.session, \
                        "database integrity violation"
                    self.attached[rec.id].append(key)
                    return 'attached', None

            if rec.result is not None:
                # result is found! return it
                result_value = self.registry.from_json(rec.result, deref=True)
                result = ResultMessage(
                    key, 'retrieved', result_value, None)
                return 'retrieved', result

            if rec.session == self.session:
                # still waiting for result, attach
                self.attached[rec.id].append(key)
                return 'attached', None
69546
def pending_job(job)
      update(job_class: job.class.to_s, job_id: job.job_id, status: Hyrax::Operation::PENDING)
    end
755563
def record_occupation_updates(self, updates, source, version):
        
        now = int(time.time())
        # Put it on the recordQueue and notify the worker thread.
        with self.recordCond:
            self.recordQueue.append((now, updates, source))
            self.recordCond.notify()
933963
def store_job(self):
        '''
        
        '''
        try:
            class_name = self.__class__.__name__.lower()
        except AttributeError:
            log.warning(
                'Unable to determine class name',
                exc_info_on_loglevel=logging.DEBUG
            )
            return True

        try:
            return self.opts['{0}_returns'.format(class_name)]
        except KeyError:
            # No such option, assume this isn't one we care about gating and
            # just return True.
            return True
346722
async def send_job_result(self, job_id: BackendJobId, result: str, text: str = "", grade: float = None, problems: Dict[str, SPResult] = None,
                              tests: Dict[str, Any] = None, custom: Dict[str, Any] = None, state: str = "", archive: Optional[bytes] = None,
                              stdout: Optional[str] = None, stderr: Optional[str] = None):
        
        if job_id not in self.__running_job:
            raise JobNotRunningException()
        del self.__running_job[job_id]

        if grade is None:
            if result == "success":
                grade = 100.0
            else:
                grade = 0.0
        if problems is None:
            problems = {}
        if custom is None:
            custom = {}
        if tests is None:
            tests = {}

        await ZMQUtils.send(self.__backend_socket, AgentJobDone(job_id, (result, text), round(grade, 2), problems, tests, custom, state, archive, stdout, stderr))
486831
def _process_job_and_get_successors(self, job_info):
        

        job = job_info.job

        successors = self._get_successors(job)

        all_new_jobs = [ ]

        for successor in successors:
            new_jobs = self._handle_successor(job, successor, successors)

            if new_jobs:
                all_new_jobs.extend(new_jobs)

                for new_job in new_jobs:
                    self._insert_job(new_job)

        self._post_job_handling(job, all_new_jobs, successors)