Save the load for a given job id
509509
def save_load(jid, load, minions=None):
    '''
    
    '''
    conn, mdb = _get_conn(ret=None)
    to_save = _safe_copy(load)

    if PYMONGO_VERSION > _LooseVersion('2.3'):
        #using .copy() to ensure original data for load is unchanged
        mdb.jobs.insert_one(to_save)
    else:
        mdb.jobs.insert(to_save)
353927
def save_cache(data, filename):
    """"""
    with open(filename, 'wb') as handle:
        pickle.dump(data, handle)
7211
def upload(self, *args, **kwargs):
    """"""
    for job in self.jobs:
      job.upload(*args, **kwargs)
998446
private void setJobId(String id) {
    if (id != null && id.startsWith("job_") && id.length() > 4) {
      this.jobNumber = id.substring(4);
    }
  }
33633
def put(self, job, result):
        ""
        self.job.put(job)
        r = result.get()
        return r
695859
def save_load(jid, load, minions=None):
    '''
    
    '''
    signaled = dispatch.Signal(
        providing_args=['jid', 'load']).send(
            sender='save_load', jid=jid, load=load)

    for signal in signaled:
        log.debug(
            'Django returner function \'save_load\' signaled %s '
            'which responded with %s', signal[0], signal[1]
        )
758083
def save_load(jid, clear_load, minion=None):
    '''
    
    '''
    cb_ = _get_connection()

    try:
        jid_doc = cb_.get(six.text_type(jid))
    except couchbase.exceptions.NotFoundError:
        cb_.add(six.text_type(jid), {}, ttl=_get_ttl())
        jid_doc = cb_.get(six.text_type(jid))

    jid_doc.value['load'] = clear_load
    cb_.replace(six.text_type(jid), jid_doc.value, cas=jid_doc.cas, ttl=_get_ttl())

    # if you have a tgt, save that for the UI etc
    if 'tgt' in clear_load and clear_load['tgt'] != '':
        ckminions = salt.utils.minions.CkMinions(__opts__)
        # Retrieve the minions list
        _res = ckminions.check_minions(
            clear_load['tgt'],
            clear_load.get('tgt_type', 'glob')
            )
        minions = _res['minions']
        save_minions(jid, minions)
177249
private void writeJsonUpdatesForJob(PrintWriter wrt, JobID jobId) {
		
		try {
			
			List<AbstractEvent> events = jobmanager.getEvents(jobId);
			
			//Serialize job to json
			wrt.write("{");
			wrt.write("\"jobid\": \"" + jobId + "\",");
			wrt.write("\"timestamp\": \"" + System.currentTimeMillis() + "\",");
			wrt.write("\"recentjobs\": [");
				
			boolean first = true;
			for(RecentJobEvent rje: jobmanager.getRecentJobs()) {
				if(first) {
					first = false;
				} else {
					wrt.write(","); }
				
				wrt.write("\""+rje.getJobID().toString()+"\""); 
			}
					
			wrt.write("],");
			
			wrt.write("\"vertexevents\": [");
		
			first = true;
			for(AbstractEvent event: events) {
				
				if(event instanceof ExecutionStateChangeEvent) {
					
					if(first) {
						first = false;
					} else {
						wrt.write(","); }
					
					ExecutionStateChangeEvent vertexevent = (ExecutionStateChangeEvent) event;
					wrt.write("{");
					wrt.write("\"vertexid\": \"" + vertexevent.getVertexID() + "\",");
					wrt.write("\"newstate\": \"" + vertexevent.getNewExecutionState() + "\",");
					wrt.write("\"timestamp\": \"" + vertexevent.getTimestamp() + "\"");
					wrt.write("}");
				}
			}
			
			wrt.write("],");
			
			wrt.write("\"jobevents\": [");
			
			first = true;
			for(AbstractEvent event: events) {
				
				if( event instanceof JobEvent) {
					
					if(first) {
						first = false;
					} else {
						wrt.write(","); }
					
					JobEvent jobevent = (JobEvent) event;
					wrt.write("{");
					wrt.write("\"newstate\": \"" + jobevent.getCurrentJobStatus() + "\",");
					wrt.write("\"timestamp\": \"" + jobevent.getTimestamp() + "\"");
					wrt.write("}");
				}
			}
			
			wrt.write("]");
			
			wrt.write("}");
			
		
		} catch (EofException eof) { // Connection closed by client
			LOG.info("Info server for jobmanager: Connection closed by client, EofException");
		} catch (IOException ioe) { // Connection closed by client	
			LOG.info("Info server for jobmanager: Connection closed by client, IOException");
		} 
		
	}
930552
def put_job(self, id, body, params=None):
        
        for param in (id, body):
            if param in SKIP_IN_PATH:
                raise ValueError("Empty value passed for a required argument.")
        return self.transport.perform_request(
            "PUT", _make_path("_rollup", "job", id), params=params, body=body
        )
139116
def submit(self, *items):
		""""""
		with self.lock:
			if self.closed:
				raise BrokenPipe('Job submission has been closed.')
			id = self.jobcount
			self._status += ['SUBMITTED'] * len(items)
			self.jobcount += len(items)
			for item in items:
				self.waitqueue.put((id, item))
				id += 1
		if len(items) == 1:
			return id - 1
		else:
			return range(id - len(items), id)
717718
def _create_job(self, mapping):
        """"""
        job_id = self.bulk.create_insert_job(mapping["sf_object"], contentType="CSV")
        self.logger.info("  Created bulk job {}".format(job_id))

        # Upload batches
        local_ids_for_batch = {}
        for batch_file, local_ids in self._get_batches(mapping):
            batch_id = self.bulk.post_batch(job_id, batch_file)
            local_ids_for_batch[batch_id] = local_ids
            self.logger.info("    Uploaded batch {}".format(batch_id))

        self.bulk.close_job(job_id)
        return job_id, local_ids_for_batch
459251
private void loadJobFile() {
    try {
      String jobFile = getJobFile();
      FileSystem fs = location.getDFS();
      File tmp = File.createTempFile(getJobID().toString(), ".xml");
      if (FileUtil.copy(fs, new Path(jobFile), tmp, false, location
          .getConfiguration())) {
        this.jobConf = new JobConf(tmp.toString());

        this.totalMaps = jobConf.getNumMapTasks();
        this.totalReduces = jobConf.getNumReduceTasks();
      }

    } catch (IOException ioe) {
      ioe.printStackTrace();
    }
  }
655616
def get_load(jid):
    '''
    
    '''
    jid = _escape_jid(jid)
    conn = _get_conn()
    if conn is None:
        return None
    cur = conn.cursor()
    sql = '''SELECT jid, tgt_type, cmd, tgt, kwargs, ret, username, arg,''' \
          ''' fun FROM jids WHERE jid = %s'''
    cur.execute(sql, (jid,))
    data = cur.fetchone()
    if data:
        return _build_dict(data)
    _close_conn(conn)
    return {}
532435
def communicate(self, job_ids = None):
    """"""
    self.lock()
    # iterate over all jobs
    jobs = self.get_jobs(job_ids)
    for job in jobs:
      job.refresh()
      if job.status in ('queued', 'executing', 'waiting') and job.queue_name != 'local':
        status = qstat(job.id, context=self.context)
        if len(status) == 0:
          job.status = 'failure'
          job.result = 70 # ASCII: 'F'
          logger.warn("The job '%s' was not executed successfully (maybe a time-out happened). Please check the log files." % job)
          for array_job in job.array:
            if array_job.status in ('queued', 'executing'):
              array_job.status = 'failure'
              array_job.result = 70 # ASCII: 'F'


    self.session.commit()
    self.unlock()
607400
def download(self, job_id, destination=None, timeout=DEFAULT_TIMEOUT, retries=DEFAULT_RETRIES):
        
        self._retries_num = 0
        sleep(timeout)
        self.save_many(job_id, destination, timeout, retries)
        return self._cache
1117250
private void insertJobMetadata() {
    final String baseURL = this.azkabanProps.get(AZKABAN_WEBSERVER_URL);
    if (baseURL != null) {
      final String flowName = this.node.getParentFlow().getFlowId();
      final String projectName = this.node.getParentFlow().getProjectName();

      this.props.put(CommonJobProperties.AZKABAN_URL, baseURL);
      this.props.put(CommonJobProperties.EXECUTION_LINK,
          String.format("%s/executor?execid=%d", baseURL, this.executionId));
      this.props.put(CommonJobProperties.JOBEXEC_LINK, String.format(
          "%s/executor?execid=%d&job=%s", baseURL, this.executionId, this.node.getNestedId()));
      this.props.put(CommonJobProperties.ATTEMPT_LINK, String.format(
          "%s/executor?execid=%d&job=%s&attempt=%d", baseURL, this.executionId,
          this.node.getNestedId(), this.node.getAttempt()));
      this.props.put(CommonJobProperties.WORKFLOW_LINK, String.format(
          "%s/manager?project=%s&flow=%s", baseURL, projectName, flowName));
      this.props.put(CommonJobProperties.JOB_LINK, String.format(
          "%s/manager?project=%s&flow=%s&job=%s", baseURL, projectName,
          flowName, this.jobId));
    } else {
      if (this.logger != null) {
        this.logger.info(AZKABAN_WEBSERVER_URL + " property was not set");
      }
    }
    // out nodes
    this.props.put(CommonJobProperties.OUT_NODES,
        StringUtils.join2(this.node.getOutNodes(), ","));

    // in nodes
    this.props.put(CommonJobProperties.IN_NODES,
        StringUtils.join2(this.node.getInNodes(), ","));
  }
1136790
def run(self, jid=None):
        '''
        
        '''
        if self.opts.get('list_hosts'):
            self._get_roster()
            ret = {}
            for roster_file in self.__parsed_rosters:
                if roster_file.startswith('#'):
                    continue
                ret[roster_file] = {}
                for host_id in self.__parsed_rosters[roster_file]:
                    hostname = self.__parsed_rosters[roster_file][host_id]['host']
                    ret[roster_file][host_id] = hostname
            salt.output.display_output(ret, 'nested', self.opts)
            sys.exit()

        fstr = '{0}.prep_jid'.format(self.opts['master_job_cache'])
        jid = self.returners[fstr](passed_jid=jid or self.opts.get('jid', None))

        # Save the invocation information
        argv = self.opts['argv']

        if self.opts.get('raw_shell', False):
            fun = 'ssh._raw'
            args = argv
        else:
            fun = argv[0] if argv else ''
            args = argv[1:]

        job_load = {
            'jid': jid,
            'tgt_type': self.tgt_type,
            'tgt': self.opts['tgt'],
            'user': self.opts['user'],
            'fun': fun,
            'arg': args,
            }

        # save load to the master job cache
        try:
            if isinstance(jid, bytes):
                jid = jid.decode('utf-8')
            if self.opts['master_job_cache'] == 'local_cache':
                self.returners['{0}.save_load'.format(self.opts['master_job_cache'])](jid, job_load, minions=self.targets.keys())
            else:
                self.returners['{0}.save_load'.format(self.opts['master_job_cache'])](jid, job_load)
        except Exception as exc:
            log.exception(exc)
            log.error(
                'Could not save load with returner %s: %s',
                self.opts['master_job_cache'], exc
            )

        if self.opts.get('verbose'):
            msg = 'Executing job with jid {0}'.format(jid)
            print(msg)
            print('-' * len(msg) + '\n')
            print('')
        sret = {}
        outputter = self.opts.get('output', 'nested')
        final_exit = 0
        for ret in self.handle_ssh():
            host = next(six.iterkeys(ret))
            if isinstance(ret[host], dict):
                host_ret = ret[host].get('retcode', 0)
                if host_ret != 0:
                    final_exit = 1
            else:
                # Error on host
                final_exit = 1

            self.cache_job(jid, host, ret[host], fun)
            ret = self.key_deploy(host, ret)

            if isinstance(ret[host], dict) and (ret[host].get('stderr') or '').startswith('ssh:'):
                ret[host] = ret[host]['stderr']

            if not isinstance(ret[host], dict):
                p_data = {host: ret[host]}
            elif 'return' not in ret[host]:
                p_data = ret
            else:
                outputter = ret[host].get('out', self.opts.get('output', 'nested'))
                p_data = {host: ret[host].get('return', {})}
            if self.opts.get('static'):
                sret.update(p_data)
            else:
                salt.output.display_output(
                        p_data,
                        outputter,
                        self.opts)
            if self.event:
                id_, data = next(six.iteritems(ret))
                if isinstance(data, six.text_type):
                    data = {'return': data}
                if 'id' not in data:
                    data['id'] = id_
                data['jid'] = jid  # make the jid in the payload the same as the jid in the tag
                self.event.fire_event(
                    data,
                    salt.utils.event.tagify(
                        [jid, 'ret', host],
                        'job'))
        if self.opts.get('static'):
            salt.output.display_output(
                    sret,
                    outputter,
                    self.opts)
        if final_exit:
            sys.exit(salt.defaults.exitcodes.EX_AGGREGATE)