Runs a new job. It works exactly like the Client class, instead that there is no callback
773828
def new_job(self, task, inputdata, launcher_name="Unknown", debug=False):
        
        bjobid = uuid.uuid4()
        self._waiting_jobs.append(str(bjobid))
        self._client.new_job(task, inputdata,
                             (lambda result, grade, problems, tests, custom, archive, stdout, stderr:
                              self._callback(bjobid, result, grade, problems, tests, custom, archive, stdout, stderr)),
                             launcher_name, debug)
        return bjobid
803276
def start(self):
        
        if self.running:
            raise RuntimeError("client already running")

        self._main_task = asyncio.ensure_future(
            self._main(),
            loop=self._loop
        )
        self._main_task.add_done_callback(self._on_main_done)
1365874
public static @Nonnull JobLauncher newJobLauncher(Properties sysProps, Properties jobProps) throws Exception {
    return newJobLauncher(sysProps, jobProps, null);
  }
233204
def run_job(self, project_id, body, dataset=None):
        

        if dataset and not self.dataset_exists(dataset):
            self.make_dataset(dataset)

        new_job = self.client.jobs().insert(projectId=project_id, body=body).execute()
        job_id = new_job['jobReference']['jobId']
        logger.info('Started import job %s:%s', project_id, job_id)
        while True:
            status = self.client.jobs().get(projectId=project_id, jobId=job_id).execute(num_retries=10)
            if status['status']['state'] == 'DONE':
                if status['status'].get('errorResult'):
                    raise Exception('BigQuery job failed: {}'.format(status['status']['errorResult']))
                return

            logger.info('Waiting for job %s:%s to complete...', project_id, job_id)
            time.sleep(5)
407342
protected void jobStarting()
    {
        this.jobContext.pushCurrentJob(this);

        this.observationManager.notify(new JobStartedEvent(getRequest().getId(), getType(), this.request), this);

        if (this.status instanceof AbstractJobStatus) {
            ((AbstractJobStatus<R>) this.status).setStartDate(new Date());
            ((AbstractJobStatus<R>) this.status).setState(JobStatus.State.RUNNING);

            ((AbstractJobStatus) this.status).startListening();
        }

        if (getRequest().isVerbose()) {
            if (getStatus().getRequest().getId() != null) {
                this.logger.info(LOG_BEGIN_ID, "Starting job of type [{}] with identifier [{}]", getType(),
                    getStatus().getRequest().getId());
            } else {
                this.logger.info(LOG_BEGIN, "Starting job of type [{}]", getType());
            }
        }
    }
656402
function jobCallback(job, worker, index) {

        if (index < ALEXA_SIZE) {
            var line = lines[index].trim();

            var split = line.split(',');
            var id = parseInt(split[0]);
            var url = split[1];
            job({
                id : id,
                url : url
            }, function(err) {
                // Lets log if it worked
                total++;
                if (err) {
                    console.log('Problem  #' + worker.id + ': ' + err.message + ' for line: ' + line);
                } else {
                    console.log('    #' + worker.id + '  DONE: ' + line);
                    successful++;
                }
                if (total % 10 === 0) {
                    console.log('################## ' + successful + '/' + total + ' (success/crawled) ################################');
                }
            });
        } else {
            // no more content!
            job(null);
        }
    }
473629
def _create_job(self, func, args=None, kwargs=None, commit=True,
                    result_ttl=None, ttl=None, id=None, description=None,
                    queue_name=None, timeout=None, meta=None):
        
        if args is None:
            args = ()
        if kwargs is None:
            kwargs = {}
        job = self.job_class.create(
                func, args=args, connection=self.connection,
                kwargs=kwargs, result_ttl=result_ttl, ttl=ttl, id=id,
                description=description, timeout=timeout, meta=meta)
        if self._queue is not None:
            job.origin = self._queue.name
        else:
            job.origin = queue_name or self.queue_name
        if commit:
            job.save()
        return job
484191
def run(self):
        ''''''
        # Register our signal handlers
        self.signals()

        with self.listener():
            for job in self.jobs():
                # If there was no job to be had, we should sleep a little bit
                if not job:
                    self.jid = None
                    self.title('Sleeping for %fs' % self.interval)
                    time.sleep(self.interval)
                else:
                    self.jid = job.jid
                    self.title('Working on %s (%s)' % (job.jid, job.klass_name))
                    with Worker.sandbox(self.sandbox):
                        job.sandbox = self.sandbox
                        job.process()
                if self.shutdown:
                    break
79151
def _run_raw(self, *args, **kwargs):
    """"""
    for job in self.jobs:
      job._run_raw(*args, **kwargs)
53509
def run(self):
        
        task = self.task
        self._outputs = self.output()

        # create the job dashboard interface
        self.dashboard = task.create_job_dashboard() or NoJobDashboard()

        # read submission data and reset some values
        submitted = not task.ignore_submission and self._outputs["submission"].exists()
        if submitted:
            self.submission_data.update(self._outputs["submission"].load(formatter="json"))
            task.tasks_per_job = self.submission_data.tasks_per_job
            self.dashboard.apply_config(self.submission_data.dashboard_config)

        # when the branch outputs, i.e. the "collection" exists, just create dummy control outputs
        if "collection" in self._outputs and self._outputs["collection"].exists():
            self.touch_control_outputs()

        # cancel jobs?
        elif self._cancel_jobs:
            if submitted:
                self.cancel()

        # cleanup jobs?
        elif self._cleanup_jobs:
            if submitted:
                self.cleanup()

        # submit and/or wait while polling
        else:
            # maybe set a tracking url
            tracking_url = self.dashboard.create_tracking_url()
            if tracking_url:
                task.set_tracking_url(tracking_url)

            # ensure the output directory exists
            if not submitted:
                self._outputs["submission"].parent.touch()

            # at this point, when the status file exists, it is considered outdated
            if "status" in self._outputs:
                self._outputs["status"].remove()

            try:
                # instantiate the configured job file factory, not kwargs yet
                self.job_file_factory = self.create_job_file_factory()

                # submit
                if not submitted:
                    # set the initial list of unsubmitted jobs
                    branches = sorted(task.branch_map.keys())
                    branch_chunks = list(iter_chunks(branches, task.tasks_per_job))
                    self.submission_data.unsubmitted_jobs = OrderedDict(
                        (i + 1, branches) for i, branches in enumerate(branch_chunks)
                    )
                    self.submit()

                    # sleep once to give the job interface time to register the jobs
                    post_submit_delay = self._get_task_attribute("post_submit_delay")()
                    if post_submit_delay:
                        time.sleep(post_submit_delay)

                # start status polling when a) no_poll is not set, or b) the jobs were already
                # submitted so that failed jobs are resubmitted after a single polling iteration
                if not task.no_poll or submitted:
                    self.poll()

            finally:
                # in any event, cleanup the job file
                if self.job_file_factory:
                    self.job_file_factory.cleanup_dir(force=False)
113761
def WriteCronJobRun(self, run_object):
    """"""
    if run_object.cron_job_id not in self.cronjobs:
      raise db.UnknownCronJobError("Job with id %s not found." %
                                   run_object.cron_job_id)

    clone = run_object.Copy()
    clone.timestamp = rdfvalue.RDFDatetime.Now()
    self.cronjob_runs[(clone.cron_job_id, clone.run_id)] = clone
687320
static function startJob($workflow_job_key, $workflow_job_config, $start_state) {

        $workflow_key = self::createWorkflowKeyFromConfig($workflow_job_config);

        $workflow_name = $workflow_job_config["name"];
        /**
         * Checking if a flow is already running
         */
        if (self::isWorkflowRunning($workflow_key)) {
            throw new \Exception("A job for $workflow_name is already running.");
        }

        /**
         *  Check how long its since last job run... we just don't want to spam errors.
         */
        $error_ttl = Util::isDevServer() ? 60 : Moment::ONEDAY;

        if (self::isWorkflowInErrorState($workflow_key, $error_ttl)) {
            throw new \Exception("A job for $workflow_name have failed less than $error_ttl seconds ago, skipping.");
        }

        $workflow_job_config = array_merge($workflow_job_config, [
            "status" => self::STATUS_RUNNING,
            "start_state" => $start_state,
            "end_state" => $start_state,
            "finished" => new \DateTime(),
        ]);

        DataStore::saveWorkflowJob($workflow_job_key, $workflow_job_config);
        return $workflow_job_config;
    }
131915
function createJob(feed) {

  // Job skeleton
  var job = {
    id: 'Job[' + uuid.v4() + ']',
    original: feed,
    time: {
      start: 0,
      end: 0
    },
    state: {
      retrying: false,
      failing: false
    },
    req: {
      retries: 0,
      data: {}
    },
    res: {
      data: null
    }
  };

  // Handling polymorphism
  if (types.check(feed, 'string')) {
    job.req.url = feed;
    feed = {};
  }
  else {

    // Parsing url if needed
    if (feed.url)
      if (types.check(feed.url, 'object'))
        job.req.url = nodeUrl.format(extend(feed.url, {protocol: 'http'}));
      else
        job.req.url = feed.url;
    else if (feed.host || feed.hostname)
      job.req.url = nodeUrl.format(extend(feed, {protocol: 'http'}));
  }

  // Request properties
  job.req.data = feed.data || {};

  [
    'auth',
    'artoo',
    'body',
    'bodyType',
    'cheerio',
    'cookies',
    'encoding',
    'headers',
    'method',
    'phantomPage',
    'proxy',
    'timeout'
  ].forEach(function(p) {
    if (feed[p]) {

      // Solving object proxy
      if (p === 'proxy' && types.check(feed[p], 'object'))
        job.req[p] = nodeUrl.format(extend(feed[p], {protocol: 'http'}));

      job.req[p] = feed[p];
    }
  });

  return job;
}
1340646
public static RunningJob runJob(JobConf job) throws IOException {
    JobClient jc = new JobClient(job);
    RunningJob rj = jc.submitJob(job);
    try {
      if (!jc.monitorAndPrintJob(job, rj)) {
        throw new IOException("Job failed!");
      }
    } catch (InterruptedException ie) {
      Thread.currentThread().interrupt();
    }
    return rj;
  }
666753
public static <T> void runWithCallback(Callable<T> job, final FutureCallback<T> callback)
  {
    final UI ui = UI.getCurrent();
    
    ListeningExecutorService exec = MoreExecutors.listeningDecorator(Executors.newSingleThreadExecutor());
    ListenableFuture<T> future = exec.submit(job);
    if(callback != null)
    {
      Futures.addCallback(future, new FutureCallback<T>()
      {

        @Override
        public void onSuccess(final T result)
        {
          ui.access(new Runnable()
          {

            @Override
            public void run()
            {
              callback.onSuccess(result);
            }
          });
        }

        @Override
        public void onFailure(final Throwable t)
        {
          ui.access(new Runnable()
          {

            @Override
            public void run()
            {
              callback.onFailure(t);
            }
          });
        }
      });
    }
  }
304858
def run_task(task_cls, task_data):
    
    task = instantiate(task_cls)
    task_callable = get_callable(task)
    return task_callable(TaskData(task_data))
808031
private void runJobExecutionLauncher() throws JobException {
    long startTime = 0;
    String newPlanningId;
    String jobName = jobProps.getProperty(ConfigurationKeys.JOB_NAME_KEY);
    Closer closer = Closer.create();
    try {
      HelixManager planningJobManager = this.taskDriverHelixManager.isPresent()?
          this.taskDriverHelixManager.get() : this.jobHelixManager;

      String builderStr = jobProps.getProperty(GobblinClusterConfigurationKeys.DISTRIBUTED_JOB_LAUNCHER_BUILDER,
          GobblinHelixDistributeJobExecutionLauncher.Builder.class.getName());

      // Check if any existing planning job is running
      Optional<String> planningJobIdFromStore = jobsMapping.getPlanningJobId(jobName);
      boolean nonblocking = false;
      // start of critical section to check if a job with same job name is running
      Lock jobLock = locks.get(jobName);
      jobLock.lock();

      try {
        if (planningJobIdFromStore.isPresent()) {
          String previousPlanningJobId = planningJobIdFromStore.get();

          if (HelixUtils.isJobFinished(previousPlanningJobId, previousPlanningJobId, planningJobManager)) {
            log.info("Previous planning job {} has reached to the final state. Start a new one.", previousPlanningJobId);
          } else {
            boolean killDuplicateJob = PropertiesUtils
                .getPropAsBoolean(this.jobProps, GobblinClusterConfigurationKeys.KILL_DUPLICATE_PLANNING_JOB, String.valueOf(GobblinClusterConfigurationKeys.DEFAULT_KILL_DUPLICATE_PLANNING_JOB));

            if (!killDuplicateJob) {
              log.info("Previous planning job {} has not finished yet. Skip this job.", previousPlanningJobId);
              return;
            } else {
              log.info("Previous planning job {} has not finished yet. Kill it.", previousPlanningJobId);
              long timeOut = PropertiesUtils.getPropAsLong(sysProps, GobblinClusterConfigurationKeys.HELIX_WORKFLOW_DELETE_TIMEOUT_SECONDS,
                  GobblinClusterConfigurationKeys.DEFAULT_HELIX_WORKFLOW_DELETE_TIMEOUT_SECONDS) * 1000;
              try {
                HelixUtils.deleteWorkflow(previousPlanningJobId, planningJobManager, timeOut);
              } catch (HelixException e) {
                log.info("Helix cannot delete previous planning job id {} within {} seconds.", previousPlanningJobId,
                    timeOut / 1000);
                throw new JobException("Helix cannot delete previous planning job id " + previousPlanningJobId, e);
              }
            }
          }
        } else {
          log.info("Planning job for {} does not exist. First time run.", jobName);
        }

        GobblinHelixDistributeJobExecutionLauncher.Builder builder = GobblinConstructorUtils.<GobblinHelixDistributeJobExecutionLauncher.Builder>invokeLongestConstructor(
            new ClassAliasResolver(GobblinHelixDistributeJobExecutionLauncher.Builder.class).resolveClass(builderStr));

        // Make a separate copy because we could update some of attributes in job properties (like adding planning id).
        Properties jobPlanningProps = new Properties();
        jobPlanningProps.putAll(this.jobProps);

        // Inject planning id and start time
        newPlanningId = HelixJobsMapping.createPlanningJobId(jobPlanningProps);
        jobPlanningProps.setProperty(GobblinClusterConfigurationKeys.PLANNING_ID_KEY, newPlanningId);
        jobPlanningProps.setProperty(GobblinClusterConfigurationKeys.PLANNING_JOB_CREATE_TIME, String.valueOf(System.currentTimeMillis()));

        builder.setSysProps(this.sysProps);
        builder.setJobPlanningProps(jobPlanningProps);
        builder.setJobHelixManager(this.jobHelixManager);
        builder.setTaskDriverHelixManager(this.taskDriverHelixManager);
        builder.setAppWorkDir(this.appWorkDir);
        builder.setJobsMapping(this.jobsMapping);
        builder.setPlanningJobLauncherMetrics(this.planningJobLauncherMetrics);
        builder.setHelixMetrics(this.helixMetrics);

        // if the distributed job launcher should wait for planning job completion

        Config combined = ConfigUtils.propertiesToConfig(jobPlanningProps)
            .withFallback(ConfigUtils.propertiesToConfig(sysProps));

        nonblocking = ConfigUtils.getBoolean(combined, GobblinClusterConfigurationKeys.NON_BLOCKING_PLANNING_JOB_ENABLED,
            GobblinClusterConfigurationKeys.DEFAULT_NON_BLOCKING_PLANNING_JOB_ENABLED);

        log.info("Planning job {} started.", newPlanningId);
        GobblinHelixDistributeJobExecutionLauncher launcher = builder.build();
        closer.register(launcher);
        this.jobsMapping.setPlanningJobId(jobName, newPlanningId);
        startTime = System.currentTimeMillis();
        this.currentJobMonitor = launcher.launchJob(null);

        // make sure the planning job is initialized (or visible) to other parallel running threads,
        // so that the same critical section check (querying Helix for job completeness)
        // can be applied.
        HelixUtils.waitJobInitialization(planningJobManager, newPlanningId, newPlanningId, 300_000);

      } finally {
        // end of the critical section to check if a job with same job name is running
        jobLock.unlock();
      }

      // we can remove the job spec from the catalog because Helix will drive this job to the end.
      this.deleteJobSpec();

      // If we are using non-blocking mode, this get() only guarantees the plannning job is submitted.
      // It doesn't guarantee the job will finish because internally we won't wait for Helix completion.
      this.currentJobMonitor.get();
      this.currentJobMonitor = null;
      if (nonblocking) {
        log.info("Planning job {} submitted successfully.", newPlanningId);
      } else {
        log.info("Planning job {} finished.", newPlanningId);
        this.planningJobLauncherMetrics.updateTimeForCompletedPlanningJobs(startTime);
      }
    } catch (Exception e) {
      if (startTime != 0) {
        this.planningJobLauncherMetrics.updateTimeForFailedPlanningJobs(startTime);
      }
      log.error("Failed to run planning job {}", jobName, e);
      throw new JobException("Failed to run planning job " + jobName, e);
    } finally {
      try {
        closer.close();
      } catch (IOException e) {
        throw new JobException("Cannot properly close planning job " + jobName, e);
      }
    }
  }