Cleans up job state if the last run is stuck.
9594
def TerminateStuckRunIfNeeded(self, job):
    """"""
    if job.current_run_id and job.last_run_time and job.lifetime:
      now = rdfvalue.RDFDatetime.Now()
      # We add additional 10 minutes to give the job run a chance to kill itself
      # during one of the HeartBeat calls (HeartBeat checks if a cron job is
      # run is running too long and raises if it is).
      expiration_time = (
          job.last_run_time + job.lifetime + rdfvalue.Duration("10m"))
      if now > expiration_time:
        run = data_store.REL_DB.ReadCronJobRun(job.cron_job_id,
                                               job.current_run_id)
        run.status = "LIFETIME_EXCEEDED"
        run.finished_at = now
        data_store.REL_DB.WriteCronJobRun(run)
        data_store.REL_DB.UpdateCronJob(
            job.cron_job_id, current_run_id=None, last_run_status=run.status)
        stats_collector_instance.Get().RecordEvent(
            "cron_job_latency", (now - job.last_run_time).seconds,
            fields=[job.cron_job_id])
        stats_collector_instance.Get().IncrementCounter(
            "cron_job_timeout", fields=[job.cron_job_id])

        return True

    return False
152702
private void cleanup() {
        if ((++counter & 0xFF) != 0) { // (++counter % 256) != 0
            return;
        }

        final long currentTimeNanos = System.nanoTime();
        if (currentTimeNanos - lastCleanupTimeNanos < CLEANUP_INTERVAL_NANOS) {
            return;
        }

        for (final Iterator<State> i = map.values().iterator(); i.hasNext();) {
            final State state = i.next();
            final boolean remove;

            synchronized (state) {
                remove = state.allActiveRequests == 0 &&
                         currentTimeNanos - state.lastActivityTimeNanos >= CLEANUP_INTERVAL_NANOS;
            }

            if (remove) {
                i.remove();
            }
        }

        lastCleanupTimeNanos = System.nanoTime();
    }
1110932
def _cleanup(self):
        """"""
        current_time = time.time()
        timeout = self._config.timeout
        if current_time - self._last_cleanup_time > timeout:
            self.store.cleanup(timeout)
            self._last_cleanup_time = current_time
882573
private void cleanJobDirectory() {
    try {
      FileSystem jobfs = jobDirectory.getFileSystem(conf);
      jobfs.delete(jobDirectory, true);
    } catch(IOException ioe) {
      LOG.warn("Unable to clean tmp directory " + jobDirectory, ioe);
    }
  }
749108
def _maybe_finish_job(self, job_id):
        """"""
        job = brozzler.Job.load(self.rr, job_id)
        if not job:
            return False
        if job.status.startswith("FINISH"):
            self.logger.warn("%s is already %s", job, job.status)
            return True

        results = self.rr.table("sites").get_all(job_id, index="job_id").run()
        n = 0
        for result in results:
            site = brozzler.Site(self.rr, result)
            if not site.status.startswith("FINISH"):
                results.close()
                return False
            n += 1

        self.logger.info(
                "all %s sites finished, job %s is FINISHED!", n, job.id)
        job.finish()
        job.save()
        return True
579207
private void fix() {
        Set<String> nodeGroups = appContext.getTaskTrackerManager().getNodeGroups();
        if (CollectionUtils.isEmpty(nodeGroups)) {
            return;
        }
        for (String nodeGroup : nodeGroups) {
            List<JobPo> deadJobPo = appContext.getExecutableJobQueue().getDeadJob(nodeGroup, SystemClock.now() - MAX_TIME_OUT);
            if (CollectionUtils.isNotEmpty(deadJobPo)) {
                for (JobPo jobPo : deadJobPo) {
                    appContext.getExecutableJobQueue().resume(jobPo);
                    LOGGER.info("Fix executable job : {} ", JSON.toJSONString(jobPo));
                }
            }
        }
    }
604044
def cleanup(self):
        """"""
        self.num_puts = 0
        cached = []
        for i, url_data in enumerate(self.queue):
            key = url_data.cache_url
            cache = url_data.aggregate.result_cache
            if cache.has_result(key):
                cached.append(i)
        for pos in cached:
            self._move_to_top(pos)
480129
protected function discardJob()
    {
        $this->skipped = true;

        // If we do not have dirty records, then assume that these dirty records were committed
        // already this request (but probably another job), so we don't need to commit anything else.
        // This could occur if we completed multiple searchupdate jobs in a prior request, and
        // we only need one commit job to commit all of them in the current request.
        if (empty(static::$dirty_indexes)) {
            $this->addMessage("Indexing already completed this request: Discarding this job");
            return;
        }


        // If any commit has run, but some (or all) indexes are un-comitted, we must re-schedule this task.
        // This could occur if we completed a searchupdate job in a prior request, as well as in
        // the current request
        $cooldown = Config::inst()->get(__CLASS__, 'cooldown');
        $now = new DateTime(DBDatetime::now()->getValue());
        $now->add(new DateInterval('PT' . $cooldown . 'S'));
        $runat = $now->Format('Y-m-d H:i:s');

        $this->addMessage("Indexing already run this request, but incomplete. Re-scheduling for {$runat}");

        // Queue after the given cooldown
        static::queue(false, $runat);
    }
1253141
def _process_state_final_run(self, job_record):
        """"""
        uow = self.uow_dao.get_one(job_record.related_unit_of_work)
        if uow.is_processed:
            self.update_job(job_record, uow, job.STATE_PROCESSED)
        elif uow.is_noop:
            self.update_job(job_record, uow, job.STATE_NOOP)
        elif uow.is_canceled:
            self.update_job(job_record, uow, job.STATE_SKIPPED)
        elif uow.is_invalid:
            msg = 'Job {0}: UOW for {1}@{2} is in {3}; ' \
                  'relying on the Garbage Collector to either recycle or cancel the UOW.' \
                  .format(job_record.db_id, job_record.process_name, job_record.timeperiod, uow.state)
            self._log_message(INFO, job_record.process_name, job_record.timeperiod, msg)
        else:
            msg = 'Suppressed creating UOW for {0}@{1}; Job is in {2}; uow is in {3}' \
                  .format(job_record.process_name, job_record.timeperiod, job_record.state, uow.state)
            self._log_message(INFO, job_record.process_name, job_record.timeperiod, msg)

        timetable_tree = self.timetable.get_tree(job_record.process_name)
        timetable_tree.build_tree()
661986
synchronized void finalizeJob(JobInProgress job) {
    // Mark the 'non-running' tasks for pruning
    markCompletedJob(job);

    JobEndNotifier.registerNotification(job.getJobConf(), job.getStatus());

    // start the merge of log files
    JobID id = job.getStatus().getJobID();

    // mark the job as completed
    try {
      JobHistory.JobInfo.markCompleted(id);
    } catch (IOException ioe) {
      LOG.info("Failed to mark job " + id + " as completed!", ioe);
    }

    final JobTrackerInstrumentation metrics = getInstrumentation();
    metrics.finalizeJob(conf, id);

    long now = getClock().getTime();

    // mark the job for cleanup at all the trackers
    addJobForCleanup(id);

    try {
      File userFileForJob =
        new File(lDirAlloc.getLocalPathToRead(SUBDIR + "/" + id,
                                              conf).toString());
      if (userFileForJob != null) {
        userFileForJob.delete();
      }
    } catch (IOException ioe) {
      LOG.info("Failed to delete job id mapping for job " + id, ioe);
    }

    // add the blacklisted trackers to potentially faulty list
    if (job.getStatus().getRunState() == JobStatus.SUCCEEDED) {
      if (job.getNoOfBlackListedTrackers() > 0) {
        for (Entry<String, List<String>> e : job.getBlackListedTrackers().entrySet()) {
          String tt = e.getKey();
          String jobName = job.getJobID().toString();
          String[] exceptions = e.getValue().toArray(new String[0]);

          faultyTrackers.incrementFaults(tt, new JobFault(tt, jobName, exceptions));
        }
      }
    }

    String jobUser = job.getProfile().getUser();
    //add to the user to jobs mapping
    synchronized (userToJobsMap) {
      ArrayList<JobInProgress> userJobs = userToJobsMap.get(jobUser);
      if (userJobs == null) {
        userJobs =  new ArrayList<JobInProgress>();
        userToJobsMap.put(jobUser, userJobs);
      }
      userJobs.add(job);
    }
  }
1202888
def Flush(self):
    """"""
    self._CheckLeaseAndFlush()
    self.Load()
    super(FlowBase, self).Flush()
    # Writing the messages queued in the queue_manager of the runner always has
    # to be the last thing that happens or we will have a race condition.
    self.FlushMessages()
291657
public void cleanup ()
    {
        // tell any pending object subscribers that they're not getting their bits
        for (PendingRequest<?> req : _penders.values()) {
            for (Subscriber<?> sub : req.targets) {
                sub.requestFailed(req.oid, new ObjectAccessException("Client connection closed"));
            }
        }
        _penders.clear();
        _flusher.cancel();
        _flushes.clear();
        _dead.clear();
        _client.getRunQueue().postRunnable(new Runnable() {
            public void run () {
                _ocache.clear();
            }
        });
    }
221268
private boolean jobHasFinishedStatus() {

		final Iterator<ExecutionVertex> it = new ExecutionGraphIterator(this, true);

		while (it.hasNext()) {

			if (it.next().getExecutionState() != ExecutionState.FINISHED) {
				return false;
			}
		}

		return true;
	}
1238130
def _clean_workers(self):
        """"""
        while self._bag_collector:
            self._bag_collector.popleft()
        self._timer_worker_delete.stop()
985223
def cleanup(api_instance=None):
    
    api = api_instance or get_api()
    r = api.list_job_for_all_namespaces()
    delete_opts = kubernetes.client.V1DeleteOptions(
            propagation_policy="Background")
    for job in r.items:
        if job.status.succeeded == job.spec.completions:
            print(job.metadata.name, "finished!")
            api.delete_namespaced_job(
                job.metadata.name, 'default', body=delete_opts)
1215003
def clean(self):

        

        self.closing = True # schedule for closing.

        if not self.fds:
            self.data.close()
969199
def delete_running(self, timeout_last_refresh=0, dry_run=False):
        
        running_all = self.handle.jobs_running()
        running_timeout = [job for job in running_all
                           if coarse_utcnow() > job["refresh_time"] +
                           timedelta(seconds=timeout_last_refresh)]
        if len(running_timeout) == 0:
            # Nothing to stop
            self.refresh_tids(None)
            return None

        if dry_run:
            logger.warning("Dry run. Not removing anything.")

        logger.info("Removing {0}/{1} running jobs. # all jobs: {2} ".
                    format(len(running_timeout), len(running_all), len(self)))

        now = coarse_utcnow()
        logger.info("Current utc time: {0}".format(now))
        logger.info("Time horizont: {0}".format(now - timedelta(seconds=timeout_last_refresh)))
        for job in running_timeout:
            logger.info("Removing job: ")
            pjob = job.to_dict()
            del pjob["misc"]  # ignore misc when printing
            logger.info(pprint.pformat(pjob))
            if not dry_run:
                self.handle.delete(job)
                logger.info("Job deleted")
        self.refresh_tids(None)