Run a task on the scheduled executor so that we can try to interrupt it and time out if it fails
175752
void runWithinPeriod(Runnable runnable, ExecuteStepMessage executeStepMessage, int timeout, TimeUnit unit) {
        if ( ! isRunningAStep.getAndSet(true)) {
            this.currentlyExecutingStep = executeStepMessage;
            Future<String> future = null;
            try {
                future = scheduledExecutorService.submit(runStepAndResetIsRunning(runnable), "OK");
                future.get(timeout, unit);
            } catch (TimeoutException e) {
                //Timed out waiting for the step to run
                //We should try to cancel and interrupt the thread which is running the step - although this isn't
                //guaranteed to succeed.
                future.cancel(true);
                log.warn("A step failed to execute within " + timeout + " " + unit + ", attempting to cancel the step");
                //Here the step server should have timed out the step and proceed already - we don't need to send a failure message
            } catch (Exception e) {
                String ms = "Exception while executing step [" + e.getMessage() + "]";
                log.error(ms, e);
                stepFailureConsumer.accept(ms, executeStepMessage);
            }
        } else {
            //server will time out this step
            String message = "Cannot execute a test step, a step is already in progress [" + currentlyExecutingStep.getStepId() + ", " + currentlyExecutingStep.getPattern() + "]";
            log.error(message);
            stepFailureConsumer.accept(message, executeStepMessage);
        }
    }
1261652
@Override
    public ScheduledFuture<?> schedule(Runnable task, Trigger trigger) {
        ScheduledExecutorService executor = getScheduledExecutor();
        try {
            ErrorHandler errorHandlerLocal = this.errorHandler != null ? this.errorHandler : TaskUtils
                    .getDefaultErrorHandler(true);
            return new ReschedulingRunnable(task, trigger, executor, errorHandlerLocal).schedule();
        } catch (RejectedExecutionException ex) {
            throw new TaskRejectedException("Executor [" + executor + "] did not accept task: " + task, ex);
        }
    }
1201982
async def run_task(self) -> None:
        ''''''

        self.last_run = 0.0
        self.target = self.time() + self.DELAY

        while self.running:
            try:
                now = self.time()

                if now < self.target:
                    sleep = self.target - now
                    await self.sleep(sleep)

                elif self.last_run < self.target:
                    Log.debug('executing timer task %s', self.name)
                    self.last_run = self.time()
                    await self.run()
                    total = self.time() - self.last_run
                    Log.debug('finished timer task %s in %.1f seconds',
                              self.name, total)

                else:
                    sleep = min(5.0, self.DELAY)
                    await self.sleep(sleep)

            except CancelledError:
                Log.debug('cancelled timer task %s', self.name)
                raise

            except Exception:
                Log.exception('exception in timer task %s', self.name)
600069
@Override
    public void run() {
        // Check for a pending exception.  A CancellationException could be set if a cancel() was driven while this Runnable
        // was being re-scheduled.
        if (this.m_pendingException != null) {
            return;
        }

        // NOTE: Already running in Default Executor thread.
        try {
            this.m_runnable.run();
        } catch (Exception e) {
            this.m_pendingException = new ExecutionException(e);
            this.m_coordinationLatch.countDown();
            return;
        }

        try {
            long scheduleTime = this.m_periodInterval;
            if (!this.m_scheduledWithDelay) {
                this.m_myNextExecutionTime = this.m_myNextExecutionTime + this.m_periodInterval;
                long currentTime = System.nanoTime();

                scheduleTime = (this.m_myNextExecutionTime > currentTime) ? (this.m_myNextExecutionTime - currentTime) : 0;
            }

            m_schedFuture = this.m_scheduledExecutor.schedule(this, scheduleTime, TimeUnit.NANOSECONDS);
        } catch (Exception e) {
            this.m_pendingException = e;
            this.m_coordinationLatch.countDown();
        }
    }
1165857
def run(self):
        '''
        '''
        while True:
            next_task = self._task_queue.get()
            # If there is any error, only consume data but not run the job
            if self._error_queue.qsize() > 0:
                self._task_queue.task_done()
                continue
            if next_task is None:
                # Poison pill means shutdown
                self._task_queue.task_done()
                break
            try:
                answer = self._func(next_task, **self._args)
                self._task_queue.task_done()
                self._result_queue.put(answer)
            except Exception as e:
                self._task_queue.task_done()
                self._error_queue.put((os.getpid(), e))
                logger.error(e)
                continue
1155661
def _fire_task(self, exclude=None):
        """"""
        connection = mail.get_connection(fail_silently=True)
        # Warning: fail_silently swallows errors thrown by the generators, too.
        connection.open()
        for m in self._mails(self._users_watching(exclude=exclude)):
            connection.send_messages([m])
558592
def run_in_executor(f):
    

    @wraps(f)
    def new_f(self, *args, **kwargs):

        if self.is_shutdown:
            return
        try:
            future = self.executor.submit(f, self, *args, **kwargs)
            future.add_done_callback(_future_completed)
        except Exception:
            log.exception("Failed to submit task to executor")

    return new_f
335652
private synchronized void maybeRun() {
        if (inprogress==null && pending!=null) {
            base.submit(new Callable<Void>() {
                @Override
                public Void call() throws Exception {
                    synchronized (AtmostOneTaskExecutor.this) {
                        // everyone who submits after this should form a next batch
                        inprogress = pending;
                        pending = null;
                    }

                    try {
                        inprogress.set(task.call());
                    } catch (Throwable t) {
                        LOGGER.log(Level.WARNING, null, t);
                        inprogress.setException(t);
                    } finally {
                        synchronized (AtmostOneTaskExecutor.this) {
                            // if next one is pending, get that scheduled
                            inprogress = null;
                            maybeRun();
                        }
                    }
                    return null;
                }
            });
        }
    }
379507
public void stop() {
    ScheduledExecutorService service = executor.get();
    if (service != null && executor.compareAndSet(service, null)) {
      service.shutdown();
    } else {
      throw new IllegalStateException("scheduler must be started before you stop it");
    }
  }
496384
public void execute(final FifoTask<E> task) throws InterruptedException {
        final int id;
        synchronized (this) {
            id = idCounter++;
            taskMap.put(id, task);
            while (activeCounter >= maxThreads) {
                wait();
            }
            activeCounter++;
        }

        this.threadPoolExecutor.execute(new Runnable() {
            public void run() {
                try {
                    try {
                        final E outcome = task.runParallel();
                        synchronized (resultMap) {
                            resultMap.put(id, new Result(outcome));
                        }
                    } catch (Throwable th) {
                        synchronized (resultMap) {
                            resultMap.put(id, new Result(null, th));
                        }
                    } finally {
                        processResults();
                        synchronized (FifoTaskExecutor.this) {
                            activeCounter--;
                            FifoTaskExecutor.this.notifyAll();
                        }
                    }
                } catch (Exception ex) {
                    Logger.getLogger(FifoTaskExecutor.class.getName()).log(Level.SEVERE, ex.getMessage(), ex);
                }
            }
        });
    }
281277
protected void checkNext ()
    {
        _executingNow = !_queue.isEmpty();
        if (_executingNow) {
            // start up a thread to execute the task in question
            ExecutorTask task = _queue.remove(0);
            final ExecutorThread thread = new ExecutorThread(task);
            thread.start();

            // start up a timer that will abort this thread after the specified timeout
            new Interval(Interval.RUN_DIRECT) {
                @Override public void expired () {
                    // this will NOOP if the task has already completed
                    thread.abort();
                }
            }.schedule(task.getTimeout());
        }
    }
682055
public static void scheduleWithFixedDelay(ScheduledExecutorService exec, Duration delay, Callable<Signal> callable)
  {
    scheduleWithFixedDelay(exec, delay, delay, callable);
  }
591904
private static void doExecute(Executor executor, Runnable runnable) {
    try {
      executor.execute(runnable);
    } catch (Throwable th) {
      log.log(Level.SEVERE, "Failed to execute PingCallback", th);
    }
  }
802941
def run_scheduler(self, parallel_jobs = 1, job_ids = None, sleep_time = 0.1, die_when_finished = False, no_log = False, nice = None, verbosity = 0):
    """"""
    running_tasks = []
    finished_tasks = set()
    try:

      # keep the scheduler alive until every job is finished or the KeyboardInterrupt is caught
      while True:
        # Flag that might be set in some rare cases, and that prevents the scheduler to die
        repeat_execution = False
        # FIRST, try if there are finished processes
        for task_index in range(len(running_tasks)-1, -1, -1):
          task = running_tasks[task_index]
          process = task[0]

          if process.poll() is not None:
            # process ended
            job_id = task[1]
            array_id = task[2] if len(task) > 2 else None
            self.lock()
            job, array_job = self._job_and_array(job_id, array_id)
            if job is not None:
              jj = array_job if array_job is not None else job
              result = "%s (%d)" % (jj.status, jj.result) if jj.result is not None else "%s (?)" % jj.status
              if jj.status not in ('success', 'failure'):
                logger.error("Job '%s' (%s) finished with status '%s' instead of 'success' or 'failure'. Usually this means an internal error. Check your wrapper_script parameter!", job.name, self._format_log(job_id, array_id), jj.status)
                raise StopIteration("Job did not finish correctly.")
              logger.info("Job '%s' (%s) finished execution with result '%s'", job.name, self._format_log(job_id, array_id), result)
            self.unlock()
            finished_tasks.add(job_id)
            # in any case, remove the job from the list
            del running_tasks[task_index]

        # SECOND, check if new jobs can be submitted; THIS NEEDS TO LOCK THE DATABASE
        if len(running_tasks) < parallel_jobs:
          # get all unfinished jobs:
          self.lock()
          jobs = self.get_jobs(job_ids)
          # put all new jobs into the queue
          for job in jobs:
            if job.status == 'submitted' and job.queue_name == 'local':
              job.queue()

          # get all unfinished jobs that are submitted to the local queue
          unfinished_jobs = [job for job in jobs if job.status in ('queued', 'executing') and job.queue_name == 'local']
          for job in unfinished_jobs:
            if job.array:
              # find array jobs that can run
              queued_array_jobs = [array_job for array_job in job.array if array_job.status == 'queued']
              if not len(queued_array_jobs):
                job.finish(0, -1)
                repeat_execution = True
              else:
                # there are new array jobs to run
                for i in range(min(parallel_jobs - len(running_tasks), len(queued_array_jobs))):
                  array_job = queued_array_jobs[i]
                  # start a new job from the array
                  process = self._run_parallel_job(job.unique, array_job.id, no_log=no_log, nice=nice, verbosity=verbosity)
                  if process is None:
                    continue
                  running_tasks.append((process, job.unique, array_job.id))
                  # we here set the status to executing manually to avoid jobs to be run twice
                  # e.g., if the loop is executed while the asynchronous job did not start yet
                  array_job.status = 'executing'
                  job.status = 'executing'
                  if len(running_tasks) == parallel_jobs:
                    break
            else:
              if job.status == 'queued':
                # start a new job
                process = self._run_parallel_job(job.unique, no_log=no_log, nice=nice, verbosity=verbosity)
                if process is None:
                  continue
                running_tasks.append((process, job.unique))
                # we here set the status to executing manually to avoid jobs to be run twice
                # e.g., if the loop is executed while the asynchronous job did not start yet
                job.status = 'executing'
            if len(running_tasks) == parallel_jobs:
              break

          self.session.commit()
          self.unlock()

        # if after the submission of jobs there are no jobs running, we should have finished all the queue.
        if die_when_finished and not repeat_execution and len(running_tasks) == 0:
          logger.info("Stopping task scheduler since there are no more jobs running.")
          break

        # THIRD: sleep the desired amount of time before re-checking
        time.sleep(sleep_time)

    # This is the only way to stop: you have to interrupt the scheduler
    except (KeyboardInterrupt, StopIteration):
      if hasattr(self, 'session'):
        self.unlock()
      logger.info("Stopping task scheduler due to user interrupt.")
      for task in running_tasks:
        logger.warn("Killing job '%s' that was still running.", self._format_log(task[1], task[2] if len(task) > 2 else None))
        try:
          task[0].kill()
        except OSError as e:
          logger.error("Killing job '%s' was not successful: '%s'", self._format_log(task[1], task[2] if len(task) > 2 else None), e)
        self.stop_job(task[1])
      # stop all jobs that are currently running or queued
      self.stop_jobs(job_ids)

    # check the result of the jobs that we have run, and return the list of failed jobs
    self.lock()
    jobs = self.get_jobs(finished_tasks)
    failures = [job.unique for job in jobs if job.status != 'success']
    self.unlock()
    return sorted(failures)
69164
def _run(self):
        
        for node in self.node.relatives:
            launch_node_task(node)
        for node in self.node.relatives:
            self.wait_and_join(node.task)
        if self.node.parent:
            while not self.node.parent.task.siblings_permission:
                time.sleep(self._polling_time)
        self.has_started = True
        self.main()
        self.siblings_permission = True
        for node in self.node.siblings:
            launch_node_task(node)
        for node in self.node.siblings:
            self.wait_and_join(node.task)
        self.finished_at = time.time()
        self.scheduler.notify_execution(self)
        self.has_finished = True
786334
@FFDCIgnore(Throwable.class)
    @Override
    public T call() throws Exception {
        final boolean trace = TraceComponent.isAnyTracingEnabled();

        if (future.isCancelled()) {
            if (trace && tc.isDebugEnabled())
                Tr.debug(this, tc, "canceled - not running the task");
            return null;
        }

        Result result = resultRef.get(), resultForThisExecution = result;
        Status<T> skipped = null;
        Status<T> status;
        boolean done = false;
        T taskResult = null;
        ArrayList<ThreadContext> contextAppliedToThread = null;
        resultForThisExecution.executionThread = Thread.currentThread();
        try {
            // EE Concurrency 3.2.6.1: All tasks submitted to an executor must not run if task's component is not started.
            // ThreadContextDescriptor.taskStarting covers this requirement for us.
            contextAppliedToThread = threadContextDescriptor.taskStarting();

            // Determine if task should be skipped
            if (trigger != null)
                try {
                    if (trigger.skipRun(lastExecution, new Date(nextExecutionTime)))
                        skipped = new Status<T>(Status.Type.SKIPPED, null, null, false);
                } catch (Throwable x) {
                    // spec requires skip when skipRun fails
                    Tr.error(tc, "CWWKC1103.skip.run.failed", getName(), managedExecSvc.name, x);
                    skipped = new Status<T>(Status.Type.SKIPPED, null, x, false);
                }

            Date nextExecutionDate = null;

            // Run task if it wasn't skipped
            if (skipped == null) {
                // notify listener: taskStarting
                if (listener != null)
                    try {
                        if (TraceComponent.isAnyTracingEnabled() && tc.isEventEnabled())
                            Tr.event(this, tc, "taskStarting", managedExecSvc, task);
                        listener.taskStarting(future, managedExecSvc, task);
                    } finally {
                        done = result.getStatus().type == Status.Type.CANCELED;
                    }

                // run the task if the listener didn't cancel it
                status = result.getStatus();
                if (status.type == Status.Type.SUBMITTED && result.compareAndSet(status, new Status<T>(Status.Type.STARTED, null, null, false))) {
                    try {
                        if (trigger == null)
                            if (isCallable)
                                taskResult = ((Callable<T>) task).call();
                            else
                                ((Runnable) task).run();
                        else {
                            long startTime = System.currentTimeMillis();

                            if (isCallable)
                                taskResult = ((Callable<T>) task).call();
                            else
                                ((Runnable) task).run();

                            long endTime = System.currentTimeMillis();
                            String identityName = threadContextDescriptor.getExecutionProperties().get(ManagedTask.IDENTITY_NAME);
                            lastExecution = new LastExecutionImpl(identityName, nextExecutionTime, startTime, endTime, taskResult);
                        }
                    } catch (Throwable x) {
                        Tr.error(tc, "CWWKC1101.task.failed", getName(), managedExecSvc.name, x);
                        status = result.getStatus();
                        if (status.type == Status.Type.CANCELED) // include the failure in the result so it will be available to taskDone
                            result.compareAndSet(status, new Status<T>(Status.Type.CANCELED, null, x, true));
                        else if (status.type == Status.Type.STARTED)
                            result.compareAndSet(status, new Status<T>(Status.Type.DONE, null, x, true));
                        result.latch.countDown();
                    }

                    status = result.getStatus();
                    if (status.type == Status.Type.STARTED) {
                        // calculate next execution
                        if (trigger == null)
                            result.compareAndSet(status, new Status<T>(Status.Type.DONE, taskResult, null, fixedDelay == null && fixedRate == null));
                        else {
                            nextExecutionDate = trigger.getNextRunTime(lastExecution, taskScheduledTime);
                            if (TraceComponent.isAnyTracingEnabled() && tc.isDebugEnabled())
                                Tr.debug(this, tc, "getNextRunTime", trigger, lastExecution,
                                         "taskScheduled " + Utils.toString(taskScheduledTime),
                                         "nextRunTime = " + Utils.toString(nextExecutionDate));

                            result.compareAndSet(status, new Status<T>(Status.Type.DONE, taskResult, null, nextExecutionDate == null));
                        }

                        result.latch.countDown();
                    }

                    done = true;

                    if (listener != null)
                        try {
                            if (status.type == Status.Type.CANCELED)
                                try {
                                    CancellationException cancelX = new CancellationException(Tr.formatMessage(tc, "CWWKC1110.task.canceled", getName(), managedExecSvc.name));
                                    if (trace && tc.isEventEnabled())
                                        Tr.event(this, tc, "taskCanceled", managedExecSvc, task, cancelX);
                                    listener.taskAborted(future, managedExecSvc, task, cancelX);
                                } catch (Throwable x) {
                                    Tr.error(tc, "CWWKC1102.listener.failed", getName(), managedExecSvc.name, x);
                                }

                            Throwable failure = result.getStatus().failure;
                            if (trace && tc.isEventEnabled())
                                Tr.event(this, tc, "taskDone", managedExecSvc, task, failure);
                            listener.taskDone(future, managedExecSvc, task, failure);
                        } catch (Throwable x) {
                            Tr.error(tc, "CWWKC1102.listener.failed", getName(), managedExecSvc.name, x);
                        }
                }
            } else {
                // Skip this execution (only possible if using a trigger)
                try {
                    status = result.getStatus();
                    if (status.type == Status.Type.SUBMITTED)
                        result.compareAndSet(status, skipped);

                    // calculate next execution
                    nextExecutionDate = trigger.getNextRunTime(lastExecution, taskScheduledTime);
                    if (TraceComponent.isAnyTracingEnabled() && tc.isDebugEnabled())
                        Tr.debug(this, tc, "getNextRunTime", trigger, lastExecution,
                                 "taskScheduled " + Utils.toString(taskScheduledTime),
                                 "nextRunTime = " + Utils.toString(nextExecutionDate));

                    // No next execution
                    if (nextExecutionDate == null)
                        result.compareAndSet(skipped, new Status<T>(Status.Type.SKIPPED, null, skipped.failure, true));
                } finally {
                    result.latch.countDown();

                    // notify listener: taskAborted
                    if (listener != null) {
                        if (TraceComponent.isAnyTracingEnabled() && tc.isEventEnabled())
                            Tr.event(this, tc, "taskAborted(skipped)", managedExecSvc, task);
                        listener.taskAborted(future, managedExecSvc, task, new SkippedException(skipped.failure));
                    }
                }

                // notify listener: taskDone
                if (listener != null) {
                    if (TraceComponent.isAnyTracingEnabled() && tc.isEventEnabled())
                        Tr.event(this, tc, "taskDone(skipped)", managedExecSvc, task);
                    listener.taskDone(future, managedExecSvc, task, null);
                }
            }

            // Resubmit this task to run at the next scheduled time
            status = result.getStatus();
            if (!status.finalExecutionIsComplete && (status.type == Status.Type.DONE || status.type == Status.Type.SKIPPED)) {
                if (trace && tc.isEventEnabled())
                    Tr.event(this, tc, "DONE-->NONE (reset for next result)");
                resultRef.set(result = new Result());
                done = false;

                // compute the delay and estimate the next execution time
                long delay;
                if (fixedDelay != null) {
                    delay = fixedDelay;
                    nextExecutionTime = System.nanoTime() + unit.toNanos(delay);
                } else if (fixedRate != null) {
                    // Time elapsed from when the task should have started for the first time
                    long nanoTime = System.nanoTime();
                    long elapsed = unit.convert(nanoTime - taskScheduledNanos, TimeUnit.NANOSECONDS) - initialDelay;
                    delay = ((elapsed / fixedRate) + 1) * fixedRate - elapsed;
                    nextExecutionTime = nanoTime + unit.toNanos(delay);
                } else {
                    nextExecutionTime = nextExecutionDate.getTime();
                    delay = nextExecutionTime - System.currentTimeMillis();
                }

                if (delay < 0)
                    delay = 0;

                // notify listener: taskSubmitted
                if (listener != null)
                    try {
                        result.executionThread = Thread.currentThread();
                        if (TraceComponent.isAnyTracingEnabled() && tc.isEventEnabled())
                            Tr.event(this, tc, "taskSubmitted", managedExecSvc, task);
                        listener.taskSubmitted(future, managedExecSvc, task);
                    } finally {
                        result.executionThread = null;
                    }

                // reschedule the task if the listener didn't cancel it
                status = result.getStatus();
                if (status.type == Status.Type.NONE && result.compareAndSet(status, new Status<T>(Status.Type.SUBMITTED, null, null, false))) {
                    if (trace && tc.isDebugEnabled())
                        Tr.debug(this, tc, "reschedule " + delay + ' ' + unit + " from now");
                    ScheduledFuture<?> scheduledFuture = managedExecSvc.scheduledExecSvc.schedule(this, delay, unit);
                    future.scheduledFutureRef.set(scheduledFuture);
                }
            }
        } catch (Throwable x) {
            // Some cases where this can happen:
            // component that scheduled the task is no longer available
            // listener.taskAborted (for skipped task) fails
            // trigger.getNextRunTime (for reschedule after skip) fails
            // listener.taskStarting fails
            // trigger.getNextRunTime (for reschedule after success) fails
            // taskAborted or taskDone fails
            // Liberty scheduled executor unavailable, or it fails to schedule

            if (contextAppliedToThread == null && x instanceof IllegalStateException && FrameworkState.isStopping()) {
                if (trace && tc.isDebugEnabled())
                    Tr.debug(this, tc, "Task not started due to server shutdown", getName(), x);
            } else
                Tr.error(tc, "CWWKC1101.task.failed", getName(), managedExecSvc.name, x);

            status = result.getStatus();
            if (!status.finalExecutionIsComplete) {
                Status<T> newStatus = status.type == Status.Type.STARTED ? new Status<T>(Status.Type.DONE, null, x, true) : new Status<T>(Status.Type.ABORTED, null, x, true);
                result.compareAndSet(status, newStatus);
            }
            result.latch.countDown();

            if (listener != null && !done)
                try {
                    try {
                        if (skipped == null && status.type != Status.Type.STARTED) {
                            if (TraceComponent.isAnyTracingEnabled() && tc.isEventEnabled())
                                Tr.event(this, tc, "taskAborted", managedExecSvc);
                            listener.taskAborted(future, managedExecSvc, task, new AbortedException(x));
                        }
                    } finally {
                        if (TraceComponent.isAnyTracingEnabled() && tc.isEventEnabled())
                            Tr.event(this, tc, "taskDone", managedExecSvc, task, x);
                        listener.taskDone(future, managedExecSvc, task, x);
                    }
                } catch (Throwable t) {
                    // Log message, but otherwise ignore because we want to the original failure to be raised
                    Tr.error(tc, "CWWKC1102.listener.failed", getName(), managedExecSvc.name, x);
                }

            if (x instanceof Exception)
                throw (Exception) x;
            else
                throw (Error) x;
        } finally {
            resultForThisExecution.executionThread = null;
            if (contextAppliedToThread != null)
                threadContextDescriptor.taskStopping(contextAppliedToThread);
        }

        return taskResult;
    }
756537
@Override
  public void execute(Runnable r) {
    Preconditions.checkNotNull(r, "'r' must not be null.");
    boolean scheduleTaskRunner = false;
    synchronized (internalLock) {
      waitQueue.add(r);

      if (!isThreadScheduled) {
        isThreadScheduled = true;
        scheduleTaskRunner = true;
      }
    }
    if (scheduleTaskRunner) {
      boolean threw = true;
      try {
        executor.execute(taskRunner);
        threw = false;
      } finally {
        if (threw) {
          synchronized (internalLock) {
            // It is possible that at this point that there are still tasks in
            // the queue, it would be nice to keep trying but the error may not
            // be recoverable.  So we update our state and propogate so that if
            // our caller deems it recoverable we won't be stuck.
            isThreadScheduled = false;
          }
        }
      }
    }
  }