Print information about one specific job.
@param jobExecutionInfoOptional Job info to print
12675
public static void printJob(Optional<JobExecutionInfo> jobExecutionInfoOptional) {
        if (!jobExecutionInfoOptional.isPresent()) {
            System.err.println("Job id not found.");
            return;
        }

        JobExecutionInfo jobExecutionInfo = jobExecutionInfoOptional.get();
        List<List<String>> data = new ArrayList<>();
        List<String> flags = Arrays.asList("", "-");

        data.add(Arrays.asList("Job Name", jobExecutionInfo.getJobName()));
        data.add(Arrays.asList("Job Id", jobExecutionInfo.getJobId()));
        data.add(Arrays.asList("State", jobExecutionInfo.getState().toString()));
        data.add(Arrays.asList("Completed/Launched Tasks",
                String.format("%d/%d", jobExecutionInfo.getCompletedTasks(), jobExecutionInfo.getLaunchedTasks())));
        data.add(Arrays.asList("Start Time", dateTimeFormatter.print(jobExecutionInfo.getStartTime())));
        data.add(Arrays.asList("End Time", dateTimeFormatter.print(jobExecutionInfo.getEndTime())));
        data.add(Arrays.asList("Duration", jobExecutionInfo.getState() == JobStateEnum.COMMITTED ? periodFormatter
                .print(new Period(jobExecutionInfo.getDuration().longValue())) : "-"));
        data.add(Arrays.asList("Tracking URL", jobExecutionInfo.getTrackingUrl()));
        data.add(Arrays.asList("Launcher Type", jobExecutionInfo.getLauncherType().name()));

        new CliTablePrinter.Builder()
                .data(data)
                .flags(flags)
                .delimiterWidth(2)
                .build()
                .printTable();

        JobInfoPrintUtils.printMetrics(jobExecutionInfo.getMetrics());
    }
137362
private void readJobInformation(JobConf jobConf, JobInfo jobInfo) throws Exception {
  
    /*
     * Convert the input strings to URL
     */
    URL jobConfFileUrl = new URL(this._jobConfFile);
    URL jobHistoryFileUrl = new URL (this._jobHistoryFile);
    
    /*
     * Read the Job Configuration from the jobConfFile url
     */  
    jobConf.addResource(jobConfFileUrl);
    
    /* 
     * Read JobHistoryFile and build job counters to evaluate diagnostic rules
     */
    if (jobHistoryFileUrl.getProtocol().equals("hdfs")) {
      DefaultJobHistoryParser.parseJobTasks (jobHistoryFileUrl.getPath(), jobInfo, FileSystem.get(jobConf));
    } else if (jobHistoryFileUrl.getProtocol().equals("file")) {
      DefaultJobHistoryParser.parseJobTasks (jobHistoryFileUrl.getPath(), jobInfo, FileSystem.getLocal(jobConf));
    } else {
      throw new Exception("Malformed URL. Protocol: "+jobHistoryFileUrl.getProtocol());
    }
  }
436460
@Override
	public JobExecutionResult execute(String jobName) throws Exception {
		if (executor == null) {
			startNewSession();
		}

		Plan p = createProgramPlan(jobName);

		// Session management is disabled, revert this commit to enable
		//p.setJobId(jobID);
		//p.setSessionTimeout(sessionTimeout);

		JobExecutionResult result = executor.executePlan(p);

		this.lastJobExecutionResult = result;
		return result;
	}
839453
public void setJobDefinitions(java.util.Collection<JobDefinition> jobDefinitions) {
        if (jobDefinitions == null) {
            this.jobDefinitions = null;
            return;
        }

        this.jobDefinitions = new java.util.ArrayList<JobDefinition>(jobDefinitions);
    }
980725
public void display() {
    System.out.printf("Job id=%s, name=%s\n", getJobID(), getJobName());
    System.out.printf("Configuration file: %s\n", getJobID());
    System.out.printf("Tracking URL: %s\n", getTrackingURL());

    System.out.printf("Completion: map: %f reduce %f\n",
        100.0 * this.mapProgress, 100.0 * this.reduceProgress);

    System.out.println("Job total maps = " + totalMaps);
    System.out.println("Job completed maps = " + completedMaps);
    System.out.println("Map percentage complete = " + mapProgress);
    System.out.println("Job total reduces = " + totalReduces);
    System.out.println("Job completed reduces = " + completedReduces);
    System.out.println("Reduce percentage complete = " + reduceProgress);
    System.out.flush();
  }
893479
def printJobChildren(self):
        """"""
        for job in self.jobsToReport:
            children = "CHILDREN_OF_JOB:%s " % job
            for level, jobList in enumerate(job.stack):
                for childJob in jobList:
                    children += "\t(CHILD_JOB:%s,PRECEDENCE:%i)" % (childJob, level)
            print(children)
1324208
private void printHelpAndExit() {
    System.out.println("Common usages:");
    System.out.println("  gobblin-admin.sh jobs --list");
    System.out.println("  gobblin-admin.sh jobs --list --name JobName");
    System.out.println("  gobblin-admin.sh jobs --details --id job_id");
    System.out.println("  gobblin-admin.sh jobs --properties --<id|name> <job_id|JobName>");
    System.out.println();

    printHelpAndExit(0);
  }
321646
public static void qdel(String jobId) {
        final CommandLine cmdLine = new CommandLine(COMMAND_QDEL);
        cmdLine.addArgument(jobId);

        final OutputStream out = new ByteArrayOutputStream();
        final OutputStream err = new ByteArrayOutputStream();

        DefaultExecuteResultHandler resultHandler;
        try {
            resultHandler = execute(cmdLine, null, out, err);
            resultHandler.waitFor(DEFAULT_TIMEOUT);
        } catch (ExecuteException e) {
            throw new PBSException("Failed to execute qdel command: " + e.getMessage(), e);
        } catch (IOException e) {
            throw new PBSException("Failed to execute qdel command: " + e.getMessage(), e);
        } catch (InterruptedException e) {
            throw new PBSException("Failed to execute qdel command: " + e.getMessage(), e);
        }

        final int exitValue = resultHandler.getExitValue();
        LOGGER.info("qdel exit value: " + exitValue);

        if (exitValue != 0)
            throw new PBSException("Failed to delete job " + jobId + ". Error output: " + err.toString());
    }
806490
def printJobLog(self):
        """"""
        for job in self.jobsToReport:
            if job.logJobStoreFileID is not None:
                msg = "LOG_FILE_OF_JOB:%s LOG: =======>\n" % job
                with job.getLogFileHandle(self.jobStore) as fH:
                    msg += fH.read()
                msg += "<========="
            else:
                msg = "LOG_FILE_OF_JOB:%s LOG: Job has no log file" % job
            print(msg)
619841
private <Q, R> void recordFinalStatus(JobIdentifier<Q, R> jobId, JobStatus<Q, R> jobStatus) {
        try {
            _jobStatusDAO.updateJobStatus(jobId, jobStatus);
        } catch (Exception e) {
            _log.error("Failed to record final status for job: [id={}, status={}]", jobId, jobStatus.getStatus(), e);
        }
    }
1046156
Long cleanUpLongJobIfItHasFinishedExecuting(long currentTime, Job job) {
		if(longRunningJobs.containsKey(job)
			&& longRunningJobs.get(job).executionsCount != job.executionsCount()) {
			Long jobLastExecutionTimeInMillis = job.lastExecutionEndedTimeInMillis();
			int jobExecutionsCount = job.executionsCount();
			LongRunningJobInfo jobRunningInfo = longRunningJobs.get(job);

			long jobExecutionDuration = 0L;
			if(jobExecutionsCount == jobRunningInfo.executionsCount + 1) {
				jobExecutionDuration = jobLastExecutionTimeInMillis - jobRunningInfo.jobStartedtimeInMillis;
				logger.info(
					"Job '{}' has finished executing after {}ms",
					job.name(),
					jobExecutionDuration
				);
			} else {
				jobExecutionDuration = currentTime - jobRunningInfo.jobStartedtimeInMillis;
				logger.info(
					"Job '{}' has finished executing after about {}ms",
					job.name(),
					jobExecutionDuration
				);
			}

			longRunningJobs.remove(job);
			return jobExecutionDuration;
		}

		return null;
	}
808031
private void runJobExecutionLauncher() throws JobException {
    long startTime = 0;
    String newPlanningId;
    String jobName = jobProps.getProperty(ConfigurationKeys.JOB_NAME_KEY);
    Closer closer = Closer.create();
    try {
      HelixManager planningJobManager = this.taskDriverHelixManager.isPresent()?
          this.taskDriverHelixManager.get() : this.jobHelixManager;

      String builderStr = jobProps.getProperty(GobblinClusterConfigurationKeys.DISTRIBUTED_JOB_LAUNCHER_BUILDER,
          GobblinHelixDistributeJobExecutionLauncher.Builder.class.getName());

      // Check if any existing planning job is running
      Optional<String> planningJobIdFromStore = jobsMapping.getPlanningJobId(jobName);
      boolean nonblocking = false;
      // start of critical section to check if a job with same job name is running
      Lock jobLock = locks.get(jobName);
      jobLock.lock();

      try {
        if (planningJobIdFromStore.isPresent()) {
          String previousPlanningJobId = planningJobIdFromStore.get();

          if (HelixUtils.isJobFinished(previousPlanningJobId, previousPlanningJobId, planningJobManager)) {
            log.info("Previous planning job {} has reached to the final state. Start a new one.", previousPlanningJobId);
          } else {
            boolean killDuplicateJob = PropertiesUtils
                .getPropAsBoolean(this.jobProps, GobblinClusterConfigurationKeys.KILL_DUPLICATE_PLANNING_JOB, String.valueOf(GobblinClusterConfigurationKeys.DEFAULT_KILL_DUPLICATE_PLANNING_JOB));

            if (!killDuplicateJob) {
              log.info("Previous planning job {} has not finished yet. Skip this job.", previousPlanningJobId);
              return;
            } else {
              log.info("Previous planning job {} has not finished yet. Kill it.", previousPlanningJobId);
              long timeOut = PropertiesUtils.getPropAsLong(sysProps, GobblinClusterConfigurationKeys.HELIX_WORKFLOW_DELETE_TIMEOUT_SECONDS,
                  GobblinClusterConfigurationKeys.DEFAULT_HELIX_WORKFLOW_DELETE_TIMEOUT_SECONDS) * 1000;
              try {
                HelixUtils.deleteWorkflow(previousPlanningJobId, planningJobManager, timeOut);
              } catch (HelixException e) {
                log.info("Helix cannot delete previous planning job id {} within {} seconds.", previousPlanningJobId,
                    timeOut / 1000);
                throw new JobException("Helix cannot delete previous planning job id " + previousPlanningJobId, e);
              }
            }
          }
        } else {
          log.info("Planning job for {} does not exist. First time run.", jobName);
        }

        GobblinHelixDistributeJobExecutionLauncher.Builder builder = GobblinConstructorUtils.<GobblinHelixDistributeJobExecutionLauncher.Builder>invokeLongestConstructor(
            new ClassAliasResolver(GobblinHelixDistributeJobExecutionLauncher.Builder.class).resolveClass(builderStr));

        // Make a separate copy because we could update some of attributes in job properties (like adding planning id).
        Properties jobPlanningProps = new Properties();
        jobPlanningProps.putAll(this.jobProps);

        // Inject planning id and start time
        newPlanningId = HelixJobsMapping.createPlanningJobId(jobPlanningProps);
        jobPlanningProps.setProperty(GobblinClusterConfigurationKeys.PLANNING_ID_KEY, newPlanningId);
        jobPlanningProps.setProperty(GobblinClusterConfigurationKeys.PLANNING_JOB_CREATE_TIME, String.valueOf(System.currentTimeMillis()));

        builder.setSysProps(this.sysProps);
        builder.setJobPlanningProps(jobPlanningProps);
        builder.setJobHelixManager(this.jobHelixManager);
        builder.setTaskDriverHelixManager(this.taskDriverHelixManager);
        builder.setAppWorkDir(this.appWorkDir);
        builder.setJobsMapping(this.jobsMapping);
        builder.setPlanningJobLauncherMetrics(this.planningJobLauncherMetrics);
        builder.setHelixMetrics(this.helixMetrics);

        // if the distributed job launcher should wait for planning job completion

        Config combined = ConfigUtils.propertiesToConfig(jobPlanningProps)
            .withFallback(ConfigUtils.propertiesToConfig(sysProps));

        nonblocking = ConfigUtils.getBoolean(combined, GobblinClusterConfigurationKeys.NON_BLOCKING_PLANNING_JOB_ENABLED,
            GobblinClusterConfigurationKeys.DEFAULT_NON_BLOCKING_PLANNING_JOB_ENABLED);

        log.info("Planning job {} started.", newPlanningId);
        GobblinHelixDistributeJobExecutionLauncher launcher = builder.build();
        closer.register(launcher);
        this.jobsMapping.setPlanningJobId(jobName, newPlanningId);
        startTime = System.currentTimeMillis();
        this.currentJobMonitor = launcher.launchJob(null);

        // make sure the planning job is initialized (or visible) to other parallel running threads,
        // so that the same critical section check (querying Helix for job completeness)
        // can be applied.
        HelixUtils.waitJobInitialization(planningJobManager, newPlanningId, newPlanningId, 300_000);

      } finally {
        // end of the critical section to check if a job with same job name is running
        jobLock.unlock();
      }

      // we can remove the job spec from the catalog because Helix will drive this job to the end.
      this.deleteJobSpec();

      // If we are using non-blocking mode, this get() only guarantees the plannning job is submitted.
      // It doesn't guarantee the job will finish because internally we won't wait for Helix completion.
      this.currentJobMonitor.get();
      this.currentJobMonitor = null;
      if (nonblocking) {
        log.info("Planning job {} submitted successfully.", newPlanningId);
      } else {
        log.info("Planning job {} finished.", newPlanningId);
        this.planningJobLauncherMetrics.updateTimeForCompletedPlanningJobs(startTime);
      }
    } catch (Exception e) {
      if (startTime != 0) {
        this.planningJobLauncherMetrics.updateTimeForFailedPlanningJobs(startTime);
      }
      log.error("Failed to run planning job {}", jobName, e);
      throw new JobException("Failed to run planning job " + jobName, e);
    } finally {
      try {
        closer.close();
      } catch (IOException e) {
        throw new JobException("Cannot properly close planning job " + jobName, e);
      }
    }
  }
924355
public Optional<JobExecutionInfo> queryByJobId(String id) throws RemoteInvocationException {
    JobExecutionQuery query = new JobExecutionQuery();
    query.setIdType(QueryIdTypeEnum.JOB_ID);
    query.setId(JobExecutionQuery.Id.create(id));
    query.setLimit(1);

    List<JobExecutionInfo> results = executeQuery(query);
    return getFirstFromQueryResults(results);
  }
1284606
@Override
	public JobExecutionResult execute(String jobName) throws Exception {
		// transform the streaming program into a JobGraph
		StreamGraph streamGraph = getStreamGraph();
		streamGraph.setJobName(jobName);

		JobGraph jobGraph = streamGraph.getJobGraph();
		jobGraph.setAllowQueuedScheduling(true);

		Configuration configuration = new Configuration();
		configuration.addAll(jobGraph.getJobConfiguration());
		configuration.setString(TaskManagerOptions.MANAGED_MEMORY_SIZE, "0");

		// add (and override) the settings with what the user defined
		configuration.addAll(this.configuration);

		if (!configuration.contains(RestOptions.BIND_PORT)) {
			configuration.setString(RestOptions.BIND_PORT, "0");
		}

		int numSlotsPerTaskManager = configuration.getInteger(TaskManagerOptions.NUM_TASK_SLOTS, jobGraph.getMaximumParallelism());

		MiniClusterConfiguration cfg = new MiniClusterConfiguration.Builder()
			.setConfiguration(configuration)
			.setNumSlotsPerTaskManager(numSlotsPerTaskManager)
			.build();

		if (LOG.isInfoEnabled()) {
			LOG.info("Running job on local embedded Flink mini cluster");
		}

		MiniCluster miniCluster = new MiniCluster(cfg);

		try {
			miniCluster.start();
			configuration.setInteger(RestOptions.PORT, miniCluster.getRestAddress().get().getPort());

			return miniCluster.executeJobBlocking(jobGraph);
		}
		finally {
			transformations.clear();
			miniCluster.close();
		}
	}
728658
@Override
	public JobExecutionResult execute(String jobName) throws Exception {
		PlanExecutor executor = getExecutor();

		Plan p = createProgramPlan(jobName);

		// Session management is disabled, revert this commit to enable
		//p.setJobId(jobID);
		//p.setSessionTimeout(sessionTimeout);

		JobExecutionResult result = executor.executePlan(p);

		this.lastJobExecutionResult = result;
		return result;
	}
129114
public void setJobClass (final Class <? extends IJob> jobClass)
  {
    if (jobClass == null)
    {
      throw new IllegalArgumentException ("Job class cannot be null.");
    }

    if (!IJob.class.isAssignableFrom (jobClass))
    {
      throw new IllegalArgumentException ("Job class must implement the Job interface.");
    }

    m_aJobClass = jobClass;
  }
732088
protected boolean jobTooOld(final JobInfo jobInfo, final JobDefinition jobDefinition) {
        final Optional<OffsetDateTime> stopped = jobInfo.getStopped();
        if (stopped.isPresent() && jobDefinition.maxAge().isPresent()) {
            final OffsetDateTime deadlineToRerun = stopped.get().plus(jobDefinition.maxAge().get());
            return deadlineToRerun.isBefore(now());
        }

        return false;
    }