Starts a {@link MapReduceJob} with the given parameters in a new Pipeline.
Returns the pipeline id.
15765
public static <I, K, V, O, R> String start(
      MapReduceSpecification<I, K, V, O, R> specification, MapReduceSettings settings) {
    if (settings.getWorkerQueueName() == null) {
      settings = new MapReduceSettings.Builder(settings).setWorkerQueueName("default").build();
    }
    PipelineService pipelineService = PipelineServiceFactory.newPipelineService();
    return pipelineService.startNewPipeline(
        new MapReduceJob<>(specification, settings), settings.toJobSettings());
  }
251732
@Override
  public int compareTo(org.apache.hadoop.mapreduce.ID o) {
    JVMId that = (JVMId)o;
    int jobComp = this.jobId.compareTo(that.jobId);
    if(jobComp == 0) {
      if(this.isMap == that.isMap) {
        return this.id - that.id;
      } else {
        return this.isMap ? -1 : 1;
      }
    } else {
      return jobComp;
    }
  }
1178094
public static PipelineObjects queryFullPipeline(String rootJobHandle) {
    Key rootJobKey = KeyFactory.createKey(JobRecord.DATA_STORE_KIND, rootJobHandle);
    return backEnd.queryFullPipeline(rootJobKey);
  }
155912
public CreateThumbnailJobResponse createThumbnailJob(CreateThumbnailJobRequest request) {
        checkStringNotEmpty(request.getPipelineName(), 
                "The parameter pipelineName should NOT be null or empty string.");
        checkNotNull(request.getSource(), "The parameter source should NOT be null.");
        checkStringNotEmpty(request.getSource().getKey(),
                "The parameter source key should NOT be null or empty string.");
        InternalRequest internalRequest =
                createRequest(HttpMethodName.POST, request, THUMBNAIL);
        
        return invokeHttpClient(internalRequest, CreateThumbnailJobResponse.class);
    }
22593
public CreateThumbnailJobResponse createThumbnailJob(String pipelineName, String sourceKey) {
        ThumbnailSource source = new ThumbnailSource();
        source.setKey(sourceKey);
        
        CreateThumbnailJobRequest request =
                new CreateThumbnailJobRequest().withPipelineName(pipelineName).withSource(source);
        
        return createThumbnailJob(request);
    }
1184536
public JobDetails getJobByJobID(QualifiedJobId jobId, boolean populateTasks)
      throws IOException {
    JobDetails job = null;
    JobKey key = idService.getJobKeyById(jobId);
    if (key != null) {
      byte[] historyKey = jobKeyConv.toBytes(key);
      Table historyTable =
          hbaseConnection.getTable(TableName.valueOf(Constants.HISTORY_TABLE));
      Result result = historyTable.get(new Get(historyKey));
      historyTable.close();
      if (result != null && !result.isEmpty()) {
        job = new JobDetails(key);
        job.populate(result);
        if (populateTasks) {
          populateTasks(job);
        }
      }
    }
    return job;
  }
40426
public static boolean init(JobHistoryObserver jobTracker, JobConf conf,
             String hostname, long jobTrackerStartTime){
    try {
      LOG_DIR = conf.get("hadoop.job.history.location" ,
        "file:///" + new File(
        System.getProperty("hadoop.log.dir", "/tmp")).getAbsolutePath()
        + File.separator + "history");
      JOBTRACKER_UNIQUE_STRING = hostname + "_" + 
                                    String.valueOf(jobTrackerStartTime) + "_";
      jobtrackerHostname = hostname;
      Path logDir = new Path(LOG_DIR);
      LOGDIR_FS = logDir.getFileSystem(conf);
      if (!LOGDIR_FS.exists(logDir)){
        if (!LOGDIR_FS.mkdirs(logDir, new FsPermission(HISTORY_DIR_PERMISSION))) {
          throw new IOException("Mkdirs failed to create " + logDir.toString());
        }
      }
      conf.set("hadoop.job.history.location", LOG_DIR);
      disableHistory = false;
      // set the job history block size (default is 3MB)
      jobHistoryBlockSize = 
        conf.getLong("mapred.jobtracker.job.history.block.size", 
                     3 * 1024 * 1024);
      jtConf = conf;

      // initialize the file manager
      fileManager = new JobHistoryFilesManager(conf, jobTracker, logDir);
    } catch(IOException e) {
        LOG.error("Failed to initialize JobHistory log file", e); 
        disableHistory = true;
    }
    fileManager.startIOExecutor();
    return !(disableHistory);
  }
254173
@Override
  public final void commitJob(org.apache.hadoop.mapreduce.JobContext context
                               ) throws IOException {
    commitJob((JobContext) context);
  }
673347
public ListThumbnailJobsResponse listThumbnailJobs(String pipelineName) {
        ListThumbnailJobsRequest request = new ListThumbnailJobsRequest().withPipeline(pipelineName);
        
        return listThumbnailJobs(request);
    }
1013952
public boolean startJob(final Long jobId) {

		final ScheduledJob job = removeFromQueueInternal(jobId);

		if (job != null) {
			activeJobs.put(jobId, job);
			job.startJob();
			return true;
		} else {
			return false;
		}
	}
584234
private static int getMapperId(long key, int numReducers, int maxKeySpace) {
    key = key - getFirstSumKey(numReducers, maxKeySpace);
    return (int) (key / numReducers);
  }
855546
public static <K, V> HadoopInputFormat<K, V> readHadoopFile(org.apache.hadoop.mapred.FileInputFormat<K, V> mapredInputFormat, Class<K> key, Class<V> value, String inputPath, JobConf job) {
		// set input path in JobConf
		org.apache.hadoop.mapred.FileInputFormat.addInputPath(job, new org.apache.hadoop.fs.Path(inputPath));
		// return wrapping InputFormat
		return createHadoopInput(mapredInputFormat, key, value, job);
	}
307957
@Override
    public Entry<Long, Future<?>> start(WSJobInstance jobInstance, Properties jobParameters, long execId) {

        if (authService != null) {
            authService.authorizedJobSubmission();
        }

        traceJobStart(jobInstance.getJobXMLName(), jobParameters);

        IJobXMLSource jobXML;
        if (!StringUtils.isEmpty(jobInstance.getJobXml())) {
            jobXML = new JobXMLSource(jobInstance.getJobXml());
        } else {
            jobXML = jslLoaderService.loadJSL(jobInstance.getJobXMLName());
        }

        Entry<Long, Future<?>> execIdFutureEntry = batchKernelService.startJob(jobInstance, jobXML, jobParameters, execId);

        long executionId = execIdFutureEntry.getKey();
        if (logger.isLoggable(Level.FINE) && executionId > 0) {
            logger.fine("Started job with instanceId: " + jobInstance.getInstanceId() + ", executionId: " + executionId);
        }

        return execIdFutureEntry;
    }
1076772
@Override
    public Entry<Long, Future<?>> startJob(WSJobInstance jobInstance,
                                           IJobXMLSource jobXML,
                                           Properties jobParameters,
                                           long executionId) throws JobStartException {

        traceJobXML(jobXML);
        // Check BatchStatus instead of InstanceState since this narrows the window
        // of a restart changing the status after a stop has occurred.  InstanceState
        // gets changed early; whereas BatchStatus changes right before the job is executed.
        JobInstanceEntity instanceEntity = getPersistenceManagerService().getJobInstance(jobInstance.getInstanceId());
        if (instanceEntity.getBatchStatus() == BatchStatus.STOPPED) {
            // Do not continue with the execution.  Instance has been stopped.
            throw new JobStoppedOnStartException();
        }

        RuntimeJobExecution jobExecution = jobExecutionHelper.createJobStartExecution(jobInstance, jobXML, jobParameters, executionId);

        BatchJobWorkUnit batchWork = new BatchJobWorkUnit(this, jobExecution, jobExecutionStartCallbacks, jobExecutionEndCallbacks, tranMgr);
        registerExecutingJob(jobExecution.getTopLevelExecutionId(), batchWork);

        Future<?> futureWork = null;

        try {
            // Set server id and rest URL since this is now a viable execution.
            getPersistenceManagerService().updateJobExecutionServerIdAndRestUrlForStartingJob(jobExecution.getTopLevelExecutionId());
            futureWork = executorService.executeTask(batchWork, null);
        } catch (RuntimeException e) {
            workUnitCompleted(batchWork);
            throw e;
        } catch (JobStoppedException e) {
            workUnitCompleted(batchWork);
            throw new BatchContainerRuntimeException(e);
        }

        return new AbstractMap.SimpleEntry<Long, Future<?>>(jobExecution.getTopLevelExecutionId(), futureWork);
    }
187769
public boolean startDistRaid() throws IOException {
    assert(raidPolicyPathPairList.size() > 0);
    if (setup()) {
      this.jobClient = new JobClient(jobconf);
      this.runningJob = this.jobClient.submitJob(jobconf);
      LOG.info("Job Started: " + runningJob.getID());
      this.startTime = System.currentTimeMillis();
      return true;
    }
    return false;
  }
1182254
public static MapReduceExecutor getMapReduceExecutor() {
    if (executor == null) {
      // Default to using a local executor with one thread per CPU.
      executor = new LocalMapReduceExecutor(
          Runtime.getRuntime().availableProcessors(), 20);
    }
    return executor;
  }
920540
public Job createJob(FileSystemDataset dataset) throws IOException {
    Configuration conf = HadoopUtils.getConfFromState(state);

    // Turn on mapreduce output compression by default
    if (conf.get("mapreduce.output.fileoutputformat.compress") == null && conf.get("mapred.output.compress") == null) {
      conf.setBoolean("mapreduce.output.fileoutputformat.compress", true);
    }

    // Disable delegation token cancellation by default
    if (conf.get("mapreduce.job.complete.cancel.delegation.tokens") == null) {
      conf.setBoolean("mapreduce.job.complete.cancel.delegation.tokens", false);
    }

    addJars(conf, this.state, fs);
    Job job = Job.getInstance(conf);
    job.setJobName(MRCompactorJobRunner.HADOOP_JOB_NAME);
    boolean emptyDirectoryFlag = this.configureInputAndOutputPaths(job, dataset);
    if (emptyDirectoryFlag) {
      this.state.setProp(HiveRegistrationPolicy.MAPREDUCE_JOB_INPUT_PATH_EMPTY_KEY, true);
    }
    this.configureMapper(job);
    this.configureReducer(job);
    if (emptyDirectoryFlag || !this.shouldDeduplicate) {
      job.setNumReduceTasks(0);
    }
    // Configure schema at the last step because FilesInputFormat will be used internally
    this.configureSchema(job);
    this.isJobCreated = true;
    this.configuredJob = job;
    return job;
  }