Sets up the actual job.

@param conf The current configuration.
@param args The command line parameters.
@return The newly created job.
@throws java.io.IOException When setting up the job fails.
176385
public static Job createSubmittableJob(Configuration conf, String[] args)
  throws IOException {
    conf.setIfUnset("hbase.client.connection.impl",
      BigtableConfiguration.getConnectionClass().getName());
    conf.setIfUnset(BigtableOptionsFactory.BIGTABLE_RPC_TIMEOUT_MS_KEY, "60000");

    TableName tableName = TableName.valueOf(args[0]);
    conf.set(TABLE_NAME, tableName.getNameAsString());
    Path inputDir = new Path(args[1]);
    Job job = Job.getInstance(conf, conf.get(JOB_NAME_CONF_KEY, NAME + "_" + tableName));
    job.setJarByClass(Importer.class);
    FileInputFormat.setInputPaths(job, inputDir);
    // Randomize the splits to avoid hot spotting a single tablet server
    job.setInputFormatClass(ShuffledSequenceFileInputFormat.class);
    // Give the mappers enough work to do otherwise each split will be dominated by spinup time
    ShuffledSequenceFileInputFormat.setMinInputSplitSize(job, 1L*1024*1024*1024);
    String hfileOutPath = conf.get(BULK_OUTPUT_CONF_KEY);

    // make sure we get the filter in the jars
    try {
      Class<? extends Filter> filter = conf.getClass(FILTER_CLASS_CONF_KEY, null, Filter.class);
      if (filter != null) {
        TableMapReduceUtil.addDependencyJars(conf, filter);
      }
    } catch (Exception e) {
      throw new IOException(e);
    }

    if (hfileOutPath != null) {
      job.setMapperClass(KeyValueImporter.class);
      try (Connection conn = ConnectionFactory.createConnection(conf); 
          Table table = conn.getTable(tableName);
          RegionLocator regionLocator = conn.getRegionLocator(tableName)){
        job.setReducerClass(KeyValueSortReducer.class);
        Path outputDir = new Path(hfileOutPath);
        FileOutputFormat.setOutputPath(job, outputDir);
        job.setMapOutputKeyClass(ImmutableBytesWritable.class);
        job.setMapOutputValueClass(KeyValue.class);
        HFileOutputFormat2.configureIncrementalLoad(job, table, regionLocator);
        TableMapReduceUtil.addDependencyJars(job.getConfiguration(),
            com.google.common.base.Preconditions.class);
      }
    } else {
      // No reducers.  Just write straight to table.  Call initTableReducerJob
      // because it sets up the TableOutputFormat.
      job.setMapperClass(Importer.class);
      //TableMapReduceUtil.initTableReducerJob(tableName.getNameAsString(), null, job);
      TableMapReduceUtil.initTableReducerJob(tableName.getNameAsString(), null, job, null, null, null, null, false);
      job.setNumReduceTasks(0);
    }
    return job;
  }
623597
public static void main(final String[] args) throws BindException, InjectionException {

    final Tang tang = Tang.Factory.getTang();

    final JavaConfigurationBuilder cb = tang.newConfigurationBuilder();

    try{
      new CommandLine(cb)
          .registerShortNameOfClass(NodeListFilePath.class)
          .registerShortNameOfClass(SshPortNum.class)
          .processCommandLine(args);
    } catch(final IOException ex) {
      LOG.log(Level.SEVERE, "Missing parameter 'nodelist' or wrong parameter input.");
      throw new RuntimeException("Missing parameter 'nodelist' or wrong parameter input: ", ex);
    }

    final Injector injector = tang.newInjector(cb.build());

    final String nodeListFilePath = injector.getNamedInstance(NodeListFilePath.class);
    final int sshPortNum = injector.getNamedInstance(SshPortNum.class);

    final Configuration runtimeConf = getRuntimeConfiguration(nodeListFilePath, sshPortNum);
    final Configuration driverConf = getDriverConfiguration();

    final LauncherStatus status = DriverLauncher
        .getLauncher(runtimeConf)
        .run(driverConf, JOB_TIMEOUT);
    LOG.log(Level.INFO, "REEF job completed: {0}", status);
  }
979062
private void symLinkAndConfigureFiles(JobConf job) throws IOException {
    if (!(job.getBoolean("mapred.used.genericoptionsparser", false))) {
      LOG.warn("Use GenericOptionsParser for parsing the arguments. " +
               "Applications should implement Tool for the same.");
    }

    // get all the command line arguments into the
    // jobconf passed in by the user conf
    String files = job.get("tmpfiles");
    String archives = job.get("tmparchives");
    // "tmpjars" are not needed because its in the classpath

    List<String> filesToSymLink = new ArrayList<String>();
    splitAndAdd(files, filesToSymLink);
    splitAndAdd(archives, filesToSymLink);

    for (String file : filesToSymLink) {
      String target = new Path(file).toUri().getPath();
      String basename = new File(target).getName();
      String linkName = new File(".").getAbsolutePath() + File.separator + basename;
      File toLink = new File(linkName);
      if (toLink.exists()) {
        LOG.info("Symlink " + linkName + " already exists. Delete it.");
        toLink.delete();
      }
      int ret = FileUtil.symLink(target, linkName);
      LOG.info("Creating symlink " + linkName + " -> " + target +
          " returns " + ret + ".");
    }
    // Configure job name
    String originalJar = job.getJar();
    if (originalJar != null) {
      // use jar name if job is not named.
      if ("".equals(job.getJobName())) {
        job.setJobName(new Path(originalJar).getName());
      }
    }
    // Configure username
    configureUserName(job);
  }
51884
@Override
  JobDesc create(QualifiedJobId qualifiedJobId, long submitTimeMillis,
      Configuration jobConf) {
    // TODO: Get the actual values appropriate for the plain Hadoop jobs.

    String appId = getAppId(jobConf);

    long appSubmitTimeMillis = jobConf.getLong(Constants.MR_RUN_CONF_KEY,
        submitTimeMillis);


    return create(qualifiedJobId, jobConf, appId, Constants.UNKNOWN,
        Framework.NONE, appSubmitTimeMillis);
  }
599299
public static void main(String[] args) {
    if (args.length != 0) {
      LOG.info("java -cp {} {}", RuntimeConstants.ALLUXIO_JAR,
          AlluxioJobMaster.class.getCanonicalName());
      System.exit(-1);
    }

    CommonUtils.PROCESS_TYPE.set(alluxio.util.CommonUtils.ProcessType.JOB_MASTER);
    AlluxioJobMasterProcess process;
    try {
      process = AlluxioJobMasterProcess.Factory.create();
    } catch (Throwable t) {
      LOG.error("Failed to create job master process", t);
      // Exit to stop any non-daemon threads.
      System.exit(-1);
      throw t;
    }
    ProcessUtils.run(process);
  }
1289247
public static void main(String argv[]
                          ) throws IOException, InterruptedException {
    StringUtils.startupShutdownMessage(JobTracker.class, argv, LOG);

    try {
      if (argv.length == 0) {
        JobTracker tracker = startTracker(new JobConf());
        tracker.offerService();
        return;
      }
      if ("-instance".equals(argv[0]) && argv.length == 2) {
        int instance = Integer.parseInt(argv[1]);
        if (instance == 0 || instance == 1) {
          JobConf conf = new JobConf();
          JobConf.overrideConfiguration(conf, instance);
          JobTracker tracker = startTracker(conf);
          tracker.offerService();
          return;
        }
      }
      if ("-dumpConfiguration".equals(argv[0]) && argv.length == 1) {
        dumpConfiguration(new PrintWriter(System.out));
        return;
      }
      System.out.println("usage: JobTracker [-dumpConfiguration]");
      System.out.println("       JobTracker [-instance <0|1>]");
      System.exit(-1);

    } catch (Throwable e) {
      LOG.fatal(StringUtils.stringifyException(e));
      System.exit(-1);
    }
  }
953808
public static void main(final String[] args) throws IOException {
        ShardingConfiguration shardingConfig = new ShardingConfigurationLoader().load();
        int port = getPort(args);
        if (null == shardingConfig.getServerConfiguration().getOrchestration()) {
            startWithoutRegistryCenter(shardingConfig.getRuleConfigurationMap(), shardingConfig.getServerConfiguration().getAuthentication(), shardingConfig.getServerConfiguration().getProps(), port);
        } else {
            startWithRegistryCenter(shardingConfig.getServerConfiguration(), shardingConfig.getRuleConfigurationMap().keySet(), shardingConfig.getRuleConfigurationMap(), port);
        }
    }
74638
public int run(String[] args) throws Exception {
    JobConf jobConf = new JobConf(getConf(), Sort.class);
    jobConf.setJobName("join");

    jobConf.setMapperClass(IdentityMapper.class);        
    jobConf.setReducerClass(IdentityReducer.class);

    JobClient client = new JobClient(jobConf);
    ClusterStatus cluster = client.getClusterStatus();
    int num_maps = cluster.getTaskTrackers() * 
                   jobConf.getInt("test.sort.maps_per_host", 10);
    int num_reduces = (int) (cluster.getMaxReduceTasks() * 0.9);
    String sort_reduces = jobConf.get("test.sort.reduces_per_host");
    if (sort_reduces != null) {
       num_reduces = cluster.getTaskTrackers() * 
                       Integer.parseInt(sort_reduces);
    }
    Class<? extends InputFormat> inputFormatClass = 
      SequenceFileInputFormat.class;
    Class<? extends OutputFormat> outputFormatClass = 
      SequenceFileOutputFormat.class;
    Class<? extends WritableComparable> outputKeyClass = BytesWritable.class;
    Class<? extends Writable> outputValueClass = TupleWritable.class;
    String op = "inner";
    List<String> otherArgs = new ArrayList<String>();
    for(int i=0; i < args.length; ++i) {
      try {
        if ("-m".equals(args[i])) {
          num_maps = Integer.parseInt(args[++i]);
        } else if ("-r".equals(args[i])) {
          num_reduces = Integer.parseInt(args[++i]);
        } else if ("-inFormat".equals(args[i])) {
          inputFormatClass = 
            Class.forName(args[++i]).asSubclass(InputFormat.class);
        } else if ("-outFormat".equals(args[i])) {
          outputFormatClass = 
            Class.forName(args[++i]).asSubclass(OutputFormat.class);
        } else if ("-outKey".equals(args[i])) {
          outputKeyClass = 
            Class.forName(args[++i]).asSubclass(WritableComparable.class);
        } else if ("-outValue".equals(args[i])) {
          outputValueClass = 
            Class.forName(args[++i]).asSubclass(Writable.class);
        } else if ("-joinOp".equals(args[i])) {
          op = args[++i];
        } else {
          otherArgs.add(args[i]);
        }
      } catch (NumberFormatException except) {
        System.out.println("ERROR: Integer expected instead of " + args[i]);
        return printUsage();
      } catch (ArrayIndexOutOfBoundsException except) {
        System.out.println("ERROR: Required parameter missing from " +
            args[i-1]);
        return printUsage(); // exits
      }
    }

    // Set user-supplied (possibly default) job configs
    jobConf.setNumMapTasks(num_maps);
    jobConf.setNumReduceTasks(num_reduces);

    if (otherArgs.size() < 2) {
      System.out.println("ERROR: Wrong number of parameters: ");
      return printUsage();
    }

    FileOutputFormat.setOutputPath(jobConf, 
      new Path(otherArgs.remove(otherArgs.size() - 1)));
    List<Path> plist = new ArrayList<Path>(otherArgs.size());
    for (String s : otherArgs) {
      plist.add(new Path(s));
    }

    jobConf.setInputFormat(CompositeInputFormat.class);
    jobConf.set("mapred.join.expr", CompositeInputFormat.compose(
          op, inputFormatClass, plist.toArray(new Path[0])));
    jobConf.setOutputFormat(outputFormatClass);

    jobConf.setOutputKeyClass(outputKeyClass);
    jobConf.setOutputValueClass(outputValueClass);

    Date startTime = new Date();
    System.out.println("Job started: " + startTime);
    JobClient.runJob(jobConf);
    Date end_time = new Date();
    System.out.println("Job ended: " + end_time);
    System.out.println("The job took " + 
        (end_time.getTime() - startTime.getTime()) /1000 + " seconds.");
    return 0;
  }
424845
public void configure(JobConf jobConf) {
    String prefix = getPrefix(isMap);
    chainJobConf = jobConf;
    SerializationFactory serializationFactory =
      new SerializationFactory(chainJobConf);
    int index = jobConf.getInt(prefix + CHAIN_MAPPER_SIZE, 0);
    for (int i = 0; i < index; i++) {
      Class<? extends Mapper> klass =
        jobConf.getClass(prefix + CHAIN_MAPPER_CLASS + i, null, Mapper.class);
      JobConf mConf =
        getChainElementConf(jobConf, prefix + CHAIN_MAPPER_CONFIG + i);
      Mapper mapper = ReflectionUtils.newInstance(klass, mConf);
      mappers.add(mapper);

      if (mConf.getBoolean(MAPPER_BY_VALUE, true)) {
        mappersKeySerialization.add(serializationFactory.getSerialization(
          mConf.getClass(MAPPER_OUTPUT_KEY_CLASS, null)));
        mappersValueSerialization.add(serializationFactory.getSerialization(
          mConf.getClass(MAPPER_OUTPUT_VALUE_CLASS, null)));
      } else {
        mappersKeySerialization.add(null);
        mappersValueSerialization.add(null);
      }
    }
    Class<? extends Reducer> klass =
      jobConf.getClass(prefix + CHAIN_REDUCER_CLASS, null, Reducer.class);
    if (klass != null) {
      JobConf rConf =
        getChainElementConf(jobConf, prefix + CHAIN_REDUCER_CONFIG);
      reducer = ReflectionUtils.newInstance(klass, rConf);
      if (rConf.getBoolean(REDUCER_BY_VALUE, true)) {
        reducerKeySerialization = serializationFactory
          .getSerialization(rConf.getClass(REDUCER_OUTPUT_KEY_CLASS, null));
        reducerValueSerialization = serializationFactory
          .getSerialization(rConf.getClass(REDUCER_OUTPUT_VALUE_CLASS, null));
      } else {
        reducerKeySerialization = null;
        reducerValueSerialization = null;
      }
    }
  }
671737
private Job createJob() {
    Job job = new Job();
    JobConfiguration jobConfig = new JobConfiguration();
    JobConfigurationLoad loadConfig = new JobConfigurationLoad();
    jobConfig.setLoad(loadConfig);
    job.setConfiguration(jobConfig);

    loadConfig.setAllowQuotedNewlines(false);
    loadConfig.setSourceFormat("NEWLINE_DELIMITED_JSON");

    List<String> sources = new ArrayList<String>();
    for (GcsFilename file : fileSet) {
      sources.add(getFileUri(file));
    }

    loadConfig.setSourceUris(sources);

    TableReference tableRef = new TableReference();
    tableRef.setDatasetId(dataset);
    tableRef.setTableId(tableName);
    tableRef.setProjectId(projectId);
    loadConfig.setDestinationTable(tableRef);
    loadConfig.setSchema(schema.getValue());
    return job;
  }
22619
public static void main(final String[] args)
	{

		if (args.length != 1) {
			throw new IllegalArgumentException(
					"Configuration File ist missing.");
		}

		try {

			// Reads the configuration
			ConfigSettings config = readConfiguration(args[0]);
			new DiffToolThread(config).run();
		}
		catch (Exception e) {
			e.printStackTrace();
		}
	}
102892
public static void main(String[] args) {
		PrintStream out = System.out;
		if (args.length > 0) {
			try {
				out = new PrintStream(new File(args[0]));
			} catch (FileNotFoundException e) {
				System.out.printf(ERR_FILE, args[0]);
			}
		}
		MBeanScanner mBeanScanner = new MBeanScanner();
		List<Config> configs = mBeanScanner.scan();
		ConfigWriter cw;
		cw = new ConfigWriter(out, configs);
		cw.write();
	}
753046
private static Configuration parseCommandLine(final String[] args)
      throws BindException, IOException {
    final JavaConfigurationBuilder confBuilder = Tang.Factory.getTang().newConfigurationBuilder();
    final CommandLine cl = new CommandLine(confBuilder);
    cl.registerShortNameOfClass(Local.class);
    cl.registerShortNameOfClass(Piggyback.class);
    cl.registerShortNameOfClass(NumEvaluators.class);
    cl.registerShortNameOfClass(NumTasks.class);
    cl.registerShortNameOfClass(Delay.class);
    cl.registerShortNameOfClass(JobId.class);
    cl.processCommandLine(args);
    return confBuilder.build();
  }
1327112
private boolean setup() throws IOException {
    estimateSavings();

    final String randomId = getRandomId();
    JobClient jClient = new JobClient(jobconf);
    Path jobdir = new Path(jClient.getSystemDir(), NAME + "_" + randomId);

    LOG.info(JOB_DIR_LABEL + "=" + jobdir);
    jobconf.set(JOB_DIR_LABEL, jobdir.toString());
    Path log = new Path(jobdir, "_logs");

    // The control file should have small size blocks. This helps
    // in spreading out the load from mappers that will be spawned.
    jobconf.setInt("dfs.blocks.size",  OP_LIST_BLOCK_SIZE);

    FileOutputFormat.setOutputPath(jobconf, log);
    LOG.info("log=" + log);

    // create operation list
    FileSystem fs = jobdir.getFileSystem(jobconf);
    Path opList = new Path(jobdir, "_" + OP_LIST_LABEL);
    jobconf.set(OP_LIST_LABEL, opList.toString());
    int opCount = 0, synCount = 0;
    SequenceFile.Writer opWriter = null;

    try {
      opWriter = SequenceFile.createWriter(fs, jobconf, opList, Text.class,
          PolicyInfo.class, SequenceFile.CompressionType.NONE);
      for (RaidPolicyPathPair p : raidPolicyPathPairList) {
        // If a large set of files are Raided for the first time, files
        // in the same directory that tend to have the same size will end up
        // with the same map. This shuffle mixes things up, allowing a better
        // mix of files.
        java.util.Collections.shuffle(p.srcPaths);
        for (EncodingCandidate ec : p.srcPaths) {
          opWriter.append(new Text(ec.toString()), p.policy);
          opCount++;
          if (++synCount > SYNC_FILE_MAX) {
            opWriter.sync();
            synCount = 0;
          }
        }
      }

    } finally {
      if (opWriter != null) {
        opWriter.close();
      }
      fs.setReplication(opList, OP_LIST_REPLICATION); // increase replication for control file
    }
    raidPolicyPathPairList.clear();
    
    jobconf.setInt(OP_COUNT_LABEL, opCount);
    LOG.info("Number of files=" + opCount);
    jobconf.setNumMapTasks(getMapCount(opCount));
    LOG.info("jobName= " + jobName + " numMapTasks=" + jobconf.getNumMapTasks());
    return opCount != 0;

  }
1125427
public static void main(final String args[]) {
        int exitCode = 1;
        final Setup setup = new Setup();
        try {
            setup.execute(args);
            exitCode = 0;
        } catch (Throwable exc) {
            System.err.println("ERROR: " + exc.getMessage());
        }
        System.exit(exitCode);
    }
769905
public static void main(String[] args) throws Exception {
        if (args.length < 1 || args.length > 2) {
            throw new IOException("One or two arguments expected.");
        }
        ServerConfiguration config =
                new ServerConfigurationParser(new FileInputStream(new File(args[0])))
                        .parse();
        if (args.length == 2) {
            Properties props = new Properties();
            props.load(new FileInputStream(new File(args[1])));
            config.applyProperties(props);
        }
        ByteArrayOutputStream out = new ByteArrayOutputStream();
        config.serialize(out);
        String content = new String(out.toByteArray(), "UTF-8");
        System.out.println(content);
    }
459146
public static void main(final String[] args)
    throws InjectionException, IOException, ParseException {
    final Configuration runtimeConfiguration =
        YarnClientConfiguration.CONF.build();
    runTaskScheduler(runtimeConfiguration, args);
  }