Packages and submits a job, which is defined in a YAML file, to Spark.

    Parameters
    ----------
    args (List): Command-line arguments
2010
def _package_and_submit(args):
    
    args = _parse_and_validate_args(args)

    logging.debug(args)
    dist = __make_tmp_dir()
    try:
        __package_dependencies(dist_dir=dist, additional_reqs=args['requirements'],
                               silent=args['silent'])
        __package_app(tasks_pkg=args['package'],
                      dist_dir=dist,
                      custom_main=args['main'],
                      extra_data=args['extra_data'])
        __run_spark_submit(lane_yaml=args['yaml'],
                           dist_dir=dist,
                           spark_home=args['spark_home'],
                           spark_args=args['spark_args'],
                           silent=args['silent'])

    except Exception as exc:
        __clean_up(dist)
        raise exc
    __clean_up(dist)
674821
def main(argv=None):  # suppress(unused-function)
    
    argv = argv or sys.argv

    if "--" not in argv:
        sys.stderr.write("""Must specify command after '--'.\n""")
        return 1

    cmd_index = argv.index("--")
    args, cmd = (argv[1:cmd_index], argv[cmd_index + 1:])

    parser = argparse.ArgumentParser(description="""Cache results from jobs""")
    parser.add_argument("--dependencies",
                        metavar="PATH",
                        nargs="*",
                        help="""A list of paths which, if more recent than the last time this job was invoked, will cause the job to be re-invoked.""")
    parser.add_argument("--output-files",
                        metavar="PATH",
                        nargs="*",
                        help="""A list of expected output paths form this command, which, if they do not exist, will cause the job to be re-invoked.""")
    parser.add_argument("--stamp-directory",
                        metavar="DIRECTORY",
                        type=str,
                        help="""A directory to store cached results from this command. If a matching invocation is used and the files specified in --dependencies and --output-files are up-to-date, then the cached stdout, stderr and return code is used and the command is not run again.""")
    parser.add_argument("--use-hashes",
                        action="store_true",
                        help="""Use hash comparison in order to determine if dependencies have changed since the last invocation of the job. This method is slower, but can withstand files being copied or moved.""")
    namespace = parser.parse_args(args)
    stamp_directory = namespace.stamp_directory
    if namespace.use_hashes:
        method = jobstamp.HashMethod
    else:
        method = jobstamp.MTimeMethod

    result = jobstamp.run(_run_cmd,
                          cmd,
                          jobstamps_dependencies=namespace.dependencies,
                          jobstamps_output_files=namespace.output_files,
                          jobstamps_cache_output_directory=stamp_directory,
                          jobstamps_method=method)

    sys.stdout.write(result["stdout"].decode())
    sys.stderr.write(result["stderr"].decode())
    return result["code"]
673022
def submit(args=None):
    
    streamsx._streams._version._mismatch_check('streamsx.topology.context')
    cmd_args = _parse_args(args)
    if cmd_args.topology is not None:
        app = _get_topology_app(cmd_args)
    elif cmd_args.main_composite is not None:
        app = _get_spl_app(cmd_args)
    elif cmd_args.bundle is not None:
        app = _get_bundle(cmd_args)
    _job_config_args(cmd_args, app)
    sr = _submit(cmd_args, app)
    if 'return_code' not in sr:
        sr['return_code'] = 1;
    print(sr)
    return sr
664315
def submit_job(job_ini, username, hazard_job_id=None):
    
    job_id = logs.init('job')
    oq = engine.job_from_file(
        job_ini, job_id, username, hazard_calculation_id=hazard_job_id)
    pik = pickle.dumps(oq, protocol=0)  # human readable protocol
    code = RUNCALC % dict(job_id=job_id, hazard_job_id=hazard_job_id, pik=pik,
                          username=username)
    tmp_py = gettemp(code, suffix='.py')
    # print(code, tmp_py)  # useful when debugging
    devnull = subprocess.DEVNULL
    popen = subprocess.Popen([sys.executable, tmp_py],
                             stdin=devnull, stdout=devnull, stderr=devnull)
    threading.Thread(target=popen.wait).start()
    logs.dbcmd('update_job', job_id, {'pid': popen.pid})
    return job_id, popen.pid
493826
public static void main(final String[] args) {
    LOG.log(Level.INFO, "Entering Launch at :::" + new Date());
    try {
      if (args == null || args.length == 0) {
        throw new IllegalArgumentException("No arguments provided, at least a clrFolder should be supplied.");
      }
      final File dotNetFolder = new File(args[0]).getAbsoluteFile();
      final String[] removedArgs = Arrays.copyOfRange(args, 1, args.length);

      final Configuration config = getClientConfiguration(removedArgs);
      final Injector commandLineInjector = Tang.Factory.getTang().newInjector(parseCommandLine(removedArgs));
      final int waitTime = commandLineInjector.getNamedInstance(WaitTimeForDriver.class);
      final int driverMemory = commandLineInjector.getNamedInstance(DriverMemoryInMb.class);
      final boolean isLocal = commandLineInjector.getNamedInstance(Local.class);
      final String driverIdentifier = commandLineInjector.getNamedInstance(DriverIdentifier.class);
      final String jobSubmissionDirectory = commandLineInjector.getNamedInstance(DriverJobSubmissionDirectory.class);
      final boolean submit = commandLineInjector.getNamedInstance(Submit.class);
      final Injector injector = Tang.Factory.getTang().newInjector(config);
      final JobClient client = injector.getInstance(JobClient.class);
      client.setDriverInfo(driverIdentifier, driverMemory, jobSubmissionDirectory);

      if (submit) {
        client.submit(dotNetFolder, true, isLocal, null);
        client.waitForCompletion(waitTime);
      } else {
        client.submit(dotNetFolder, false, isLocal, config);
        client.waitForCompletion(0);
      }


      LOG.info("Done!");
    } catch (final BindException | InjectionException | IOException ex) {
      LOG.log(Level.SEVERE, "Job configuration error", ex);
    }
  }
807958
public int run(final String[] args) throws Exception {

        String input = args[0];
        String output = args[1];

        Configuration conf = super.getConf();

        writeInput(conf, new Path(input));

        setupSecondarySort(conf);

        Job job = new Job(conf);
        job.setJarByClass(SecondarySort.class);
        job.setMapperClass(Map.class);
        job.setReducerClass(Reduce.class);

        job.setMapOutputKeyClass(Tuple.class);
        job.setMapOutputValueClass(Text.class);

        Path outputPath = new Path(output);

        FileInputFormat.setInputPaths(job, input);
        FileOutputFormat.setOutputPath(job, outputPath);

        if (job.waitForCompletion(true)) {
            return 0;
        }
        return 1;
    }
463835
def submit(args):
    gpus = args.gpus.strip().split(',')
    """"""
    def mthread_submit(nworker, nserver, envs):
        """
        customized submit script, that submit nslave jobs, each must contain args as parameter
        note this can be a lambda function containing additional parameters in input

        Parameters
        ----------
        nworker: number of slave process to start up
        nserver: number of server nodes to start up
        envs: enviroment variables to be added to the starting programs
        """
        procs = {}
        for i, gpu in enumerate(gpus):
            for j in range(args.num_threads):
                procs[i] = Thread(target=exec_cmd, args=(args.command + ['--gpus=%s'%gpu], 'worker', i*args.num_threads+j, envs))
                procs[i].setDaemon(True)
                procs[i].start()
        for i in range(len(gpus)*args.num_threads, len(gpus)*args.num_threads + nserver):
            procs[i] = Thread(target=exec_cmd, args=(args.command, 'server', i, envs))
            procs[i].setDaemon(True)
            procs[i].start()

    # call submit, with nslave, the commands to run each job and submit function
    tracker.submit(args.num_threads*len(gpus), args.num_servers, fun_submit=mthread_submit,
                   pscmd=(' '.join(args.command)))
1078587
def build_job_configs(self, args):
        
        job_configs = {}

        components = Component.build_from_yamlfile(args['comp'])
        NAME_FACTORY.update_base_dict(args['data'])

        ret_dict = make_catalog_comp_dict(library=args['library'],
                                          basedir=NAME_FACTORY.base_dict['basedir'])
        catalog_info_dict = ret_dict['catalog_info_dict']

        for catalog_name  in catalog_info_dict:
            for comp in components:
                zcut = "zmax%i" % comp.zmax
                key = comp.make_key('{ebin_name}_{evtype_name}')
                name_keys = dict(zcut=zcut,
                                 sourcekey=catalog_name,
                                 ebin=comp.ebin_name,
                                 psftype=comp.evtype_name,
                                 coordsys=comp.coordsys,
                                 irf_ver=NAME_FACTORY.irf_ver(),
                                 mktime='none',
                                 fullpath=True)

                outfile = NAME_FACTORY.srcmaps(**name_keys)
                outfile_tokens = os.path.splitext(outfile)
                infile_regexp = "%s_*.fits" % outfile_tokens[0]
                logfile = make_nfs_path(outfile.replace('.fits', '.log'))
                job_configs[key] = dict(output=outfile,
                                        args=infile_regexp,
                                        logfile=logfile)

        return job_configs
835539
def _parse_update_snapshot_schedule(cls, args):
        
        argparser = ArgumentParser(prog="cluster snapshot_schedule")

        group = argparser.add_mutually_exclusive_group(required=True)
        group.add_argument("--id", dest="cluster_id",
                          help="execute on cluster with this id")
        group.add_argument("--label", dest="label",
                          help="execute on cluster with this label")

        argparser.add_argument("--frequency-num",
                          help="frequency number")
        argparser.add_argument("--frequency-unit",
                          help="frequency unit")
        argparser.add_argument("--s3-location",
                          help="s3_location about where to store snapshots")
        argparser.add_argument("--status",
                          help="status of periodic job you want to change to", choices = ["RUNNING", "SUSPENDED"])

        arguments = argparser.parse_args(args)

        return arguments
68816
def set_args(self, args, unknown_args=None):
        
        if unknown_args is None:
            unknown_args = []
        self.logger.setLevel(getattr(logging, args.log_level))

        parent = hdfs.path.dirname(hdfs.path.abspath(args.output.rstrip("/")))
        self.remote_wd = hdfs.path.join(
            parent, utils.make_random_str(prefix="pydoop_submit_")
        )
        self.remote_exe = hdfs.path.join(self.remote_wd, str(uuid.uuid4()))
        self.properties[JOB_NAME] = args.job_name or 'pydoop'
        self.properties[IS_JAVA_RR] = (
            'false' if args.do_not_use_java_record_reader else 'true'
        )
        self.properties[IS_JAVA_RW] = (
            'false' if args.do_not_use_java_record_writer else 'true'
        )
        self.properties[JOB_REDUCES] = args.num_reducers
        if args.job_name:
            self.properties[JOB_NAME] = args.job_name
        self.properties.update(args.job_conf or {})
        self.__set_files_to_cache(args)
        self.__set_archives_to_cache(args)
        self.requested_env = self._env_arg_to_dict(args.set_env or [])
        self.args = args
        self.unknown_args = unknown_args
1005571
def build_job_configs(self, args):
        
        job_configs = {}

        components = Component.build_from_yamlfile(args['comp'])
        NAME_FACTORY.update_base_dict(args['data'])

        # FIXME
        mktime = args['mktimefilter']

        for comp in components:
            for sourcekey in args['sourcekeys']:
                zcut = "zmax%i" % comp.zmax
                key = comp.make_key('{ebin_name}_{evtype_name}') + "_%s" % sourcekey
                name_keys = dict(zcut=zcut,
                                 ebin=comp.ebin_name,
                                 psftype=comp.evtype_name,
                                 irf_ver=NAME_FACTORY.irf_ver(),
                                 sourcekey=sourcekey,
                                 mktime=mktime,
                                 coordsys=comp.coordsys,
                                 fullpath=True)
                outfile = NAME_FACTORY.template_sunmoon(**name_keys)
                job_configs[key] = dict(expsun=NAME_FACTORY.bexpcube_sun(**name_keys),
                                        avgexp=NAME_FACTORY.bexpcube(**name_keys),
                                        sunprof=NAME_FACTORY.angprofile(**name_keys),
                                        cmap='none',
                                        outfile=outfile,
                                        irfs=NAME_FACTORY.irfs(**name_keys),
                                        evtype=comp.evtype,
                                        emin=comp.emin,
                                        emax=comp.emax,
                                        enumbins=comp.enumbins,
                                        logfile=outfile.replace('.fits', '.log'))

        return job_configs
1238823
def main():
    '''
    
    '''

    parser = argparse.ArgumentParser()

    # add parser arguments
    parser.add_argument('--input_path',
                        help='The full path to the input SAM/BAM/ADAM/FASTQ file.')
    parser.add_argument('--output-path',
                        help='full path where final results will be output.')
    parser.add_argument('--kmer-length',
                        help='Length to use for k-mer counting. Defaults to 20.',
                        default=20,
                        type=int)
    parser.add_argument('--spark-conf',
                        help='Optional configuration to pass to Spark commands. Either this or --workers must be specified.',
                        default=None)
    parser.add_argument('--memory',
                        help='Optional memory configuration for Spark workers/driver. This must be specified if --workers is specified.',
                        default=None,
                        type=int)
    parser.add_argument('--cores',
                        help='Optional core configuration for Spark workers/driver. This must be specified if --workers is specified.',
                        default=None,
                        type=int)
    parser.add_argument('--workers',
                        help='Number of workers to spin up in Toil. Either this or --spark-conf must be specified. If this is specified, --memory and --cores must be specified.',
                        default=None,
                        type=int)
    parser.add_argument('--sudo',
                        help='Run docker containers with sudo. Defaults to False.',
                        default=False,
                        action='store_true')

    Job.Runner.addToilOptions(parser)
    args = parser.parse_args()
    Job.Runner.startToil(Job.wrapJobFn(kmer_dag,
                                       args.kmer_length,
                                       args.input_path,
                                       args.output_path,
                                       args.spark_conf,
                                       args.workers,
                                       args.cores,
                                       args.memory,
                                       args.sudo,
                                       checkpoint=True), args)
807133
private int run(String[] args) throws Exception {
    Configuration conf = new Configuration();
    String numMaps =  new GenericOptionsParser(conf, args).getRemainingArgs()[0];
    conf.set(MRJobConfig.NUM_MAPS, numMaps);
    createHdfsFilesystem(conf);

    Job job = Job.getInstance(conf, "MapReduceIntegrationChecker");
    job.setJarByClass(MapReduceIntegrationChecker.class);
    job.setMapperClass(CheckerMapper.class);
    job.setCombinerClass(CheckerReducer.class);
    job.setReducerClass(CheckerReducer.class);
    job.setOutputKeyClass(Text.class);
    job.setOutputValueClass(Text.class);
    job.setInputFormatClass(EmptyInputFormat.class);
    FileOutputFormat.setOutputPath(job, mOutputFilePath);

    try {
      if (!job.waitForCompletion(true)) {
        return 1;
      }
      Status resultStatus = generateReport();
      return resultStatus.equals(Status.SUCCESS) ? 0
          : (resultStatus.equals(Status.FAIL_TO_FIND_CLASS) ? 2 : 1);
    } finally {
      if (mFileSystem.exists(mOutputFilePath)) {
        mFileSystem.delete(mOutputFilePath, true);
      }
      mFileSystem.close();
    }
  }
844055
def build_job_configs(self, args):
        
        job_configs = {}

        components = Component.build_from_yamlfile(args['comp'])
        NAME_FACTORY.update_base_dict(args['data'])

        if self._comp_dict is None or self._comp_dict_file != args['library']:
            self._comp_dict_file = args['library']
            self._comp_dict = make_catalog_comp_dict(sources=self._comp_dict_file,
                                                     basedir=NAME_FACTORY.base_dict['basedir'])
        else:
            print ("Using cached catalog dict from %s" % args['library'])

        catalog_info_dict = self._comp_dict['catalog_info_dict']
        comp_info_dict = self._comp_dict['comp_info_dict']

        n_src_per_job = args['nsrc']

        if args['make_xml']:
            SrcmapsCatalog_SG._make_xml_files(catalog_info_dict, comp_info_dict)

        for catalog_name, catalog_info in catalog_info_dict.items():

            n_cat_src = len(catalog_info.catalog.table)
            n_job = int(math.ceil(float(n_cat_src) / n_src_per_job))

            for comp in components:
                zcut = "zmax%i" % comp.zmax
                key = comp.make_key('{ebin_name}_{evtype_name}')
                name_keys = dict(zcut=zcut,
                                 sourcekey=catalog_name,
                                 ebin=comp.ebin_name,
                                 psftype=comp.evtype_name,
                                 coordsys=comp.coordsys,
                                 irf_ver=NAME_FACTORY.irf_ver(),
                                 mktime='none',
                                 fullpath=True)

                for i_job in range(n_job):
                    full_key = "%s_%02i" % (key, i_job)
                    srcmin = i_job * n_src_per_job
                    srcmax = min(srcmin + n_src_per_job, n_cat_src)
                    outfile = NAME_FACTORY.srcmaps(
                        **name_keys).replace('.fits', "_%02i.fits" % (i_job))
                    logfile = make_nfs_path(outfile.replace('.fits', '.log'))
                    job_configs[full_key] = dict(cmap=NAME_FACTORY.ccube(**name_keys),
                                                 expcube=NAME_FACTORY.ltcube(**name_keys),
                                                 irfs=NAME_FACTORY.irfs(**name_keys),
                                                 bexpmap=NAME_FACTORY.bexpcube(**name_keys),
                                                 outfile=outfile,
                                                 logfile=logfile,
                                                 srcmdl=catalog_info.srcmdl_name,
                                                 evtype=comp.evtype,
                                                 srcmin=srcmin,
                                                 srcmax=srcmax)

        return job_configs
221046
def execute(self, *, args: Union[list, tuple], options: dict) -> tuple:
        ''''''
        cmd = self._build_cmd(args)
        process = subprocess.Popen(cmd, stdout=PIPE, stderr=PIPE, stdin=PIPE,
                                   encoding='utf-8', shell=options.get('shell', False), env=options.get('env'))
        return process
695199
def _parse_args(args):
    
    cmd_parser = argparse.ArgumentParser(description='Execute a Streams application using a Streaming Analytics service.')

    ctx_group = cmd_parser.add_mutually_exclusive_group(required=True)
    ctx_group.add_argument('--service-name', help='Submit to Streaming Analytics service')
    ctx_group.add_argument('--create-bundle', action='store_true', help='Create a bundle using a local IBM Streams install. No job submission occurs.')

    app_group = cmd_parser.add_mutually_exclusive_group(required=True)
    app_group.add_argument('--topology', help='Topology to call')
    app_group.add_argument('--main-composite', help='SPL main composite')
    app_group.add_argument('--bundle', help="Streams application bundle (sab file) to submit to service")

    bld_group = cmd_parser.add_argument_group('Build options', 'Application build options')
    bld_group.add_argument('--toolkits', nargs='+', help='SPL toolkit containing the main composite and any other required SPL toolkits.')

    _define_jco_args(cmd_parser)

    cmd_args = cmd_parser.parse_args(args)
    return cmd_args
305672
def get_args(job):
    
    return tuple(_get_args_loop(job, INPUT_FIELD)), \
        tuple(_get_args_loop(job, OPTIONAL_FIELD))