Main Hebcal function
503020
function Hebcal(year, month) {
	var me = this; // whenever this is done, it is for optimizations.
	if (!year) {
		year = (new HDate())[getFullYear](); // this year;
	}
	if (typeof year !== 'number') {
		throw new TE('year to Hebcal() is not a number');
	}
	me.year = year;
	if (month) {
		if (typeof month == 'string') {
			month = c.monthFromName(month);
		}
		if (typeof month == 'number') {
			month = [month];
		}

		if (Array.isArray(month)) {
			me.months = month[map](function(i){
				var m = new Month(i, year);
				defProp(m, '__year', {
					configurable: true,
					writable: true,
					value: me
				});
				return m;
			});

			me.holidays = holidays.year(year);
		} else {
			throw new TE('month to Hebcal is not a valid type');
		}
	} else {
		return new Hebcal(year, c.range(1, c.MONTH_CNT(year)));
	}

	me[length] = c.daysInYear(year);

	defProp(me, 'il', getset(function() {
		return me[getMonth](1).il;
	}, function(il) {
		me.months.forEach(function(m){
			m.il = il;
		});
	}));

	defProp(me, 'lat', getset(function() {
		return me[getMonth](1).lat;
	}, function(lat) {
		me.months.forEach(function(m){
			m.lat = lat;
		});
	}));
	defProp(me, 'long', getset(function() {
		return me[getMonth](1).long;
	}, function(lon) {
		me.months.forEach(function(m){
			m.long = lon;
		});
	}));
}
5515
public function main()
    {
        LibraryLoader::includeAll();

        $this->doc = GeneralUtility::makeInstance(DocumentTemplate::class);
        $this->doc->backPath = $GLOBALS['BACK_PATH'];

        if ($GLOBALS['BE_USER']->user['admin']) {
            $this->doc->bodyTagAdditions = 'id="doc3"';

            $this->addAdditionalHeaderData();

            $this->cleanOutputBuffers();
            $this->outputService->output(
                $this->doc->startPage($this->translate('title')) .
                $this->doc->header(\PHPUnit_Runner_Version::getVersionString())
            );

            $this->renderRunTests();

            $this->outputService->output(
                $this->doc->section(
                    $this->translate('shortcuts.title'),
                    '<p>' . $this->translate('shortcuts.text') . '</p>
                    <p>' . $this->translate('shortcuts.browser_dependency') . '</p>
                    <ul>
                    <li>' . $this->translate('shortcuts.browser_safari_ie') . '</li>
                    <li>' . $this->translate('shortcuts.browser_firefox') . '</li>
                    </ul>'
                )
            );
        } else {
            $this->outputService->output(
                $this->doc->startPage($this->translate('title')) .
                $this->doc->header($this->translate('title')) .
                $this->translate('admin_rights_needed')
            );
        }

        $this->outputService->output($this->doc->endPage());
    }
1194980
public function main()
    {
        $this->validateAttributes();

        $filesToExtract = [];
        if ($this->file !== null) {
            if ($this->forceExtract || !$this->isDestinationUpToDate($this->file)) {
                $filesToExtract[] = $this->file;
            } else {
                $this->log(
                    'Nothing to do: ' . $this->todir->getAbsolutePath() . ' is up to date for ' . $this->file->getCanonicalPath(),
                    Project::MSG_INFO
                );
            }
        }

        foreach ($this->filesets as $compressedArchiveFileset) {
            $compressedArchiveDirScanner = $compressedArchiveFileset->getDirectoryScanner($this->project);
            $compressedArchiveFiles = $compressedArchiveDirScanner->getIncludedFiles();
            $compressedArchiveDir = $compressedArchiveFileset->getDir($this->project);

            foreach ($compressedArchiveFiles as $compressedArchiveFilePath) {
                $compressedArchiveFile = new PhingFile($compressedArchiveDir, $compressedArchiveFilePath);
                if ($compressedArchiveFile->isDirectory()) {
                    throw new BuildException(
                        $compressedArchiveFile->getAbsolutePath() . ' compressed archive cannot be a directory.'
                    );
                }

                if ($this->forceExtract || !$this->isDestinationUpToDate($compressedArchiveFile)) {
                    $filesToExtract[] = $compressedArchiveFile;
                } else {
                    $this->log(
                        'Nothing to do: ' . $this->todir->getAbsolutePath() . ' is up to date for ' . $compressedArchiveFile->getCanonicalPath(),
                        Project::MSG_INFO
                    );
                }
            }
        }

        foreach ($filesToExtract as $compressedArchiveFile) {
            $this->extractArchive($compressedArchiveFile);
        }
    }
1218815
def main(pub_port=None, sub_port=None):
    '''
    '''
    try:
        if sub_port is None:
            sub_port = get_sub_port()
        if pub_port is None:
            pub_port = get_pub_port()
        context = zmq.Context(1)
        frontend = context.socket(zmq.SUB)
        backend = context.socket(zmq.PUB)

        frontend.bind('tcp://*:{pub_port}'.format(pub_port=pub_port))
        frontend.setsockopt(zmq.SUBSCRIBE, b'')
        backend.bind('tcp://*:{sub_port}'.format(sub_port=sub_port))
        zmq.device(zmq.FORWARDER, frontend, backend)
    except KeyboardInterrupt:
        pass
    finally:
        frontend.close()
        backend.close()
        context.term()
1286769
def main():

    parser = argparse.ArgumentParser(description="An interface to CarbonBlack environments")

    #profiles = auth.CredentialStore("response").get_profiles()
    parser.add_argument('-e', '--environment', choices=auth.CredentialStore("response").get_profiles(),
                        help='specify a specific instance you want to work with. If not defined \'-t production\' will be used implicitly.')
    parser.add_argument('-t', '--envtypes', type=str, 
                        help='specify any combination of envtypes. Default=All \'production\' envtypes. Ignored if -e is set.',
                        default='production')
    #parser.add_argument('--debug', action='store_true', help='print debugging info')
    #parser.add_argument('--warnings', action='store_true',
    #                         help="Warn before printing large executions")

    subparsers = parser.add_subparsers(dest='command') #title='subcommands', help='additional help')
    cbinterface_commands = [ 'query', 'proc', 'collect', 'remediate', 'enumerate_usb', 'vxdetect']

    parser_vx = subparsers.add_parser('vxdetect', help="search cbsandbox for processes in vxstream report, show detections")
    parser_vx.add_argument('vxstream_report', help='path to vxstream report')
    parser_vx.add_argument('-p', '--print-process-tree', action='store_true', help='print the process tree')

    parser_usb = subparsers.add_parser('enumerate_usb', help="Show recent removable drive activity on the sensor")
    parser_usb.add_argument('sensor', help='hostname of the sensor')
    parser_usb.add_argument('-s', '--start-time', action='store',
                            help='how far back to query (default:ALL time)')

    parser_proc = subparsers.add_parser('proc', help="analyze a process GUID. 'proc -h' for more")
    parser_proc.add_argument('process', help="the process GUID to analyze")
    parser_proc.add_argument('--warnings', action='store_true',
                             help="Warn before printing large executions")
    parser_proc.add_argument('-w', '--walk-tree', action='store_true',
                             help="walk and analyze the process tree")
    parser_proc.add_argument('-wp', '--walk-parents', action='store_true',
                             help="print details on the process ancestry")
    #parser_proc.add_argument('-d', '--detection', action='store_true',
    #                         help="show detections that would result in ACE alerts")
    parser_proc.add_argument('-i', '--proc-info', action='store_true',
                             help="show binary and process information")
    parser_proc.add_argument('-c','--show-children', action='store_true',
                             help="only print process children event details")
    parser_proc.add_argument('-nc', '--netconns', action='store_true',
                             help="print network connections")
    parser_proc.add_argument('-fm', '--filemods', action='store_true',
                             help="print file modifications")
    parser_proc.add_argument('-rm', '--regmods', action='store_true',
                             help="print registry modifications")
    parser_proc.add_argument('-um', '--unsigned-modloads', action='store_true',
                             help="print unsigned modloads")
    parser_proc.add_argument('-ml', '--modloads', action='store_true',
                             help="print modloads")
    parser_proc.add_argument('-cp', '--crossprocs', action='store_true',
                             help="print crossprocs")
    #parser_proc.add_argument('-intel', '--intel-hits', action='store_true',
    #                         help="show intel (feed/WL) hits that do not result in ACE alerts")
    parser_proc.add_argument('--no-analysis', action='store_true',
                             help="Don't fetch and print process activity")
    parser_proc.add_argument('--json', action='store_true', help='output process summary in json')
    parser_proc.add_argument('--segment-limit', action='store', type=int, default=None,
                             help='stop processing events into json after this many process segments')

    facet_args = [
        'process_name', 'childproc_name', 'username', 'parent_name', 'path', 'hostname',
        'parent_pid', 'comms_ip', 'process_md5', 'start', 'group', 'interface_ip',
        'modload_count', 'childproc_count', 'cmdline', 'regmod_count', 'process_pid',
        'parent_id', 'os_type', 'rocessblock_count', 'crossproc_count', 'netconn_count',
        'parent_md5', 'host_type', 'last_update', 'filemod_count'
        ]
 
    parser_query = subparsers.add_parser('query',
                                         help="execute a process search query. 'query -h' for more")
    parser_query.add_argument('query', help="the process search query you'd like to execute")
    parser_query.add_argument('-s', '--start-time', action='store',
                              help="Only return processes with events after given date/time stamp\
 (server’s clock). Format:'Y-m-d H:M:S' eastern time")
    parser_query.add_argument('-e', '--end-time', action='store',
                              help="Set the maximum last update time. Format:'Y-m-d H:M:S' eastern time")
    parser_query.add_argument('--facet', action='store', choices=facet_args,
                              help='stats info on single field accross query results (ex. process_name)')
    parser_query.add_argument('--no-warnings', action='store_true',
                             help="Don't warn before printing large query results")
    parser_query.add_argument('-lh', '--logon-history', action='store_true', help="Display available logon history for given username or hostname")

    parser_collect = subparsers.add_parser('collect', help='perform LR collection tasks on a host')
    parser_collect.add_argument('sensor', help="the hostname/sensor to collect from")
    parser_collect.add_argument('-f', '--filepath', action='store', help='collect file')
    parser_collect.add_argument('-c', '--command-exec', action='store', help='command to execute')
    parser_collect.add_argument('-p', '--process-list', action='store_true', 
                                help='show processes running on sensor')
    parser_collect.add_argument('-m', '--memdump', action='store', const='ALLMEM', nargs='?',
                                help='dump memory on a specific process-id')
    parser_collect.add_argument('-lr', '--regkeypath', action='store',
                                help='List all registry values from the specified registry key.')
    parser_collect.add_argument('-r', '--regkeyvalue', action='store',
                                help='Returns the associated value of the specified registry key.')
    parser_collect.add_argument('-i', '--info', action='store_true', help='print sensor information')
    parser_collect.add_argument('-gst', '--get-task', action='store_true', help='get scheduled tasks or specifc task')
    parser_collect.add_argument('-mc', '--multi-collect', action='store', help='path to ini file listing files and regs to collect')

    remediate_file_example = """Example remediate ini file:
    [files]
    file1=C:\\Users\\user\\Desktop\\testfile.txt 

    [process_names]
    proc1=cmd.exe
    proc2=notepad++.exe
     
    [directories]
    directory1=C:\\Users\\user\\Desktop\\nanocore
     
    [scheduled_tasks]
    task1=\\monkey_task
    task1=\\Microsoft\\windows\\some\\flake\\task

    [pids]
    pid1=10856
     
    [registry_paths]
    reg1=HKLM\\Software\\Microsoft\\Windows\\CurrentVersion\\Run\\calc
    reg2=HKLM\\Software\\Microsoft\\Windows\\CurrentVersion\\Run\\hippo"""

    parser_remediate = subparsers.add_parser('remediate', help='remediate a host')
    parser_remediate.add_argument('sensor', help="the hostname/sensor needing remediation")
    parser_remediate.add_argument('-i', '--isolate', help='toggle host isolation', default=False, action='store_true')
    parser_remediate.add_argument('-f', '--remediation-filepath',
                                  help="path to the remediation ini file; 'help' as the filepath for example")
    parser_remediate.add_argument('-dst', '--delete-scheduled-task',
                                  help="path of scheduled task to delete")
    parser_remediate.add_argument('-kpname', '--kill-process-name', help="kill all processes with this name")
    parser_remediate.add_argument('-kpid', '--kill-pid', help="a process id to kill")
    parser_remediate.add_argument('-df', '--delete-file', help="path to file needing deletion")
    parser_remediate.add_argument('-dr', '--delete-regkey', help="path to regkey value needing deletion")
    parser_remediate.add_argument('-dd', '--delete-directory', help="path to directory needing deletion")
    args = parser.parse_args()

    if args.command == 'remediate' and args.remediation_filepath == 'help':
        print(remediate_file_example)
        parser.parse_args(['remediate', '-h'])

    if args.command is None:
        print("\n\n*****")
        print("You must specify one of the following commands:\n")
        print(cbinterface_commands)
        print("\n*****\n\n")
        parser.parse_args(['-h'])

    #args.debug = True
    #if args.debug:
    # configure some more logging
    root = logging.getLogger()
    root.addHandler(logging.StreamHandler())
    logging.getLogger("cbapi").setLevel(logging.ERROR)
    logging.getLogger("lerc_api").setLevel(logging.WARNING)

    '''  '''
    if args.command == 'vxdetect':
        cb = CbResponseAPI(profile='vxstream')
        process_list = parse_vxstream_report(cb, args.vxstream_report)
        if args.print_process_tree:
            print()
            print(process_list)
        print()
        return 0


    # Set up environment profiles
    profile = None
    profiles = []
    if args.environment: 
        print("Using {} environment ..".format(args.environment))
        profiles.append(args.environment)
    else:
        # a little hack for getting our environment type variable defined
        default_profile = auth.default_profile
        default_profile['envtype'] = 'production'
        query_envtype = set(args.envtypes.lower().split(','))
        for profile in auth.CredentialStore("response").get_profiles():
            credentials = auth.CredentialStore("response").get_credentials(profile=profile)
            profile_envtype = set(credentials['envtype'].lower().split(','))
            if(query_envtype.issubset(profile_envtype)):
                profiles.append(profile)
        
        


    # Process Quering #
    if args.command == 'query':
        for profile in profiles:
            handle_proxy(profile)
            print("\nSearching {} environment..".format(profile))
            q = CBquery(profile=profile)
            q.process_query(args)
        return 0


    # Select correct environment by sensor hostname and get the sensor object
    sensor = None
    if args.command == 'collect' or args.command == 'remediate' or args.command == 'enumerate_usb':
        cb_results = sensor_search(profiles, args.sensor)
        if not isinstance(cb_results, list):
            # an error occured
            return cb_results
        else:
            if not cb_results:
                LOGGER.info("A sensor with hostname {} wasn't found in any environments".format(args.sensor))
                return 0
            elif len(cb_results) > 1:
                LOGGER.error("A sensor by hostname {} was found in multiple environments".format(args.sensor))
                for r in cb_results:
                    print("Results:")
                    print("Profile {}: {} (SID:{})".format(r[1],r[0].hostname,r[0].id))
                return 1
            results = cb_results[0]
            profile = results[1]
            sensor = results[0]


    # Show USB Regmod activity
    if args.command == 'enumerate_usb':
        enumerate_usb(sensor, args.start_time)


    # lerc install arguments can differ by company/environment
    # same lazy hack to define in cb config
    config = {}
    try:
        default_profile = auth.default_profile
        default_profile['lerc_install_cmd'] = None
        config = auth.CredentialStore("response").get_credentials(profile=profile)
    except:
        pass


    # Collection #
    if args.command == 'collect':
        hyper_lr = hyperLiveResponse(sensor)

        if args.info:
            print(hyper_lr)
            return True

        # start a cb lr session
        lr_session = hyper_lr.go_live()

        if args.multi_collect:
            filepaths = regpaths = full_collect = None
            config = ConfigParser()
            config.read(args.multi_collect)
            try:
                filepaths = config.items("files")
            except:
                filepaths = []
            try:
                regpaths = config.items("registry_paths")
            except:
                regpaths = []
            try:
                full_collect = config.get('full_collect', 'action')
            except:
                pass

            if regpaths is not None:
                for regpath in regpaths:
                    if isinstance(regpath, tuple):
                        regpath = regpath[1]
                    print("~ Trying to get {}".format(regpath))
                    try:
                        result = lr_session.get_registry_value(regpath)
                        if result:
                            localfname = args.sensor + '_regkey_' + result['value_name'] + ".txt"
                            with open(localfname,'wb') as f:
                                f.write(bytes(result['value_data'], 'UTF-8'))
                            print("\t+ Data written to: {}".format(localfname))
                    except Exception as e:
                        print("[!] Error: {}".format(str(e)))
            if filepaths is not None:
                for filepath in filepaths:
                    try:
                        hyper_lr.getFile_with_timeout(filepath[1])
                    except Exception as e:
                        print("[!] Error: {}".format(str(e)))
            if full_collect == 'True':
               return False #LR_collection(hyper_lr, args)
            return True

        elif args.filepath:
            hyper_lr.getFile_with_timeout(args.filepath)

        elif args.process_list:
            hyper_lr.print_processes()

        elif args.memdump:
            # get config items
            config = ConfigParser()
            config.read(CONFIG_PATH)
            #if config.has_section('memory'):
            #    if 
            cb_compress = config['memory'].getboolean('cb_default_compress')
            custom_compress = config['memory'].getboolean('custom_compress')
            custom_compress_file = config['memory']['custom_compress_file']
            auto_collect_mem = config['memory'].getboolean('auto_collect_mem_file')
            lerc_collect_mem = config['memory'].getboolean('lerc_collect_mem')
            path_to_procdump = config['memory']['path_to_procdump']
            
            if args.memdump == "ALLMEM":
                return hyper_lr.dump_sensor_memory(cb_compress=cb_compress, custom_compress=custom_compress,
                                                   custom_compress_file=custom_compress_file,
                                                   auto_collect_result=auto_collect_mem)
            else:
                return hyper_lr.dump_process_memory(args.memdump, path_to_procdump=path_to_procdump)

        elif args.command_exec:
            print("executing '{}' on {}".format(args.command_exec, args.sensor))
            result = lr_session.create_process(args.command_exec, wait_timeout=60, wait_for_output=True)
            print("\n-------------------------")
            result = result.decode('utf-8')
            print(result + "\n-------------------------")
            print()

        elif args.regkeypath:
            print("\n\t{}".format(args.regkeypath))
            results = lr_session.list_registry_keys(args.regkeypath)
            for result in results:
                print("\t-------------------------")
                print("\tName: {}".format(result['value_name']))
                print("\tType: {}".format(result['value_type']))
                print("\tData: {}".format(result['value_data']))
                print()

        elif args.regkeyvalue:
            print("\n\t{}".format(args.regkeyvalue))
            result = lr_session.get_registry_value(args.regkeyvalue)
            print("\t-------------------------")
            print("\tName: {}".format(result['value_name']))
            print("\tType: {}".format(result['value_type']))
            print("\tData: {}".format(result['value_data']))
            print()

        elif args.get_task:
            return hyper_lr.get_scheduled_tasks()

        else:
            # perform full live response collection
            if config['lerc_install_cmd']:
                result = hyper_lr.get_lerc_status()
                if not result or result == 'UNINSTALLED' or result == 'UNKNOWN':
                   if not hyper_lr.deploy_lerc(config['lerc_install_cmd']):
                       LOGGER.warn("LERC deployment failed")
            else:
                LOGGER.info("{} environment is not configrued for LERC deployment".format(profile))
            return LR_collection(hyper_lr, args)

    # Remediation #
    if args.command == 'remediate':
        return Remediation(sensor, args)


    # Process Investigation #
    process_tree = None
    if args.command == 'proc':
        proc = proc_search_environments(profiles, args.process)
        if not proc:
            return 1
        sp = SuperProcess(proc)
        if args.proc_info:
            print(sp)
        elif args.walk_tree:
            sp.walk_process_tree()
            print()
            print(sp.process_tree)

            for process in sp.process_tree:
                if process.is_suppressed:
                    print("+  [DATA SUPPRESSED] {} (PID:{}) - {}".format(process.name, process.pid,
                                                                         process.id))
                    continue

                print("+  {} (PID:{}) - {}".format(process.name, process.pid, process.id))
                if args.filemods:
                    process.print_filemods()
                    args.no_analysis = True
                if args.netconns:
                    process.print_netconns()
                    args.no_analysis = True
                if args.regmods:
                    process.print_regmods()
                    args.no_analysis = True
                if args.unsigned_modloads:
                    process.print_unsigned_modloads()
                    args.no_analysis = True
                if args.modloads:
                    process.print_modloads()
                    args.no_analysis = True
                if args.crossprocs:
                    process.print_crossprocs()
                    args.no_analysis = True
                if args.walk_parents:
                    sp.show_ancestry()
                    args.no_analysis = True
                if args.no_analysis != True:
                    if args.json:
                        if args.segment_limit:
                            print(process.events_to_json(segment_limit=args.segment_limit))
                        else:
                            print(process.events_to_json())
                    else:
                        process.default_print()
        else:
            print()
            print(sp.process_tree)
            if args.walk_parents:
                sp.show_ancestry()
                args.no_analysis = True
            if args.filemods:
                sp.print_filemods()
                args.no_analysis = True
            if args.netconns:
                sp.print_netconns()
                args.no_analysis = True
            if args.regmods:
                sp.print_regmods()
                args.no_analysis = True
            if args.unsigned_modloads:
                sp.print_unsigned_modloads()
                args.no_analysis = True
            if args.modloads:
                sp.print_modloads()
                args.no_analysis = True
            if args.crossprocs:
                sp.print_crossprocs()
                args.no_analysis = True
            if args.show_children:
                sp.print_child_events()
                args.no_analysis = True

            if args.no_analysis != True:
                if args.json:
                    if args.segment_limit:
                        print(sp.events_to_json(segment_limit=args.segment_limit))
                    else:
                        print(sp.events_to_json())
                else:
                    sp.default_print()

        
    print()
    return True
552037
def main():
    
    mlperf_log.ROOT_DIR_GNMT = os.path.dirname(os.path.abspath(__file__))
    mlperf_log.LOGGER.propagate = False

    args = parse_args()
    device = utils.set_device(args.cuda, args.local_rank)
    distributed = utils.init_distributed(args.cuda)
    gnmt_print(key=mlperf_log.RUN_START, sync=True)
    args.rank = utils.get_rank()

    if not args.cudnn:
        torch.backends.cudnn.enabled = False

    # create directory for results
    save_path = os.path.join(args.results_dir, args.save)
    args.save_path = save_path
    os.makedirs(save_path, exist_ok=True)

    # setup logging
    log_filename = f'log_rank_{utils.get_rank()}.log'
    utils.setup_logging(os.path.join(save_path, log_filename))

    if args.env:
        utils.log_env_info()

    logging.info(f'Saving results to: {save_path}')
    logging.info(f'Run arguments: {args}')

    # automatically set train_iter_size based on train_global_batch_size,
    # world_size and per-worker train_batch_size
    if args.train_global_batch_size is not None:
        global_bs = args.train_global_batch_size
        bs = args.train_batch_size
        world_size = utils.get_world_size()
        assert global_bs % (bs * world_size) == 0
        args.train_iter_size = global_bs // (bs * world_size)
        logging.info(f'Global batch size was set in the config, '
                     f'Setting train_iter_size to {args.train_iter_size}')

    worker_seeds, shuffling_seeds = utils.setup_seeds(args.seed, args.epochs,
                                                      device)
    worker_seed = worker_seeds[args.rank]
    logging.info(f'Worker {args.rank} is using worker seed: {worker_seed}')
    torch.manual_seed(worker_seed)

    # build tokenizer
    pad_vocab = utils.pad_vocabulary(args.math)
    tokenizer = Tokenizer(os.path.join(args.dataset_dir, config.VOCAB_FNAME),
                          pad_vocab)

    # build datasets
    gnmt_print(key=mlperf_log.PREPROC_TOKENIZE_TRAINING, sync=False)
    gnmt_print(key=mlperf_log.TRAIN_HP_MAX_SEQ_LEN,
               value=args.max_length_train, sync=False)

    train_data = LazyParallelDataset(
        src_fname=os.path.join(args.dataset_dir, config.SRC_TRAIN_FNAME),
        tgt_fname=os.path.join(args.dataset_dir, config.TGT_TRAIN_FNAME),
        tokenizer=tokenizer,
        min_len=args.min_length_train,
        max_len=args.max_length_train,
        sort=False,
        max_size=args.max_size)

    gnmt_print(key=mlperf_log.PREPROC_NUM_TRAIN_EXAMPLES,
               value=len(train_data), sync=False)

    val_data = ParallelDataset(
        src_fname=os.path.join(args.dataset_dir, config.SRC_VAL_FNAME),
        tgt_fname=os.path.join(args.dataset_dir, config.TGT_VAL_FNAME),
        tokenizer=tokenizer,
        min_len=args.min_length_val,
        max_len=args.max_length_val,
        sort=True)

    gnmt_print(key=mlperf_log.PREPROC_TOKENIZE_EVAL, sync=False)

    test_data = TextDataset(
        src_fname=os.path.join(args.dataset_dir, config.SRC_TEST_FNAME),
        tokenizer=tokenizer,
        min_len=args.min_length_test,
        max_len=args.max_length_test,
        sort=True)

    gnmt_print(key=mlperf_log.PREPROC_NUM_EVAL_EXAMPLES,
               value=len(test_data), sync=False)

    vocab_size = tokenizer.vocab_size
    gnmt_print(key=mlperf_log.PREPROC_VOCAB_SIZE,
               value=vocab_size, sync=False)

    # build GNMT model
    model_config = {'hidden_size': args.hidden_size,
                    'num_layers': args.num_layers,
                    'dropout': args.dropout, 'batch_first': False,
                    'share_embedding': args.share_embedding}
    model = GNMT(vocab_size=vocab_size, **model_config)
    logging.info(model)

    batch_first = model.batch_first

    # define loss function (criterion) and optimizer
    criterion = build_criterion(vocab_size, config.PAD, args.smoothing)

    opt_config = {'optimizer': args.optimizer, 'lr': args.lr}
    opt_config.update(literal_eval(args.optimizer_extra))
    logging.info(f'Training optimizer config: {opt_config}')

    scheduler_config = {'warmup_steps': args.warmup_steps,
                        'remain_steps': args.remain_steps,
                        'decay_interval': args.decay_interval,
                        'decay_steps': args.decay_steps,
                        'decay_factor': args.decay_factor}

    logging.info(f'Training LR schedule config: {scheduler_config}')

    num_parameters = sum([l.nelement() for l in model.parameters()])
    logging.info(f'Number of parameters: {num_parameters}')

    batching_opt = {'shard_size': args.shard_size,
                    'num_buckets': args.num_buckets}
    # get data loaders
    train_loader = train_data.get_loader(batch_size=args.train_batch_size,
                                         seeds=shuffling_seeds,
                                         batch_first=batch_first,
                                         shuffle=True,
                                         batching=args.batching,
                                         batching_opt=batching_opt,
                                         num_workers=args.train_loader_workers)

    gnmt_print(key=mlperf_log.INPUT_BATCH_SIZE,
               value=args.train_batch_size * utils.get_world_size(),
               sync=False)
    gnmt_print(key=mlperf_log.INPUT_SIZE,
               value=train_loader.sampler.num_samples, sync=False)

    val_loader = val_data.get_loader(batch_size=args.val_batch_size,
                                     batch_first=batch_first,
                                     shuffle=False,
                                     num_workers=args.val_loader_workers)

    test_loader = test_data.get_loader(batch_size=args.test_batch_size,
                                       batch_first=batch_first,
                                       shuffle=False,
                                       pad=True,
                                       num_workers=args.test_loader_workers)

    gnmt_print(key=mlperf_log.EVAL_SIZE,
               value=len(test_loader.dataset), sync=False)

    translator = Translator(model=model,
                            tokenizer=tokenizer,
                            loader=test_loader,
                            beam_size=args.beam_size,
                            max_seq_len=args.max_length_test,
                            len_norm_factor=args.len_norm_factor,
                            len_norm_const=args.len_norm_const,
                            cov_penalty_factor=args.cov_penalty_factor,
                            cuda=args.cuda,
                            print_freq=args.print_freq,
                            dataset_dir=args.dataset_dir,
                            target_bleu=args.target_bleu,
                            save_path=args.save_path)

    # create trainer
    total_train_iters = len(train_loader) // args.train_iter_size * args.epochs
    save_info = {'model_config': model_config, 'config': args, 'tokenizer':
                 tokenizer.get_state()}
    trainer_options = dict(
        criterion=criterion,
        grad_clip=args.grad_clip,
        iter_size=args.train_iter_size,
        save_path=save_path,
        save_freq=args.save_freq,
        save_info=save_info,
        opt_config=opt_config,
        scheduler_config=scheduler_config,
        train_iterations=total_train_iters,
        batch_first=batch_first,
        keep_checkpoints=args.keep_checkpoints,
        math=args.math,
        print_freq=args.print_freq,
        cuda=args.cuda,
        distributed=distributed,
        intra_epoch_eval=args.intra_epoch_eval,
        translator=translator)

    trainer_options['model'] = model
    trainer = trainers.Seq2SeqTrainer(**trainer_options)

    # optionally resume from a checkpoint
    if args.resume:
        checkpoint_file = args.resume
        if os.path.isdir(checkpoint_file):
            checkpoint_file = os.path.join(
                checkpoint_file, 'model_best.pth')
        if os.path.isfile(checkpoint_file):
            trainer.load(checkpoint_file)
        else:
            logging.error(f'No checkpoint found at {args.resume}')

    # training loop
    best_loss = float('inf')
    break_training = False
    test_bleu = None
    gnmt_print(key=mlperf_log.TRAIN_LOOP, sync=True)
    for epoch in range(args.start_epoch, args.epochs):
        logging.info(f'Starting epoch {epoch}')
        gnmt_print(key=mlperf_log.TRAIN_EPOCH,
                   value=epoch, sync=True)

        train_loader.sampler.set_epoch(epoch)

        trainer.epoch = epoch
        train_loss, train_perf = trainer.optimize(train_loader)

        # evaluate on validation set
        if args.eval:
            logging.info(f'Running validation on dev set')
            val_loss, val_perf = trainer.evaluate(val_loader)

            # remember best prec@1 and save checkpoint
            gnmt_print(key=mlperf_log.TRAIN_CHECKPOINT, sync=False)
            if args.rank == 0:
                is_best = val_loss < best_loss
                best_loss = min(val_loss, best_loss)
                trainer.save(save_all=args.save_all, is_best=is_best)

        if args.eval:
            gnmt_print(key=mlperf_log.EVAL_START, value=epoch, sync=True)
            test_bleu, break_training = translator.run(calc_bleu=True,
                                                       epoch=epoch)
            gnmt_print(key=mlperf_log.EVAL_ACCURACY,
                       value={"epoch": epoch, "value": round(test_bleu, 2)},
                       sync=False)
            gnmt_print(key=mlperf_log.EVAL_TARGET,
                       value=args.target_bleu, sync=False)
            gnmt_print(key=mlperf_log.EVAL_STOP, sync=True)

        acc_log = []
        acc_log += [f'Summary: Epoch: {epoch}']
        acc_log += [f'Training Loss: {train_loss:.4f}']
        if args.eval:
            acc_log += [f'Validation Loss: {val_loss:.4f}']
            acc_log += [f'Test BLEU: {test_bleu:.2f}']

        perf_log = []
        perf_log += [f'Performance: Epoch: {epoch}']
        perf_log += [f'Training: {train_perf:.0f} Tok/s']
        if args.eval:
            perf_log += [f'Validation: {val_perf:.0f} Tok/s']

        if args.rank == 0:
            logging.info('\t'.join(acc_log))
            logging.info('\t'.join(perf_log))

        logging.info(f'Finished epoch {epoch}')
        if break_training:
            break

    gnmt_print(key=mlperf_log.RUN_STOP,
               value={"success": bool(break_training)}, sync=True)
    gnmt_print(key=mlperf_log.RUN_FINAL, sync=False)
873152
def main():
    
    allowed_format = ("emf", "eps", "pdf", "png", "ps", \
                      "raw", "rgba", "svg", "svgz")
    p = OptionParser(main.__doc__)
    p.add_option("--skip", default=0, type="int",
            help="skip the first several lines [default: %default]")
    p.add_option("--col", default=0, type="int", help="Get the n-th column")
    p.set_histogram()
    p.add_option("--tags", dest="tags", default=None,
            help="tags for data if multiple input files, comma sep")
    p.add_option("--ascii", default=False, action="store_true",
            help="print ASCII text stem-leaf plot [default: %default]")
    p.add_option("--base", default="0", choices=("0", "2", "10"),
            help="use logarithm axis with base, 0 to disable [default: %default]")
    p.add_option("--facet", default=False, action="store_true",
            help="place multiple histograms side-by-side [default: %default]")
    p.add_option("--fill", default="white",
            help="color of the bin [default: %default]")
    p.add_option("--format", default="pdf", choices=allowed_format,
            help="Generate image of format [default: %default]")
    p.add_option("--quick", default=False, action="store_true",
            help="Use quick plot, assuming bins are already counted")
    p.add_option("--noprintstats", default=False, action="store_true",
            help="Write basic stats when using --quick")
    opts, args = p.parse_args()

    if len(args) < 1:
        sys.exit(not p.print_help())

    skip = opts.skip
    vmin, vmax = opts.vmin, opts.vmax
    bins = opts.bins
    xlabel, title = opts.xlabel, opts.title
    title = title or args[0]
    base = int(opts.base)
    fileno = len(args)

    if opts.quick:
        assert fileno == 1, "Single input file expected using --quick"
        filename = args[0]
        figname = filename.rsplit(".", 1)[0] + ".pdf"
        data = DictFile(filename, keycast=int, cast=int)
        quickplot(data, vmin, vmax, xlabel, title, figname=figname,
                  print_stats=(not opts.noprintstats))
        return

    if fileno == 1:
        histogram(args[0], vmin, vmax, xlabel, title, outfmt=opts.format,
                bins=bins, skip=skip, ascii=opts.ascii,
                base=base, fill=opts.fill, col=opts.col)
    else:
        histogram_multiple(args, vmin, vmax, xlabel, title, outfmt=opts.format,
                tags=opts.tags, bins=bins, skip=skip, ascii=opts.ascii,
                facet=opts.facet, fill=opts.fill)
856063
async function main () {

    let exchanges = []

    // instantiate all exchanges
    await Promise.all (ccxt.exchanges.map (async id => {
        let exchange = new (ccxt)[id] ()
        exchanges.push (exchange)
        await test (exchange, symbol)
    }))

    let succeeded = exchanges.filter (exchange => exchange.markets ? true : false).length.toString ().bright.green
    let failed = exchanges.filter (exchange => exchange.markets ? false : true).length
    let total = ccxt.exchanges.length.toString ().bright.white
    console.log (succeeded, 'of', total, 'exchanges loaded', ('(' + failed + ' errors)').red)
}
146314
def main():
    """"""
    arguments = docopt.docopt(__doc__, version=__version__)
    colorlogs()
    # Read input file file


    wrapper = BMIWrapper(
        engine=arguments['<engine>'],
        configfile=arguments['<config>'] or ''
    )


    # add logger if required
    if not arguments['--disable-logger']:
        logging.root.setLevel(logging.DEBUG)
        wrapper.set_logger(logging.root)

    with wrapper as model:
        # if siginfo is supported by OS (BSD)
        def handler(signum, frame):
            """report progress information"""
            t_start = model.get_start_time()
            t_end = model.get_end_time()
            t_current = model.get_current_time()
            total = (t_end - t_start)
            now = (t_current - t_start)
            if total > 0:
                logging.info("progress: %s%%", 100.0 * now / total)
            else:
                logging.info("progress: unknown")

        if hasattr(signal, 'SIGINFO'):
            # attach a siginfo handler (CTRL-t) to print progress
            signal.signal(signal.SIGINFO, handler)

        if arguments['--info']:
            logging.info("%s", trace(model))
        t_end = model.get_end_time()
        t = model.get_start_time()
        while t < t_end:
            model.update(-1)
            t = model.get_current_time()
        if arguments['--info']:
            logging.info("%s", trace(model))
1189245
def main():
    
    startLogging(stdout)
    checker = InMemoryUsernamePasswordDatabaseDontUse()
    checker.addUser("testuser", "examplepass")
    realm = AdditionRealm()
    factory = CredAMPServerFactory(Portal(realm, [checker]))
    reactor.listenTCP(7805, factory)
    reactor.run()
1246950
def main():
    

    logging.captureWarnings(True)
    logging.basicConfig(format=('%(asctime)s - %(name)s - %(levelname)s - ' +
                                '%(message)s'), level=logging.INFO)

    print("=====================================")
    print("========   Submit and Wait   ========")
    print("=====================================")
    print("")

    functionJobs = make_jobs()

    print("sending function jobs to cluster")
    print("")

    job_outputs = process_jobs(functionJobs, max_processes=4)

    print("results from each job")
    for (i, result) in enumerate(job_outputs):
        print("Job {0}- result: {1}".format(i, result))
1032106
def main():
    
    # Define Parser object and add to toil
    url_prefix = 'https://s3-us-west-2.amazonaws.com/cgl-pipeline-inputs/'
    parser = argparse.ArgumentParser(description=main.__doc__, formatter_class=argparse.RawTextHelpFormatter)
    parser.add_argument('--config', required=True,
                        help='Path to configuration file for samples, one per line. UUID,URL_to_bamfile. '
                             'The URL may be a standard "http://", a "file://<abs_path>", or "s3://<bucket>/<key>"')
    parser.add_argument('--gtf', help='URL to annotation GTF file',
                        default=url_prefix + 'rnaseq_cgl/gencode.v23.annotation.gtf')
    parser.add_argument('--gtf-pickle', help='Pickled GTF file',
                        default=url_prefix + 'spladder/gencode.v23.annotation.gtf.pickle')
    parser.add_argument('--gtf-m53', help='M53 preprocessing annotation table',
                        default=url_prefix + 'spladder/gencode.v23.annotation.gtf.m53')
    parser.add_argument('--positions', help='URL to SNP positions over genes file (TSV)',
                        default=url_prefix + 'spladder/positions_fixed.tsv')
    parser.add_argument('--genome', help='URL to Genome fasta',
                        default=url_prefix + 'rnaseq_cgl/hg38_no_alt.fa')
    parser.add_argument('--genome-index', help='Index file (fai) of genome',
                        default=url_prefix + 'spladder/hg38_no_alt.fa.fai')
    parser.add_argument('--ssec', default=None, help='Path to master key used for downloading encrypted files.')
    parser.add_argument('--output-s3-dir', default=None, help='S3 Directory of the form: s3://bucket/directory')
    parser.add_argument('--output-dir', default=None, help='full path where final results will be output')
    parser.add_argument('--sudo', action='store_true', default=False,
                        help='Set flag if sudo is required to run Docker.')
    parser.add_argument('--star-index', help='URL to download STAR Index built from HG38/gencodev23 annotation.',
                        default=url_prefix + 'rnaseq_cgl/starIndex_hg38_no_alt.tar.gz')
    parser.add_argument('--fwd-3pr-adapter', help="Sequence for the FWD 3' Read Adapter.", default='AGATCGGAAGAG')
    parser.add_argument('--rev-3pr-adapter', help="Sequence for the REV 3' Read Adapter.", default='AGATCGGAAGAG')
    Job.Runner.addToilOptions(parser)
    args = parser.parse_args()
    # Sanity Checks
    if args.config:
        assert os.path.isfile(args.config), 'Config not found at: {}'.format(args.config)
    if args.ssec:
        assert os.path.isfile(args.ssec), 'Encryption key not found at: {}'.format(args.config)
    if args.output_s3_dir:
        assert args.output_s3_dir.startswith('s3://'), 'Wrong format for output s3 directory'
    # Program checks
    for program in ['curl', 'docker']:
        assert which(program), 'Program "{}" must be installed on every node.'.format(program)

    Job.Runner.startToil(Job.wrapJobFn(parse_input_samples, args), args)
738073
function main() {
  config = createConfig(process.argv.slice(2));

  if (config.options.help) {
    config.options.showHelp(NODE_DEBUG_MODE);
    process.exit();
  }

  if (config.options.version) {
    config.options.showVersion();
    process.exit();
  }

  process.on('SIGINT', function() {
     process.exit();
  });

  startInspectorProcess(function(error, result) {
    if (error) {
      console.error(formatNodeInspectorError(error));
      process.exit(1);
    }

    var url = result.address.url;
    var isUnixSocket = result.address.isUnixSocket;

    startDebuggedProcess(function(error) {
      if (error) {
        console.error('Cannot start %s: %s', config.subproc.script, error.message || error);
        process.exit(2);
      }

      openBrowserAndPrintInfo(url, isUnixSocket);
    });
  });
}
447371
def do_eb(self, arg):
        
        # TODO
        # data parameter should be optional, use a child Cmd here
        pid        = self.get_process_id_from_prefix()
        token_list = self.split_tokens(arg, 2)
        address    = self.input_address(token_list[0], pid)
        data       = HexInput.hexadecimal(' '.join(token_list[1:]))
        self.write_memory(address, data, pid)
1187054
function Main() {
  const lerna = requireLerna();
  const useYarn = shouldUseYarn();

  if (!lerna) {
    if (useYarn) {
      logger.error(
        'Lerna cannot not be found. Please run `yarn --check-files`.'
      );
    } else {
      logger.error('Lerna cannot not be found.');
      logger.hint('Remove `node_modules` and run `npm install` again.');
    }

    process.exit(1);
  }

  const args = ['bootstrap'];

  if (useYarn) {
    args.push('--npm-client=yarn');
    // yarn doesn't support concurrency
    args.push('--concurrency=1');
  } else {
    // windows doesn't handle concurrency too well
    if (isWindows || !shouldUseNpmWithConcurrency()) {
      args.push('--concurrency=1');
    }
  }

  child = spawn(lerna, args, { stdio: 'inherit' });
  child.on('close', code => process.exit(code));
}
687065
public function main()
    {
        $options = (array)$this->params;
        $options['bundle'] = empty($options['bundle']) ? null : $options['bundle'];

        if (empty($options['use'])) {
            $this->err(__d('eav', 'You must indicate a table alias name using the "--use" option. Example: "Articles.Users"'));

            return false;
        }

        try {
            $table = TableRegistry::get($options['use']);
        } catch (\Exception $ex) {
            $table = false;
        }

        if (!$table) {
            $this->err(__d('eav', 'The specified table does not exists.'));

            return false;
        } elseif (!$table->behaviors()->has('Eav')) {
            $this->err(__d('eav', 'The specified table is not using EAV behavior.'));

            return false;
        }

        $columns = $table->listColumns($options['bundle']);
        ksort($columns, SORT_LOCALE_STRING);
        $rows = [
            [
                __d('eav', 'Column Name'),
                __d('eav', 'Data Type'),
                __d('eav', 'Bundle'),
                __d('eav', 'Searchable'),
            ]
        ];

        foreach ($columns as $name => $info) {
            $rows[] = [
                $name,
                $info['type'],
                (!empty($info['bundle']) ? $info['bundle'] : '---'),
                (!empty($info['searchable']) ? 'no' : 'yes'),
            ];
        }

        $this->out();
        $this->out(__d('eav', 'EAV information for table "{0}":', $options['use']));
        $this->out();
        $this->helper('table')->output($rows);

        return true;
    }
1244376
def main():
    """"""
    # Check command line arguments.
    if len(sys.argv) < 3:
        raise RuntimeError(
            'Usage: spark-submit spark_imager_test.py <settings_file> <dir> '
            '[partitions]')

    # Create log object.
    log = logging.getLogger('pyspark')
    log.setLevel(logging.INFO)
    log.addHandler(logging.StreamHandler(sys.stdout))

    # Load pipeline settings.
    with open(sys.argv[1]) as f:
        settings = json.load(f)

    # Get a list of input Measurement Sets to process.
    data_dir = str(sys.argv[2])
    inputs = glob(join(data_dir, '*.ms')) + glob(join(data_dir, '*.MS'))
    inputs = filter(None, inputs)
    log.info('Found input Measurement Sets: %s', ', '.join(inputs))

    # Get a Spark context.
    context = pyspark.SparkContext(appName="spark_imager_test")

    # Create the Spark RDD containing the input filenames,
    # suitably parallelized.
    partitions = int(sys.argv[3]) if len(sys.argv) > 3 else 2
    rdd = context.parallelize(inputs, partitions)

    # Define Spark broadcast variables.
    bc_settings = context.broadcast(settings)
    bc_grid_weights = None

    # Process coordinates first if required.
    if (settings['combine'] and (
            settings['imager']['weighting'] == 'Uniform' or
            settings['imager']['algorithm'] == 'W-projection')):
        # Create RDD to generate weights grids.
        rdd_coords = rdd.map(
            partial(node_run, coords_only=True, bc_settings=bc_settings,
                    bc_grid_weights=None))

        # Mark this RDD as persistent so it isn't computed more than once.
        rdd_coords.persist()

        # Get the maximum number of W-planes required, and update settings.
        num_w_planes = rdd_coords.map(lambda x: x[1]).max()
        settings['imager']['num_w_planes'] = num_w_planes

        # Get the combined grid of weights and broadcast it to nodes.
        output = rdd_coords.reduce(reduce_sequences)
        bc_grid_weights = context.broadcast(output[0])

        # Delete this RDD.
        rdd_coords.unpersist()

    # Re-broadcast updated settings.
    bc_settings = context.broadcast(settings)

    # Run parallel pipeline on worker nodes and combine visibility grids.
    output = rdd.map(
        partial(node_run, coords_only=False, bc_settings=bc_settings,
                bc_grid_weights=bc_grid_weights)).reduce(reduce_sequences)

    # Finalise combined visibility grids if required.
    if settings['combine']:
        # Create an imager to finalise (FFT) the gridded data.
        imager = oskar.Imager(settings['precision'])
        for key, value in settings['imager'].items():
            setattr(imager, key, value)

        # Finalise grid and save image.
        save_image(imager, output[0], output[1], settings['output_file'])
        log.info('Finished. Output file is %s', settings['output_file'])

    context.stop()