Train model using CloudML.

  See local_train() for a description of the args.
  Args:
    config: A CloudTrainingConfig object.
    job_name: Training job name. A default will be picked if None.
169403
def cloud_train(train_dataset,
                eval_dataset,
                analysis_dir,
                output_dir,
                features,
                model_type,
                max_steps,
                num_epochs,
                train_batch_size,
                eval_batch_size,
                min_eval_frequency,
                top_n,
                layer_sizes,
                learning_rate,
                epsilon,
                job_name,
                job_name_prefix,
                config):
  
  import google.datalab.ml as ml

  if len(train_dataset.input_files) != 1 or len(eval_dataset.input_files) != 1:
    raise ValueError('CsvDataSets must be built with a file pattern, not list '
                     'of files.')

  if file_io.file_exists(output_dir):
    raise ValueError('output_dir already exist. Use a new output path.')

  if isinstance(features, dict):
    # Make a features file.
    if not file_io.file_exists(output_dir):
      file_io.recursive_create_dir(output_dir)
    features_file = os.path.join(output_dir, 'features_file.json')
    file_io.write_string_to_file(
        features_file,
        json.dumps(features))
  else:
    features_file = features

  if not isinstance(config, ml.CloudTrainingConfig):
    raise ValueError('cloud should be an instance of '
                     'google.datalab.ml.CloudTrainingConfig for cloud training.')

  _assert_gcs_files([output_dir, train_dataset.input_files[0], eval_dataset.input_files[0],
                     features_file, analysis_dir])

  args = ['--train-data-paths=%s' % train_dataset.input_files[0],
          '--eval-data-paths=%s' % eval_dataset.input_files[0],
          '--preprocess-output-dir=%s' % analysis_dir,
          '--transforms-file=%s' % features_file,
          '--model-type=%s' % model_type,
          '--max-steps=%s' % str(max_steps),
          '--train-batch-size=%s' % str(train_batch_size),
          '--eval-batch-size=%s' % str(eval_batch_size),
          '--min-eval-frequency=%s' % str(min_eval_frequency),
          '--learning-rate=%s' % str(learning_rate),
          '--epsilon=%s' % str(epsilon)]
  if num_epochs:
    args.append('--num-epochs=%s' % str(num_epochs))
  if top_n:
    args.append('--top-n=%s' % str(top_n))
  if layer_sizes:
    for i in range(len(layer_sizes)):
      args.append('--layer-size%s=%s' % (i + 1, str(layer_sizes[i])))

  job_request = {
    'package_uris': [_package_to_staging(output_dir), _TF_GS_URL, _PROTOBUF_GS_URL],
    'python_module': 'mltoolbox._structured_data.trainer.task',
    'job_dir': output_dir,
    'args': args
  }
  job_request.update(dict(config._asdict()))

  if not job_name:
    job_name = job_name_prefix or 'structured_data_train'
    job_name += '_' + datetime.datetime.now().strftime('%y%m%d_%H%M%S')
  job = ml.Job.submit_training(job_request, job_name)
  print('Job request send. View status of job at')
  print('https://console.developers.google.com/ml/jobs?project=%s' %
        _default_project())

  return job
1254930
def is_training_name(name):
    
    # TODO: maybe simply check against TRAINABLE_VARIABLES and MODEL_VARIABLES?
    # TODO or use get_slot_names()
    name = get_op_tensor_name(name)[0]
    if name.endswith('/Adam') or name.endswith('/Adam_1'):
        return True
    if name.endswith('/Momentum'):
        return True
    if name.endswith('/Adadelta') or name.endswith('/Adadelta_1'):
        return True
    if name.endswith('/RMSProp') or name.endswith('/RMSProp_1'):
        return True
    if name.endswith('/Adagrad'):
        return True
    if name.startswith('EMA/') or '/EMA/' in name:  # all the moving average summaries
        return True
    if name.startswith('AccumGrad') or name.endswith('/AccumGrad'):
        return True
    if name.startswith('apply_gradients'):
        return True
    return False
220272
def build_job_configs(self, args):
        
        job_configs = {}

        ttype = args['ttype']
        (targets_yaml, sim) = NAME_FACTORY.resolve_targetfile(args)
        if targets_yaml is None:
            return job_configs

        config_yaml = 'config.yaml'
        config_override = args.get('config')
        if is_not_null(config_override):
            config_yaml = config_override

        targets = load_yaml(targets_yaml)
        nsims_job = args['nsims_job']
        first_seed = args['seed']
        nsims = args['nsims']
        last_seed = first_seed + nsims

        base_config = dict(sim_profile=args['sim_profile'],
                           roi_baseline=args['roi_baseline'],
                           non_null_src=args['non_null_src'],
                           sim=sim)

        for target_name, target_list in targets.items():
            name_keys = dict(target_type=ttype,
                             target_name=target_name,
                             sim_name=sim,
                             fullpath=True)
            simdir = NAME_FACTORY.sim_targetdir(**name_keys)
            config_path = os.path.join(simdir, config_yaml)

            job_config = base_config.copy()
            job_config.update(dict(config=config_path,
                                   profiles=target_list))

            current_seed = first_seed
            while current_seed < last_seed:
                fullkey = "%s_%06i" % (target_name, current_seed)
                logfile = make_nfs_path(os.path.join(simdir, "%s_%s_%06i.log" % (self.linkname, 
                                                                                 target_name, current_seed)))
                if nsims_job <= 0 or current_seed + nsims_job >= last_seed:
                    nsims_current = last_seed - current_seed
                else:
                    nsims_current = nsims_job
                job_config.update(dict(seed=current_seed,
                                       nsims=nsims_current,
                                       logfile=logfile))
                job_configs[fullkey] = job_config.copy()
                current_seed += nsims_current

        return job_configs
568569
def build_job_configs(self, args):
        
        job_configs = {}

        comp_file = args.get('comp', None)
        if comp_file is not None:
            comp_dict = yaml.safe_load(open(comp_file))
            coordsys = comp_dict.pop('coordsys')
            for v in comp_dict.values():
                v['coordsys'] = coordsys
        else:
            return job_configs

        datafile = args['data']
        if datafile is None or datafile == 'None':
            return job_configs
        NAME_FACTORY.update_base_dict(args['data'])

        inputfiles = create_inputlist(args['ft1file'])
        outdir_base = os.path.join(NAME_FACTORY.base_dict['basedir'], 'counts_cubes')
        data_ver = NAME_FACTORY.base_dict['data_ver']

        for idx, infile in enumerate(inputfiles):
            key = "%06i" % idx
            key_scfile = "%03i" % (idx + 1)
            output_dir = os.path.join(outdir_base, key)
            try:
                os.mkdir(output_dir)
            except OSError:
                pass
            scfile = args['ft2file'].replace('.lst', '_%s.fits' % key_scfile)
            logfile = make_nfs_path(os.path.join(output_dir,
                                                 'scatter_mk_%s_%s.log' % (data_ver, key)))

            job_configs[key] = comp_dict.copy()
            job_configs[key].update(dict(ft1file=infile,
                                         scfile=scfile,
                                         comp=args['comp'],
                                         hpx_order_max=args['hpx_order_max'],
                                         outdir=outdir_base,
                                         outkey=key,
                                         logfile=logfile,
                                         pfiles=output_dir))

        return job_configs
4280
public static org.apache.hadoop.mapred.JobConf asJobConf(Configuration cfg) {
        return cfg instanceof org.apache.hadoop.mapred.JobConf
            ? (org.apache.hadoop.mapred.JobConf) cfg
            : new org.apache.hadoop.mapred.JobConf(cfg);
    }
1006269
def train(self, data_iterator):
        
        optimizer = get_optimizer(self.master_optimizer)
        self.model = model_from_yaml(self.yaml, self.custom_objects)
        self.model.compile(optimizer=optimizer,
                           loss=self.master_loss, metrics=self.master_metrics)
        self.model.set_weights(self.parameters.value)

        feature_iterator, label_iterator = tee(data_iterator, 2)
        x_train = np.asarray([x for x, y in feature_iterator])
        y_train = np.asarray([y for x, y in label_iterator])

        self.model.compile(optimizer=self.master_optimizer,
                           loss=self.master_loss,
                           metrics=self.master_metrics)

        weights_before_training = self.model.get_weights()
        if x_train.shape[0] > self.train_config.get('batch_size'):
            self.model.fit(x_train, y_train, **self.train_config)
        weights_after_training = self.model.get_weights()
        deltas = subtract_params(
            weights_before_training, weights_after_training)
        yield deltas
1063814
def build_config(config, filename=None):
    
    # Done here since it seems not to be possible to detect if an argument
    # was supplied by the user or if it's the default argument.
    # If using type=lambda v: None if v in ('transparent', 'trans') else v
    # we cannot detect if "None" comes from "transparent" or the default value
    for clr in ('color', 'background'):
        val = config.pop(clr, None)
        if val in ('transparent', 'trans'):
            config[clr] = None
        elif val:
            config[clr] = val
    # SVG
    for name in ('svgid', 'svgclass', 'lineclass'):
        if config.get(name, None) is None:
            config.pop(name, None)
    if config.pop('no_classes', False):
        config['svgclass'] = None
        config['lineclass'] = None
    if filename is not None:
        ext = filename[filename.rfind('.') + 1:].lower()
        if ext == 'svgz':  # There is no svgz serializer, use same config as svg
            ext = 'svg'
        supported_args = _EXT_TO_KW_MAPPING.get(ext, ())
        # Drop unsupported arguments from config rather than getting a
        # "unsupported keyword" exception
        for k in list(config):
            if k not in supported_args:
                del config[k]
    return config
587284
public static boolean contains(Configuration jobConf, String name) {
		if (StringUtils.isNotBlank(jobConf.get(name))) {
			return true;
		} else {
			return false;
		}
	}
1292892
def train(self, data_iterator):
        
        feature_iterator, label_iterator = tee(data_iterator, 2)
        x_train = np.asarray([x for x, y in feature_iterator])
        y_train = np.asarray([y for x, y in label_iterator])

        if x_train.size == 0:
            return

        optimizer = get_optimizer(self.master_optimizer)
        self.model = model_from_yaml(self.yaml, self.custom_objects)
        self.model.compile(optimizer=optimizer,
                           loss=self.master_loss, metrics=self.master_metrics)
        self.model.set_weights(self.parameters.value)

        epochs = self.train_config['epochs']
        batch_size = self.train_config.get('batch_size')
        nb_train_sample = x_train.shape[0]
        nb_batch = int(np.ceil(nb_train_sample / float(batch_size)))
        index_array = np.arange(nb_train_sample)
        batches = [
            (i * batch_size, min(nb_train_sample, (i + 1) * batch_size))
            for i in range(0, nb_batch)
        ]

        if self.frequency == 'epoch':
            for epoch in range(epochs):
                weights_before_training = self.client.get_parameters()
                self.model.set_weights(weights_before_training)
                self.train_config['epochs'] = 1
                if x_train.shape[0] > batch_size:
                    self.model.fit(x_train, y_train, **self.train_config)
                self.train_config['epochs'] = epochs
                weights_after_training = self.model.get_weights()
                deltas = subtract_params(
                    weights_before_training, weights_after_training)
                self.client.update_parameters(deltas)
        elif self.frequency == 'batch':
            for epoch in range(epochs):
                if x_train.shape[0] > batch_size:
                    for (batch_start, batch_end) in batches:
                        weights_before_training = self.client.get_parameters()
                        self.model.set_weights(weights_before_training)
                        batch_ids = index_array[batch_start:batch_end]
                        x = slice_arrays(x_train, batch_ids)
                        y = slice_arrays(y_train, batch_ids)
                        self.model.train_on_batch(x, y)
                        weights_after_training = self.model.get_weights()
                        deltas = subtract_params(
                            weights_before_training, weights_after_training)
                        self.client.update_parameters(deltas)
        else:
            raise ValueError(
                'frequency parameter can be `epoch` or `batch, got {}'.format(self.frequency))
        yield []
1175245
def best_training_job(self):
        
        self._ensure_last_tuning_job()

        tuning_job_describe_result = \
            self.estimator.sagemaker_session.sagemaker_client.describe_hyper_parameter_tuning_job(
                HyperParameterTuningJobName=self.latest_tuning_job.name)

        try:
            return tuning_job_describe_result['BestTrainingJob']['TrainingJobName']
        except KeyError:
            raise Exception('Best training job not available for tuning job: {}'.format(self.latest_tuning_job.name))
1283534
def build_job_configs(self, args):
        
        job_configs = {}

        components = Component.build_from_yamlfile(args['comp'])
        NAME_FACTORY.update_base_dict(args['data'])

        mktime = args['mktimefilter']

        base_config = dict(nxpix=args['nxpix'],
                           nypix=args['nypix'],
                           binsz=args['binsz'])

        for comp in components:
            zcut = "zmax%i" % comp.zmax
            key = comp.make_key('{ebin_name}_{evtype_name}')
            name_keys = dict(zcut=zcut,
                             ebin=comp.ebin_name,
                             psftype=comp.evtype_name,
                             irf_ver=NAME_FACTORY.irf_ver(),
                             coordsys=comp.coordsys,
                             mktime=mktime,
                             fullpath=True)
            outfile = NAME_FACTORY.bexpcube(**name_keys)
            ltcube = NAME_FACTORY.ltcube(**name_keys)
            full_config = base_config.copy()
            full_config.update(dict(infile=ltcube,
                                    outfile=outfile,
                                    irfs=NAME_FACTORY.irfs(**name_keys),
                                    evtype=comp.evtype,
                                    emin=comp.emin,
                                    emax=comp.emax,
                                    enumbins=comp.enumbins,
                                    logfile=make_nfs_path(outfile.replace('.fits', '.log'))))
            job_configs[key] = full_config

        return job_configs
1028225
def build_job_configs(self, args):
        
        job_configs = {}

        ttype = args['ttype']
        (targets_yaml, sim) = NAME_FACTORY.resolve_targetfile(args)
        if targets_yaml is None:
            return job_configs

        targets = load_yaml(targets_yaml)

        for target_name, target_list in targets.items():
            for targ_prof in target_list:
                name_keys = dict(target_type=ttype,
                                 target_name=target_name,
                                 profile=targ_prof,
                                 fullpath=True)
                targ_key = "%s_%s" % (target_name, targ_prof)
                input_path = NAME_FACTORY.sedfile(**name_keys)
                output_path = input_path.replace('.fits', '.png')
                logfile = make_nfs_path(input_path.replace('.fits', '.log'))
                job_config = dict(infile=input_path,
                                  outfile=output_path,
                                  logfile=logfile)
                job_configs[targ_key] = job_config

        return job_configs
476073
def parse_config_file(job, ids, input_args):
    
    samples = []
    config = input_args['config']
    with open(config, 'r') as f:
        for line in f.readlines():
            if not line.isspace():
                sample = line.strip().split(',')
                samples.append(sample)
    for sample in samples:
        job.addChildJobFn(download_sample, ids, input_args, sample)
1243370
function configuration(self, name) {
  if (!self.__configurations__[name]) {
    throw new Error('Unknown configuration: ' + name);
  }

  return self.__configurations__[name];
}
1120932
def _load_config(self):
        
        # load the global config
        with open(self.config_global_path, "r") as f_config:
            config = ordered_yaml_load(f_config)
        self.config_global = config

        # Load the local config
        if self.config_global_local_path:
            config = ordered_yaml_load(open(self.config_global_local_path, "r"))
            self.config_global_local = config

        self.config = merge_config(
            OrderedDict(
                [
                    ("global_config", self.config_global),
                    ("global_local", self.config_global_local),
                ]
            )
        )
449292
def build_job_configs(self, args):
        
        job_configs = {}

        ttype = args['ttype']
        (targets_yaml, sim) = NAME_FACTORY.resolve_targetfile(
            args, require_sim_name=True)
        if targets_yaml is None:
            return job_configs

        write_full = args['write_full']

        targets = load_yaml(targets_yaml)

        base_config = dict(config=args['config'],
                           nsims=args['nsims'],
                           seed=args['seed'])

        first = args['seed']
        last = first + args['nsims'] - 1

        for target_name, profile_list in targets.items():
            for profile in profile_list:
                full_key = "%s:%s:%s" % (target_name, profile, sim)
                name_keys = dict(target_type=ttype,
                                 target_name=target_name,
                                 sim_name=sim,
                                 profile=profile,
                                 fullpath=True)
                sed_file = NAME_FACTORY.sim_sedfile(**name_keys)
                outfile = sed_file.replace(
                    '_SEED.fits', '_collected_%06i_%06i.fits' % (first, last))
                logfile = make_nfs_path(outfile.replace('.fits', '.log'))
                if not write_full:
                    outfile = None
                summaryfile = sed_file.replace(
                    '_SEED.fits', '_summary_%06i_%06i.fits' % (first, last))
                job_config = base_config.copy()
                job_config.update(dict(sed_file=sed_file,
                                       outfile=outfile,
                                       summaryfile=summaryfile,
                                       logfile=logfile))
                job_configs[full_key] = job_config

        return job_configs
806151
def build_job_configs(self, args):
        
        job_configs = {}

        components = Component.build_from_yamlfile(args['comp'])
        NAME_FACTORY.update_base_dict(args['data'])
        ret_dict = make_catalog_comp_dict(sources=args['library'], basedir='.')
        comp_info_dict = ret_dict['comp_info_dict']

        for split_ver, split_dict in comp_info_dict.items():
            for source_key, source_dict in split_dict.items():
                full_key = "%s_%s" % (split_ver, source_key)
                merged_name = "%s_%s" % (source_dict.catalog_info.catalog_name, source_key)
                if source_dict.model_type != 'CompositeSource':
                    continue

                for comp in components:
                    zcut = "zmax%i" % comp.zmax
                    key = "%s_%s" % (full_key, comp.make_key('{ebin_name}_{evtype_name}'))
                    name_keys = dict(zcut=zcut,
                                     sourcekey=full_key,
                                     ebin=comp.ebin_name,
                                     psftype=comp.evtype_name,
                                     coordsys=comp.coordsys,
                                     mktime='none',
                                     irf_ver=NAME_FACTORY.irf_ver())
                    nested_name_keys = dict(zcut=zcut,
                                            sourcekey=source_dict.catalog_info.catalog_name,
                                            ebin=comp.ebin_name,
                                            psftype=comp.evtype_name,
                                            coordsys=comp.coordsys,
                                            mktime='none',
                                            irf_ver=NAME_FACTORY.irf_ver())
                    outfile = NAME_FACTORY.srcmaps(**name_keys)
                    logfile = make_nfs_path(outfile.replace('.fits', '.log'))
                    job_configs[key] = dict(srcmaps=NAME_FACTORY.srcmaps(**nested_name_keys),
                                            expcube=NAME_FACTORY.ltcube(**name_keys),
                                            irfs=NAME_FACTORY.irfs(**name_keys),
                                            bexpmap=NAME_FACTORY.bexpcube(**name_keys),
                                            srcmdl=NAME_FACTORY.srcmdl_xml(**name_keys),
                                            merged=merged_name,
                                            outfile=outfile,
                                            outxml=NAME_FACTORY.nested_srcmdl_xml(**name_keys),
                                            logfile=logfile)

        return job_configs