Create an iteration for the experiment group.
500903
def create_iteration(self, num_suggestions):
        """"""
        from db.models.experiment_groups import ExperimentGroupIteration

        iteration_config = self.experiment_group.iteration_config

        if iteration_config is None:
            iteration = 0
            old_experiment_ids = None
            old_experiments_configs = None
            old_experiments_metrics = None
        else:
            iteration = iteration_config.iteration + 1
            old_experiment_ids = iteration_config.combined_experiment_ids
            old_experiments_configs = iteration_config.combined_experiments_configs
            old_experiments_metrics = iteration_config.combined_experiments_metrics

        # Create a new iteration config
        iteration_config = BOIterationConfig(
            iteration=iteration,
            num_suggestions=num_suggestions,
            old_experiment_ids=old_experiment_ids,
            old_experiments_configs=old_experiments_configs,
            old_experiments_metrics=old_experiments_metrics,
            experiment_ids=[],
            experiments_configs=[],
        )
        return ExperimentGroupIteration.objects.create(
            experiment_group=self.experiment_group,
            data=iteration_config.to_dict())
1036707
def create_experiment_fn(*args, **kwargs):
  """"""

  def experiment_fn(run_config, hparams):
    return create_experiment(run_config, hparams, *args, **kwargs)

  return experiment_fn
63858
def run(self):
        """"""
        _, test_data = self.data.load(train=False, test=True)
        try:
            self.model.fit_generator(
                self.samples_to_batches(self.generate_samples(), self.args.batch_size), steps_per_epoch=self.args.steps_per_epoch,
                epochs=self.epoch + self.args.epochs, validation_data=test_data,
                callbacks=self.callbacks, initial_epoch=self.epoch
            )
        finally:
            self.model.save(self.args.model)
            save_params(self.args.model)
1210672
def setup(self):
        ''' 
        '''

        self.selection = EXPFACTORY_EXPERIMENTS
        self.ordered = len(EXPFACTORY_EXPERIMENTS) > 0
        self.data_base = EXPFACTORY_DATA
        self.study_id = EXPFACTORY_SUBID
        self.base = EXPFACTORY_BASE
        self.randomize = EXPFACTORY_RANDOMIZE
        self.headless = EXPFACTORY_HEADLESS

        # Generate variables, if they exist
        self.vars = generate_runtime_vars() or None

        available = get_experiments("%s" % self.base)
        self.experiments = get_selection(available, self.selection)
        self.logger.debug(self.experiments)
        self.lookup = make_lookup(self.experiments)
        final = "\n".join(list(self.lookup.keys()))       

        bot.log("Headless mode: %s" % self.headless)
        bot.log("User has selected: %s" % self.selection)
        bot.log("Experiments Available: %s" %"\n".join(available))
        bot.log("Randomize: %s" % self.randomize)
        bot.log("Final Set \n%s" % final)
567622
def runExperiment( self, e ):
        """"""

        # create the parameter space
        ps = self.parameterSpace()

        # run the experiment at each point
        nb = self.notebook()
        for p in ps:
            #print "Running {p}".format(p = p)
            res = e.set(p).run()
            nb.addResult(res)

        # commit the results
        nb.commit()
610224
def experiment(self):
        """"""
        if self._experiment is None:
            self._experiment = list(self.config.experiments.keys())[-1]
        return self._experiment
110888
def copy_experiment(experiment):
    """"""
    try:
        publisher.publish_experiment_job_log(
            log_lines='Copying outputs from experiment `{}` into experiment `{}`'.format(
                experiment.original_experiment.unique_name, experiment.unique_name
            ),
            experiment_uuid=experiment.uuid.hex,
            experiment_name=experiment.unique_name,
            job_uuid='all',
        )
        stores.copy_experiment_outputs(
            persistence_outputs_from=experiment.original_experiment.persistence_outputs,
            persistence_outputs_to=experiment.persistence_outputs,
            experiment_name_from=experiment.original_experiment.unique_name,
            experiment_name_to=experiment.unique_name)

    except OSError:
        publisher.publish_experiment_job_log(
            log_lines='Could not copy the outputs of experiment `{}` into experiment `{}`'.format(
                experiment.original_experiment.unique_name, experiment.unique_name
            ),
            experiment_uuid=experiment.uuid.hex,
            experiment_name=experiment.unique_name,
            job_uuid='all',
        )
        _logger.warning(
            'Could not copy the outputs of experiment `%s` into experiment `%s`',
            experiment.original_experiment.unique_name, experiment.unique_name)
880451
def run(self):
        """"""
        while True:
            try:
                self.perform_iteration()
            except Exception:
                traceback.print_exc()
                pass

            time.sleep(ray_constants.REPORTER_UPDATE_INTERVAL_MS / 1000)
177389
def get_reduced_configs(self):
        """"""
        iteration_config = self.experiment_group.iteration_config
        if iteration_config is None:
            logger.error(
                'Experiment group `%s` attempt to update iteration, but has no iteration',
                self.experiment_group.id,
                extra={'stack': True})
            return
        search_manager = self.experiment_group.search_manager

        # Get the number of experiments to keep
        n_configs_to_keep = search_manager.get_n_config_to_keep_for_iteration(
            iteration=iteration_config.iteration,
            bracket_iteration=iteration_config.bracket_iteration)

        # Get the last group's experiments metrics
        experiments_metrics = self.experiment_group.iteration_config.experiments_metrics
        if experiments_metrics is None:
            raise ExperimentGroupException()

        # Order the experiments
        reverse = Optimization.maximize(
            self.experiment_group.hptuning_config.hyperband.metric.optimization)
        experiments_metrics = sorted(experiments_metrics, key=lambda x: x[1], reverse=reverse)

        # Keep n experiments
        return [xp[0] for xp in experiments_metrics[:n_configs_to_keep]]
775678
def launch():
    """"""
    try:
        exp = Experiment(db.init_db(drop_all=False))
    except Exception as ex:
        return error_response(
            error_text="Failed to load experiment in /launch: {}".format(str(ex)),
            status=500,
            simple=True,
        )
    try:
        exp.log("Launching experiment...", "-----")
    except IOError as ex:
        return error_response(
            error_text="IOError writing to experiment log: {}".format(str(ex)),
            status=500,
            simple=True,
        )

    try:
        recruitment_details = exp.recruiter.open_recruitment(
            n=exp.initial_recruitment_size
        )
        session.commit()
    except Exception as e:
        return error_response(
            error_text="Failed to open recruitment, check experiment server log "
            "for details: {}".format(str(e)),
            status=500,
            simple=True,
        )

    for task in exp.background_tasks:
        try:
            gevent.spawn(task)
        except Exception:
            return error_response(
                error_text="Failed to spawn task on launch: {}, ".format(task)
                + "check experiment server log for details",
                status=500,
                simple=True,
            )

    if _config().get("replay", False):
        try:
            task = ReplayBackend(exp)
            gevent.spawn(task)
        except Exception:
            return error_response(
                error_text="Failed to launch replay task for experiment."
                "check experiment server log for details",
                status=500,
                simple=True,
            )

    # If the experiment defines a channel, subscribe the experiment to the
    # redis communication channel:
    if exp.channel is not None:
        try:
            from dallinger.experiment_server.sockets import chat_backend

            chat_backend.subscribe(exp, exp.channel)
        except Exception:
            return error_response(
                error_text="Failed to subscribe to chat for channel on launch "
                + "{}".format(exp.channel)
                + ", check experiment server log for details",
                status=500,
                simple=True,
            )

    message = "\n".join(
        (
            "Initial recruitment list:\n{}".format(
                "\n".join(recruitment_details["items"])
            ),
            "Additional details:\n{}".format(recruitment_details["message"]),
        )
    )

    return success_response(recruitment_msg=message)
424881
def run(self):
        """"""
        if self.target_eff_nsamples and self.checkpoint_interval is None:
            raise ValueError("A checkpoint interval must be set if "
                             "targetting an effective number of samples")
        # get the starting number of samples:
        # "nsamples" keeps track of the number of samples we've obtained (if
        # target_eff_nsamples is not None, this is the effective number of
        # samples; otherwise, this is the total number of samples).
        # _lastclear is the number of iterations that the file already
        # contains (either due to sampler burn-in, or a previous checkpoint)
        if self.new_checkpoint:
            self._lastclear = 0
        else:
            with self.io(self.checkpoint_file, "r") as fp:
                self._lastclear = fp.niterations
        if self.target_eff_nsamples is not None:
            target_nsamples = self.target_eff_nsamples
            with self.io(self.checkpoint_file, "r") as fp:
                nsamples = fp.effective_nsamples
        elif self.target_niterations is not None:
            # the number of samples is the number of iterations times the
            # number of walkers
            target_nsamples = self.nwalkers * self.target_niterations
            nsamples = self._lastclear * self.nwalkers
        else:
            raise ValueError("must set either target_eff_nsamples or "
                             "target_niterations; see set_target")
        self._itercounter = 0
        # figure out the interval to use
        iterinterval = self.checkpoint_interval
        if iterinterval is None:
            iterinterval = self.target_niterations
        # run sampler until we have the desired number of samples
        while nsamples < target_nsamples:
            # adjust the interval if we would go past the number of iterations
            if self.target_niterations is not None and (
                    self.niterations + iterinterval > self.target_niterations):
                iterinterval = self.target_niterations - self.niterations
            # run sampler and set initial values to None so that sampler
            # picks up from where it left off next call
            logging.info("Running sampler for {} to {} iterations".format(
                self.niterations, self.niterations + iterinterval))
            # run the underlying sampler for the desired interval
            self.run_mcmc(iterinterval)
            # update the itercounter
            self._itercounter = self._itercounter + iterinterval
            # dump the current results
            self.checkpoint()
            # update nsamples for next loop
            if self.target_eff_nsamples is not None:
                nsamples = self.effective_nsamples
                logging.info("Have {} effective samples post burn in".format(
                    nsamples))
            else:
                nsamples += iterinterval * self.nwalkers
1079323
def create_group(cls, prefix: str, name: str) -> ErrorGroup:
        """"""
        group = cls.ErrorGroup(prefix, name)
        cls.groups.append(group)
        return group
647629
def _iter_grouped(self):
        """"""
        for indices in self._group_indices:
            yield self._obj.isel(**{self._group_dim: indices})
1141031
def run_experiment(experiment, roleouts, episodes, in_cloud=False,
                   dynProfile=None):
    
    def run():
        if dynProfile is None:
            maxsteps = len(experiment.profile) # episode length
        else:
            maxsteps = dynProfile.shape[1]
        na = len(experiment.agents)
        ni = roleouts * episodes * maxsteps

        all_action = zeros((na, 0))
        all_reward = zeros((na, 0))
        epsilon = zeros((na, ni)) # exploration rate

        # Converts to action vector in percentage markup values.
        vmarkup = vectorize(get_markup)

        for roleout in range(roleouts):
            if dynProfile is not None:
                # Apply new load profile before each roleout (week).
                i = roleout * episodes # index of first profile value
                experiment.profile = dynProfile[i:i + episodes, :]

#            print "PROFILE:", experiment.profile, episodes

            experiment.doEpisodes(episodes) # number of samples per learning step

            nei = episodes * maxsteps # num interactions per role
            epi_action = zeros((0, nei))
            epi_reward = zeros((0, nei))

            for i, (task, agent) in \
            enumerate(zip(experiment.tasks, experiment.agents)):
                action = copy(agent.history["action"])
                reward = copy(agent.history["reward"])

                for j in range(nei):
                    if isinstance(agent.learner, DirectSearchLearner):
                        action[j, :] = task.denormalize(action[j, :])
                        k = nei * roleout
                        epsilon[i, k:k + nei] = agent.learner.explorer.sigma[0]
                    elif isinstance(agent.learner, ValueBasedLearner):
                        action[j, :] = vmarkup(action[j, :], task)
                        k = nei * roleout
                        epsilon[i, k:k + nei] = agent.learner.explorer.epsilon
                    else:
                        action = vmarkup(action, task)

                # FIXME: Only stores action[0] for all interactions.
                epi_action = c_[epi_action.T, action[:, 0].flatten()].T
                epi_reward = c_[epi_reward.T, reward.flatten()].T

                if hasattr(agent, "module"):
                    print "PARAMS:", agent.module.params

                agent.learn()
                agent.reset()

            all_action = c_[all_action, epi_action]
            all_reward = c_[all_reward, epi_reward]

        return all_action, all_reward, epsilon

    if in_cloud:
        import cloud
        job_id = cloud.call(run, _high_cpu=False)
        result = cloud.result(job_id)
        all_action, all_reward, epsilon = result
    else:
        all_action, all_reward, epsilon = run()

    return all_action, all_reward, epsilon
1094894
def set_experiment(experiment_config, mode, port, config_file_name):
    ''''''
    request_data = dict()
    request_data['authorName'] = experiment_config['authorName']
    request_data['experimentName'] = experiment_config['experimentName']
    request_data['trialConcurrency'] = experiment_config['trialConcurrency']
    request_data['maxExecDuration'] = experiment_config['maxExecDuration']
    request_data['maxTrialNum'] = experiment_config['maxTrialNum']
    request_data['searchSpace'] = experiment_config.get('searchSpace')
    request_data['trainingServicePlatform'] = experiment_config.get('trainingServicePlatform')

    if experiment_config.get('description'):
        request_data['description'] = experiment_config['description']
    if experiment_config.get('multiPhase'):
        request_data['multiPhase'] = experiment_config.get('multiPhase')
    if experiment_config.get('multiThread'):
        request_data['multiThread'] = experiment_config.get('multiThread')
    if experiment_config.get('advisor'):
        request_data['advisor'] = experiment_config['advisor']
    else:
        request_data['tuner'] = experiment_config['tuner']
        if 'assessor' in experiment_config:
            request_data['assessor'] = experiment_config['assessor']
    #debug mode should disable version check
    if experiment_config.get('debug') is not None:
        request_data['versionCheck'] = not experiment_config.get('debug')
    if experiment_config.get('logCollection'):
        request_data['logCollection'] = experiment_config.get('logCollection')

    request_data['clusterMetaData'] = []
    if experiment_config['trainingServicePlatform'] == 'local':
        request_data['clusterMetaData'].append(
            {'key':'codeDir', 'value':experiment_config['trial']['codeDir']})
        request_data['clusterMetaData'].append(
            {'key': 'command', 'value': experiment_config['trial']['command']})
    elif experiment_config['trainingServicePlatform'] == 'remote':
        request_data['clusterMetaData'].append(
            {'key': 'machine_list', 'value': experiment_config['machineList']})
        request_data['clusterMetaData'].append(
            {'key': 'trial_config', 'value': experiment_config['trial']})
    elif experiment_config['trainingServicePlatform'] == 'pai':
        request_data['clusterMetaData'].append(
            {'key': 'pai_config', 'value': experiment_config['paiConfig']})        
        request_data['clusterMetaData'].append(
            {'key': 'trial_config', 'value': experiment_config['trial']})
    elif experiment_config['trainingServicePlatform'] == 'kubeflow':
        request_data['clusterMetaData'].append(
            {'key': 'kubeflow_config', 'value': experiment_config['kubeflowConfig']})
        request_data['clusterMetaData'].append(
            {'key': 'trial_config', 'value': experiment_config['trial']})
    elif experiment_config['trainingServicePlatform'] == 'frameworkcontroller':
        request_data['clusterMetaData'].append(
            {'key': 'frameworkcontroller_config', 'value': experiment_config['frameworkcontrollerConfig']})
        request_data['clusterMetaData'].append(
            {'key': 'trial_config', 'value': experiment_config['trial']})

    response = rest_post(experiment_url(port), json.dumps(request_data), REST_TIME_OUT, show_error=True)
    if check_response(response):
        return response
    else:
        _, stderr_full_path = get_log_path(config_file_name)
        if response is not None:
            with open(stderr_full_path, 'a+') as fout:
                fout.write(json.dumps(json.loads(response.text), indent=4, sort_keys=True, separators=(',', ':')))
            print_error('Setting experiment error, error message is {}'.format(response.text))
        return None
59336
def do_experiment(self, params):
        
        paramlist = self.expand_param_list(params)
        
        # create directories, write config files
        for pl in paramlist:
            # check for required param keys
            if ('name' in pl) and ('iterations' in pl) and ('repetitions' in pl) and ('path' in pl):
               self.create_dir(pl, self.options.delete)
            else:
                print 'Error: parameter set does not contain all required keys: name, iterations, repetitions, path'
                return False
            
        # create experiment list 
        explist = []
            
        # expand paramlist for all repetitions and add self and rep number
        for p in paramlist:
            explist.extend(zip( [self]*p['repetitions'], [p]*p['repetitions'], xrange(p['repetitions']) ))
                
        # if only 1 process is required call each experiment seperately (no worker pool)
        if self.options.ncores == 1:
            for e in explist:
                mp_runrep(e)
        else:
            # create worker processes    
            pool = Pool(processes=self.options.ncores)
            pool.map(mp_runrep, explist)
        
        return True
1066506
def iterator(self):
        """"""
        if self.combined:
            grouped_data = [[(0, datum)] for datum in self.data]
            skip_dims = {"chain"}
        else:
            grouped_data = [datum.groupby("chain") for datum in self.data]
            skip_dims = set()

        label_dict = OrderedDict()
        for name, grouped_datum in zip(self.model_names, grouped_data):
            for _, sub_data in grouped_datum:
                datum_iter = xarray_var_iter(
                    sub_data,
                    var_names=[self.var_name],
                    skip_dims=skip_dims,
                    reverse_selections=True,
                )
                for _, selection, values in datum_iter:
                    label = make_label(self.var_name, selection, position="beside")
                    if label not in label_dict:
                        label_dict[label] = OrderedDict()
                    if name not in label_dict[label]:
                        label_dict[label][name] = []
                    label_dict[label][name].append(values)

        y = self.y_start
        for label, model_data in label_dict.items():
            for model_name, value_list in model_data.items():
                if model_name:
                    row_label = "{}: {}".format(model_name, label)
                else:
                    row_label = label
                for values in value_list:
                    yield y, row_label, label, values, self.model_color[model_name]
                    y += self.chain_offset
                y += self.var_offset
            y += self.group_offset