Compute the statistics of fields of a TensorDataset

    Parameters
    ----------
    dataset_path : str
        path to the dataset
    output_dir : str
        where to save the data
    config : :obj:`YamlConfig`
        parameters for the analysis
505644
def compute_dataset_statistics(dataset_path,
                               output_path,
                               config):
    
    # parse config
    analysis_fields = config['analysis_fields']
    num_percentiles = config['num_percentiles']
    thresholds = config['thresholds']
    log_rate = config['log_rate']

    num_bins = config['num_bins']
    font_size = config['font_size']
    line_width = config['line_width']
    dpi = config['dpi']
    
    # create dataset for the aggregated results
    dataset = TensorDataset.open(dataset_path)
    num_datapoints = dataset.num_datapoints

    # allocate buffers
    analysis_data = {}
    for field in analysis_fields:
        analysis_data[field] = []

    # loop through dataset
    for i in range(num_datapoints):
        if i % log_rate == 0:
            logging.info('Reading datapoint %d of %d' %(i+1, num_datapoints))

        # read datapoint
        datapoint = dataset.datapoint(i, analysis_fields)
        for key, value in datapoint.iteritems():
            analysis_data[key].append(value)

    # create output CSV
    stats_headers = {
        'name': 'str',
        'mean': 'float',
        'median': 'float',
        'std': 'float'
    }    
    for i in range(num_percentiles):
        pctile = int((100.0 / num_percentiles) * i)
        field = '%d_pctile' %(pctile)
        stats_headers[field] = 'float'
    for t in thresholds:
        field = 'pct_above_%.3f' %(t)
        stats_headers[field] = 'float'
    
    # analyze statistics
    for field, data in analysis_data.iteritems():
        # init arrays
        data = np.array(data)

        # init filename
        stats_filename = os.path.join(output_path, '%s_stats.json' %(field))
        if os.path.exists(stats_filename):
            logging.warning('Statistics file %s exists!' %(stats_filename))
        
        # stats
        mean = np.mean(data)
        median = np.median(data)
        std = np.std(data)
        stats = {
            'name': str(field),
            'mean': float(mean),
            'median': float(median),
            'std': float(std),
        }
        for i in range(num_percentiles):
            pctile = int((100.0 / num_percentiles) * i)
            pctile_field = '%d_pctile' %(pctile)
            stats[pctile_field] = float(np.percentile(data, pctile))
        for t in thresholds:
            t_field = 'pct_above_%.3f' %(t)
            stats[t_field] = float(np.mean(1 * (data > t)))
        json.dump(stats,
                  open(stats_filename, 'w'),
                  indent=2,
                  sort_keys=True)
                  
        # histogram
        num_unique = np.unique(data).shape[0]
        nb = min(num_bins, data.shape[0], num_unique)
        bounds = (np.min(data), np.max(data))
        vis2d.figure()
        utils.histogram(data,
                        nb,
                        bounds,
                        normalized=False,
                        plot=True)
        vis2d.xlabel(field, fontsize=font_size)
        vis2d.ylabel('Count', fontsize=font_size)
        data_filename = os.path.join(output_path, 'histogram_%s.pdf' %(field))
        vis2d.show(data_filename, dpi=dpi)
659368
def init_dataset_prepare_args(self, parser):
        ''' '''
        parser.add_argument('-f', '--configuration', dest='config', default=DEFAULT_USER_CONFIG_PATH,
                            help='the path to the configuration file to use -- ./config.yaml by default')
        parser.add_argument('-c', '--corpus-name', help='the name of the corpus to process')
        parser.add_argument('-n', '--dataset-name', help='the name to assign the newly processed dataset')
1168471
def _default_output_dir():
  """"""
  try:
    dataset_name = gin.query_parameter("inputs.dataset_name")
  except ValueError:
    dataset_name = "random"
  dir_name = "{model_name}_{dataset_name}_{timestamp}".format(
      model_name=gin.query_parameter("train.model").configurable.name,
      dataset_name=dataset_name,
      timestamp=datetime.datetime.now().strftime("%Y%m%d_%H%M"),
  )
  dir_path = os.path.join("~", "trax", dir_name)
  print()
  trax.log("No --output_dir specified")
  return dir_path
320955
private static DataSet<Tuple2<String, String>> getDocumentsDataSet(ExecutionEnvironment env, ParameterTool params) {
		// Create DataSet for documents relation (URL, Doc-Text)
		if (params.has("documents")) {
			return env.readCsvFile(params.get("documents"))
						.fieldDelimiter("|")
						.types(String.class, String.class);
		} else {
			System.out.println("Executing WebLogAnalysis example with default documents data set.");
			System.out.println("Use --documents to specify file input.");
			return WebLogData.getDocumentDataSet(env);
		}
	}
990384
def average(param_paths: Iterable[str]) -> Dict[str, mx.nd.NDArray]:
    
    all_arg_params = []
    all_aux_params = []
    for path in param_paths:
        logger.info("Loading parameters from '%s'", path)
        arg_params, aux_params = utils.load_params(path)
        all_arg_params.append(arg_params)
        all_aux_params.append(aux_params)

    logger.info("%d models loaded", len(all_arg_params))
    utils.check_condition(all(all_arg_params[0].keys() == p.keys() for p in all_arg_params),
                          "arg_param names do not match across models")
    utils.check_condition(all(all_aux_params[0].keys() == p.keys() for p in all_aux_params),
                          "aux_param names do not match across models")

    avg_params = {}
    # average arg_params
    for k in all_arg_params[0]:
        arrays = [p[k] for p in all_arg_params]
        avg_params["arg:" + k] = utils.average_arrays(arrays)
    # average aux_params
    for k in all_aux_params[0]:
        arrays = [p[k] for p in all_aux_params]
        avg_params["aux:" + k] = utils.average_arrays(arrays)

    return avg_params
944038
def path(self):
        """"""
        return "/projects/%s/datasets/%s/models/%s" % (
            self._proto.project_id,
            self._proto.dataset_id,
            self._proto.model_id,
        )
1141463
def cache_to_df(dir_path):
    
    table = {}
    for attrib in glob.glob(os.path.join(dir_path, '*')):
        attrib_name, attrib_ext = os.path.splitext(os.path.basename(attrib))
        if attrib_ext == '.lf8':
            attrib_data = np.fromfile(attrib, np.float64)
            table[attrib_name] = attrib_data

        elif attrib_ext == '.lf4':
            attrib_data = np.fromfile(attrib, np.float32)
            table[attrib_name] = attrib_data

        elif attrib_ext == '.li2':
            attrib_data = np.fromfile(attrib, np.int16)
            table[attrib_name] = attrib_data

        elif attrib_ext == '.li4':
            attrib_data = np.fromfile(attrib, np.int32)
            table[attrib_name] = attrib_data

        elif attrib_ext == '.li8':
            attrib_data = np.fromfile(attrib, np.int64)
            table[attrib_name] = attrib_data

        elif attrib_ext == '.ib1':
            attrib_data = np.fromfile(attrib, np.bool_)
            table[attrib_name] = attrib_data

        elif attrib_ext.startswith('.iS'):
            length_string = int(attrib_ext[3:])
            attrib_data = np.fromfile(attrib, ('a' + str(length_string)))
            table[attrib_name] = attrib_data

        else:
            print('Array {} is not a recognized data type'.format(attrib))

    df = pd.DataFrame(table)
    return df
6324
def _get_output_nodes(self, output_path, error_path):
        
        from pymatgen.io.nwchem import NwOutput
        from aiida.orm.data.structure import StructureData
        from aiida.orm.data.array.trajectory import TrajectoryData

        ret_dict = []
        nwo = NwOutput(output_path)
        for out in nwo.data:
            molecules = out.pop('molecules', None)
            structures = out.pop('structures', None)
            if molecules:
                structlist = [StructureData(pymatgen_molecule=m)
                              for m in molecules]
                ret_dict.append(('trajectory',
                                 TrajectoryData(structurelist=structlist)))
            if structures:
                structlist = [StructureData(pymatgen_structure=s)
                              for s in structures]
                ret_dict.append(('trajectory',
                                 TrajectoryData(structurelist=structlist)))
            ret_dict.append(('output', ParameterData(dict=out)))

        # Since ParameterData rewrites it's properties (using _set_attr())
        # with keys from the supplied dictionary, ``source`` has to be
        # moved to another key. See issue #9 for details:
        # (https://bitbucket.org/epfl_theos/aiida_epfl/issues/9)
        nwo.job_info['program_source'] = nwo.job_info.pop('source', None)
        ret_dict.append(('job_info', ParameterData(dict=nwo.job_info)))
        
        return ret_dict
586647
def load(self):
        
        data = self.dict_class()
        
        for path in self.paths:
            if path in self.paths_loaded: continue
            
            try:
                with open(path, 'r') as file:
                    path_data = yaml.load(file.read())
                    data = dict_merge(data, path_data)
                self.paths_loaded.add(path)
            except IOError:
                # TODO: Log this correctly once logging is implemented
                if not path.endswith('.local.yml'):
                    print 'CONFIG NOT FOUND: %s' % (path)

        self.data = data
1359743
def dump_profiles(self, path):
        
        if self.profiler_collector is not None:
            self.profiler_collector.dump_profiles(path)
        else:
            raise RuntimeError("'spark.python.profile' configuration must be set "
                               "to 'true' to enable Python profile.")
1052428
def _get_output_nodes(self, output_path, error_path):
        
        from aiida.orm.data.array.trajectory import TrajectoryData
        import re

        state = None
        step = None
        scale = None
        with open(output_path) as f:
            lines = [x.strip('\n') for x in f.readlines()]

        result_dict = dict()
        trajectory = None
        for line in lines:
            if state is None and re.match('^\s*NWChem SCF Module\s*$',line):
                state = 'nwchem-scf-module'
                continue
            if state is None and re.match('^\s*NWChem Geometry Optimization\s*$',line):
                state = 'nwchem-geometry-optimisation'
                trajectory = TrajectoryData()
                continue
            if state == 'nwchem-scf-module' and re.match('^\s*Final RHF \s*results\s*$',line):
                state = 'final-rhf-results'
                continue
            if re.match('^\s*\-*\s*$',line):
                continue
            if state == 'final-rhf-results':
                result = re.match('^\s*([^=]+?)\s*=\s*([\-\d\.]+)$',line)
                if result:
                    key = re.sub('[^a-zA-Z0-9]+', '_', result.group(1).lower())
                    result_dict[key] = result.group(2)
                else:
                    state = 'nwchem-scf-module'
            if state == 'nwchem-geometry-optimisation' and re.match('^\s*Step\s+\d+\s*$',line):
                result = re.match('^\s*Step\s+(\d+)\s*$',line)
                step = result.group(1)
                continue
            if state == 'nwchem-geometry-optimisation' and \
                re.match('^\s*Output coordinates in a.u.',line):
                state = 'nwchem-geometry-optimisation-coordinates'
                result = re.match('scale by \s(*[\-\d\.]+)',line)
                scale = result.group(1)
                continue
        return [('parameters', ParameterData(dict=result_dict))]
1243393
public static <E> View<E> viewForPath(Dataset<E> dataset, Path path) {
    if (dataset instanceof FileSystemDataset) {
      return ((FileSystemDataset<E>) dataset).viewForUri(path.toUri());
    }
    throw new IllegalArgumentException("Not a file system dataset: " + dataset);
  }
1132383
def _get_result_paths(self, data):
        

        result = {}

        result['Output'] = ResultPath(
            Path=self.Parameters['--output'].Value,
            IsWritten=self.Parameters['--output'].isOn())

        result['ClusterFile'] = ResultPath(
            Path=self.Parameters['--uc'].Value,
            IsWritten=self.Parameters['--uc'].isOn())

        result['PairwiseAlignments'] = ResultPath(
            Path=self.Parameters['--fastapairs'].Value,
            IsWritten=self.Parameters['--fastapairs'].isOn())

        return result
709039
def _get_result_paths(self, data):
        

        result = {}

        # OTU map (mandatory output)
        result['OtuMap'] = ResultPath(Path=self.Parameters['-O'].Value,
                                      IsWritten=True)

        # SumaClust will not produce any output file if the
        # input file was empty, so we create an empty
        # output file
        if not isfile(result['OtuMap'].Path):
            otumap_f = open(result['OtuMap'].Path, 'w')
            otumap_f.close()

        return result
800095
def annotated_dataset_path(cls, project, dataset, annotated_dataset):
        """"""
        return google.api_core.path_template.expand(
            "projects/{project}/datasets/{dataset}/annotatedDatasets/{annotated_dataset}",
            project=project,
            dataset=dataset,
            annotated_dataset=annotated_dataset,
        )
14994
def load_configuration(conf_path):
    
    with open(conf_path) as f:
        conf_dict = yaml.load(f)
    validate_config(conf_dict)
    return conf_dict
534933
def create_prediction_estimator(hyper_params, model, checkpoint_path=None):
    
    if checkpoint_path is None:
        chkpts = sorted([name for name in os.listdir(hyper_params.train.checkpoint_path)])
        checkpoint_path = hyper_params.train.checkpoint_path + "/" + chkpts[-1]
        print("Latest found checkpoint: {}".format(checkpoint_path))

    estimator_spec = create_tf_estimator_spec(checkpoint_path, model, create_loss=None)

    # Create the estimator.
    estimator = tf.estimator.Estimator(estimator_spec,
                                       model_dir=checkpoint_path,
                                       params=hyper_params)

    return estimator