Build meta file for trial.

        Args:
            expr_dir (str): Directory path of the experiment.

        Return:
            A dict of trial meta info.
162251
def _build_trial_meta(cls, expr_dir):
        
        meta_file = os.path.join(expr_dir, EXPR_META_FILE)
        meta = parse_json(meta_file)

        if not meta:
            job_id = expr_dir.split("/")[-2]
            trial_id = expr_dir[-8:]
            params = parse_json(os.path.join(expr_dir, EXPR_PARARM_FILE))
            meta = {
                "trial_id": trial_id,
                "job_id": job_id,
                "status": "RUNNING",
                "type": "TUNE",
                "start_time": os.path.getctime(expr_dir),
                "end_time": None,
                "progress_offset": 0,
                "result_offset": 0,
                "params": params
            }

        if not meta.get("start_time", None):
            meta["start_time"] = os.path.getctime(expr_dir)

        if isinstance(meta["start_time"], float):
            meta["start_time"] = timestamp2date(meta["start_time"])

        if meta.get("end_time", None):
            meta["end_time"] = timestamp2date(meta["end_time"])

        meta["params"] = parse_json(os.path.join(expr_dir, EXPR_PARARM_FILE))

        return meta
406466
def print_app_meta_data(self):
        
        l_metaData  = dir(self)
        l_classVar  = [x for x in l_metaData if x.isupper() ]
        for str_var in l_classVar:
            str_val = getattr(self, str_var)
            print("%20s: %s" % (str_var, str_val))
697378
def verify_directory(verbose=True, max_size_mb=50):
    
    # Check required files
    ok = True
    mb_to_bytes = 1000 * 1000
    expected_files = ["config.txt", "experiment.py"]

    for f in expected_files:
        if os.path.exists(f):
            log("✓ {} is PRESENT".format(f), chevrons=False, verbose=verbose)
        else:
            log("✗ {} is MISSING".format(f), chevrons=False, verbose=verbose)
            ok = False

    # Check size
    max_size = max_size_mb * mb_to_bytes
    size = size_on_copy()
    if size > max_size:
        size_in_mb = round(size / mb_to_bytes)
        log(
            "✗ {}MB is TOO BIG (greater than {}MB)".format(size_in_mb, max_size_mb),
            chevrons=False,
            verbose=verbose,
        )
        ok = False

    return ok
532608
def create_paired_dir(output_dir, meta_id, static=False, needwebdir=True):
    
    # get the absolute root path
    root_path = os.path.abspath(output_dir)
    # if it's a static directory, add even and odd
    if static:
        # determine whether meta-id is odd or even
        if meta_id[-1].isdigit():
            last_character = int(meta_id[-1])
        else:
            last_character = ord(meta_id[-1])
        if last_character % 2 == 0:
            num_dir = 'even'
        else:
            num_dir = 'odd'
        # add odd or even to the path, based on the meta-id
        output_path = os.path.join(root_path, num_dir)
    # if it's a meta directory, output as normal
    else:
        output_path = root_path
    # if it doesn't already exist, create the output path (includes even/odd)
    if not os.path.exists(output_path):
        os.mkdir(output_path)
    # add the pairtree to the output path
    path_name = add_to_pairtree(output_path, meta_id)
    # add the meta-id directory to the end of the pairpath
    meta_dir = os.path.join(path_name, meta_id)
    os.mkdir(meta_dir)
    # if we are creating static output
    if static and needwebdir:
        # add the web path to the output directory
        os.mkdir(os.path.join(meta_dir, 'web'))
        static_dir = os.path.join(meta_dir, 'web')
        return static_dir
    # else we are creating meta output or don't need web directory
    else:
        return meta_dir
238309
def _get_evidence_bam(work_dir, data):
    
    evidence_bam = glob.glob(os.path.join(work_dir, "results", "evidence",
                                            "evidence_*.%s*.bam" % (dd.get_sample_name(data))))
    if evidence_bam:
        return evidence_bam[0]
254262
def copy_target_dir(cls, orig_dir, dest_dir, roi_baseline, extracopy):
        
        try:
            os.makedirs(dest_dir)
        except OSError:
            pass

        copyfiles = ['%s.fits' % roi_baseline,
                     '%s.npy' % roi_baseline,
                     '%s_*.xml' % roi_baseline] + cls.copyfiles
        if isinstance(extracopy, list):
            copyfiles += extracopy

        cls.copy_analysis_files(orig_dir, dest_dir, copyfiles)
863455
def _find_dirs(metadata):
    '''
    
    '''

    ret = []
    found = {}

    for bucket_dict in metadata:
        for bucket_name, data in six.iteritems(bucket_dict):
            dirpaths = set()
            for path in [k['Key'] for k in data]:
                prefix = ''
                for part in path.split('/')[:-1]:
                    directory = prefix + part + '/'
                    dirpaths.add(directory)
                    prefix = directory
            if bucket_name not in found:
                found[bucket_name] = True
                ret.append({bucket_name: list(dirpaths)})
            else:
                for bucket in ret:
                    if bucket_name in bucket:
                        bucket[bucket_name] += list(dirpaths)
                        bucket[bucket_name] = list(set(bucket[bucket_name]))
                        break
    return ret
863924
def get_meta_type_by_dir(dir_name):
    parent_data = get_default_metadata_data()
    child_data = get_child_metadata_data()
    data = parent_data['metadataObjects'] + child_data
    for item in data:
        if 'directoryName' in item and item['directoryName'].lower() == dir_name.lower():
            return item
        elif 'tagName' in item and item['tagName'].lower() == dir_name.lower():
            return item
    '''
        
    '''
    try:
        if config.describe_data != None:
            project_org_describe = config.describe_data
        if config.project != None and os.path.isfile(os.path.join(config.project.location,'config','.describe')):
            project_org_describe = parse_json_from_file(os.path.join(config.project.location,'config','.describe'))
        if project_org_describe != None and 'metadataObjects' in project_org_describe:
            for item in project_org_describe["metadataObjects"]:
                if 'directoryName' in item and item['directoryName'].lower() == dir_name.lower():
                    return item
                elif 'tagName' in item and item['tagName'].lower() == dir_name.lower():
                    return item
    except:
        pass
1087294
def set_trial_config(experiment_config, port, config_file_name):
    ''''''
    request_data = dict()
    request_data['trial_config'] = experiment_config['trial']
    response = rest_put(cluster_metadata_url(port), json.dumps(request_data), REST_TIME_OUT)
    if check_response(response):
        return True
    else:
        print('Error message is {}'.format(response.text))
        _, stderr_full_path = get_log_path(config_file_name)
        if response:
            with open(stderr_full_path, 'a+') as fout:
                fout.write(json.dumps(json.loads(response.text), indent=4, sort_keys=True, separators=(',', ':')))
        return False
1054961
def p_common_scalar_magic_dir(p):
    ''
    value = getattr(p.lexer, 'filename', None)
    if value is not None:
        value = os.path.dirname(value)
    p[0] = ast.MagicConstant(p[1].upper(), value, lineno=p.lineno(1))
128776
def get_exps(self, path='.'):
        
        exps = []
        for dp, dn, fn in os.walk(path):
            if 'experiment.cfg' in fn:
                subdirs = [os.path.join(dp, d) for d in os.listdir(dp) if os.path.isdir(os.path.join(dp, d))]
                if all(map(lambda s: self.get_exps(s) == [], subdirs)):       
                    exps.append(dp)
        return exps
931930
def _build_result_metainfo(args):
        
        data = dict()
        if hasattr(args, "branch") and args.branch:
            data["build_branch"] = args.branch
        if hasattr(args, "commitId") and args.commitId:
            data["buildcommit"] = args.commitId
        if hasattr(args, "gitUrl") and args.gitUrl:
            data["build_git_url"] = args.gitUrl
        if hasattr(args, "buildUrl") and args.buildUrl:
            data["build_url"] = args.buildUrl
        if hasattr(args, "campaign") and args.campaign:
            data["campaign"] = args.campaign
        if hasattr(args, "jobId") and args.jobId:
            data["job_id"] = args.jobId
        if hasattr(args, "toolchain") and args.toolchain:
            data["toolchain"] = args.toolchain
        if hasattr(args, "buildDate") and args.buildDate:
            data["build_date"] = args.buildDate
        return data
1031494
def build_sample_paths(sample):
    
    for path_name, path in sample.paths.items():
        print("{}: '{}'".format(path_name, path))
        base, ext = os.path.splitext(path)
        if ext:
            print("Skipping file-like: '[}'".format(path))
        elif not os.path.isdir(base):
            os.makedirs(base)
507192
def dataset_initialize(self, folder):
        
        if not os.path.isdir(folder):
            raise ValueError('Invalid folder: ' + folder)

        ref = self.config_values[self.CONFIG_NAME_USER] + '/INSERT_SLUG_HERE'
        licenses = []
        default_license = {'name': 'CC0-1.0'}
        licenses.append(default_license)

        meta_data = {
            'title': 'INSERT_TITLE_HERE',
            'id': ref,
            'licenses': licenses
        }
        meta_file = os.path.join(folder, self.DATASET_METADATA_FILE)
        with open(meta_file, 'w') as f:
            json.dump(meta_data, f, indent=2)

        print('Data package template written to: ' + meta_file)
        return meta_file
1207735
def parsemeta(metadataloc):
    

    # filename or directory? if several fit, use first one and warn
    if os.path.isdir(metadataloc):
        metalist = glob.glob(os.path.join(metadataloc, METAPATTERN))
        if not metalist:
            raise MTLParseError(
                "No files matching metadata file pattern in directory %s."
                % metadataloc)
        elif len(metalist) > 0:
            metadatafn = metalist[0]
            filehandle = open(metadatafn, 'r')
            if len(metalist) > 1:
                logging.warning(
                    "More than one file in directory match metadata "
                    + "file pattern. Using %s." % metadatafn)
    elif os.path.isfile(metadataloc):
        metadatafn = metadataloc
        filehandle = open(metadatafn, 'r')
        logging.info("Using file %s." % metadatafn)
    elif 'L1_METADATA_FILE' in metadataloc:
        filehandle = StringIO(metadataloc)
    else:
        raise MTLParseError(
            "File location %s is unavailable " % metadataloc
            + "or doesn't contain a suitable metadata file.")

    # Reading file line by line and inserting data into metadata dictionary
    status = 0
    metadata = {}
    grouppath = []
    dictpath = [metadata]

    for line in filehandle:
        if status == 4:
            # we reached the end in the previous iteration,
            # but are still reading lines
            logging.warning(
                "Metadata file %s appears to " % metadatafn
                + "have extra lines after the end of the metadata. "
                + "This is probably, but not necessarily, harmless.")
        status = _checkstatus(status, line)
        grouppath, dictpath = _transstat(status, grouppath, dictpath, line)

    return metadata
268003
def load_demonstration(file_path):
    

    # First 32 bytes of file dedicated to meta-data.
    INITIAL_POS = 33

    if not os.path.isfile(file_path):
        raise FileNotFoundError("The demonstration file {} does not exist.".format(file_path))
    file_extension = pathlib.Path(file_path).suffix
    if file_extension != '.demo':
        raise ValueError("The file is not a '.demo' file. Please provide a file with the "
                         "correct extension.")

    brain_params = None
    brain_infos = []
    data = open(file_path, "rb").read()
    next_pos, pos, obs_decoded = 0, 0, 0
    total_expected = 0
    while pos < len(data):
        next_pos, pos = _DecodeVarint32(data, pos)
        if obs_decoded == 0:
            meta_data_proto = DemonstrationMetaProto()
            meta_data_proto.ParseFromString(data[pos:pos + next_pos])
            total_expected = meta_data_proto.number_steps
            pos = INITIAL_POS
        if obs_decoded == 1:
            brain_param_proto = BrainParametersProto()
            brain_param_proto.ParseFromString(data[pos:pos + next_pos])
            brain_params = BrainParameters.from_proto(brain_param_proto)
            pos += next_pos
        if obs_decoded > 1:
            agent_info = AgentInfoProto()
            agent_info.ParseFromString(data[pos:pos + next_pos])
            brain_info = BrainInfo.from_agent_proto([agent_info], brain_params)
            brain_infos.append(brain_info)
            if len(brain_infos) == total_expected:
                break
            pos += next_pos
        obs_decoded += 1
    return brain_params, brain_infos, total_expected
686787
def receive_trial_result(self, parameter_id, parameters, value):
        
        value = extract_scalar_reward(value)
        if self.optimize_mode == OptimizeMode.Maximize:
            value = -value

        logger.info("Received trial result.")
        logger.info("value is :" + str(value))
        logger.info("parameter is : " + str(parameters))

        # parse parameter to sample_x
        sample_x = [0 for i in range(len(self.key_order))]
        for key in parameters:
            idx = self.key_order.index(key)
            sample_x[idx] = parameters[key]

        # parse value to sample_y
        temp_y = []
        if sample_x in self.samples_x:
            idx = self.samples_x.index(sample_x)
            temp_y = self.samples_y[idx]
            temp_y.append(value)
            self.samples_y[idx] = temp_y

            # calculate y aggregation
            median = get_median(temp_y)
            self.samples_y_aggregation[idx] = [median]
        else:
            self.samples_x.append(sample_x)
            self.samples_y.append([value])

            # calculate y aggregation
            self.samples_y_aggregation.append([value])