collect metadata into this object

        :param data: dict
        :param args: build from args instead of data
8843
def build_result_metadata(self, data=None, args=None):
        
        data = data if data else self._build_result_metainfo(args)
        if data.get("build_branch"):
            self.build_branch = data.get("build_branch")
        if data.get("buildcommit"):
            self.buildcommit = data.get("buildcommit")
        if data.get("build_git_url"):
            self.build_git_url = data.get("build_git_url")
        if data.get("build_url"):
            self.build_url = data.get("build_url")
        if data.get("campaign"):
            self.campaign = data.get("campaign")
        if data.get("job_id"):
            self.job_id = data.get("job_id")
        if data.get("toolchain"):
            self.toolchain = data.get("toolchain")
        if data.get("build_date"):
            self.build_date = data.get("build_date")
363978
def convert(self, *args, **kwargs):
        

        self.strings()
        self.metadata()

        # save file
        self.result.save(self.output())
998018
def parse_meta_data(self, line):
        """"""
        line = line.rstrip()
        logger.debug("Parsing metadata line:{0}".format(line))
        line_info = line[2:].split('=')
        match = False

        if line_info[0] == 'fileformat':
            logger.debug("Parsing fileformat")
            try:
                self.fileformat = line_info[1]
                logger.debug("Found fileformat {0}".format(self.fileformat))
            except IndexError:
                raise SyntaxError("fileformat must have a value")

        elif line_info[0] == 'INFO':
            match = self.info_pattern.match(line)
            if not match:
                raise SyntaxError("One of the INFO lines is malformed:{0}".format(line))

            matches = [
                match.group('id'), match.group('number'),
                match.group('type'), match.group('desc')
            ]

            # extra_info is a dictionary to check the metadata about the INFO values:
            self.extra_info[matches[0]] = dict(
                zip(self.header_keys['info'][1:], matches[1:])
            )

            info_line = dict(list(zip(self.header_keys['info'],matches)))

            if len(info_line['Description'].split('Format:')) > 1:
                info_line['Format'] = [
                    info.strip() for info in info_line['Description'].split('Format:')
                ][-1]
            self.info_lines.append(info_line)

            # Store the vep columns:
            if info_line['ID'] == 'CSQ':
                self.vep_columns = info_line.get('Format', '').split('|')

            if info_line['ID'] == 'ANN':
                self.snpeff_columns = [
                    annotation.strip("' ") for annotation in
                    info_line.get('Description', '').split(':')[-1].split('|')]

            self.info_dict[match.group('id')] = line

        elif line_info[0] == 'FILTER':
            match = self.filter_pattern.match(line)
            if not match:
                raise SyntaxError("One of the FILTER lines is malformed: {0}".format(line))
            matches = [match.group('id'), match.group('desc')]
            self.filter_lines.append(dict(
                list(zip(self.header_keys['filt'],matches)))
            )
            self.filter_dict[match.group('id')] = line

        elif line_info[0] == 'contig':
            match = self.contig_pattern.match(line)
            if not match:
                print()
                raise SyntaxError("One of the contig lines is malformed: {0}".format(line))

            matches = [match.group('id'), match.group('length')]
            self.contig_lines.append(dict(
                list(zip(self.header_keys['contig'],matches)))
            )
            self.contig_dict[match.group('id')] = line

        elif line_info[0] == 'FORMAT':
            match = self.format_pattern.match(line)
            if not match:
                raise SyntaxError("One of the FORMAT lines is malformed: {0}".format(line))

            matches = [
                match.group('id'), match.group('number'),
                match.group('type'), match.group('desc')
            ]
            self.format_lines.append(dict(
                list(zip(self.header_keys['form'],matches)))
            )
            self.format_dict[match.group('id')] = line

        elif line_info[0] == 'ALT':
            match = self.alt_pattern.match(line)
            if not match:
                raise SyntaxError("One of the ALT lines is malformed: {0}".format(line))

            matches = [match.group('id'), match.group('desc')]
            self.alt_lines.append(dict(
                list(zip(self.header_keys['alt'],matches)))
            )
            self.alt_dict[match.group('id')] = line

        else:
            match = self.meta_pattern.match(line)
            if not match:
                raise SyntaxError("One of the meta data lines is malformed: {0}".format(line))

            self.other_lines.append({match.group('key'): match.group('val')})
            self.other_dict[match.group('key')] = line
244248
def run(args):
    
    html_metadata_extractor = HTMLMetadataExtractor()

    with warnings.catch_warnings():
        warnings.simplefilter('ignore')

        extractions = html_metadata_extractor.extract(html_text=args.input_file)
        for e in extractions:
            print(e.value)
845043
def collect_metadata(self, name, obj):
        
        # Look through each subgroup
        base_name = name + "/" if name else ""
        for group_name, group_obj in obj.groups.items():
            self.collect_metadata(base_name + group_name, group_obj)
        for var_name, var_obj in obj.variables.items():
            var_name = base_name + var_name
            self.file_content[var_name] = var_obj
            self.file_content[var_name + "/dtype"] = var_obj.dtype
            self.file_content[var_name + "/shape"] = var_obj.shape
            self._collect_attrs(var_name, var_obj)
        self._collect_attrs(name, obj)
1257806
def _collect_data(self):
        
        all_data = []
        for line in self.engine.run_engine():
            logging.debug("Adding {} to all_data".format(line))
            all_data.append(line.copy())
            logging.debug("all_data is now {}".format(all_data))

        return all_data
345005
def _pack_data(X, Y, metadata):
    
    if not any(metadata):
        # legacy list of list format is acceptable
        return list(zip(X, Y))

    else:
        # newer dictionary-based format is required in order to save metadata
            return [
                {
                    'data': x,
                    'target': y,
                    'metadata': meta
                }
                for x, y, meta in zip(X, Y, metadata)
            ]
694132
def import_data(self, data: dict):
        '''
        
        '''
        if not isinstance(data, dict):
            raise ValueError('Cannot import data not as dict')
        self._data.update(data)

        for name, field in self._fields.items():
            self._data[name] = field._import(self._data.get(name)) or field.default
658584
def combine_meta_data(files_dict, meta_data_v2=True):
    
    if len(files_dict) > 10:
        logging.info("Combine the meta data from %d files", len(files_dict))
    # determine total length needed for the new combined array, thats the fastest way to combine arrays
    total_length = 0  # the total length of the new table
    for file_name in files_dict.iterkeys():
        with tb.open_file(file_name, mode="r") as in_file_h5:  # open the actual file
            total_length += in_file_h5.root.meta_data.shape[0]

    if meta_data_v2:
        meta_data_combined = np.empty((total_length, ), dtype=[
            ('index_start', np.uint32),
            ('index_stop', np.uint32),
            ('data_length', np.uint32),
            ('timestamp_start', np.float64),
            ('timestamp_stop', np.float64),
            ('error', np.uint32)])
    else:
        meta_data_combined = np.empty((total_length, ), dtype=[
            ('start_index', np.uint32),
            ('stop_index', np.uint32),
            ('length', np.uint32),
            ('timestamp', np.float64),
            ('error', np.uint32)])

    if len(files_dict) > 10:
        progress_bar = progressbar.ProgressBar(widgets=['', progressbar.Percentage(), ' ', progressbar.Bar(marker='*', left='|', right='|'), ' ', progressbar.AdaptiveETA()], maxval=total_length, term_width=80)
        progress_bar.start()

    index = 0

    # fill actual result array
    for file_name in files_dict.iterkeys():
        with tb.open_file(file_name, mode="r") as in_file_h5:  # open the actual file
            array_length = in_file_h5.root.meta_data.shape[0]
            meta_data_combined[index:index + array_length] = in_file_h5.root.meta_data[:]
            index += array_length
            if len(files_dict) > 10:
                progress_bar.update(index)
    if len(files_dict) > 10:
        progress_bar.finish()
    return meta_data_combined
1246623
def create_metadata(self, **params):
        
        params = json.dumps(params)
        return self.post("https://upload.twitter.com/1.1/media/metadata/create.json", params=params)
443710
def listTagged(self, *args, **kwargs):
        
        data = yield self.call('listTagged', *args, **kwargs)
        builds = []
        for bdata in data:
            build = Build.fromDict(bdata)
            build.connection = self
            builds.append(build)
        defer.returnValue(builds)
1084813
def generatemetadata(self, parameters, parentfile, relevantinputfiles, provenancedata = None):
        """"""
        assert isinstance(provenancedata,CLAMProvenanceData) or provenancedata is None

        data = {}

        if self.copymetadata:
            #Copy parent metadata
            for key, value in parentfile.metadata.items():
                data[key] = value

        for metafield in self.metafields:
            if isinstance(metafield, ParameterCondition):
                metafield = metafield.evaluate(parameters)
                if not metafield:
                    continue
            assert isinstance(metafield, AbstractMetaField)
            metafield.resolve(data, parameters, parentfile, relevantinputfiles)

        if provenancedata:
            data['provenance'] = provenancedata

        return self.formatclass(None, **data)
1245731
def update_metadata(ctx, archive_name):
    '''
    
    '''

    _generate_api(ctx)
    args, kwargs = _parse_args_and_kwargs(ctx.args)
    assert len(args) == 0, 'Unrecognized arguments: "{}"'.format(args)

    var = ctx.obj.api.get_archive(archive_name)

    var.update_metadata(metadata=kwargs)
219372
def data(self, data: numpy.ndarray) -> None:
        
        self.__data_item.set_data(numpy.copy(data))
1242636
def _loadData(self, data):
        
        self._data = data
        self.fastKey = data.attrib.get('fastKey')
        self.key = data.attrib.get('key')
        self.thumb = data.attrib.get('thumb')
        self.title = data.attrib.get('title')
        self.type = data.attrib.get('type')
234874
def meta(self, meta):
        ''''''
        data = meta.as_dict()
        data['namespace'] = self.basekey(meta)
        return data
114442
def parse_meta(self, selected_meta_data):
        
        # reads all meta data files
        files = self._get_files("meta", self.path)
        df = pd.DataFrame()
        print("Parsing the metadata files...")
        for f in tqdm(files):
            data = self.parse_single_meta(f, selected_meta_data)
            if data is not None:
                df = pd.concat([df, data], axis=0)
        df.index = df['sample']
        #
        # df = df.drop('sample', 1)  # 1 for the columns
        return df