Reads the GCMT metadata from line 2 of the ndk batch
15688
def _get_metadata_from_ndk_string(self, gcmt, ndk_string):
        
        gcmt.identifier = ndk_string[:16]
        inversion_data = re.split('[A-Z:]+', ndk_string[17:61])
        gcmt.metadata['BODY'] = [float(x) for x in inversion_data[1].split()]
        gcmt.metadata['SURFACE'] = [
            float(x) for x in inversion_data[2].split()]
        gcmt.metadata['MANTLE'] = [float(x) for x in inversion_data[3].split()]
        further_meta = re.split('[: ]+', ndk_string[62:])
        gcmt.metadata['CMT'] = int(further_meta[1])
        gcmt.metadata['FUNCTION'] = {'TYPE': further_meta[2],
                                     'DURATION': float(further_meta[3])}
        return gcmt
62092
def read_parameters(self):
        
        param_fname = join(self.out_dir, "PARAMETERS.OUT")

        if not exists(param_fname):
            raise FileNotFoundError("No PARAMETERS.OUT found")

        with open(param_fname) as nml_file:
            parameters = dict(f90nml.read(nml_file))
            for group in ["nml_years", "nml_allcfgs", "nml_outputcfgs"]:
                parameters[group] = dict(parameters[group])
                for k, v in parameters[group].items():
                    parameters[group][k] = _clean_value(v)
                parameters[group.replace("nml_", "")] = parameters.pop(group)
            self.config = parameters
        return parameters
959023
def read_dataset_metadata(self):
    """"""
    if self.dataset_meta:
      return
    shell_call(['gsutil', 'cp',
                'gs://' + self.storage_client.bucket_name + '/'
                + 'dataset/' + self.dataset_name + '_dataset.csv',
                LOCAL_DATASET_METADATA_FILE])
    with open(LOCAL_DATASET_METADATA_FILE, 'r') as f:
      self.dataset_meta = eval_lib.DatasetMetadata(f)
531342
def _read_data_legacy(prefix, batch_size):
    
    prefix = prefix.replace("\\", "/")
    folder = "/".join(prefix.split("/")[:-1])
    phase = prefix.split("/")[-1]
    config = json.load(open(prefix + '_config.json'))
    num_threads = config["num_threads"]

    filenames = [folder + "/" + f for f in listdir(folder) if isfile(join(folder, f)) and phase in f and not "config.json" in f]

    # Create a tf object for the filename list and the readers.
    filename_queue = tf.train.string_input_producer(filenames)
    readers = [_read_tf_record(filename_queue, config) for _ in range(num_threads)]

    batch_dict = tf.train.shuffle_batch_join(
        readers,
        batch_size=batch_size,
        capacity=10 * batch_size,
        min_after_dequeue=5 * batch_size
    )

    # Add batch dimension to feature and label shape

    feature_batch = {}
    label_batch = {}
    for k in batch_dict.keys():
        shape = tuple([batch_size] + list(config[k]["shape"]))
        tensor = tf.reshape(batch_dict[k], shape, name="input/"+phase+"/" + k + "_reshape")
        if "feature_" in k:
            feature_batch["_".join(k.split("_")[1:])] = tensor
        if "label_" in k:
            label_batch["_".join(k.split("_")[1:])] = tensor

    return feature_batch, label_batch
413957
def _prt_txt_desc2nts(self, prt, desc2nts, prtfmt=None):
        """"""
        if prtfmt is None:
            prtfmt = self.get_prtfmt("fmta")
        if self.ver_list is not None:
            prt.write("# Versions:\n#    {VER}\n".format(VER="\n#    ".join(self.ver_list)))
        self.prt_txt_desc2nts(prt, desc2nts, prtfmt)
10514
def _read_mat_mnu0(filename):
    
    print('read_mag_single_file: {0}'.format(filename))

    mat = sio.loadmat(filename, squeeze_me=True)
    # check the version
    version = mat['MP']['Version'].item()
    if version != 'FZJ-EZ-2017':
        raise Exception(
            'This data format is not supported (expected: FZJ-EZ-2017)' +
            ' got: {}'.format(version)
        )

    df_emd = _extract_emd(mat, filename=filename)
    df_md = _extract_md(mat)

    return df_emd, df_md
1138219
def get_metadata(self, filename):
        ''''''
        dest = self.path(filename)
        with open(dest, 'rb', buffering=0) as f:
            checksum = 'sha1:{0}'.format(sha1(f))
        return {
            'checksum': checksum,
            'size': os.path.getsize(dest),
            'mime': files.mime(filename),
            'modified': datetime.fromtimestamp(os.path.getmtime(dest)),
        }
998018
def parse_meta_data(self, line):
        """"""
        line = line.rstrip()
        logger.debug("Parsing metadata line:{0}".format(line))
        line_info = line[2:].split('=')
        match = False

        if line_info[0] == 'fileformat':
            logger.debug("Parsing fileformat")
            try:
                self.fileformat = line_info[1]
                logger.debug("Found fileformat {0}".format(self.fileformat))
            except IndexError:
                raise SyntaxError("fileformat must have a value")

        elif line_info[0] == 'INFO':
            match = self.info_pattern.match(line)
            if not match:
                raise SyntaxError("One of the INFO lines is malformed:{0}".format(line))

            matches = [
                match.group('id'), match.group('number'),
                match.group('type'), match.group('desc')
            ]

            # extra_info is a dictionary to check the metadata about the INFO values:
            self.extra_info[matches[0]] = dict(
                zip(self.header_keys['info'][1:], matches[1:])
            )

            info_line = dict(list(zip(self.header_keys['info'],matches)))

            if len(info_line['Description'].split('Format:')) > 1:
                info_line['Format'] = [
                    info.strip() for info in info_line['Description'].split('Format:')
                ][-1]
            self.info_lines.append(info_line)

            # Store the vep columns:
            if info_line['ID'] == 'CSQ':
                self.vep_columns = info_line.get('Format', '').split('|')

            if info_line['ID'] == 'ANN':
                self.snpeff_columns = [
                    annotation.strip("' ") for annotation in
                    info_line.get('Description', '').split(':')[-1].split('|')]

            self.info_dict[match.group('id')] = line

        elif line_info[0] == 'FILTER':
            match = self.filter_pattern.match(line)
            if not match:
                raise SyntaxError("One of the FILTER lines is malformed: {0}".format(line))
            matches = [match.group('id'), match.group('desc')]
            self.filter_lines.append(dict(
                list(zip(self.header_keys['filt'],matches)))
            )
            self.filter_dict[match.group('id')] = line

        elif line_info[0] == 'contig':
            match = self.contig_pattern.match(line)
            if not match:
                print()
                raise SyntaxError("One of the contig lines is malformed: {0}".format(line))

            matches = [match.group('id'), match.group('length')]
            self.contig_lines.append(dict(
                list(zip(self.header_keys['contig'],matches)))
            )
            self.contig_dict[match.group('id')] = line

        elif line_info[0] == 'FORMAT':
            match = self.format_pattern.match(line)
            if not match:
                raise SyntaxError("One of the FORMAT lines is malformed: {0}".format(line))

            matches = [
                match.group('id'), match.group('number'),
                match.group('type'), match.group('desc')
            ]
            self.format_lines.append(dict(
                list(zip(self.header_keys['form'],matches)))
            )
            self.format_dict[match.group('id')] = line

        elif line_info[0] == 'ALT':
            match = self.alt_pattern.match(line)
            if not match:
                raise SyntaxError("One of the ALT lines is malformed: {0}".format(line))

            matches = [match.group('id'), match.group('desc')]
            self.alt_lines.append(dict(
                list(zip(self.header_keys['alt'],matches)))
            )
            self.alt_dict[match.group('id')] = line

        else:
            match = self.meta_pattern.match(line)
            if not match:
                raise SyntaxError("One of the meta data lines is malformed: {0}".format(line))

            self.other_lines.append({match.group('key'): match.group('val')})
            self.other_dict[match.group('key')] = line
703594
def read_micromanager_metadata(fh):
    
    fh.seek(0)
    try:
        byteorder = {b'II': '<', b'MM': '>'}[fh.read(2)]
    except IndexError:
        raise ValueError('not a MicroManager TIFF file')

    result = {}
    fh.seek(8)
    (index_header, index_offset, display_header, display_offset,
     comments_header, comments_offset, summary_header, summary_length
     ) = struct.unpack(byteorder + 'IIIIIIII', fh.read(32))

    if summary_header != 2355492:
        raise ValueError('invalid MicroManager summary header')
    result['Summary'] = read_json(fh, byteorder, None, summary_length, None)

    if index_header != 54773648:
        raise ValueError('invalid MicroManager index header')
    fh.seek(index_offset)
    header, count = struct.unpack(byteorder + 'II', fh.read(8))
    if header != 3453623:
        raise ValueError('invalid MicroManager index header')
    data = struct.unpack(byteorder + 'IIIII'*count, fh.read(20*count))
    result['IndexMap'] = {'Channel': data[::5],
                          'Slice': data[1::5],
                          'Frame': data[2::5],
                          'Position': data[3::5],
                          'Offset': data[4::5]}

    if display_header != 483765892:
        raise ValueError('invalid MicroManager display header')
    fh.seek(display_offset)
    header, count = struct.unpack(byteorder + 'II', fh.read(8))
    if header != 347834724:
        raise ValueError('invalid MicroManager display header')
    result['DisplaySettings'] = read_json(fh, byteorder, None, count, None)

    if comments_header != 99384722:
        raise ValueError('invalid MicroManager comments header')
    fh.seek(comments_offset)
    header, count = struct.unpack(byteorder + 'II', fh.read(8))
    if header != 84720485:
        raise ValueError('invalid MicroManager comments header')
    result['Comments'] = read_json(fh, byteorder, None, count, None)

    return result
1209607
def read_lines_from_file(cls_name, filename):
  """"""
  with tf.io.gfile.GFile(filename, "rb") as f:
    lines = [tf.compat.as_text(line)[:-1] for line in f]
  header_line = "%s%s" % (_HEADER_PREFIX, cls_name)
  if lines[0] != header_line:
    raise ValueError("File {fname} does not seem to have been created from "
                     "{name}.save_to_file.".format(
                         fname=filename, name=cls_name))
  metadata_dict = json.loads(lines[1][len(_METADATA_PREFIX):])
  return lines[2:], metadata_dict
333948
private static void extractIccData(@NotNull final Metadata metadata, @NotNull SequentialReader reader) throws IOException
    {
        byte[] buffer = decodeHexCommentBlock(reader);

        if (buffer != null)
            new IccReader().extract(new ByteArrayReader(buffer), metadata);
    }
748
def _readstream(self, nml_file, nml_patch_in=None):
        """"""
        nml_patch = nml_patch_in if nml_patch_in is not None else Namelist()

        tokenizer = Tokenizer()
        f90lex = []
        for line in nml_file:
            toks = tokenizer.parse(line)
            while tokenizer.prior_delim:
                new_toks = tokenizer.parse(next(nml_file))

                # Skip empty lines
                if not new_toks:
                    continue

                # The tokenizer always pre-tokenizes the whitespace (leftover
                # behaviour from Fortran source parsing) so this must be added
                # manually.
                if new_toks[0].isspace():
                    toks[-1] += new_toks.pop(0)

                # Append the rest of the string (if present)
                if new_toks:
                    toks[-1] += new_toks[0]

                    # Attach the rest of the tokens
                    toks.extend(new_toks[1:])

            toks.append('\n')
            f90lex.extend(toks)

        self.tokens = iter(f90lex)

        nmls = Namelist()

        # Attempt to get first token; abort on empty file
        try:
            self._update_tokens(write_token=False)
        except StopIteration:
            return nmls

        # TODO: Replace "while True" with an update_token() iterator
        while True:
            try:
                # Check for classic group terminator
                if self.token == 'end':
                    self._update_tokens()

                # Ignore tokens outside of namelist groups
                while self.token not in ('&', '$'):
                    self._update_tokens()

            except StopIteration:
                break

            # Create the next namelist
            self._update_tokens()
            g_name = self.token

            g_vars = Namelist()
            v_name = None

            # TODO: Edit `Namelist` to support case-insensitive `get` calls
            grp_patch = nml_patch.get(g_name.lower(), Namelist())

            # Populate the namelist group
            while g_name:

                if self.token not in ('=', '%', '('):
                    self._update_tokens()

                # Set the next active variable
                if self.token in ('=', '(', '%'):

                    v_name, v_values = self._parse_variable(
                        g_vars,
                        patch_nml=grp_patch
                    )

                    if v_name in g_vars:
                        v_prior_values = g_vars[v_name]
                        v_values = merge_values(v_prior_values, v_values)

                    g_vars[v_name] = v_values

                    # Deselect variable
                    v_name = None
                    v_values = []

                # Finalise namelist group
                if self.token in ('/', '&', '$'):

                    # Append any remaining patched variables
                    for v_name, v_val in grp_patch.items():
                        g_vars[v_name] = v_val
                        v_strs = nmls._var_strings(v_name, v_val)
                        for v_str in v_strs:
                            self.pfile.write(
                                '{0}{1}\n'.format(nml_patch.indent, v_str)
                            )

                    # Append the grouplist to the namelist
                    if g_name in nmls:
                        g_update = nmls[g_name]

                        # Update to list of groups
                        if not isinstance(g_update, list):
                            g_update = [g_update]

                        g_update.append(g_vars)

                    else:
                        g_update = g_vars

                    nmls[g_name] = g_update

                    # Reset state
                    g_name, g_vars = None, None

            try:
                self._update_tokens()
            except StopIteration:
                break

        return nmls
1081328
function parseSnmpData(snmpData) {
        var lines = snmpData.trim().split('\n');
        return _.compact(_.map(lines, function(line) {
            return parseSnmpLine(line);
        }));
    }
317391
def _read_with_sitk(datapath):
        
        try:
            import SimpleITK as Sitk
        except ImportError as e:
            logger.error("Unable to import SimpleITK. On Windows try version 1.0.1")
        image = Sitk.ReadImage(datapath)
        data3d = dcmtools.get_pixel_array_from_sitk(image)
        # data3d, original_dtype = dcmreaddata.get_pixel_array_from_dcmobj(image)
        metadata = _metadata(image, datapath)
        return data3d, metadata
877595
def _read_mat_mnu0(filename):
    
    print('read_mag_single_file')

    mat = sio.loadmat(filename)

    df_emd = _extract_emd(mat)

    return df_emd
372046
def read_metadata(self, key):
        
        if getattr(getattr(self.group, 'meta', None), key, None) is not None:
            return self.parent.select(self._get_metadata_path(key))
        return None
459704
def readfmt(stream, fmt):
    """"""
    size = struct.calcsize(fmt)
    blob = stream.read(size)
    return struct.unpack(fmt, blob)