Convert MagIC specimen headers to short/readable
    headers for a figure (used by ipmag.specimens_extract)
511407
def convert_specimen_dm3_table(spec_df):
    
    from pmagpy import data_model3 as dm3
    # get the data model
    dm = dm3.DataModel()
    # get the specimen absolute intensity column headers
    meas_group = ['meas_step_min', 'meas_step_max', 'meas_step_unit']
    pint_group = dm.get_group_headers('specimens', 'Paleointensity')
    arai_group = dm.get_group_headers(
        'specimens', 'Paleointensity Arai Statistics')
    # filter out the relative intensity stuff
    dm3_columns = list(meas_group)+list(pint_group)+list(arai_group)
    dm3_columns = filter(lambda x: '_rel' not in x, dm3_columns)
    # apply to specimen dataframe
    meas_group_columns = ['meas_step_min', 'meas_step_max', 'meas_step_unit']
    pint_group_columns = list(spec_df.columns.intersection(pint_group))
    arai_group_columns = list(spec_df.columns.intersection(arai_group))
    columns = ['specimen', 'sample']+meas_group_columns + \
        pint_group_columns+arai_group_columns
    spec_df = spec_df.copy()[columns]
    muT_list = ['int_abs', 'int_abs_sigma', 'int_treat_dc_field']
    meas_list = ['meas_step_min', 'meas_step_max']
    for el in muT_list:
        if el in columns:
            spec_df[el] = 1e6*spec_df[el]
    if 'meas_step_unit' in columns:
        for el in meas_list:
            spec_df.loc[spec_df['meas_step_unit'] == 'K', el] = spec_df[el]-273
            spec_df.loc[spec_df['meas_step_unit'] == 'T', el] = 1e3*spec_df[el]
        spec_df.loc[spec_df['meas_step_unit'] == 'K', 'meas_step_unit'] = 'C'
        spec_df.loc[spec_df['meas_step_unit'] == 'T', 'meas_step_unit'] = 'mT'
        spec_df['meas_step_min'] = spec_df['meas_step_min'].astype('int')
        spec_df['meas_step_max'] = spec_df['meas_step_max'].astype('int')
    dm3_columns = ['meas_step_min', 'meas_step_max', 'meas_step_unit', 'int_abs', 'int_abs_sigma',
                   'int_abs_sigma_perc', 'int_n_measurements', 'int_corr', 'int_corr_cooling_rate',
                   'int_corr_aniso', 'int_corr_nlt', 'int_corr_arm', 'int_viscosity_index', 'int_treat_dc_field',
                   'int_b', 'int_b_sigma', 'int_b_beta', 'int_rsc', 'int_f', 'int_fvds', 'int_frac',
                   'int_g', 'int_gmax', 'int_q', 'int_w', 'int_k', 'int_k_sse', 'int_k_prime', 'int_k_prime_sse',
                   'int_scat', 'int_r2_corr', 'int_r2_det', 'int_z', 'int_z_md']
    table_columns = ['Min', 'Max', 'Units', 'B (uT)', 'sigma', 'percent', 'N', 'c/u', 'CR', 'Aniso.', 'NLT', 'AARM', 'VI', 'Lab Field',
                     'b', 'b sigma', 'beta', 'R2', 'f', 'fvds', 'frac', 'g', 'gap max', 'q', 'w', 'k', 'k sse', 'k prime', 'k prime sse',
                     'scat', 'r2 corr', 'r2 det', 'Z', 'Z md']
    spec_mapping = dict(list(zip(dm3_columns, table_columns)))
    spec_df_out = spec_df.rename(columns=spec_mapping)
    if 'N' in spec_df_out.columns:
        spec_df_out['N'] = spec_df_out['N'].astype('int')
    if 'Lab Field' in spec_df_out.columns:
        spec_df_out['Lab Field'] = spec_df_out['Lab Field'].round().astype('int')
    return spec_df_out
1119645
def _read_image_data( mhd_file):
    
    tempdir = get_tempdir()
    
    # Load description from mhd file
    fname = tempdir + '/' + mhd_file
    des = open(fname, 'r').read()
    
    # Get data filename and load raw data
    match = re.findall('ElementDataFile = (.+?)\n', des)
    fname = tempdir + '/' + match[0]
    data = open(fname, 'rb').read()
    
    # Determine dtype
    match = re.findall('ElementType = (.+?)\n', des)
    dtype_itk = match[0].upper().strip()
    dtype = DTYPE_ITK2NP.get(dtype_itk, None)
    if dtype is None:
        raise RuntimeError('Unknown ElementType: ' + dtype_itk)
    
    # Create numpy array
    a = np.frombuffer(data, dtype=dtype)
    
    # Determine shape, sampling and origin of the data
    match = re.findall('DimSize = (.+?)\n', des)
    shape = [int(i) for i in match[0].split(' ')]
    #
    match = re.findall('ElementSpacing = (.+?)\n', des)
    sampling = [float(i) for i in match[0].split(' ')]
    #
    match = re.findall('Offset = (.+?)\n', des)
    origin = [float(i) for i in match[0].split(' ')]
    
    # Reverse shape stuff to make z-y-x order
    shape = [s for s in reversed(shape)]
    sampling = [s for s in reversed(sampling)]
    origin = [s for s in reversed(origin)]
    
    # Take vectors/colours into account
    N = np.prod(shape)
    if N != a.size:
        extraDim = int( a.size / N )
        shape = tuple(shape) + (extraDim,)
        sampling = tuple(sampling) + (1.0,)
        origin = tuple(origin) + (0,)
    
    # Check shape
    N = np.prod(shape)
    if N != a.size:
        raise RuntimeError('Cannot apply shape to data.')
    else:
        a.shape = shape
        a = Image(a)
        a.sampling = sampling
        a.origin = origin
    return a
483877
def _parseImageNtHeaders(self, data, imageDosHeader):
        """"""
        inth = self._classes.IMAGE_NT_HEADERS.from_buffer(data, imageDosHeader.header.e_lfanew)

        if inth.Signature != b'PE':
            raise BinaryError('No valid PE/COFF file')

        return ImageNtHeaderData(header=inth)
269064
def _header_string(basis_dict):
    '''
    '''

    tw = textwrap.TextWrapper(initial_indent='', subsequent_indent=' ' * 20)

    header = '-' * 70 + '\n'
    header += ' Basis Set Exchange\n'
    header += ' Version ' + version() + '\n'
    header += ' ' + _main_url + '\n'
    header += '-' * 70 + '\n'
    header += '   Basis set: ' + basis_dict['name'] + '\n'
    header += tw.fill(' Description: ' + basis_dict['description']) + '\n'
    header += '        Role: ' + basis_dict['role'] + '\n'
    header += tw.fill('     Version: {}  ({})'.format(basis_dict['version'],
                                                      basis_dict['revision_description'])) + '\n'
    header += '-' * 70 + '\n'

    return header
1037195
def to_image_header(img):
    '''
    
    '''
    if not img.__module__.startswith('nibabel.'):
        raise ValueError('to_image_header: only nibabel obejcts can be coerced to headers')
    if type(img).__name__.endswith('Header'): return img
    # if not a header given, must be an image given:
    try: return img.header
    except Exception:
        raise ValueError('to_image_header: can only convert nibabel image or header objects')
255662
def extract_images(filename):
    """"""
    with gzip.open(filename) as bytestream:
        magic = _read32(bytestream)
        if magic != 2051:
            raise ValueError(
                'Invalid magic number %d in MNIST image file: %s' %
                (magic, filename))
        num_images = _read32(bytestream)
        rows = _read32(bytestream)
        cols = _read32(bytestream)
        buf = bytestream.read(rows * cols * num_images)
        data = numpy.frombuffer(buf, dtype=numpy.uint8)
        data = data.reshape(num_images, rows, cols, 1)
        data = data.astype('float32') / 255.0
        return data
44278
def figure_buffer(figs):
    ''''''
    assert len(figs) > 0, 'No figure buffers given. Forgot to return from draw call?'
    buffers = []
    w, h = figs[0].canvas.get_width_height()
    for f in figs:
        wf, hf = f.canvas.get_width_height()
        assert wf == w and hf == h, 'All canvas objects need to have same size'
        buffers.append(np.fromstring(f.canvas.tostring_rgb(), dtype=np.uint8).reshape(h, w, 3))

    return np.stack(buffers)
484127
function extractHeaders(options) {
    let headers = {};
    if (options && options.headers) {
        headers = options.headers;
        delete options.headers;
    }
    return headers;
}
244498
def svg_html_from_pyplot_figure(fig: "Figure") -> str:
    
    if fig is None:
        return ""
    memfile = io.BytesIO()  # StringIO doesn't like mixing str/unicode
    fig.savefig(memfile, format="svg")
    return memfile.getvalue().decode("utf-8")
1358151
def stripped_photo_to_jpg(stripped):
    
    if len(stripped) < 3 or stripped[0] != 1:
        return stripped

    header = bytearray(b'\xff\xd8\xff\xe0\x00\x10JFIF\x00\x01\x01\x00\x00\x01\x00\x01\x00\x00\xff\xdb\x00C\x00(\x1c\x1e#\x1e\x19(#!#-+(0<dA<77<{X]Id\x91\x80\x99\x96\x8f\x80\x8c\x8a\xa0\xb4\xe6\xc3\xa0\xaa\xda\xad\x8a\x8c\xc8\xff\xcb\xda\xee\xf5\xff\xff\xff\x9b\xc1\xff\xff\xff\xfa\xff\xe6\xfd\xff\xf8\xff\xdb\x00C\x01+--<5<vAAv\xf8\xa5\x8c\xa5\xf8\xf8\xf8\xf8\xf8\xf8\xf8\xf8\xf8\xf8\xf8\xf8\xf8\xf8\xf8\xf8\xf8\xf8\xf8\xf8\xf8\xf8\xf8\xf8\xf8\xf8\xf8\xf8\xf8\xf8\xf8\xf8\xf8\xf8\xf8\xf8\xf8\xf8\xf8\xf8\xf8\xf8\xf8\xf8\xf8\xf8\xf8\xf8\xf8\xf8\xff\xc0\x00\x11\x08\x00\x00\x00\x00\x03\x01"\x00\x02\x11\x01\x03\x11\x01\xff\xc4\x00\x1f\x00\x00\x01\x05\x01\x01\x01\x01\x01\x01\x00\x00\x00\x00\x00\x00\x00\x00\x01\x02\x03\x04\x05\x06\x07\x08\t\n\x0b\xff\xc4\x00\xb5\x10\x00\x02\x01\x03\x03\x02\x04\x03\x05\x05\x04\x04\x00\x00\x01}\x01\x02\x03\x00\x04\x11\x05\x12!1A\x06\x13Qa\x07"q\x142\x81\x91\xa1\x08#B\xb1\xc1\x15R\xd1\xf0$3br\x82\t\n\x16\x17\x18\x19\x1a%&\'()*456789:CDEFGHIJSTUVWXYZcdefghijstuvwxyz\x83\x84\x85\x86\x87\x88\x89\x8a\x92\x93\x94\x95\x96\x97\x98\x99\x9a\xa2\xa3\xa4\xa5\xa6\xa7\xa8\xa9\xaa\xb2\xb3\xb4\xb5\xb6\xb7\xb8\xb9\xba\xc2\xc3\xc4\xc5\xc6\xc7\xc8\xc9\xca\xd2\xd3\xd4\xd5\xd6\xd7\xd8\xd9\xda\xe1\xe2\xe3\xe4\xe5\xe6\xe7\xe8\xe9\xea\xf1\xf2\xf3\xf4\xf5\xf6\xf7\xf8\xf9\xfa\xff\xc4\x00\x1f\x01\x00\x03\x01\x01\x01\x01\x01\x01\x01\x01\x01\x00\x00\x00\x00\x00\x00\x01\x02\x03\x04\x05\x06\x07\x08\t\n\x0b\xff\xc4\x00\xb5\x11\x00\x02\x01\x02\x04\x04\x03\x04\x07\x05\x04\x04\x00\x01\x02w\x00\x01\x02\x03\x11\x04\x05!1\x06\x12AQ\x07aq\x13"2\x81\x08\x14B\x91\xa1\xb1\xc1\t#3R\xf0\x15br\xd1\n\x16$4\xe1%\xf1\x17\x18\x19\x1a&\'()*56789:CDEFGHIJSTUVWXYZcdefghijstuvwxyz\x82\x83\x84\x85\x86\x87\x88\x89\x8a\x92\x93\x94\x95\x96\x97\x98\x99\x9a\xa2\xa3\xa4\xa5\xa6\xa7\xa8\xa9\xaa\xb2\xb3\xb4\xb5\xb6\xb7\xb8\xb9\xba\xc2\xc3\xc4\xc5\xc6\xc7\xc8\xc9\xca\xd2\xd3\xd4\xd5\xd6\xd7\xd8\xd9\xda\xe2\xe3\xe4\xe5\xe6\xe7\xe8\xe9\xea\xf2\xf3\xf4\xf5\xf6\xf7\xf8\xf9\xfa\xff\xda\x00\x0c\x03\x01\x00\x02\x11\x03\x11\x00?\x00')
    footer = b"\xff\xd9"
    header[164] = stripped[1]
    header[166] = stripped[2]
    return bytes(header) + stripped[3:] + footer
1038030
def from_httplib(cls, message):  # Python 2
        """"""
        # python2.7 does not expose a proper API for exporting multiheaders
        # efficiently. This function re-reads raw lines from the message
        # object and extracts the multiheaders properly.
        headers = []

        for line in message.headers:
            if line.startswith((' ', '\t')):
                key, value = headers[-1]
                headers[-1] = (key, value + '\r\n' + line.rstrip())
                continue

            key, value = line.split(':', 1)
            headers.append((key, value.strip()))

        return cls(headers)
1118699
def figure_cleanup(docxml)
      docxml.xpath(FIGURE_WITH_FOOTNOTES).each do |f|
        key = figure_get_or_make_dl(f)
        f.xpath(".//aside").each do |aside|
          figure_aside_process(f, aside, key)
        end
      end
      docxml
    end
426203
def read_tvips_header(fh, byteorder, dtype, count, offsetsize):
    """"""
    result = {}
    header = fh.read_record(TIFF.TVIPS_HEADER_V1, byteorder=byteorder)
    for name, typestr in TIFF.TVIPS_HEADER_V1:
        result[name] = header[name].tolist()
    if header['Version'] == 2:
        header = fh.read_record(TIFF.TVIPS_HEADER_V2, byteorder=byteorder)
        if header['Magic'] != int(0xaaaaaaaa):
            log.warning('read_tvips_header: invalid TVIPS v2 magic number')
            return {}
        # decode utf16 strings
        for name, typestr in TIFF.TVIPS_HEADER_V2:
            if typestr.startswith('V'):
                s = header[name].tostring().decode('utf16', errors='ignore')
                result[name] = stripnull(s, null='\0')
            else:
                result[name] = header[name].tolist()
        # convert nm to m
        for axis in 'XY':
            header['PhysicalPixelSize' + axis] /= 1e9
            header['PixelSize' + axis] /= 1e9
    elif header.version != 1:
        log.warning('read_tvips_header: unknown TVIPS header version')
        return {}
    return result
502123
def get_header(results):
    ''''''
    ret = ['name', ]
    values = next(iter(results.values()))
    for k, v in values.items():
        if isinstance(v, dict):
            for metric in v.keys():
                ret.append('%s:%s' % (k, metric))
        else:
            ret.append(k)
    return ret
534495
def pillar_format(ret, keys, value, expand_keys):
    '''
    
    '''
    # if value is empty in Consul then it's None here - skip it
    if value is None:
        return ret

    # If value is not None then it's a string
    # YAML strips whitespaces unless they're surrounded by quotes
    # If expand_keys is true, deserialize the YAML data
    if expand_keys:
        pillar_value = salt.utils.yaml.safe_load(value)
    else:
        pillar_value = value

    keyvalue = keys.pop()
    pil = {keyvalue: pillar_value}
    keys.reverse()
    for k in keys:
        pil = {k: pil}

    return dict_merge(ret, pil)
265846
def read_nih_image_header(fh, byteorder, dtype, count, offsetsize):
    """"""
    a = fh.read_record(TIFF.NIH_IMAGE_HEADER, byteorder=byteorder)
    a = a.newbyteorder(byteorder)
    a = recarray2dict(a)
    a['XUnit'] = a['XUnit'][:a['XUnitSize']]
    a['UM'] = a['UM'][:a['UMsize']]
    return a
727070
def short_repr(obj, noneAsNA=False):
    ''''''
    if obj is None:
        return 'unspecified' if noneAsNA else 'None'
    elif isinstance(obj, str) and len(obj) > 80:
        return '{}...{}'.format(obj[:60].replace('\n', '\\n'),
                                obj[-20:].replace('\n', '\\n'))
    elif isinstance(obj, (str, int, float, bool)):
        return repr(obj)
    elif hasattr(obj, '__short_repr__'):
        return obj.__short_repr__()
    elif isinstance(obj, Sequence):  # should be a list or tuple
        if len(obj) == 0:
            return '[]'
        elif len(obj) == 1:
            return f'{short_repr(obj[0])}'
        elif len(obj) == 2:
            return f'{short_repr(obj[0])}, {short_repr(obj[1])}'
        else:
            return f'{short_repr(obj[0])}, {short_repr(obj[1])}, ... ({len(obj)} items)'
    elif isinstance(obj, dict):
        if not obj:
            return ''
        elif len(obj) == 1:
            first_key = list(obj.keys())[0]
            return f'{short_repr(first_key)!r}:{short_repr(obj[first_key])!r}'
        else:
            first_key = list(obj.keys())[0]
            return f'{short_repr(first_key)}:{short_repr(obj[first_key])}, ... ({len(obj)} items)'
    elif isinstance(obj, KeysView):
        if not obj:
            return ''
        elif len(obj) == 1:
            return short_repr(next(iter(obj)))
        else:
            return f'{short_repr(next(iter(obj)))}, ... ({len(obj)} items)'
    #elif hasattr(obj, 'target_name'):
    #    return obj.target_name()
    else:
        ret = str(obj)
        if len(ret) > 40:
            return f'{repr(obj)[:35]}...'
        else:
            return ret