parses the trained .caffemodel file

    filepath: /path/to/trained-model.caffemodel

    returns: layers
355154
def parse_caffemodel(file_path):
    
    f = open(file_path, 'rb')
    contents = f.read()

    net_param = caffe_pb2.NetParameter()
    net_param.ParseFromString(contents)

    layers = find_layers(net_param)
    return layers
36248
def get_models(model_dat_path):
    

    # Check first if we already have a model data file in the data directory


    with open(model_dat_path) as f:

        # model.dat is a text file, no size issues here (will fit in memory)

        model_dat = f.read()

    # Replace \r or \r\n with \n (the former two might be in the file if
    # it originates from Windows)
    if "\n" in model_dat:

        model_dat = model_dat.replace("\r", "")

    else:

        model_dat = model_dat.replace("\r", "\n")

    # Loop through the file and build the model definition
    # dictionary

    lines = model_dat.split("\n")

    model_definitions = collections.OrderedDict()

    for line in lines:

        match = re.match('''(.+(add|mul|con|acn).+)''', line)

        if match is not None:

            # This is a model definition

            tokens = line.split()

            if len(tokens) == 7:

                (model_name, n_parameters,
                 min_energy, max_energy,
                 library_function,
                 model_type,
                 flag) = line.split()

            else:

                (model_name, n_parameters,
                 min_energy, max_energy,
                 library_function,
                 model_type,
                 flag,
                 flag_2) = line.split()

            this_model = collections.OrderedDict()

            this_model['description'] = 'The %s model from XSpec (https://heasarc.gsfc.nasa.gov/xanadu/' \
                                                   'xspec/manual/XspecModels.html)' % model_name

            this_model['parameters'] = collections.OrderedDict()

            model_definitions[(model_name, library_function, model_type)] = this_model

        else:

            # This is a parameter definition

            if len(line.split()) == 0:
                # Empty line
                continue

            # Parameters are free by default, unless they have delta < 0
            # they are switch parameters or scale parameters

            free = True

            if line[0] == '$':

                # Probably a switch parameter

                free = False

                tokens = line.split()

                if len(tokens) == 2:

                    par_name = tokens[0][1:]

                    default_value = tokens[1]

                    par_unit = ""

                    hard_minimum, soft_minimum, soft_maximum, hard_maximum = (0, 0, 1e9, 1e9)

                elif len(tokens) == 3:

                    par_name = tokens[0][1:]

                    default_value = tokens[2]

                    par_unit = ""

                    hard_minimum, soft_minimum, soft_maximum, hard_maximum = (0, 0, 1e9, 1e9)

                else:

                    match = re.match('(\S+)\s+(\".+\"|[a-zA-Z]+)?(.+)*', line[1:])

                    par_name, par_unit, par_spec = match.groups()

                    tokens = par_spec.split()

                    if len(tokens) == 1:

                        default_value = tokens[0]
                        par_unit = ""
                        hard_minimum, soft_minimum, soft_maximum, hard_maximum = (0, 0, 1e9, 1e9)

                    else:

                        par_unit = ""

                        (default_value,
                         hard_minimum, soft_minimum,
                         soft_maximum, hard_maximum,
                         delta) = par_spec.split()


            else:

                # A normal parameter

                match = re.match('(\S+)\s+(\".+\"|\S+)(.+)', line)

                if match is None:

                    raise RuntimeError("Cannot parse parameter %s" % line)

                par_name, par_unit, par_spec = match.groups()

                if par_name[0] == '*':

                    # Scale parameter (always frozen)

                    par_name = par_name[1:]

                    free = False

                    tokens = par_spec.split()

                    if len(tokens) == 1:

                        default_value = tokens[0]
                        (hard_minimum, soft_minimum,
                         soft_maximum, hard_maximum,
                         delta) = (None, None, None, None, 0.1)

                    else:

                        (default_value,
                         hard_minimum, soft_minimum,
                         soft_maximum, hard_maximum,
                         delta) = par_spec.split()

                        delta = abs(float(delta))

                else:

                    (default_value,
                     hard_minimum, soft_minimum,
                     soft_maximum, hard_maximum,
                     delta) = par_spec.split()

                    delta = float(delta)

                    if delta <= 0:
                        free = False

                        delta = abs(delta)

            # Now fix the parameter name removing illegal characters
            # (for example 'log(A)' is not a legal name)

            par_name = re.sub('[\(,\)]', '_', par_name)
            par_name = re.sub('<', '_less_', par_name)
            par_name = re.sub('>', '_more_', par_name)
            par_name = re.sub('/', '_div_', par_name)
            par_name = re.sub('\-', '_minus_', par_name)
            par_name = re.sub('\+', '_plus_', par_name)
            par_name = re.sub('\.', '_dot_', par_name)
            par_name = re.sub('@', '_at_', par_name)

            # Parameter names must be lower case
            par_name = par_name.lower()

            # Some parameters are enclosed between ", like "z"
            par_name = par_name.replace('"', '')

            # "z" is a protected name in astromodels.
            if par_name == "z":

                par_name = "redshift"

            # Check that the parameter name is not an illegal Python name
            if par_name in illegal_variable_names:

                par_name = "xs_%s" % par_name

            # Sometimes the unit is " " which is not recognized by astropy
            if par_unit:

                par_unit = par_unit.replace("\"",'')

            # There are some errors in model.dat , like KeV instead of keV, and ergcm/s instead of erg cm /s
            # Let's correct them
            if par_unit == "KeV":

                par_unit = "keV"

            elif par_unit == "ergcm/s":

                par_unit = "erg cm / s"

            elif par_unit == "days":

                par_unit = "day"

            elif par_unit == "z" and par_name.lower() == "redshift":

                par_unit = ""

            # There are funny units in model.dat, like "Rs" (which means Schwarzschild radius) or other things
            # so let's try to convert the par_unit into an astropy.Unit instance. If that fails, use a unitless unit
            try:

                _ = u.Unit(par_unit)

            except ValueError:

                # Unit not recognized

                #warnings.warn("Unit %s is not recognized by astropy." % par_unit)

                par_unit = ''

            # Make sure that this is a valid python identifier
            # by matching it with the relative regexp
            if re.match('([a-zA-Z_][a-zA-Z0-9_]*)$', par_name) is None:

                raise ValueError("Illegal identifier name %s" % (par_name))

            this_model['parameters'][par_name] = {'initial value': float(default_value),
                                                  'desc': '(see https://heasarc.gsfc.nasa.gov/xanadu/xspec/manual/'
                                                          'XspecModels.html)',
                                                  'min': hard_minimum,
                                                  'max': hard_maximum,
                                                  'delta': float(delta),
                                                  'unit': par_unit,
                                                  'free': free}

    return model_definitions
576402
def load_metafile(filepath):
    
    try:
        with open(filepath, 'r', encoding='utf-8') as file:
            return email.message_from_file(file)
    except FileNotFoundError:
        logger.warning("Category file %s not found", filepath)
        orm.delete(c for c in model.Category if c.file_path == filepath)
        orm.commit()

    return None
1019215
def download_caffe_model(model_name, meta_info, dst_dir='./model'):
    
    if not os.path.isdir(dst_dir):
        os.mkdir(dst_dir)
    model_name = os.path.join(dst_dir, model_name)

    assert 'prototxt' in meta_info, "missing prototxt url"
    proto_url, proto_sha1 = meta_info['prototxt']
    prototxt = mx.gluon.utils.download(proto_url,
                                       model_name+'_deploy.prototxt',
                                       sha1_hash=proto_sha1)

    assert 'caffemodel' in meta_info, "mssing caffemodel url"
    caffemodel_url, caffemodel_sha1 = meta_info['caffemodel']
    caffemodel = mx.gluon.utils.download(caffemodel_url,
                                         model_name+'.caffemodel',
                                         sha1_hash=caffemodel_sha1)
    assert 'mean' in meta_info, 'no mean info'
    mean = meta_info['mean']
    if isinstance(mean[0], str):
        mean_url, mean_sha1 = mean
        mean = mx.gluon.utils.download(mean_url,
                                       model_name+'_mean.binaryproto',
                                       sha1_hash=mean_sha1)
    return (prototxt, caffemodel, mean)
1054867
def ft1file(self, **kwargs):
        
        kwargs_copy = self.base_dict.copy()
        kwargs_copy.update(**kwargs)
        kwargs_copy['dataset'] = kwargs.get('dataset', self.dataset(**kwargs))
        self._replace_none(kwargs_copy)        
        localpath = NameFactory.ft1file_format.format(**kwargs_copy)
        if kwargs.get('fullpath', False):
            return self.fullpath(localpath=localpath)
        return localpath
839139
def load_model(self, fname):
        
        if isinstance(fname, STRING_TYPES):  # assume file name
            if os.path.exists(fname):
                _LIB.XGBoosterLoadModel(self.handle, c_str(fname))
            else:
                raise ValueError("No such file: {0}".format(fname))
        else:
            buf = fname
            length = ctypes.c_ulong(len(buf))
            ptr = (ctypes.c_char * len(buf)).from_buffer(buf)
            _check_call(_LIB.XGBoosterLoadModelFromBuffer(self.handle, ptr, length))
1137211
def parse_file(file_path)
      node = parse_xml(file_path)
      node.xpath('*').each do |node_child|
        case node_child.name
        when 'Relationships'
          parse(node_child)
        end
      end
      self
    end
388304
def parse_file(path, target_ext=nil)
      return nil if path.nil?
      return nil if File.directory? path
      _load_from_file path, target_ext
      # if File.directory? path
      #   _load_from_directory path
      # else
      #   _load_from_file path, target_ext
      # end
    end
1163639
def load_batch(self, fn_batch):
        

        # TODO Assumes targets are available, which is how its distinct from
        # utils.load_batch_x(). These functions need to change names to be
        # clearer.

        inverse = list(zip(*fn_batch))
        feat_fn_batch = inverse[0]
        target_fn_batch = inverse[1]

        batch_inputs, batch_inputs_lens = utils.load_batch_x(feat_fn_batch,
                                                             flatten=False)
        batch_targets_list = []
        for targets_path in target_fn_batch:
            with open(targets_path, encoding=ENCODING) as targets_f:
                target_indices = self.corpus.labels_to_indices(targets_f.readline().split())
                batch_targets_list.append(target_indices)
        batch_targets = utils.target_list_to_sparse_tensor(batch_targets_list)

        return batch_inputs, batch_inputs_lens, batch_targets
443335
def from_file(filepath: str):
        
        trace = TrainingTrace()
        trace._trace = load_config(filepath)
        return trace
705415
def _parse_proto(prototxt_fname):
    
    proto = caffe_parser.read_prototxt(prototxt_fname)

    # process data layer
    input_name, input_dim, layers = _get_input(proto)
    # only support single input, so always use `data` as the input data
    mapping = {input_name: 'data'}
    need_flatten = {input_name: False}
    symbol_string = "import mxnet as mx\ndata = mx.symbol.Variable(name='data')\n"

    flatten_count = 0
    output_name = ""
    prev_name = None

    # convert reset layers one by one
    for i, layer in enumerate(layers):
        type_string = ''
        param_string = ''
        skip_layer = False
        bottom_order = []
        name = re.sub('[-/]', '_', layer.name)
        if layer.type == 'Convolution' or layer.type == 4:
            type_string = 'mx.symbol.Convolution'
            param_string = _convert_conv_param(layer.convolution_param)
            need_flatten[name] = True
        if layer.type == 'Deconvolution' or layer.type == 39:
            type_string = 'mx.symbol.Deconvolution'
            param_string = _convert_conv_param(layer.convolution_param)
            need_flatten[name] = True
        if layer.type == 'Pooling' or layer.type == 17:
            type_string = 'mx.symbol.Pooling'
            param_string = _convert_pooling_param(layer.pooling_param)
            need_flatten[name] = True
        if layer.type == 'ReLU' or layer.type == 18:
            type_string = 'mx.symbol.Activation'
            param_string = "act_type='relu'"
            need_flatten[name] = need_flatten[mapping[layer.bottom[0]]]
        if layer.type == 'TanH' or layer.type == 23:
            type_string = 'mx.symbol.Activation'
            param_string = "act_type='tanh'"
            need_flatten[name] = need_flatten[mapping[layer.bottom[0]]]
        if layer.type == 'Sigmoid' or layer.type == 19:
            type_string = 'mx.symbol.Activation'
            param_string = "act_type='sigmoid'"
            need_flatten[name] = need_flatten[mapping[layer.bottom[0]]]
        if layer.type == 'LRN' or layer.type == 15:
            type_string = 'mx.symbol.LRN'
            param = layer.lrn_param
            param_string = "alpha=%f, beta=%f, knorm=%f, nsize=%d" % (
                param.alpha, param.beta, param.k, param.local_size)
            need_flatten[name] = True
        if layer.type == 'InnerProduct' or layer.type == 14:
            type_string = 'mx.symbol.FullyConnected'
            param = layer.inner_product_param
            param_string = "num_hidden=%d, no_bias=%s" % (
                param.num_output, not param.bias_term)
            need_flatten[name] = False
        if layer.type == 'Dropout' or layer.type == 6:
            type_string = 'mx.symbol.Dropout'
            param = layer.dropout_param
            param_string = "p=%f" % param.dropout_ratio
            need_flatten[name] = need_flatten[mapping[layer.bottom[0]]]
        if layer.type == 'Softmax' or layer.type == 20:
            if layer.softmax_param.axis == 2:
                symbol_string += "%s = mx.symbol.transpose(%s, axes=(0,2,1))\n" %\
                    (mapping[layer.bottom[0]], mapping[layer.bottom[0]])
                type_string = 'mx.symbol.SoftmaxActivation'
                param_string = "mode='channel'"
                need_flatten[name] = False
            else:
                type_string = 'mx.symbol.SoftmaxOutput'
        if layer.type == 'Flatten' or layer.type == 8:
            if 'softmax' in layer.bottom[0]:
                prev_name = re.sub('[-/]', '_', layers[i-1].name)
                skip_layer = True
            else:
                type_string = 'mx.symbol.Flatten'
            need_flatten[name] = False
        if layer.type == 'Split' or layer.type == 22:
            type_string = 'split'  # will process later
        if layer.type == 'Concat' or layer.type == 3:
            type_string = 'mx.symbol.Concat'
            need_flatten[name] = True
        if layer.type == 'Crop':
            type_string = 'mx.symbol.Crop'
            need_flatten[name] = True
            param_string = 'center_crop=True'
        if layer.type == 'BatchNorm':
            type_string = 'mx.symbol.BatchNorm'
            param = layer.batch_norm_param
            # CuDNN requires eps to be greater than 1e-05
            # We compensate for this change in convert_model
            epsilon = param.eps
            if (epsilon <= 1e-05):
                epsilon = 1e-04
            # if next layer is scale, don't fix gamma
            fix_gamma = layers[i+1].type != 'Scale'
            param_string = 'use_global_stats=%s, fix_gamma=%s, eps=%f' % (
                param.use_global_stats, fix_gamma, epsilon)
            need_flatten[name] = need_flatten[mapping[layer.bottom[0]]]
        if layer.type == 'Scale':
            assert layers[i-1].type == 'BatchNorm'
            need_flatten[name] = need_flatten[mapping[layer.bottom[0]]]
            skip_layer = True
            prev_name = re.sub('[-/]', '_', layers[i-1].name)
        if layer.type == 'PReLU':
            type_string = 'mx.symbol.LeakyReLU'
            param = layer.prelu_param
            param_string = "act_type='prelu', slope=%f" % param.filler.value
            need_flatten[name] = need_flatten[mapping[layer.bottom[0]]]
        if layer.type == 'Eltwise':
            type_string = 'mx.symbol.broadcast_add'
            param_string = ""
            need_flatten[name] = False
        if layer.type == 'Reshape':
            type_string = 'mx.symbol.Reshape'
            param = layer.reshape_param
            param_string = 'shape=(' + ','.join([str(x) for x in list(param.shape.dim)]) + ')'
            need_flatten[name] = True
        if layer.type == 'AbsVal':
            type_string = 'mx.symbol.abs'
            need_flatten[name] = need_flatten[mapping[layer.bottom[0]]]
        if layer.type == 'Normalize':
            bottom = re.sub('[-/]', '_', layer.bottom[0])
            conv_layer = _find_layer(layers, bottom)
            assert conv_layer is not None
            param = layer.norm_param
            assert not param.across_spatial and not param.channel_shared
            assert param.scale_filler.type == 'constant'
            if conv_layer.type == 'Convolution':
                scale_name = "%s_scale" % name
                symbol_string += "%s=mx.sym.Variable(name='%s', shape=(1, %d, 1, 1), init=mx.init.Constant(%f))\n" % \
                    (scale_name, scale_name, conv_layer.convolution_param.num_output,
                    param.scale_filler.value)
                symbol_string += "%s=mx.symbol.L2Normalization(name='%s', data=%s, mode='channel')\n" %\
                    (name, name, mapping[layer.bottom[0]])
                symbol_string += "%s=mx.symbol.broadcast_mul(lhs=%s, rhs=%s)\n" %\
                    (name, scale_name, name)
                type_string = 'split'
                need_flatten[name] = True
            else:
                raise ValueError('Unknown/Invalid normalize layer!')
        if layer.type == 'Permute':
            type_string = 'mx.symbol.transpose'
            param_string = "axes=(%s)" % (','.join([str(x) for x in layer.permute_param.order]))
            need_flatten[name] = True
            from_name = ''
        if layer.type == 'PriorBox':
            param = layer.prior_box_param
            if layer.bottom[0] == 'data':
                bottom_order = [1]
            else:
                bottom_order = [0]
            try:
                import math
                min_size = param.min_size[0] / input_dim[2]
                max_size = math.sqrt(param.min_size[0] * param.max_size[0]) / input_dim[2]
                sizes = '(%f, %f)' %(min_size, max_size)
            except AttributeError:
                min_size = param.min_size[0] / input_dim[2]
                sizes = '(%f)' %(min_size)
            ars = list(param.aspect_ratio)
            ratios = [1.]
            for ar in ars:
                ratios.append(ar)
                if param.flip:
                    ratios.append(1. / ar)
            ratios_string = '(' + ','.join(str(x) for x in ratios) + ')'
            clip = param.clip
            if (param.step_h > 0 or param.step_w > 0):
                step_h = param.step_h
                step_w = param.step_w
            elif param.step > 0:
                step_h = param.step
                step_w = param.step
            else:
                step_h = -1
                step_w = -1
            finput_dimh = float(input_dim[2])
            finput_dimw = float(input_dim[3])
            step = '(%f, %f)' % (step_h / finput_dimh, step_w / finput_dimw)
            assert param.offset == 0.5, "currently only support offset = 0.5"
            symbol_string += '%s = mx.contrib.symbol.MultiBoxPrior(%s, sizes=%s, ratios=%s, clip=%s, steps=%s, name="%s")\n' % \
                (name, mapping[layer.bottom[0]], sizes, ratios_string, clip, step, name)
            symbol_string += '%s = mx.symbol.Flatten(data=%s)\n' % (name, name)
            type_string = 'split'
            need_flatten[name] = False
        if layer.type == 'DetectionOutput':
            bottom_order = [1, 0, 2]
            param = layer.detection_output_param
            assert param.share_location == True
            assert param.background_label_id == 0
            nms_param = param.nms_param
            type_string = 'mx.contrib.symbol.MultiBoxDetection'
            param_string = "nms_threshold=%f, nms_topk=%d, clip=False" % \
                (nms_param.nms_threshold, nms_param.top_k)
        if skip_layer:
            assert len(layer.bottom) == 1
            symbol_string += "%s = %s\n" % (name, prev_name)
        elif type_string == '':
            raise ValueError('Unknown layer %s!' % layer.type)
        elif type_string != 'split':
            bottom = layer.bottom
            if param_string != "":
                param_string = ", " + param_string
            if len(bottom) == 1:
                # print(need_flatten)
                if need_flatten[mapping[bottom[0]]] and type_string == 'mx.symbol.FullyConnected':
                    flatten_name = "flatten_%d" % flatten_count
                    symbol_string += "%s=mx.symbol.Flatten(name='%s', data=%s)\n" % (
                        flatten_name, flatten_name, mapping[bottom[0]])
                    flatten_count += 1
                    need_flatten[flatten_name] = False
                    bottom[0] = flatten_name
                    mapping[bottom[0]] = bottom[0]
                symbol_string += "%s = %s(name='%s', data=%s %s)\n" % (
                    name, type_string, name, mapping[bottom[0]], param_string)
            else:
                if not bottom_order:
                    bottom_order = range(len(bottom))
                symbol_string += "%s = %s(name='%s', *[%s] %s)\n" % \
                                 (name, type_string, name, ','.join([mapping[bottom[x]] for x in bottom_order]), param_string)
                if layer.type == 'Concat' and layer.concat_param.axis == 2:
                    symbol_string += "%s = mx.symbol.Reshape(data=%s, shape=(0, -1, 4), name='%s')\n" %\
                        (name, name, name)
        for j in range(len(layer.top)):
            mapping[layer.top[j]] = name
        output_name = name
    return symbol_string, output_name, input_dim
660626
def load_model(f, format=None, load_external_data=True):  # type: (Union[IO[bytes], Text], Optional[Any], bool) -> ModelProto
    '''
    
    '''
    s = _load_bytes(f)
    model = load_model_from_string(s, format=format)

    if load_external_data:
        model_filepath = _get_file_path(f)
        if model_filepath:
            base_dir = os.path.dirname(model_filepath)
            load_external_data_for_model(model, base_dir)

    return model
212695
def load_neurite_features(filepath):
    ''''''
    stuff = defaultdict(lambda: defaultdict(list))
    nrns = nm.load_neurons(filepath)
    # unpack data into arrays
    for nrn in nrns:
        for t in NEURITES_:
            for feat in FEATURES:
                stuff[feat][str(t).split('.')[1]].extend(
                    nm.get(feat, nrn, neurite_type=t)
                )
    return stuff
479069
def dump_to_path(self, cnf, filepath, **kwargs):
        
        with self.wopen(filepath) as out:
            self.dump_to_stream(cnf, out, **kwargs)
557208
def convert_caffe_model(model_name, meta_info, dst_dir='./model'):
    """"""

    (prototxt, caffemodel, mean) = download_caffe_model(model_name, meta_info, dst_dir)
    model_name = os.path.join(dst_dir, model_name)
    convert_model(prototxt, caffemodel, model_name)
    if isinstance(mean, str):
        mx_mean = model_name + '-mean.nd'
        convert_mean(mean, mx_mean)
        mean = mx_mean
    return (model_name, mean)
950208
def load(cls, path):
        
        data = json.load(open(path))

        weights = data['weights']
        weights = np.asarray(weights, dtype=np.float64)

        s = cls(data['num_neurons'],
                data['data_dimensionality'],
                data['params']['lr']['orig'],
                neighborhood=data['params']['infl']['orig'],
                valfunc=data['valfunc'],
                argfunc=data['argfunc'],
                lr_lambda=data['params']['lr']['factor'],
                nb_lambda=data['params']['nb']['factor'])

        s.weights = weights
        s.trained = True

        return s
539583
def get_model_path(self, name):
        """"""

        name, ext = os.path.splitext(name)
        ext = '.xml'
        xmlfile = name + self.config['file_suffix'] + ext
        xmlfile = utils.resolve_path(xmlfile,
                                     workdir=self.config['fileio']['workdir'])

        return xmlfile