Transfer ownership from global (shared) model to local model which will be worked on
12920
@Override protected void setupLocal(){
//    long start = System.currentTimeMillis();
    assert(_localmodel == null);
    _localmodel = _sharedmodel;
    _sharedmodel = null;
    _localmodel.set_processed_local(0);
    final int weightIdx =_fr.find(_localmodel.get_params()._weights_column);
    final int respIdx =_fr.find(_localmodel.get_params()._response_column);
    final int batchSize = _localmodel.get_params()._mini_batch_size;

//    long nativetime = 0;
    DeepWaterIterator iter = null;
    long seed = 0xDECAF + 0xD00D * _localmodel.get_processed_global();
    Random rng = RandomUtils.getRNG(seed);

    if (_fr.numRows()>Integer.MAX_VALUE) {
      throw H2O.unimpl("Need to implement batching into int-sized chunks.");
    }
    int len = (int)_fr.numRows();
    int j=0;
    Futures fs = new Futures();
    ArrayList trainLabels = new ArrayList<>();
    ArrayList trainData = new ArrayList<>();

    try {
      // Binary data (Images/Documents/etc.)
      if (_localmodel.get_params()._problem_type == DeepWaterParameters.ProblemType.image ||
          _localmodel.get_params()._problem_type == DeepWaterParameters.ProblemType.text) {
        int dataIdx = 0; //must be the first column //FIXME
        Log.debug("Using column " + _fr.name(dataIdx) + " for " +
            ((_localmodel.get_params()._problem_type == DeepWaterParameters.ProblemType.image) ? "path to image data"
                :((_localmodel.get_params()._problem_type == DeepWaterParameters.ProblemType.text) ? "text data"
                : "path to arbitrary bytes")));
        // full passes over the data
        BufferedString bs = new BufferedString();
        int fullpasses = (int)_useFraction; // Example: train_samples_per_iteration = 4700, and train.numRows()=1000 -> _useFraction = 4.7 -> fullpasses = 4
        while (j++ < fullpasses) {
          for (int i=0; i<_fr.numRows(); ++i) {
            double weight = weightIdx == -1 ? 1 : _fr.vec(weightIdx).at(i);
            if (weight == 0)
              continue;
            BufferedString file = _fr.vec(dataIdx).atStr(bs, i);
            if (file!=null)
              trainData.add(file.toString());
            float response = (float) _fr.vec(respIdx).at(i);
            trainLabels.add(response);
          }
        }

        // fractional passes // 0.7
        while (trainData.size() < _useFraction*len || trainData.size() % batchSize != 0) {
          assert(_shuffle);
          int i = rng.nextInt(len);
          double weight = weightIdx == -1 ? 1 : _fr.vec(weightIdx).at(i);
          if (weight == 0)
            continue;
          BufferedString file = _fr.vec(dataIdx).atStr(bs, i);
          if (file!=null)
            trainData.add(file.toString());
          float response = (float) _fr.vec(respIdx).at(i);
          trainLabels.add(response);
        }
      }

      // Numeric data (H2O Frame full with numeric columns)
      else if (_localmodel.get_params()._problem_type == DeepWaterParameters.ProblemType.dataset) {
        double mul = _localmodel._dataInfo._normRespMul!=null ? _localmodel._dataInfo._normRespMul[0] : 1;
        double sub = _localmodel._dataInfo._normRespSub!=null ? _localmodel._dataInfo._normRespSub[0] : 0;

        // full passes over the data
        int fullpasses = (int) _useFraction;
        while (j++ < fullpasses) {
          for (int i = 0; i < _fr.numRows(); ++i) {
            double weight = weightIdx == -1 ? 1 : _fr.vec(weightIdx).at(i);
            if (weight == 0)
              continue;
            float response = (float)((_fr.vec(respIdx).at(i) - sub) / mul);
            trainData.add(i);
            trainLabels.add(response);
          }
        }

        // fractional passes
        while (trainData.size() < _useFraction * len || trainData.size() % batchSize != 0) {
          int i = rng.nextInt(len);
          double weight = weightIdx == -1 ? 1 : _fr.vec(weightIdx).at(i);
          if (weight == 0)
            continue;
          float response = (float)((_fr.vec(respIdx).at(i) - sub) / mul);
          trainData.add(i);
          trainLabels.add(response);
        }
      }

      // shuffle the (global) list
      if (_shuffle) {
        rng.setSeed(seed);
        Collections.shuffle(trainLabels, rng);
        rng.setSeed(seed);
        Collections.shuffle(trainData, rng);
      }
      if (_localmodel.get_params()._problem_type == DeepWaterParameters.ProblemType.image) {
        iter = new DeepWaterImageIterator(trainData, trainLabels, _localmodel._meanData, batchSize, _localmodel._width, _localmodel._height, _localmodel._channels, _localmodel.get_params()._cache_data);
      }
      else if (_localmodel.get_params()._problem_type == DeepWaterParameters.ProblemType.dataset) {
        assert (_localmodel._dataInfo != null);
        iter = new DeepWaterDatasetIterator(trainData, trainLabels, _localmodel._dataInfo, batchSize, _localmodel.get_params()._cache_data);
      }
      else if (_localmodel.get_params()._problem_type == DeepWaterParameters.ProblemType.text) {
        iter = new DeepWaterTextIterator(trainData, trainLabels, batchSize, 56/*FIXME*/, _localmodel.get_params()._cache_data);
      }

      NativeTrainTask ntt;
      while (iter.Next(fs) && !_job.isStopping()) {
//        if (ntt != null) nativetime += ntt._timeInMillis;
        long n = _localmodel.get_processed_total();

//        if(!_localmodel.get_params()._quiet_mode)
//          Log.info("Trained " + n + " samples. Training on " + Arrays.toString(((DeepWaterImageIterator)iter).getFiles()));

        _localmodel._backend.setParameter(_localmodel.getModel().get(), "learning_rate", _localmodel.get_params().learningRate((double) n));
        _localmodel._backend.setParameter(_localmodel.getModel().get(), "momentum", _localmodel.get_params().momentum((double) n));

        //fork off GPU work, but let the iterator.Next() wait on completion before swapping again
        //System.err.println("data: " + Arrays.toString(iter.getData()));
        /*
        float[] preds = _localmodel._backend.predict(_localmodel._model, iter.getData());
        if (Float.isNaN(ArrayUtils.sum(preds))) {
          Log.err(DeepWaterModel.unstable_msg);
          throw new UnsupportedOperationException(DeepWaterModel.unstable_msg);
        }
        */
//        System.err.println("pred: " + Arrays.toString(preds));
        ntt = new NativeTrainTask(_localmodel._backend, _localmodel.getModel().get(), iter.getData(), iter.getLabel());
        fs.add(H2O.submitTask(ntt));
        _localmodel.add_processed_local(iter._batch_size);
      }
      fs.blockForPending();
//      nativetime += ntt._timeInMillis;
    } catch (IOException e) {
      e.printStackTrace(); //gracefully continue if we can't find files etc.
    }
//    long end = System.currentTimeMillis();
//    if (!_localmodel.get_params()._quiet_mode) {
//      Log.info("Time for one iteration: " + PrettyPrint.msecs(end - start, true));
//      Log.info("Time for native training : " + PrettyPrint.msecs(nativetime, true));
//    }
  }
350079
def clone(cls, objective, model=None, **kwargs):
        
        return cls(cls._substitute_variables(objective, model=model), name=objective.name,
                   direction=objective.direction, sloppy=True, **kwargs)
588512
def install_global_objects(self):
        
        import uliweb
        for k, v in settings.GLOBAL_OBJECTS.items():
            setattr(uliweb, k, import_attr(v))
103960
def save(self, model_fname='model.pkl'):
        
        with open(model_fname, 'wb') as fh:
            pickle.dump(self.stmts, fh, protocol=4)
586352
def set_model(self, m):
        
        self._model = m
        self.new_root.emit(QtCore.QModelIndex())
        self.model_changed(m)
151476
def update_rel_to(self, klass):
        

        if isinstance(self.remote_field.to, basestring):
            relation = self.remote_field.to
            try:
                app_label, model_name = relation.split(".")
            except ValueError:
                # If we can't split, assume a model in current app
                app_label = klass._meta.app_label
                model_name = relation

            model = None
            try:
                model = klass._meta.apps.get_registered_model(app_label, model_name)
            # For django < 1.6
            except AttributeError:
                model = models.get_model(
                    app_label, model_name,
                    seed_cache=False, only_installed=False)
            except LookupError:
                pass

            if model:
                self.remote_field.model = model
36079
function initGlobal (args) {
    // do nothing if global data already defined
    if (global.__immutable_core_model__) {
        return
    }
    // create new global object
    var GLOBAL = global.__immutable_core_model__ = {
        // number of concurrent related queries to allow per query
        concurrency: 5,
        // default to use binary columns for ids
        defaultBinaryIds: true,
        // default character set
        defaultCharset: 'utf8',
        // default compression setting
        defaultCompression: true,
        // default elastic search client
        defaultElasticsearch: undefined,
        // default database engine
        defaultEngine: 'InnoDB',
        // default insert delayed option for myisam tables
        defaultInsertDelayed: false,
        // map of models indexed by name
        models: {},
    }
    // create validator as global object so that schemas will be shared
    // between all model classes
    GLOBAL.validator = new Ajv({
        allErrors: true,
        coerceTypes: 'array',
        removeAdditional: true,
        useDefaults: true,
        v5: true,
    })
}
988063
def _update_new_linked_model(self, internal, linked_mdl_ins, link):
        

        # If there is a link between two sides (A and B), if a link from A to B,
        # link should be saved at B but it is not necessary to control again data in A.
        # If internal field is True, data control is not done and passes.
        if not internal:

            for lnk in linked_mdl_ins.get_links():
                mdl = lnk['mdl']
                if not isinstance(self, mdl) or lnk['reverse'] != link['field']:
                    continue
                local_field_name = lnk['field']
                # remote_name = lnk['reverse']
                remote_field_name = un_camel(mdl.__name__)
                if not link['o2o']:
                    if '.' in local_field_name:
                        local_field_name, remote_field_name = local_field_name.split('.')
                    remote_set = getattr(linked_mdl_ins, local_field_name)

                    if remote_set._TYPE == 'ListNode' and self not in remote_set:
                        remote_set(**{remote_field_name: self._root_node})
                        if linked_mdl_ins._exists is False:
                            raise ObjectDoesNotExist('Linked %s on field %s with key %s doesn\'t exist' % (
                                linked_mdl_ins.__class__.__name__,
                                remote_field_name,
                                linked_mdl_ins.key,
                            ))
                        linked_mdl_ins.save(internal=True)
                else:
                    linked_mdl_ins.setattr(remote_field_name, self._root_node)
                    if linked_mdl_ins._exists is False:
                        raise ObjectDoesNotExist('Linked object %s on field %s with key %s doesn\'t exist' % (
                            linked_mdl_ins.__class__.__name__,
                            remote_field_name,
                            linked_mdl_ins.key,
                        ))
                    linked_mdl_ins.save(internal=True)
912475
def _setup_global_state_for_execution(laser_evm, transaction) -> None:
    
    # TODO: Resolve circular import between .transaction and ..svm to import LaserEVM here
    global_state = transaction.initial_global_state()
    global_state.transaction_stack.append((transaction, None))

    new_node = Node(
        global_state.environment.active_account.contract_name,
        function_name=global_state.environment.active_function_name,
    )
    if laser_evm.requires_statespace:
        laser_evm.nodes[new_node.uid] = new_node

    if transaction.world_state.node:
        if laser_evm.requires_statespace:
            laser_evm.edges.append(
                Edge(
                    transaction.world_state.node.uid,
                    new_node.uid,
                    edge_type=JumpType.Transaction,
                    condition=None,
                )
            )

        global_state.mstate.constraints += transaction.world_state.node.constraints
        new_node.constraints = global_state.mstate.constraints.as_list

    global_state.world_state.transaction_sequence.append(transaction)
    global_state.node = new_node
    new_node.states.append(global_state)
    laser_evm.work_list.append(global_state)
2315
def copy_fields(self, model):
        
        fields = {'__module__' : model.__module__}

        for field in model._meta.fields:

            if not field.name in self._exclude:

                field  = copy.deepcopy(field)

                if isinstance(field, models.AutoField):
                    #we replace the AutoField of the original model
                    #with an IntegerField because a model can
                    #have only one autofield.

                    field.__class__ = models.IntegerField

                if field.primary_key:
                    field.serialize = True

                #OneToOne fields should really be tracked
                #as ForeignKey fields
                if isinstance(field, models.OneToOneField):
                    field.__class__ = models.ForeignKey


                if field.primary_key or field.unique:
                    #unique fields of the original model
                    #can not be guaranteed to be unique
                    #in the audit log entry but they
                    #should still be indexed for faster lookups.

                    field.primary_key = False
                    field._unique = False
                    field.db_index = True


                if field.remote_field and field.remote_field.related_name:
                    field.remote_field.related_name = '_auditlog_{}_{}'.format(
                        model._meta.model_name,
                        field.remote_field.related_name
                    )
                elif field.remote_field:
                    try:
                        if field.remote_field.get_accessor_name():
                            field.remote_field.related_name = '_auditlog_{}_{}'.format(
                                model._meta.model_name,
                                field.remote_field.get_accessor_name()
                            )
                    except e:
                        pass

                fields[field.name] = field

        return fields
1219830
def to_model
      vm = value ? value.to_model : nil
      HQMF::SubsetOperator.new(type, vm)
    end
964697
def import_model(self, source):
        """"""
        if not hasattr(source, 'read'):  # Not a File-like object
            with open(self._resolve_source(source), 'r') as f:
                return self._import(f)
        else:
            return self._import(source)
1206926
public void copyFrom(Model other) {
        if (!metaModelLocal.getTableName().equals(other.metaModelLocal.getTableName())) {
            throw new IllegalArgumentException("can only copy between the same types");
        }
        Map<String, Object> otherAttributes = other.getAttributes();
        for (String name : metaModelLocal.getAttributeNamesSkipId()) {
            attributes.put(name, otherAttributes.get(name));
            dirtyAttributeNames.add(name);
            // Why not use setRaw() here? Does the same and avoids duplication of code... (Garagoth)
            // other.setRaw(name, getRaw(name));
        }
    }
1363201
def internal_model_from_file(
            self, file_name, encoding='utf-8', debug=None,
            pre_ref_resolution_callback=None, is_main_model=True):
        
        model = None
        callback = pre_ref_resolution_callback

        if hasattr(self, "_tx_model_repository"):
            # metamodel has a global repo
            if not callback:
                def _pre_ref_resolution_callback(other_model):
                    from textx.scoping import GlobalModelRepository
                    filename = other_model._tx_filename
                    assert filename
                    # print("METAMODEL PRE-CALLBACK{}".format(filename))
                    other_model._tx_model_repository = GlobalModelRepository(
                        self._tx_model_repository.all_models)
                    self._tx_model_repository.all_models\
                        .filename_to_model[filename] = other_model

                callback = _pre_ref_resolution_callback
            if self._tx_model_repository.all_models.has_model(file_name):
                model = self._tx_model_repository.all_models\
                    .filename_to_model[file_name]

        if not model:
            # model not present (from global repo) -> load it
            model = self._parser_blueprint.clone().get_model_from_file(
                file_name, encoding, debug=debug,
                pre_ref_resolution_callback=callback,
                is_main_model=is_main_model)

        for p in self._model_processors:
            p(model, self)
        return model
79411
def _transfer_result(fut1, fut2):
  """"""
  exc = fut1.get_exception()
  if exc is not None:
    tb = fut1.get_traceback()
    fut2.set_exception(exc, tb)
  else:
    val = fut1.get_result()
    fut2.set_result(val)
208218
def create_many_to_many_intermediary_model(field, klass):
    
    managed = True
    if (isinstance(field.remote_field.to, basestring) and
            field.remote_field.to != related.RECURSIVE_RELATIONSHIP_CONSTANT):
        to_model = field.remote_field.to
        to = to_model.split('.')[-1]

        def set_managed(field, model, cls):
            managed = model._meta.managed or cls._meta.managed
            if issubclass(cls, VersionView):
                managed = False
            field.remote_field.through._meta.managed = managed
        related.add_lazy_relation(klass, field, to_model, set_managed)
    elif isinstance(field.remote_field.to, basestring):
        to = klass._meta.object_name
        to_model = klass
        managed = klass._meta.managed
    else:
        to = field.remote_field.to._meta.object_name
        to_model = field.remote_field.to
        managed = klass._meta.managed or to_model._meta.managed
        if issubclass(klass, VersionView):
            managed = False

    name = '%s_%s' % (klass._meta.object_name, field.name)
    if (field.remote_field.to == related.RECURSIVE_RELATIONSHIP_CONSTANT or
            to == klass._meta.object_name):
        from_ = 'from_%s' % to.lower()
        to = 'to_%s' % to.lower()
    else:
        from_ = klass._meta.object_name.lower()
        to = to.lower()
    meta = type('Meta', (object,), {
        'db_table': field._get_m2m_db_table(klass._meta),
        'managed': managed,
        'auto_created': klass,
        'app_label': klass._meta.app_label,
        'db_tablespace': klass._meta.db_tablespace,
        'unique_together': ('from', 'to'),
        'verbose_name': '%(from)s-%(to)s relationship' % {
            'from': from_, 'to': to},
        'verbose_name_plural': '%(from)s-%(to)s relationships' % {
            'from': from_, 'to': to},
        'apps': field.model._meta.apps,
    })

    # Construct and return the new class.
    return type(str(name), (models.Model,), {
        'Meta': meta,
        '__module__': klass.__module__,
        'from': FKToVersion(klass, related_name='%s+' % name,
                            db_tablespace=field.db_tablespace,
                            db_constraint=field.remote_field.db_constraint),
        'to': models.ForeignKey(to_model, related_name='%s+' % name,
                                db_tablespace=field.db_tablespace,
                                db_constraint=field.remote_field.db_constraint)
    })
661100
def global_lppooling(attrs, inputs, proto_obj):
    """"""
    p_value = attrs.get('p', 2)
    new_attrs = translation_utils._add_extra_attributes(attrs, {'global_pool': True,
                                                                'kernel': (1, 1),
                                                                'pool_type': 'lp',
                                                                'p_value': p_value})
    new_attrs = translation_utils._remove_attributes(new_attrs, ['p'])
    return 'Pooling', new_attrs, inputs