(Re/Un)Index Mongo document on post_save
165393
def reindex_model_on_save(sender, document, **kwargs):
    ''''''
    if current_app.config.get('AUTO_INDEX'):
        reindex.delay(document)
1351639
def update_doc(doc_id, params = {}, update_limit = 10)
      resp = {'ok' => false}
      last_fail = nil

      until resp['ok'] or update_limit <= 0
        doc = self.get(doc_id, params)
        yield doc
        begin
          resp = self.save_doc doc
        rescue CouchRest::RequestFailed => e
          if e.http_code == 409 # Update collision
            update_limit -= 1
            last_fail = e
          else
            raise e
          end
        end
      end

      raise last_fail unless resp['ok']
      doc
    end
1130788
function(dbOrUri, stream, callback) {
    if (arguments.length === 2) {
      callback = stream;
      stream = undefined;
    }
    if (!stream) {
      stream = process.stdout;
    }
    var db;
    var out = stream;
    var endOfCollection = crypto.pseudoRandomBytes(8).toString('base64');
    write({
      type: 'mongo-dump-stream',
      version: '2',
      endOfCollection: endOfCollection
    });
    return async.series({
      connect: function(callback) {
        if (typeof(dbOrUri) !== 'string') {
          // Already a mongodb connection
          db = dbOrUri;
          return setImmediate(callback);
        }
        return mongodb.MongoClient.connect(dbOrUri, function(err, _db) {
          if (err) {
            return callback(err);
          }
          db = _db;
          return callback(null);
        });
      },
      getCollections: function(callback) {
        return db.collections(function(err, _collections) {
          if (err) {
            return callback(err);
          }
          collections = _collections;
          return callback(null);
        });
      },
      dumpCollections: function(callback) {
        return async.eachSeries(collections, function(collection, callback) {
          if (collection.collectionName.match(/^system\./)) {
            return setImmediate(callback);
          }
          return async.series({
            getIndexes: function(callback) {
              return collection.indexInformation({ full: true }, function(err, info) {
                if (err) {
                  return callback(err);
                }
                write({
                  type: 'collection',
                  name: collection.collectionName,
                  indexes: info
                });
                return callback(null);
              });
            },
            getDocuments: function(callback) {
              var cursor = collection.find({}, { raw: true });
              iterate();
              function iterate() {
                return cursor.nextObject(function(err, item) {
                  if (err) {
                    return callback(err);
                  }
                  if (!item) {
                    write({
                      // Ensures we don't confuse this with
                      // a legitimate database object
                      endOfCollection: endOfCollection
                    });
                    return callback(null);
                  }
                  // v2: just a series of actual data documents
                  out.write(item);

                  // If we didn't have the raw BSON document,
                  // we could do this instead, but it would be very slow
                  // write({
                  //   type: 'document',
                  //   document: item
                  // });

                  return setImmediate(iterate);
                });
              }
            },
          }, callback);
        }, callback);
      },
      endDatabase: function(callback) {
        write({
          type: 'endDatabase'
        }, callback);
      }
    }, function(err) {
      if (err) {
        return callback(err);
      }
      return callback(null);
    });
    function write(o, callback) {
      out.write(BSON.serialize(o, false, true, false), callback);
    }
  }
1269830
def reindex(self, fq= [], **kwargs):
        '''
        
        '''
        for items in self._getter(fq=fq, **kwargs):
            self._putter(items)
        if type(self._dest) is SolrClient and self._dest_coll:
            self.log.info("Finished Indexing, sending a commit")
            self._dest.commit(self._dest_coll, openSearcher=True)
1291479
private void esIndex(String index, String id, String json, String pipeline) {
        logger.debug("Indexing {}/{}?pipeline={}", index, id, pipeline);
        logger.trace("JSon indexed : {}", json);

        if (!closed) {
            esClient.index(index, id, json, pipeline);
        } else {
            logger.warn("trying to add new file while closing crawler. Document [{}]/[{}] has been ignored", index, id);
        }
    }
794132
def _ensure_index_cache(self, db_uri, db_name, collection_name):
        """"""
        if not self._check_indexes or db_uri is None:
            return {'indexes': None}
        if db_name not in self.get_cache():
            self._internal_map[db_name] = {}
        if collection_name not in self._internal_map[db_name]:
            indexes = []
            try:
                if self._index_cache_connection is None:
                    self._index_cache_connection = pymongo.MongoClient(db_uri,
                                                                       document_class=OrderedDict,
                                                                       read_preference=pymongo.ReadPreference.PRIMARY_PREFERRED)

                db = self._index_cache_connection[db_name]
                indexes = db[collection_name].index_information()
            except:
                warning = 'Warning: unable to connect to ' + db_uri + "\n"
            else:
                internal_map_entry = {'indexes': indexes}
                self.get_cache()[db_name][collection_name] = internal_map_entry
        return self.get_cache()[db_name][collection_name]
353355
public void ensureIndex(final DBObject keys, final DBObject optionsIN) throws MongoException {
        dbCollection.ensureIndex(keys, optionsIN);
    }
369100
def search_update_index(self):
        
        doc_id = self.search_get_document_id(self.key)

        fields = [search.AtomField('class_name', name) for name in self.search_get_class_names()]

        index = self.search_get_index()

        if self.searchable_fields is None:
            searchable_fields = []

            for field, prop in self._properties.items():
                if field == 'class':
                    continue
                for class_, field_type in SEARCHABLE_PROPERTY_TYPES.items():
                    if isinstance(prop, class_):
                        searchable_fields.append(field)
        else:
            searchable_fields = self.searchable_fields

        for f in set(searchable_fields):
            prop = self._properties[f]
            value = getattr(self, f)
            field = None
            field_found = False
            for class_, field_type in SEARCHABLE_PROPERTY_TYPES.items():
                if isinstance(prop, class_):
                    field_found = True
                    if value is not None:
                        if isinstance(value, list) or isinstance(value, tuple) or isinstance(value, set):
                            for v in value:
                                field = field_type(name=f, value=v)
                        elif isinstance(value, ndb.Key):
                            field = field_type(name=f, value=value.urlsafe())
                        else:
                            field = field_type(name=f, value=value)
            if not field_found:
                raise ValueError('Cannot find field type for %r on %r' % (prop, self.__class__))

            if field is not None:
                fields.append(field)

        document = search.Document(doc_id, fields=fields)
        index.put(document)
493445
def patch(self, keepNull = True, **docArgs) :
        """"""

        if self.URL is None :
            raise ValueError("Cannot patch a document that was not previously saved")

        payload = self._store.getPatches()
        
        if self.collection._validation['on_save'] :
            self.validate()

        if len(payload) > 0 :
            params = dict(docArgs)
            params.update({'collection': self.collection.name, 'keepNull' : keepNull})
            payload = json.dumps(payload, default=str)

            r = self.connection.session.patch(self.URL, params = params, data = payload)
            data = r.json()
            if (r.status_code == 201 or r.status_code == 202) and "error" not in data :
                self._rev = data['_rev']
            else :
                raise UpdateError(data['errorMessage'], data)

            self.modified = False

        self._store.resetPatch()
502329
def index(self, corpus=None, clear_buffer=True):
        
        if not self.model:
            msg = 'must initialize model for %s before indexing documents' % self.basename
            logger.error(msg)
            raise AttributeError(msg)

        if corpus is not None:
            # use the supplied corpus only (erase existing buffer, if any)
            self.flush(clear_buffer=True)
            self.buffer(corpus)

        if not self.fresh_docs:
            msg = "index called but no indexing corpus specified for %s" % self
            logger.error(msg)
            raise ValueError(msg)

        if not self.fresh_index:
            logger.info("starting a new fresh index for %s" % self)
            self.fresh_index = SimIndex(self.location('index_fresh'), self.model.num_features)
        self.fresh_index.index_documents(self.fresh_docs, self.model)
        if self.opt_index is not None:
            self.opt_index.delete(self.fresh_docs.keys())
        logger.info("storing document payloads")
        for docid in self.fresh_docs:
            payload = self.fresh_docs[docid].get('payload', None)
            if payload is None:
                # HACK: exit on first doc without a payload (=assume all docs have payload, or none does)
                break
            self.payload[docid] = payload
        self.flush(save_index=True, clear_buffer=clear_buffer)
280303
def query_mongo(
        database_name,
        collection_name,
        query={},
        include_num_results="0",
        skip=0,
        sort=None,
        limit=getattr(
            settings,
            'MONGO_LIMIT',
            200),
        cast_strings_to_integers=False,
        return_keys=()):
    """"""

    l = []
    response_dict = {}

    try:
        mongodb_client_url = getattr(settings, 'MONGODB_CLIENT',
                                 'mongodb://localhost:27017/')
        mc = MongoClient(mongodb_client_url,document_class=OrderedDict)

        db = mc[str(database_name)]
        collection = db[str(collection_name)]

        # Cast the query to integers
        if cast_strings_to_integers:
            query = cast_number_strings_to_integers(query)

        # print query
        if return_keys:
            return_dict = {}
            for k in return_keys:
                return_dict[k] = 1
            # print "returndict=",return_dict
            mysearchresult = collection.find(
                query, return_dict).skip(skip).limit(limit)
        else:
            mysearchresult = collection.find(query).skip(skip).limit(limit)

        if sort:
            mysearchresult.sort(sort)

        response_dict['code'] = 200
        if include_num_results == "1":
            response_dict['num_results'] = response_dict['num_results'] = int(
                mysearchresult.count(with_limit_and_skip=False))

        if include_num_results == "2":
            response_dict['num_results'] = response_dict['num_results'] = int(
                mysearchresult.count(with_limit_and_skip=True))

        response_dict['type'] = "search-results"
        for d in mysearchresult:
            d['id'] = d['_id'].__str__()
            del d['_id']
            l.append(d)
        response_dict['results'] = l

    except:
        print("Error reading from Mongo")
        print(str(sys.exc_info()))
        response_dict['num_results'] = 0
        response_dict['code'] = 500
        response_dict['type'] = "Error"
        response_dict['results'] = []
        response_dict['message'] = str(sys.exc_info())

    return response_dict
44222
function upsertDocumentController(req, res) {
                    var modelName = req.params.modelName,
                        id = req.params.documentId,
                        user = req.admin_user,
                        data = _.extend({}, req.body, req.files),
                        modelConfig = registry.models[modelName],
                        form;

                    polymorphGetDocument(modelConfig.model, modelConfig.is_single, id).then(
                        function createTheForm(document) {
                            var action = document ? 'update' : 'create';
                            var docType = document && document._doc.__t;
                            if (docType) modelConfig = registry.models[docType];
                            var FormType = modelConfig.options.form;
                            if (!user.hasPermissions(modelConfig.modelName, action)) throw new Error('unauthorized');
                            form = new FormType(modelConfig.options, modelConfig.model, document, data);
                            return form.save();
                        }
                    ).then(
                        function (document) {
                            modelConfig.options.post(document);
                            return form;
                        },
                        function (err) {
                            err.modelConfig = modelConfig;
                            err.form = form;
                            if (!form) throw err;

                            form.errors = form.errors || {};
                            if (err.name == 'ValidationError') {
                                form.errors = _.merge(form.errors, err.errors);
                            } else if (err.name === 'MongoError' && err.code == 11000) {
                                form.errors.exception = err;
                                err.name = "Duplicate";
                                var msg = '{' + err.err.split('$').pop().split('{').pop();
                                var ObjectID = /ObjectId\(\'(\w+)\'\)/.exec(msg);
                                err.fields = '\n<br />';
                                err.fields += (ObjectID.length == 2) ? '<a href="' + ObjectID[1] + '" target="_blank">Click to open duplicate document</a>' :
                                    msg;
                            } else {
                                form.errors.exception = err;
                            }
                            if (_.isEmpty(form.errors)) return form;
                            throw err;
                        }
                    ).then(
                        function (form) {
                            res._debug_form = form;
                            var response = form.instance.toJSON();
                            response.id = response.id || form.instance._id;
                            response.label = response.label || form.instance.name || form.instance.title || form.instance.toString();
                            return response;
                        }
                    ).then(
                        function (response) {
                            res.json(200, response);
                        },
                        function (err) {
                            err.form = res._debug_form;
                            res.json(422, err);
                        }
                    ).end();
                }
1127516
def params_to_mongo(query_params):
    
    if not query_params:
        return {}
    for key, value in query_params.items():
        if isinstance(value, list):
            query_params[key] = {'$in': value}
    return query_params
290936
def index(self, refresh=False):
        """"""
        es = connections.get_connection("default")
        index = self.__class__.search_objects.mapping.index
        doc_type = self.__class__.search_objects.mapping.doc_type
        es.index(index, doc_type,
                 id=self.pk,
                 body=self.to_dict(),
                 refresh=refresh)
682023
def flush(self, save_index=False, save_model=False, clear_buffer=False):
        """"""
        if save_index:
            if self.fresh_index is not None:
                self.fresh_index.save(self.location('index_fresh'))
            if self.opt_index is not None:
                self.opt_index.save(self.location('index_opt'))
        if save_model:
            if self.model is not None:
                self.model.save(self.location('model'))
        self.payload.commit()
        if clear_buffer:
            if hasattr(self, 'fresh_docs'):
                try:
                    self.fresh_docs.terminate() # erase all buffered documents + file on disk
                except:
                    pass
            self.fresh_docs = SqliteDict(journal_mode=JOURNAL_MODE) # buffer defaults to a random location in temp
        self.fresh_docs.sync()
977853
def add_resource(self, resource):
        ''''''
        resource.validate()
        self.update(__raw__={
            '$push': {
                'resources': {
                    '$each': [resource.to_mongo()],
                    '$position': 0
                }
            }
        })
        self.reload()
        post_save.send(self.__class__, document=self,
                       resource_added=resource.id)
975924
def _update(self, sock_info, criteria, document, upsert=False,
                check_keys=True, multi=False, manipulate=False,
                write_concern=None, op_id=None, ordered=True,
                bypass_doc_val=False, collation=None, array_filters=None,
                session=None, retryable_write=False):
        """"""
        common.validate_boolean("upsert", upsert)
        if manipulate:
            document = self.__database._fix_incoming(document, self)
        collation = validate_collation_or_none(collation)
        write_concern = write_concern or self.write_concern
        acknowledged = write_concern.acknowledged
        update_doc = SON([('q', criteria),
                          ('u', document),
                          ('multi', multi),
                          ('upsert', upsert)])
        if collation is not None:
            if sock_info.max_wire_version < 5:
                raise ConfigurationError(
                    'Must be connected to MongoDB 3.4+ to use collations.')
            elif not acknowledged:
                raise ConfigurationError(
                    'Collation is unsupported for unacknowledged writes.')
            else:
                update_doc['collation'] = collation
        if array_filters is not None:
            if sock_info.max_wire_version < 6:
                raise ConfigurationError(
                    'Must be connected to MongoDB 3.6+ to use array_filters.')
            elif not acknowledged:
                raise ConfigurationError(
                    'arrayFilters is unsupported for unacknowledged writes.')
            else:
                update_doc['arrayFilters'] = array_filters
        command = SON([('update', self.name),
                       ('ordered', ordered),
                       ('updates', [update_doc])])
        if not write_concern.is_server_default:
            command['writeConcern'] = write_concern.document

        if not sock_info.op_msg_enabled and not acknowledged:
            # Legacy OP_UPDATE.
            return self._legacy_write(
                sock_info, 'update', command, op_id,
                bypass_doc_val, message.update, self.__full_name, upsert,
                multi, criteria, document, False, write_concern.document,
                check_keys, self.__write_response_codec_options)

        # Update command.
        if bypass_doc_val and sock_info.max_wire_version >= 4:
            command['bypassDocumentValidation'] = True

        # The command result has to be published for APM unmodified
        # so we make a shallow copy here before adding updatedExisting.
        result = sock_info.command(
            self.__database.name,
            command,
            write_concern=write_concern,
            codec_options=self.__write_response_codec_options,
            session=session,
            client=self.__database.client,
            retryable_write=retryable_write).copy()
        _check_write_command_response(result)
        # Add the updatedExisting field for compatibility.
        if result.get('n') and 'upserted' not in result:
            result['updatedExisting'] = True
        else:
            result['updatedExisting'] = False
            # MongoDB >= 2.6.0 returns the upsert _id in an array
            # element. Break it out for backward compatibility.
            if 'upserted' in result:
                result['upserted'] = result['upserted'][0]['_id']

        if not acknowledged:
            return None
        return result