We are currently solving the following issue within our repository. Here is the issue text:
--- BEGIN ISSUE ---
Document reload cause clash between reference with same id but different collection
Hi there,

Let say I have a document with two `ReferenceField` on two different collections with incremental ids.
Given the ids are incremental and not uuid, an id can point on a valid document in both collections (pretty obvious for id `1` ^^)

That being said, the `reload` handle reference reloading in around here: https://github.com/MongoEngine/mongoengine/blob/d4a8ae5743fe335ac4cea957537be34de45a440f/mongoengine/dereference.py#L77

The idea is (if I'm not wrong) to collect the document's references in a dict something like that

``` python
>>> print(self.reference_map)
{
    MyDocumentA: [DBref('MyDocumentA', 'id-1'), DBref('MyDocumentA', 'id-2') ],
    MyDocumentB: [DBref('MyDocumentB', 'id-1'),]
}
```

Then use this dict to retrieve all those document from database

``` python
>>> print(self.object_map)
{
    'id-1': <MyDocumentB: MyDocumentB object>,
    'id-2': <MyDocumentA: MyDocumentA object>
}
```

As you can see, given object_map only use id as key, the document `MyDocumentA` with key 'id-1' has been shadowed by the `MyDocumentB` with similar id !

Thus, the final `_attach_objects` aiming at refreshing the main document's references will replace a `MyDocumentA` reference with a `MyDocumentB`...

I think to solve this trouble, we should replace the key in object_map from this single to a `DBref`, thus the check would be on both collection and id.

What do you think ? Given I've spend some time tracking this nasty bug, I think I can provide a fix myself if you ask for ;-)


--- END ISSUE ---

Below are some code segments, each from a relevant file. One or more of these files may contain bugs.
--- BEGIN FILES ---
Path: `mongoengine/dereference.py`
Content:
```
1 from bson import DBRef, SON
2 
3 from base import (
4     BaseDict, BaseList, EmbeddedDocumentList,
5     TopLevelDocumentMetaclass, get_document
6 )
7 from fields import (ReferenceField, ListField, DictField, MapField)
8 from connection import get_db
9 from queryset import QuerySet
10 from document import Document, EmbeddedDocument
11 
12 
13 class DeReference(object):
14 
15     def __call__(self, items, max_depth=1, instance=None, name=None):
16         """
17         Cheaply dereferences the items to a set depth.
18         Also handles the conversion of complex data types.
19 
20         :param items: The iterable (dict, list, queryset) to be dereferenced.
21         :param max_depth: The maximum depth to recurse to
22         :param instance: The owning instance used for tracking changes by
23             :class:`~mongoengine.base.ComplexBaseField`
24         :param name: The name of the field, used for tracking changes by
25             :class:`~mongoengine.base.ComplexBaseField`
26         :param get: A boolean determining if being called by __get__
27         """
28         if items is None or isinstance(items, basestring):
29             return items
30 
31         # cheapest way to convert a queryset to a list
32         # list(queryset) uses a count() query to determine length
33         if isinstance(items, QuerySet):
34             items = [i for i in items]
35 
36         self.max_depth = max_depth
37         doc_type = None
38 
39         if instance and isinstance(instance, (Document, EmbeddedDocument,
40                                               TopLevelDocumentMetaclass)):
41             doc_type = instance._fields.get(name)
42             while hasattr(doc_type, 'field'):
43                 doc_type = doc_type.field
44 
45             if isinstance(doc_type, ReferenceField):
46                 field = doc_type
47                 doc_type = doc_type.document_type
48                 is_list = not hasattr(items, 'items')
49 
50                 if is_list and all([i.__class__ == doc_type for i in items]):
51                     return items
52                 elif not is_list and all([i.__class__ == doc_type
53                                          for i in items.values()]):
54                     return items
55                 elif not field.dbref:
56                     if not hasattr(items, 'items'):
57 
58                         def _get_items(items):
59                             new_items = []
60                             for v in items:
61                                 if isinstance(v, list):
62                                     new_items.append(_get_items(v))
63                                 elif not isinstance(v, (DBRef, Document)):
64                                     new_items.append(field.to_python(v))
65                                 else:
66                                     new_items.append(v)
67                             return new_items
68 
69                         items = _get_items(items)
70                     else:
71                         items = dict([
72                             (k, field.to_python(v))
73                             if not isinstance(v, (DBRef, Document)) else (k, v)
74                             for k, v in items.iteritems()]
75                         )
76 
77         self.reference_map = self._find_references(items)
78         self.object_map = self._fetch_objects(doc_type=doc_type)
79         return self._attach_objects(items, 0, instance, name)
80 
81     def _find_references(self, items, depth=0):
82         """
83         Recursively finds all db references to be dereferenced
84 
85         :param items: The iterable (dict, list, queryset)
86         :param depth: The current depth of recursion
87         """
88         reference_map = {}
89         if not items or depth >= self.max_depth:
90             return reference_map
91 
92         # Determine the iterator to use
93         if not hasattr(items, 'items'):
94             iterator = enumerate(items)
95         else:
96             iterator = items.iteritems()
97 
98         # Recursively find dbreferences
99         depth += 1
100         for k, item in iterator:
101             if isinstance(item, (Document, EmbeddedDocument)):
102                 for field_name, field in item._fields.iteritems():
103                     v = item._data.get(field_name, None)
104                     if isinstance(v, (DBRef)):
105                         reference_map.setdefault(field.document_type, set()).add(v.id)
106                     elif isinstance(v, (dict, SON)) and '_ref' in v:
107                         reference_map.setdefault(get_document(v['_cls']), set()).add(v['_ref'].id)
108                     elif isinstance(v, (dict, list, tuple)) and depth <= self.max_depth:
109                         field_cls = getattr(getattr(field, 'field', None), 'document_type', None)
110                         references = self._find_references(v, depth)
111                         for key, refs in references.iteritems():
112                             if isinstance(field_cls, (Document, TopLevelDocumentMetaclass)):
113                                 key = field_cls
114                             reference_map.setdefault(key, set()).update(refs)
115             elif isinstance(item, (DBRef)):
116                 reference_map.setdefault(item.collection, set()).add(item.id)
117             elif isinstance(item, (dict, SON)) and '_ref' in item:
118                 reference_map.setdefault(get_document(item['_cls']), set()).add(item['_ref'].id)
119             elif isinstance(item, (dict, list, tuple)) and depth - 1 <= self.max_depth:
120                 references = self._find_references(item, depth - 1)
121                 for key, refs in references.iteritems():
122                     reference_map.setdefault(key, set()).update(refs)
123 
124         return reference_map
125 
126     def _fetch_objects(self, doc_type=None):
127         """Fetch all references and convert to their document objects
128         """
129         object_map = {}
130         for collection, dbrefs in self.reference_map.iteritems():
131             refs = [dbref for dbref in dbrefs
132                     if unicode(dbref).encode('utf-8') not in object_map]
133             if hasattr(collection, 'objects'):  # We have a document class for the refs
134                 references = collection.objects.in_bulk(refs)
135                 for key, doc in references.iteritems():
136                     object_map[key] = doc
137             else:  # Generic reference: use the refs data to convert to document
138                 if isinstance(doc_type, (ListField, DictField, MapField,)):
139                     continue
140 
141                 if doc_type:
142                     references = doc_type._get_db()[collection].find({'_id': {'$in': refs}})
143                     for ref in references:
144                         doc = doc_type._from_son(ref)
145                         object_map[doc.id] = doc
146                 else:
147                     references = get_db()[collection].find({'_id': {'$in': refs}})
148                     for ref in references:
149                         if '_cls' in ref:
150                             doc = get_document(ref["_cls"])._from_son(ref)
151                         elif doc_type is None:
152                             doc = get_document(
153                                 ''.join(x.capitalize()
154                                     for x in collection.split('_')))._from_son(ref)
155                         else:
156                             doc = doc_type._from_son(ref)
157                         object_map[doc.id] = doc
158         return object_map
159 
160     def _attach_objects(self, items, depth=0, instance=None, name=None):
161         """
162         Recursively finds all db references to be dereferenced
163 
164         :param items: The iterable (dict, list, queryset)
165         :param depth: The current depth of recursion
166         :param instance: The owning instance used for tracking changes by
167             :class:`~mongoengine.base.ComplexBaseField`
168         :param name: The name of the field, used for tracking changes by
169             :class:`~mongoengine.base.ComplexBaseField`
170         """
171         if not items:
172             if isinstance(items, (BaseDict, BaseList)):
173                 return items
174 
175             if instance:
176                 if isinstance(items, dict):
177                     return BaseDict(items, instance, name)
178                 else:
179                     return BaseList(items, instance, name)
180 
181         if isinstance(items, (dict, SON)):
182             if '_ref' in items:
183                 return self.object_map.get(items['_ref'].id, items)
184             elif '_cls' in items:
185                 doc = get_document(items['_cls'])._from_son(items)
186                 _cls = doc._data.pop('_cls', None)
187                 del items['_cls']
188                 doc._data = self._attach_objects(doc._data, depth, doc, None)
189                 if _cls is not None:
190                     doc._data['_cls'] = _cls
191                 return doc
192 
193         if not hasattr(items, 'items'):
194             is_list = True
195             list_type = BaseList
196             if isinstance(items, EmbeddedDocumentList):
197                 list_type = EmbeddedDocumentList
198             as_tuple = isinstance(items, tuple)
199             iterator = enumerate(items)
200             data = []
201         else:
202             is_list = False
203             iterator = items.iteritems()
204             data = {}
205 
206         depth += 1
207         for k, v in iterator:
208             if is_list:
209                 data.append(v)
210             else:
211                 data[k] = v
212 
213             if k in self.object_map and not is_list:
214                 data[k] = self.object_map[k]
215             elif isinstance(v, (Document, EmbeddedDocument)):
216                 for field_name, field in v._fields.iteritems():
217                     v = data[k]._data.get(field_name, None)
218                     if isinstance(v, (DBRef)):
219                         data[k]._data[field_name] = self.object_map.get(v.id, v)
220                     elif isinstance(v, (dict, SON)) and '_ref' in v:
221                         data[k]._data[field_name] = self.object_map.get(v['_ref'].id, v)
222                     elif isinstance(v, (dict, list, tuple)) and depth <= self.max_depth:
223                         item_name = "{0}.{1}.{2}".format(name, k, field_name)
224                         data[k]._data[field_name] = self._attach_objects(v, depth, instance=instance, name=item_name)
225             elif isinstance(v, (dict, list, tuple)) and depth <= self.max_depth:
226                 item_name = '%s.%s' % (name, k) if name else name
227                 data[k] = self._attach_objects(v, depth - 1, instance=instance, name=item_name)
228             elif hasattr(v, 'id'):
229                 data[k] = self.object_map.get(v.id, v)
230 
231         if instance and name:
232             if is_list:
233                 return tuple(data) if as_tuple else list_type(data, instance, name)
234             return BaseDict(data, instance, name)
235         depth += 1
236         return data
237
```
--- END FILES ---

Please first localize the bug based on the issue statement, and then generate a patch according to the `git diff` format fenced by three backticks.

Here is an example:

```diff
diff --git a/examples/server_async.py b/examples/server_async.py
--- a/examples/server_async.py
+++ b/examples/server_async.py
@@ -313,4 +313,4 @@
 
 
 if __name__ == "__main__":
-    asyncio.run(run_async_server("."), debug=True)
+    asyncio.run(run_async_server(), debug=True)
diff --git a/examples/server_sync.py b/examples/server_sync.py
--- a/examples/server_sync.py
+++ b/examples/server_sync.py
@@ -313,5 +313,5 @@
 
 
 if __name__ == "__main__":
-    server = run_sync_server(".")
+    server = run_sync_server()
     server.shutdown()

```

diff --git a/mongoengine/dereference.py b/mongoengine/dereference.py
--- a/mongoengine/dereference.py
+++ b/mongoengine/dereference.py
@@ -128,21 +128,25 @@
         """
         object_map = {}
         for collection, dbrefs in self.reference_map.iteritems():
-            refs = [dbref for dbref in dbrefs
-                    if unicode(dbref).encode('utf-8') not in object_map]
             if hasattr(collection, 'objects'):  # We have a document class for the refs
+                col_name = collection._get_collection_name()
+                refs = [dbref for dbref in dbrefs
+                        if (col_name, dbref) not in object_map]
                 references = collection.objects.in_bulk(refs)
                 for key, doc in references.iteritems():
-                    object_map[key] = doc
+                    object_map[(col_name, key)] = doc
             else:  # Generic reference: use the refs data to convert to document
                 if isinstance(doc_type, (ListField, DictField, MapField,)):
                     continue
 
+                refs = [dbref for dbref in dbrefs
+                        if (collection, dbref) not in object_map]
+
                 if doc_type:
                     references = doc_type._get_db()[collection].find({'_id': {'$in': refs}})
                     for ref in references:
                         doc = doc_type._from_son(ref)
-                        object_map[doc.id] = doc
+                        object_map[(collection, doc.id)] = doc
                 else:
                     references = get_db()[collection].find({'_id': {'$in': refs}})
                     for ref in references:
@@ -154,7 +158,7 @@
                                     for x in collection.split('_')))._from_son(ref)
                         else:
                             doc = doc_type._from_son(ref)
-                        object_map[doc.id] = doc
+                        object_map[(collection, doc.id)] = doc
         return object_map
 
     def _attach_objects(self, items, depth=0, instance=None, name=None):
@@ -180,7 +184,8 @@
 
         if isinstance(items, (dict, SON)):
             if '_ref' in items:
-                return self.object_map.get(items['_ref'].id, items)
+                return self.object_map.get(
+                    (items['_ref'].collection, items['_ref'].id), items)
             elif '_cls' in items:
                 doc = get_document(items['_cls'])._from_son(items)
                 _cls = doc._data.pop('_cls', None)
@@ -216,9 +221,11 @@
                 for field_name, field in v._fields.iteritems():
                     v = data[k]._data.get(field_name, None)
                     if isinstance(v, (DBRef)):
-                        data[k]._data[field_name] = self.object_map.get(v.id, v)
+                        data[k]._data[field_name] = self.object_map.get(
+                            (v.collection, v.id), v)
                     elif isinstance(v, (dict, SON)) and '_ref' in v:
-                        data[k]._data[field_name] = self.object_map.get(v['_ref'].id, v)
+                        data[k]._data[field_name] = self.object_map.get(
+                            (v['_ref'].collection , v['_ref'].id), v)
                     elif isinstance(v, (dict, list, tuple)) and depth <= self.max_depth:
                         item_name = "{0}.{1}.{2}".format(name, k, field_name)
                         data[k]._data[field_name] = self._attach_objects(v, depth, instance=instance, name=item_name)
@@ -226,7 +233,7 @@
                 item_name = '%s.%s' % (name, k) if name else name
                 data[k] = self._attach_objects(v, depth - 1, instance=instance, name=item_name)
             elif hasattr(v, 'id'):
-                data[k] = self.object_map.get(v.id, v)
+                data[k] = self.object_map.get((v.collection, v.id), v)
 
         if instance and name:
             if is_list:

{"golden_diff": "diff --git a/mongoengine/dereference.py b/mongoengine/dereference.py\n--- a/mongoengine/dereference.py\n+++ b/mongoengine/dereference.py\n@@ -128,21 +128,25 @@\n         \"\"\"\n         object_map = {}\n         for collection, dbrefs in self.reference_map.iteritems():\n-            refs = [dbref for dbref in dbrefs\n-                    if unicode(dbref).encode('utf-8') not in object_map]\n             if hasattr(collection, 'objects'):  # We have a document class for the refs\n+                col_name = collection._get_collection_name()\n+                refs = [dbref for dbref in dbrefs\n+                        if (col_name, dbref) not in object_map]\n                 references = collection.objects.in_bulk(refs)\n                 for key, doc in references.iteritems():\n-                    object_map[key] = doc\n+                    object_map[(col_name, key)] = doc\n             else:  # Generic reference: use the refs data to convert to document\n                 if isinstance(doc_type, (ListField, DictField, MapField,)):\n                     continue\n \n+                refs = [dbref for dbref in dbrefs\n+                        if (collection, dbref) not in object_map]\n+\n                 if doc_type:\n                     references = doc_type._get_db()[collection].find({'_id': {'$in': refs}})\n                     for ref in references:\n                         doc = doc_type._from_son(ref)\n-                        object_map[doc.id] = doc\n+                        object_map[(collection, doc.id)] = doc\n                 else:\n                     references = get_db()[collection].find({'_id': {'$in': refs}})\n                     for ref in references:\n@@ -154,7 +158,7 @@\n                                     for x in collection.split('_')))._from_son(ref)\n                         else:\n                             doc = doc_type._from_son(ref)\n-                        object_map[doc.id] = doc\n+                        object_map[(collection, doc.id)] = doc\n         return object_map\n \n     def _attach_objects(self, items, depth=0, instance=None, name=None):\n@@ -180,7 +184,8 @@\n \n         if isinstance(items, (dict, SON)):\n             if '_ref' in items:\n-                return self.object_map.get(items['_ref'].id, items)\n+                return self.object_map.get(\n+                    (items['_ref'].collection, items['_ref'].id), items)\n             elif '_cls' in items:\n                 doc = get_document(items['_cls'])._from_son(items)\n                 _cls = doc._data.pop('_cls', None)\n@@ -216,9 +221,11 @@\n                 for field_name, field in v._fields.iteritems():\n                     v = data[k]._data.get(field_name, None)\n                     if isinstance(v, (DBRef)):\n-                        data[k]._data[field_name] = self.object_map.get(v.id, v)\n+                        data[k]._data[field_name] = self.object_map.get(\n+                            (v.collection, v.id), v)\n                     elif isinstance(v, (dict, SON)) and '_ref' in v:\n-                        data[k]._data[field_name] = self.object_map.get(v['_ref'].id, v)\n+                        data[k]._data[field_name] = self.object_map.get(\n+                            (v['_ref'].collection , v['_ref'].id), v)\n                     elif isinstance(v, (dict, list, tuple)) and depth <= self.max_depth:\n                         item_name = \"{0}.{1}.{2}\".format(name, k, field_name)\n                         data[k]._data[field_name] = self._attach_objects(v, depth, instance=instance, name=item_name)\n@@ -226,7 +233,7 @@\n                 item_name = '%s.%s' % (name, k) if name else name\n                 data[k] = self._attach_objects(v, depth - 1, instance=instance, name=item_name)\n             elif hasattr(v, 'id'):\n-                data[k] = self.object_map.get(v.id, v)\n+                data[k] = self.object_map.get((v.collection, v.id), v)\n \n         if instance and name:\n             if is_list:\n", "issue": "Document reload cause clash between reference with same id but different collection\nHi there,\n\nLet say I have a document with two `ReferenceField` on two different collections with incremental ids.\nGiven the ids are incremental and not uuid, an id can point on a valid document in both collections (pretty obvious for id `1` ^^)\n\nThat being said, the `reload` handle reference reloading in around here: https://github.com/MongoEngine/mongoengine/blob/d4a8ae5743fe335ac4cea957537be34de45a440f/mongoengine/dereference.py#L77\n\nThe idea is (if I'm not wrong) to collect the document's references in a dict something like that\n\n``` python\n>>> print(self.reference_map)\n{\n    MyDocumentA: [DBref('MyDocumentA', 'id-1'), DBref('MyDocumentA', 'id-2') ],\n    MyDocumentB: [DBref('MyDocumentB', 'id-1'),]\n}\n```\n\nThen use this dict to retrieve all those document from database\n\n``` python\n>>> print(self.object_map)\n{\n    'id-1': <MyDocumentB: MyDocumentB object>,\n    'id-2': <MyDocumentA: MyDocumentA object>\n}\n```\n\nAs you can see, given object_map only use id as key, the document `MyDocumentA` with key 'id-1' has been shadowed by the `MyDocumentB` with similar id !\n\nThus, the final `_attach_objects` aiming at refreshing the main document's references will replace a `MyDocumentA` reference with a `MyDocumentB`...\n\nI think to solve this trouble, we should replace the key in object_map from this single to a `DBref`, thus the check would be on both collection and id.\n\nWhat do you think ? Given I've spend some time tracking this nasty bug, I think I can provide a fix myself if you ask for ;-)\n\n", "before_files": [{"content": "from bson import DBRef, SON\n\nfrom base import (\n    BaseDict, BaseList, EmbeddedDocumentList,\n    TopLevelDocumentMetaclass, get_document\n)\nfrom fields import (ReferenceField, ListField, DictField, MapField)\nfrom connection import get_db\nfrom queryset import QuerySet\nfrom document import Document, EmbeddedDocument\n\n\nclass DeReference(object):\n\n    def __call__(self, items, max_depth=1, instance=None, name=None):\n        \"\"\"\n        Cheaply dereferences the items to a set depth.\n        Also handles the conversion of complex data types.\n\n        :param items: The iterable (dict, list, queryset) to be dereferenced.\n        :param max_depth: The maximum depth to recurse to\n        :param instance: The owning instance used for tracking changes by\n            :class:`~mongoengine.base.ComplexBaseField`\n        :param name: The name of the field, used for tracking changes by\n            :class:`~mongoengine.base.ComplexBaseField`\n        :param get: A boolean determining if being called by __get__\n        \"\"\"\n        if items is None or isinstance(items, basestring):\n            return items\n\n        # cheapest way to convert a queryset to a list\n        # list(queryset) uses a count() query to determine length\n        if isinstance(items, QuerySet):\n            items = [i for i in items]\n\n        self.max_depth = max_depth\n        doc_type = None\n\n        if instance and isinstance(instance, (Document, EmbeddedDocument,\n                                              TopLevelDocumentMetaclass)):\n            doc_type = instance._fields.get(name)\n            while hasattr(doc_type, 'field'):\n                doc_type = doc_type.field\n\n            if isinstance(doc_type, ReferenceField):\n                field = doc_type\n                doc_type = doc_type.document_type\n                is_list = not hasattr(items, 'items')\n\n                if is_list and all([i.__class__ == doc_type for i in items]):\n                    return items\n                elif not is_list and all([i.__class__ == doc_type\n                                         for i in items.values()]):\n                    return items\n                elif not field.dbref:\n                    if not hasattr(items, 'items'):\n\n                        def _get_items(items):\n                            new_items = []\n                            for v in items:\n                                if isinstance(v, list):\n                                    new_items.append(_get_items(v))\n                                elif not isinstance(v, (DBRef, Document)):\n                                    new_items.append(field.to_python(v))\n                                else:\n                                    new_items.append(v)\n                            return new_items\n\n                        items = _get_items(items)\n                    else:\n                        items = dict([\n                            (k, field.to_python(v))\n                            if not isinstance(v, (DBRef, Document)) else (k, v)\n                            for k, v in items.iteritems()]\n                        )\n\n        self.reference_map = self._find_references(items)\n        self.object_map = self._fetch_objects(doc_type=doc_type)\n        return self._attach_objects(items, 0, instance, name)\n\n    def _find_references(self, items, depth=0):\n        \"\"\"\n        Recursively finds all db references to be dereferenced\n\n        :param items: The iterable (dict, list, queryset)\n        :param depth: The current depth of recursion\n        \"\"\"\n        reference_map = {}\n        if not items or depth >= self.max_depth:\n            return reference_map\n\n        # Determine the iterator to use\n        if not hasattr(items, 'items'):\n            iterator = enumerate(items)\n        else:\n            iterator = items.iteritems()\n\n        # Recursively find dbreferences\n        depth += 1\n        for k, item in iterator:\n            if isinstance(item, (Document, EmbeddedDocument)):\n                for field_name, field in item._fields.iteritems():\n                    v = item._data.get(field_name, None)\n                    if isinstance(v, (DBRef)):\n                        reference_map.setdefault(field.document_type, set()).add(v.id)\n                    elif isinstance(v, (dict, SON)) and '_ref' in v:\n                        reference_map.setdefault(get_document(v['_cls']), set()).add(v['_ref'].id)\n                    elif isinstance(v, (dict, list, tuple)) and depth <= self.max_depth:\n                        field_cls = getattr(getattr(field, 'field', None), 'document_type', None)\n                        references = self._find_references(v, depth)\n                        for key, refs in references.iteritems():\n                            if isinstance(field_cls, (Document, TopLevelDocumentMetaclass)):\n                                key = field_cls\n                            reference_map.setdefault(key, set()).update(refs)\n            elif isinstance(item, (DBRef)):\n                reference_map.setdefault(item.collection, set()).add(item.id)\n            elif isinstance(item, (dict, SON)) and '_ref' in item:\n                reference_map.setdefault(get_document(item['_cls']), set()).add(item['_ref'].id)\n            elif isinstance(item, (dict, list, tuple)) and depth - 1 <= self.max_depth:\n                references = self._find_references(item, depth - 1)\n                for key, refs in references.iteritems():\n                    reference_map.setdefault(key, set()).update(refs)\n\n        return reference_map\n\n    def _fetch_objects(self, doc_type=None):\n        \"\"\"Fetch all references and convert to their document objects\n        \"\"\"\n        object_map = {}\n        for collection, dbrefs in self.reference_map.iteritems():\n            refs = [dbref for dbref in dbrefs\n                    if unicode(dbref).encode('utf-8') not in object_map]\n            if hasattr(collection, 'objects'):  # We have a document class for the refs\n                references = collection.objects.in_bulk(refs)\n                for key, doc in references.iteritems():\n                    object_map[key] = doc\n            else:  # Generic reference: use the refs data to convert to document\n                if isinstance(doc_type, (ListField, DictField, MapField,)):\n                    continue\n\n                if doc_type:\n                    references = doc_type._get_db()[collection].find({'_id': {'$in': refs}})\n                    for ref in references:\n                        doc = doc_type._from_son(ref)\n                        object_map[doc.id] = doc\n                else:\n                    references = get_db()[collection].find({'_id': {'$in': refs}})\n                    for ref in references:\n                        if '_cls' in ref:\n                            doc = get_document(ref[\"_cls\"])._from_son(ref)\n                        elif doc_type is None:\n                            doc = get_document(\n                                ''.join(x.capitalize()\n                                    for x in collection.split('_')))._from_son(ref)\n                        else:\n                            doc = doc_type._from_son(ref)\n                        object_map[doc.id] = doc\n        return object_map\n\n    def _attach_objects(self, items, depth=0, instance=None, name=None):\n        \"\"\"\n        Recursively finds all db references to be dereferenced\n\n        :param items: The iterable (dict, list, queryset)\n        :param depth: The current depth of recursion\n        :param instance: The owning instance used for tracking changes by\n            :class:`~mongoengine.base.ComplexBaseField`\n        :param name: The name of the field, used for tracking changes by\n            :class:`~mongoengine.base.ComplexBaseField`\n        \"\"\"\n        if not items:\n            if isinstance(items, (BaseDict, BaseList)):\n                return items\n\n            if instance:\n                if isinstance(items, dict):\n                    return BaseDict(items, instance, name)\n                else:\n                    return BaseList(items, instance, name)\n\n        if isinstance(items, (dict, SON)):\n            if '_ref' in items:\n                return self.object_map.get(items['_ref'].id, items)\n            elif '_cls' in items:\n                doc = get_document(items['_cls'])._from_son(items)\n                _cls = doc._data.pop('_cls', None)\n                del items['_cls']\n                doc._data = self._attach_objects(doc._data, depth, doc, None)\n                if _cls is not None:\n                    doc._data['_cls'] = _cls\n                return doc\n\n        if not hasattr(items, 'items'):\n            is_list = True\n            list_type = BaseList\n            if isinstance(items, EmbeddedDocumentList):\n                list_type = EmbeddedDocumentList\n            as_tuple = isinstance(items, tuple)\n            iterator = enumerate(items)\n            data = []\n        else:\n            is_list = False\n            iterator = items.iteritems()\n            data = {}\n\n        depth += 1\n        for k, v in iterator:\n            if is_list:\n                data.append(v)\n            else:\n                data[k] = v\n\n            if k in self.object_map and not is_list:\n                data[k] = self.object_map[k]\n            elif isinstance(v, (Document, EmbeddedDocument)):\n                for field_name, field in v._fields.iteritems():\n                    v = data[k]._data.get(field_name, None)\n                    if isinstance(v, (DBRef)):\n                        data[k]._data[field_name] = self.object_map.get(v.id, v)\n                    elif isinstance(v, (dict, SON)) and '_ref' in v:\n                        data[k]._data[field_name] = self.object_map.get(v['_ref'].id, v)\n                    elif isinstance(v, (dict, list, tuple)) and depth <= self.max_depth:\n                        item_name = \"{0}.{1}.{2}\".format(name, k, field_name)\n                        data[k]._data[field_name] = self._attach_objects(v, depth, instance=instance, name=item_name)\n            elif isinstance(v, (dict, list, tuple)) and depth <= self.max_depth:\n                item_name = '%s.%s' % (name, k) if name else name\n                data[k] = self._attach_objects(v, depth - 1, instance=instance, name=item_name)\n            elif hasattr(v, 'id'):\n                data[k] = self.object_map.get(v.id, v)\n\n        if instance and name:\n            if is_list:\n                return tuple(data) if as_tuple else list_type(data, instance, name)\n            return BaseDict(data, instance, name)\n        depth += 1\n        return data\n", "path": "mongoengine/dereference.py"}], "after_files": [{"content": "from bson import DBRef, SON\n\nfrom base import (\n    BaseDict, BaseList, EmbeddedDocumentList,\n    TopLevelDocumentMetaclass, get_document\n)\nfrom fields import (ReferenceField, ListField, DictField, MapField)\nfrom connection import get_db\nfrom queryset import QuerySet\nfrom document import Document, EmbeddedDocument\n\n\nclass DeReference(object):\n\n    def __call__(self, items, max_depth=1, instance=None, name=None):\n        \"\"\"\n        Cheaply dereferences the items to a set depth.\n        Also handles the conversion of complex data types.\n\n        :param items: The iterable (dict, list, queryset) to be dereferenced.\n        :param max_depth: The maximum depth to recurse to\n        :param instance: The owning instance used for tracking changes by\n            :class:`~mongoengine.base.ComplexBaseField`\n        :param name: The name of the field, used for tracking changes by\n            :class:`~mongoengine.base.ComplexBaseField`\n        :param get: A boolean determining if being called by __get__\n        \"\"\"\n        if items is None or isinstance(items, basestring):\n            return items\n\n        # cheapest way to convert a queryset to a list\n        # list(queryset) uses a count() query to determine length\n        if isinstance(items, QuerySet):\n            items = [i for i in items]\n\n        self.max_depth = max_depth\n        doc_type = None\n\n        if instance and isinstance(instance, (Document, EmbeddedDocument,\n                                              TopLevelDocumentMetaclass)):\n            doc_type = instance._fields.get(name)\n            while hasattr(doc_type, 'field'):\n                doc_type = doc_type.field\n\n            if isinstance(doc_type, ReferenceField):\n                field = doc_type\n                doc_type = doc_type.document_type\n                is_list = not hasattr(items, 'items')\n\n                if is_list and all([i.__class__ == doc_type for i in items]):\n                    return items\n                elif not is_list and all([i.__class__ == doc_type\n                                         for i in items.values()]):\n                    return items\n                elif not field.dbref:\n                    if not hasattr(items, 'items'):\n\n                        def _get_items(items):\n                            new_items = []\n                            for v in items:\n                                if isinstance(v, list):\n                                    new_items.append(_get_items(v))\n                                elif not isinstance(v, (DBRef, Document)):\n                                    new_items.append(field.to_python(v))\n                                else:\n                                    new_items.append(v)\n                            return new_items\n\n                        items = _get_items(items)\n                    else:\n                        items = dict([\n                            (k, field.to_python(v))\n                            if not isinstance(v, (DBRef, Document)) else (k, v)\n                            for k, v in items.iteritems()]\n                        )\n\n        self.reference_map = self._find_references(items)\n        self.object_map = self._fetch_objects(doc_type=doc_type)\n        return self._attach_objects(items, 0, instance, name)\n\n    def _find_references(self, items, depth=0):\n        \"\"\"\n        Recursively finds all db references to be dereferenced\n\n        :param items: The iterable (dict, list, queryset)\n        :param depth: The current depth of recursion\n        \"\"\"\n        reference_map = {}\n        if not items or depth >= self.max_depth:\n            return reference_map\n\n        # Determine the iterator to use\n        if not hasattr(items, 'items'):\n            iterator = enumerate(items)\n        else:\n            iterator = items.iteritems()\n\n        # Recursively find dbreferences\n        depth += 1\n        for k, item in iterator:\n            if isinstance(item, (Document, EmbeddedDocument)):\n                for field_name, field in item._fields.iteritems():\n                    v = item._data.get(field_name, None)\n                    if isinstance(v, (DBRef)):\n                        reference_map.setdefault(field.document_type, set()).add(v.id)\n                    elif isinstance(v, (dict, SON)) and '_ref' in v:\n                        reference_map.setdefault(get_document(v['_cls']), set()).add(v['_ref'].id)\n                    elif isinstance(v, (dict, list, tuple)) and depth <= self.max_depth:\n                        field_cls = getattr(getattr(field, 'field', None), 'document_type', None)\n                        references = self._find_references(v, depth)\n                        for key, refs in references.iteritems():\n                            if isinstance(field_cls, (Document, TopLevelDocumentMetaclass)):\n                                key = field_cls\n                            reference_map.setdefault(key, set()).update(refs)\n            elif isinstance(item, (DBRef)):\n                reference_map.setdefault(item.collection, set()).add(item.id)\n            elif isinstance(item, (dict, SON)) and '_ref' in item:\n                reference_map.setdefault(get_document(item['_cls']), set()).add(item['_ref'].id)\n            elif isinstance(item, (dict, list, tuple)) and depth - 1 <= self.max_depth:\n                references = self._find_references(item, depth - 1)\n                for key, refs in references.iteritems():\n                    reference_map.setdefault(key, set()).update(refs)\n\n        return reference_map\n\n    def _fetch_objects(self, doc_type=None):\n        \"\"\"Fetch all references and convert to their document objects\n        \"\"\"\n        object_map = {}\n        for collection, dbrefs in self.reference_map.iteritems():\n            if hasattr(collection, 'objects'):  # We have a document class for the refs\n                col_name = collection._get_collection_name()\n                refs = [dbref for dbref in dbrefs\n                        if (col_name, dbref) not in object_map]\n                references = collection.objects.in_bulk(refs)\n                for key, doc in references.iteritems():\n                    object_map[(col_name, key)] = doc\n            else:  # Generic reference: use the refs data to convert to document\n                if isinstance(doc_type, (ListField, DictField, MapField,)):\n                    continue\n\n                refs = [dbref for dbref in dbrefs\n                        if (collection, dbref) not in object_map]\n\n                if doc_type:\n                    references = doc_type._get_db()[collection].find({'_id': {'$in': refs}})\n                    for ref in references:\n                        doc = doc_type._from_son(ref)\n                        object_map[(collection, doc.id)] = doc\n                else:\n                    references = get_db()[collection].find({'_id': {'$in': refs}})\n                    for ref in references:\n                        if '_cls' in ref:\n                            doc = get_document(ref[\"_cls\"])._from_son(ref)\n                        elif doc_type is None:\n                            doc = get_document(\n                                ''.join(x.capitalize()\n                                    for x in collection.split('_')))._from_son(ref)\n                        else:\n                            doc = doc_type._from_son(ref)\n                        object_map[(collection, doc.id)] = doc\n        return object_map\n\n    def _attach_objects(self, items, depth=0, instance=None, name=None):\n        \"\"\"\n        Recursively finds all db references to be dereferenced\n\n        :param items: The iterable (dict, list, queryset)\n        :param depth: The current depth of recursion\n        :param instance: The owning instance used for tracking changes by\n            :class:`~mongoengine.base.ComplexBaseField`\n        :param name: The name of the field, used for tracking changes by\n            :class:`~mongoengine.base.ComplexBaseField`\n        \"\"\"\n        if not items:\n            if isinstance(items, (BaseDict, BaseList)):\n                return items\n\n            if instance:\n                if isinstance(items, dict):\n                    return BaseDict(items, instance, name)\n                else:\n                    return BaseList(items, instance, name)\n\n        if isinstance(items, (dict, SON)):\n            if '_ref' in items:\n                return self.object_map.get(\n                    (items['_ref'].collection, items['_ref'].id), items)\n            elif '_cls' in items:\n                doc = get_document(items['_cls'])._from_son(items)\n                _cls = doc._data.pop('_cls', None)\n                del items['_cls']\n                doc._data = self._attach_objects(doc._data, depth, doc, None)\n                if _cls is not None:\n                    doc._data['_cls'] = _cls\n                return doc\n\n        if not hasattr(items, 'items'):\n            is_list = True\n            list_type = BaseList\n            if isinstance(items, EmbeddedDocumentList):\n                list_type = EmbeddedDocumentList\n            as_tuple = isinstance(items, tuple)\n            iterator = enumerate(items)\n            data = []\n        else:\n            is_list = False\n            iterator = items.iteritems()\n            data = {}\n\n        depth += 1\n        for k, v in iterator:\n            if is_list:\n                data.append(v)\n            else:\n                data[k] = v\n\n            if k in self.object_map and not is_list:\n                data[k] = self.object_map[k]\n            elif isinstance(v, (Document, EmbeddedDocument)):\n                for field_name, field in v._fields.iteritems():\n                    v = data[k]._data.get(field_name, None)\n                    if isinstance(v, (DBRef)):\n                        data[k]._data[field_name] = self.object_map.get(\n                            (v.collection, v.id), v)\n                    elif isinstance(v, (dict, SON)) and '_ref' in v:\n                        data[k]._data[field_name] = self.object_map.get(\n                            (v['_ref'].collection , v['_ref'].id), v)\n                    elif isinstance(v, (dict, list, tuple)) and depth <= self.max_depth:\n                        item_name = \"{0}.{1}.{2}\".format(name, k, field_name)\n                        data[k]._data[field_name] = self._attach_objects(v, depth, instance=instance, name=item_name)\n            elif isinstance(v, (dict, list, tuple)) and depth <= self.max_depth:\n                item_name = '%s.%s' % (name, k) if name else name\n                data[k] = self._attach_objects(v, depth - 1, instance=instance, name=item_name)\n            elif hasattr(v, 'id'):\n                data[k] = self.object_map.get((v.collection, v.id), v)\n\n        if instance and name:\n            if is_list:\n                return tuple(data) if as_tuple else list_type(data, instance, name)\n            return BaseDict(data, instance, name)\n        depth += 1\n        return data\n", "path": "mongoengine/dereference.py"}]}