diff --git a/docs/changelog.rst b/docs/changelog.rst index 71dd9238e..fbba81685 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -19,11 +19,11 @@ Changes in 1.0.0 - Using .count() in a transaction will always use Collection.count_document (as estimated_document_count is not supported in transactions) - Add a warning that ``mongoengine.org`` is no longer controlled by the MongoEngine project and appears to be an expired domain takeover. -- Fix querying GenericReferenceField with __in operator #2886 -- Fix Document.compare_indexes() not working correctly for text indexes on multiple fields #2612 +- Bug Fix - Fix querying GenericReferenceField with __in operator #2886 +- Bug Fix - Fix Document.compare_indexes() not working correctly for text indexes on multiple fields #2612 - BREAKING CHANGE: wrap _document_registry (normally not used by end users) with _DocumentRegistry which acts as a singleton to access the registry - Log a warning in case users creates multiple Document classes with the same name as it can lead to unexpected behavior #1778 -- Fix use of $geoNear or $collStats in aggregate #2493 +- BugFix - Fix use of $geoNear or $collStats in aggregate #2493 - BREAKING CHANGE: Further to the deprecation warning, remove ability to use an unpacked list to `Queryset.aggregate(*pipeline)`, a plain list must be provided instead `Queryset.aggregate(pipeline)`, as it's closer to pymongo interface - BREAKING CHANGE: Further to the deprecation warning, remove `full_response` from `QuerySet.modify` as it wasn't supported with Pymongo 3+ - BREAKING CHANGE: Remove deprecated ``QuerySet.snapshot``, which had no effect with PyMongo 3+. Remove calls to ``.snapshot(...)``; there is no direct replacement. @@ -31,10 +31,11 @@ Changes in 1.0.0 - Fixed stacklevel of many warnings (to point places emitting the warning more accurately) - Add support for collation/hint/comment to delete/update and aggregate #2842 - BREAKING CHANGE: Remove LongField as it's equivalent to IntField since we drop support to Python2 long time ago (User should simply switch to IntField) #2309 +- Replace MongoEngine-created ``bson.SON`` objects with built-in dictionaries, SON providing no advantages since Python 3.7 as native dict preserved insertion order. #2898 - BREAKING CHANGE: The obsolete ``slaves`` and ``is_slave`` connection options were silently ignored since 2014 and will now raise ``ConnectionFailure`` if provided #2920. - BugFix - Calling .clear on a ListField wasn't being marked as changed (and flushed to db upon .save()) #2858 - Improve error message in case a document assigned to a ReferenceField wasn't saved yet #1955 -- Fix inc/dec atomic updates rejecting deltas outside a field's min_value/max_value #2339 +- BugFix - Fix inc/dec atomic updates rejecting deltas outside a field's min_value/max_value #2339 - BugFix - Take `where()` into account when using `.modify()`, as in MyDocument.objects().where("this[field] >= this[otherfield]").modify(field='new') #2044 Changes in 0.29.3 diff --git a/mongoengine/base/document.py b/mongoengine/base/document.py index 400fb7931..2fcc27aa9 100644 --- a/mongoengine/base/document.py +++ b/mongoengine/base/document.py @@ -103,7 +103,7 @@ def __init__(self, *args, **values): else: self._data = {} - self._dynamic_fields = SON() + self._dynamic_fields = {} # Assign default values for fields # not set in the constructor @@ -220,10 +220,11 @@ def __getstate__(self): if hasattr(self, k): data[k] = getattr(self, k) data["_data"] = self.to_mongo() + data["_data_is_mongo"] = True return data def __setstate__(self, data): - if isinstance(data["_data"], SON): + if data.pop("_data_is_mongo", False) or isinstance(data["_data"], SON): data["_data"] = self.__class__._from_son(data["_data"])._data for k in ( "_changed_fields", @@ -241,7 +242,7 @@ def __setstate__(self, data): _super_fields_ordered = type(self)._fields_ordered self._fields_ordered = _super_fields_ordered - dynamic_fields = data.get("_dynamic_fields") or SON() + dynamic_fields = data.get("_dynamic_fields") or {} for k in dynamic_fields.keys(): setattr(self, k, data["_data"].get(k)) @@ -331,11 +332,11 @@ def get_text_score(self): def to_mongo(self, use_db_field=True, fields=None): """ - Return as SON data ready for use with MongoDB. + Return as a dictionary ready for use with MongoDB. """ fields = fields or [] - data = SON() + data = {} data["_id"] = None data["_cls"] = self._class_name diff --git a/mongoengine/base/fields.py b/mongoengine/base/fields.py index 308530046..6da31ff85 100644 --- a/mongoengine/base/fields.py +++ b/mongoengine/base/fields.py @@ -4,7 +4,7 @@ import weakref import pymongo -from bson import SON, DBRef, ObjectId +from bson import DBRef, ObjectId from mongoengine.base.common import UPDATE_OPERATORS from mongoengine.base.datastructures import ( @@ -748,4 +748,4 @@ def _validate_multipolygon(self, value): def to_mongo(self, value): if isinstance(value, dict): return value - return SON([("type", self._type), ("coordinates", value)]) + return {"type": self._type, "coordinates": value} diff --git a/mongoengine/dereference.py b/mongoengine/dereference.py index 38da2e873..548afeb9e 100644 --- a/mongoengine/dereference.py +++ b/mongoengine/dereference.py @@ -1,4 +1,4 @@ -from bson import SON, DBRef +from bson import DBRef from mongoengine.base import ( BaseDict, @@ -130,7 +130,7 @@ def _find_references(self, items, depth=0): continue elif isinstance(v, DBRef): reference_map.setdefault(field.document_type, set()).add(v.id) - elif isinstance(v, (dict, SON)) and "_ref" in v: + elif isinstance(v, dict) and "_ref" in v: reference_map.setdefault( _DocumentRegistry.get(v["_cls"]), set() ).add(v["_ref"].id) @@ -150,7 +150,7 @@ def _find_references(self, items, depth=0): continue elif isinstance(item, DBRef): reference_map.setdefault(item.collection, set()).add(item.id) - elif isinstance(item, (dict, SON)) and "_ref" in item: + elif isinstance(item, dict) and "_ref" in item: reference_map.setdefault( _DocumentRegistry.get(item["_cls"]), set() ).add(item["_ref"].id) @@ -229,7 +229,7 @@ def _attach_objects(self, items, depth=0, instance=None, name=None): else: return BaseList(items, instance, name) - if isinstance(items, (dict, SON)): + if isinstance(items, dict): if "_ref" in items: return self.object_map.get( (items["_ref"].collection, items["_ref"].id), items @@ -272,7 +272,7 @@ def _attach_objects(self, items, depth=0, instance=None, name=None): data[k]._data[field_name] = self.object_map.get( (v.collection, v.id), v ) - elif isinstance(v, (dict, SON)) and "_ref" in v: + elif isinstance(v, dict) and "_ref" in v: data[k]._data[field_name] = self.object_map.get( (v["_ref"].collection, v["_ref"].id), v ) diff --git a/mongoengine/document.py b/mongoengine/document.py index 829c07135..ef2f5173f 100644 --- a/mongoengine/document.py +++ b/mongoengine/document.py @@ -116,7 +116,7 @@ def __setstate__(self, state): def to_mongo(self, *args, **kwargs): data = super().to_mongo(*args, **kwargs) - # remove _id from the SON if it's in it and it's None + # remove _id from the data if it's in it and it's None if "_id" in data and data["_id"] is None: del data["_id"] @@ -303,7 +303,7 @@ def to_mongo(self, *args, **kwargs): data = super().to_mongo(*args, **kwargs) # If '_id' is None, try and set it from self._data. If that - # doesn't exist either, remove '_id' from the SON completely. + # doesn't exist either, remove '_id' from the data completely. if data["_id"] is None: if self._data.get("id") is None: del data["_id"] diff --git a/mongoengine/fields.py b/mongoengine/fields.py index 0f5ee5402..37d5070a0 100644 --- a/mongoengine/fields.py +++ b/mongoengine/fields.py @@ -12,7 +12,7 @@ import gridfs import pymongo -from bson import SON, Binary, DBRef, ObjectId +from bson import Binary, DBRef, ObjectId from bson.decimal128 import Decimal128, create_decimal128_context from pymongo import ReturnDocument @@ -824,7 +824,7 @@ def to_python(self, value): return value def validate(self, value, clean=True): - if self.choices and isinstance(value, SON): + if self.choices and isinstance(value, dict): for choice in self.choices: if value["_cls"] == choice._class_name: return True @@ -1385,14 +1385,14 @@ def to_mongo(self, document, use_db_field=True, fields=None): else: self.error("Only accept a document object") - value = SON((("_id", id_field.to_mongo(id_)),)) + value = {"_id": id_field.to_mongo(id_)} if fields: new_fields = [f for f in self.fields if f in fields] else: new_fields = self.fields - value.update(dict(document.to_mongo(use_db_field, fields=new_fields))) + value.update(document.to_mongo(use_db_field, fields=new_fields)) return value def prepare_query_value(self, op, value): @@ -1506,10 +1506,10 @@ def __get__(self, instance, owner): return super().__get__(instance, owner) def validate(self, value): - if not isinstance(value, (Document, DBRef, dict, SON)): + if not isinstance(value, (Document, DBRef, dict)): self.error("GenericReferences can only contain documents") - if isinstance(value, (dict, SON)): + if isinstance(value, dict): if "_ref" not in value or "_cls" not in value: self.error("GenericReferences can only contain documents") @@ -1521,7 +1521,7 @@ def to_mongo(self, document): if document is None: return None - if isinstance(document, (dict, SON, ObjectId, DBRef)): + if isinstance(document, (dict, ObjectId, DBRef)): return document id_field_name = document.__class__._meta["id_field"] @@ -1539,7 +1539,7 @@ def to_mongo(self, document): id_ = id_field.to_mongo(id_) collection = document._get_collection_name() ref = DBRef(collection, id_) - return SON((("_cls", document._class_name), ("_ref", ref))) + return {"_cls": document._class_name, "_ref": ref} def prepare_query_value(self, op, value): if value is None: @@ -2575,7 +2575,7 @@ def build_lazyref(self, value): value.document_type, value.pk, passthrough=self.passthrough ) elif value is not None: - if isinstance(value, (dict, SON)): + if isinstance(value, dict): value = LazyReference( _DocumentRegistry.get(value["_cls"]), value["_ref"].id, @@ -2611,17 +2611,12 @@ def to_mongo(self, document): return None if isinstance(document, LazyReference): - return SON( - ( - ("_cls", document.document_type._class_name), - ( - "_ref", - DBRef( - document.document_type._get_collection_name(), document.pk - ), - ), - ) - ) + return { + "_cls": document.document_type._class_name, + "_ref": DBRef( + document.document_type._get_collection_name(), document.pk + ), + } else: return super().to_mongo(document) diff --git a/mongoengine/queryset/base.py b/mongoengine/queryset/base.py index e29b64a28..c17dc3303 100644 --- a/mongoengine/queryset/base.py +++ b/mongoengine/queryset/base.py @@ -6,7 +6,7 @@ import pymongo import pymongo.errors -from bson import SON, json_util +from bson import json_util from bson.code import Code from pymongo.collection import ReturnDocument from pymongo.common import validate_read_preference @@ -246,7 +246,7 @@ def search_text(self, text, language=None, text_score=True): if queryset._search_text: raise OperationError("It is not possible to use search_text two times.") - query_kwargs = SON({"$search": text}) + query_kwargs = {"$search": text} if language: query_kwargs["$language"] = language @@ -1509,7 +1509,7 @@ def map_reduce( if value: ordered_output.append((part, value)) - mr_args["out"] = SON(ordered_output) + mr_args["out"] = dict(ordered_output) db = queryset._document._get_db() result = db.command( diff --git a/mongoengine/queryset/transform.py b/mongoengine/queryset/transform.py index c0e58eb45..7d2c6e495 100644 --- a/mongoengine/queryset/transform.py +++ b/mongoengine/queryset/transform.py @@ -1,7 +1,7 @@ from collections import defaultdict import pymongo -from bson import SON, ObjectId +from bson import ObjectId from bson.dbref import DBRef from mongoengine.base import UPDATE_OPERATORS @@ -201,38 +201,38 @@ def query(_doc_cls=None, **kwargs): else: if isinstance(mongo_query[key], dict) and isinstance(value, dict): mongo_query[key].update(value) - # $max/minDistance needs to come last - convert to SON + # $max/minDistance needs to come last - rebuild in order value_dict = mongo_query[key] if ("$maxDistance" in value_dict or "$minDistance" in value_dict) and ( "$near" in value_dict or "$nearSphere" in value_dict ): - value_son = SON() + ordered_value = {} for k, v in value_dict.items(): if k == "$maxDistance" or k == "$minDistance": continue - value_son[k] = v + ordered_value[k] = v # Required for MongoDB >= 2.6, may fail when combining # PyMongo 3+ and MongoDB < 2.6 near_embedded = False for near_op in ("$near", "$nearSphere"): if isinstance(value_dict.get(near_op), dict): - value_son[near_op] = SON(value_son[near_op]) + ordered_value[near_op] = dict(ordered_value[near_op]) if "$maxDistance" in value_dict: - value_son[near_op]["$maxDistance"] = value_dict[ + ordered_value[near_op]["$maxDistance"] = value_dict[ "$maxDistance" ] if "$minDistance" in value_dict: - value_son[near_op]["$minDistance"] = value_dict[ + ordered_value[near_op]["$minDistance"] = value_dict[ "$minDistance" ] near_embedded = True if not near_embedded: if "$maxDistance" in value_dict: - value_son["$maxDistance"] = value_dict["$maxDistance"] + ordered_value["$maxDistance"] = value_dict["$maxDistance"] if "$minDistance" in value_dict: - value_son["$minDistance"] = value_dict["$minDistance"] - mongo_query[key] = value_son + ordered_value["$minDistance"] = value_dict["$minDistance"] + mongo_query[key] = ordered_value else: # Store for manually merging later merge_query[key].append(value) diff --git a/tests/document/test_delta.py b/tests/document/test_delta.py index e610290b6..e4d4fa7bd 100644 --- a/tests/document/test_delta.py +++ b/tests/document/test_delta.py @@ -1,7 +1,5 @@ import unittest -from bson import SON - from mongoengine import * from mongoengine.pymongo_support import list_collection_names from tests.utils import MongoDBTestCase, get_as_pymongo @@ -663,14 +661,14 @@ class Person(DynamicDocument): p = Person(name="James", age=34) assert p._delta() == ( - SON([("_cls", "Person"), ("name", "James"), ("age", 34)]), + {"_cls": "Person", "name": "James", "age": 34}, {}, ) p.doc = 123 del p.doc assert p._delta() == ( - SON([("_cls", "Person"), ("name", "James"), ("age", 34)]), + {"_cls": "Person", "name": "James", "age": 34}, {}, ) diff --git a/tests/document/test_instance.py b/tests/document/test_instance.py index 5428ad45f..118ef9315 100644 --- a/tests/document/test_instance.py +++ b/tests/document/test_instance.py @@ -9,7 +9,7 @@ import bson import pytest -from bson import DBRef, ObjectId +from bson import SON, DBRef, ObjectId from pymongo.errors import DuplicateKeyError from mongoengine import * @@ -708,17 +708,13 @@ class Person(EmbeddedDocument): class Employee(Person): salary = IntField() - assert sorted(Person(name="Bob", age=35).to_mongo().keys()) == [ - "_cls", - "age", - "name", - ] - assert sorted(Employee(name="Bob", age=35, salary=0).to_mongo().keys()) == [ - "_cls", - "age", - "name", - "salary", - ] + person_data = Person(name="Bob", age=35).to_mongo() + assert type(person_data) is dict + assert list(person_data) == ["_cls", "name", "age"] + + employee_data = Employee(name="Bob", age=35, salary=0).to_mongo() + assert type(employee_data) is dict + assert list(employee_data) == ["_cls", "name", "age", "salary"] def test_embedded_document_to_mongo_id(self): class SubDoc(EmbeddedDocument): @@ -782,12 +778,9 @@ class Embedded(EmbeddedDocument): class Doc(Document): embedded_field = ListField(EmbeddedDocumentField(Embedded)) - d = ( - Doc(embedded_field=[Embedded(string="Hi")]) - .to_mongo(use_db_field=False) - .to_dict() - ) - assert d["embedded_field"] == [{"string": "Hi"}] + data = Doc(embedded_field=[Embedded(string="Hi")]).to_mongo(use_db_field=False) + assert type(data) is dict + assert data["embedded_field"] == [{"string": "Hi"}] def test_instance_is_set_on_setattr(self): class Email(EmbeddedDocument): @@ -2764,6 +2757,44 @@ def test_dynamic_document_pickle(self): == pickle_doc.embedded._dynamic_fields.keys() ) + def test_pickle__new_and_legacy_state__restores_python_values(self): + class PickleChild(EmbeddedDocument): + value = StringField(db_field="db_value") + + class PickleParent(EmbeddedDocument): + title = StringField(db_field="db_title") + child = EmbeddedDocumentField(PickleChild, db_field="db_child") + + document = PickleParent(title="parent", child=PickleChild(value="child")) + state = document.__getstate__() + + assert type(state["_data"]) is dict + assert state["_data_is_mongo"] is True + + restored = PickleParent.__new__(PickleParent) + restored.__setstate__(copy.deepcopy(state)) + assert restored.title == "parent" + assert isinstance(restored.child, PickleChild) + assert restored.child.value == "child" + + legacy_son_state = copy.deepcopy(state) + legacy_son_state.pop("_data_is_mongo") + legacy_son_state["_data"] = SON(legacy_son_state["_data"]) + restored_from_son = PickleParent.__new__(PickleParent) + restored_from_son.__setstate__(legacy_son_state) + assert restored_from_son.title == "parent" + assert isinstance(restored_from_son.child, PickleChild) + assert restored_from_son.child.value == "child" + + legacy_raw_state = copy.deepcopy(state) + legacy_raw_state.pop("_data_is_mongo") + legacy_raw_state["_data"] = copy.deepcopy(document._data) + restored_from_raw_data = PickleParent.__new__(PickleParent) + restored_from_raw_data.__setstate__(legacy_raw_state) + assert restored_from_raw_data.title == "parent" + assert isinstance(restored_from_raw_data.child, PickleChild) + assert restored_from_raw_data.child.value == "child" + def test_picklable_on_signals(self): pickle_doc = PickleSignalsTest(number=1, string="One", lists=["1", "2"]) pickle_doc.embedded = PickleEmbedded() diff --git a/tests/fields/test_dict_field.py b/tests/fields/test_dict_field.py index c2c6ea1fd..98bf3d93a 100644 --- a/tests/fields/test_dict_field.py +++ b/tests/fields/test_dict_field.py @@ -138,10 +138,11 @@ def __init__(self, *args, **kwargs): # with a Document with a _cls field to_embed_recursive = ToEmbedChild(id=1).save() to_embed_child = ToEmbedChild( - id=2, recursive=to_embed_recursive.to_mongo().to_dict() + id=2, recursive=to_embed_recursive.to_mongo() ).save() - doc_dump_as_dict = to_embed_child.to_mongo().to_dict() + doc_dump_as_dict = to_embed_child.to_mongo() + assert type(doc_dump_as_dict) is dict doc = Doc(field=doc_dump_as_dict) assert Doc.field._auto_dereference is False assert isinstance(doc.field, dict) # depends on auto_dereference @@ -174,10 +175,8 @@ class ToEmbed(Document): recursive = DictField() to_embed_recursive = ToEmbed(id=1).save() - to_embed = ToEmbed( - id=2, recursive=to_embed_recursive.to_mongo().to_dict() - ).save() - doc = Doc(field=to_embed.to_mongo().to_dict()) + to_embed = ToEmbed(id=2, recursive=to_embed_recursive.to_mongo()).save() + doc = Doc(field=to_embed.to_mongo()) doc.save() assert isinstance(doc.field, dict) assert doc.field == {"_id": 2, "recursive": {"_id": 1, "recursive": {}}} diff --git a/tests/fields/test_embedded_document_field.py b/tests/fields/test_embedded_document_field.py index a892c0dcd..2d0bd02e5 100644 --- a/tests/fields/test_embedded_document_field.py +++ b/tests/fields/test_embedded_document_field.py @@ -280,6 +280,16 @@ class Person(Document): person = Person.objects.first() assert isinstance(person.like, Dish) + def test_generic_embedded_document_choices_accept_mongo_dict(self): + class Dish(EmbeddedDocument): + food = StringField() + + field = GenericEmbeddedDocumentField(choices=(Dish,)) + mongo_value = field.to_mongo(Dish(food="arroz")) + + assert type(mongo_value) is dict + assert field.validate(mongo_value) is True + def test_generic_list_embedded_document_choices(self): """Ensure you can limit GenericEmbeddedDocument choices inside a list field. diff --git a/tests/fields/test_reference_field.py b/tests/fields/test_reference_field.py index 55ffb6845..12d0483e2 100644 --- a/tests/fields/test_reference_field.py +++ b/tests/fields/test_reference_field.py @@ -1,5 +1,5 @@ import pytest -from bson import SON, DBRef +from bson import DBRef from mongoengine import * from tests.utils import MongoDBTestCase @@ -113,7 +113,7 @@ class Person(Document): parent = ReferenceField("self", dbref=False) p = Person(name="Steve", parent=DBRef("person", "abcdefghijklmnop")) - assert p.to_mongo() == SON([("name", "Steve"), ("parent", "abcdefghijklmnop")]) + assert p.to_mongo() == {"name": "Steve", "parent": "abcdefghijklmnop"} def test_objectid_reference_fields(self): class Person(Document): diff --git a/tests/queryset/test_transform.py b/tests/queryset/test_transform.py index db4ad8fc2..75e024b95 100644 --- a/tests/queryset/test_transform.py +++ b/tests/queryset/test_transform.py @@ -2,7 +2,6 @@ import pytest from bson.decimal128 import Decimal128 -from bson.son import SON from mongoengine import * from mongoengine.queryset import Q, transform @@ -37,6 +36,33 @@ def test_transform_query(self): "$and": [{"name": {"$in": ["Tom"]}}, {"name": "Mark"}] } + def test_query__near_with_distance__returns_ordered_dict(self): + class LegacyLocation(Document): + location = GeoPointField() + + legacy_query = transform.query( + LegacyLocation, location__near=[1, 2], location__max_distance=3 + ) + assert type(legacy_query["location"]) is dict + assert list(legacy_query["location"]) == ["$near", "$maxDistance"] + + class GeoJsonLocation(Document): + location = PointField() + + near_value = { + "$geometry": {"type": "Point", "coordinates": [1, 2]}, + } + geojson_query = transform.query( + GeoJsonLocation, location__near=near_value, location__max_distance=3 + ) + near_query = geojson_query["location"]["$near"] + assert type(geojson_query["location"]) is dict + assert type(near_query) is dict + assert list(near_query) == ["$geometry", "$maxDistance"] + assert near_value == { + "$geometry": {"type": "Point", "coordinates": [1, 2]}, + } + def test_transform_update(self): class LisDoc(Document): foo = ListField(StringField()) @@ -399,9 +425,7 @@ class MainDoc(Document): word = Word(word="abc", index=1) update = transform.update(MainDoc, pull__content__text=word) - assert update == { - "$pull": {"content.text": SON([("word", "abc"), ("index", 1)])} - } + assert update == {"$pull": {"content.text": {"word": "abc", "index": 1}}} update = transform.update(MainDoc, pull__content__heading="xyz") assert update == {"$pull": {"content.heading": "xyz"}}