diff --git a/.travis.yml b/.travis.yml index de32e6bf..ad659fca 100644 --- a/.travis.yml +++ b/.travis.yml @@ -3,7 +3,7 @@ # with a very large number of jobs, hence we only test a subset of all the # combinations: # * MongoDB v3.4 & the latest PyMongo v3.x is currently the "main" setup, -# tested against Python v2.7, v3.5, v3.6, and PyPy. +# tested against Python v2.7, v3.5, v3.6, v3.7, v3.8, PyPy and PyPy3. # * Besides that, we test the lowest actively supported Python/MongoDB/PyMongo # combination: MongoDB v3.4, PyMongo v3.4, Python v2.7. # * MongoDB v3.6 is tested against Python v3.6, and PyMongo v3.6, v3.7, v3.8. @@ -23,6 +23,7 @@ python: - 3.5 - 3.6 - 3.7 +- 3.8 - pypy - pypy3 @@ -34,14 +35,16 @@ env: global: - MONGODB_3_4=3.4.17 - MONGODB_3_6=3.6.12 - - PYMONGO_3_9=3.9 - - PYMONGO_3_6=3.6 + - MONGODB_4_0=4.0.13 + - PYMONGO_3_4=3.4 + - PYMONGO_3_6=3.6 + - PYMONGO_3_9=3.9 + - PYMONGO_3_10=3.10 matrix: - - MONGODB=${MONGODB_3_4} PYMONGO=${PYMONGO_3_9} + - MONGODB=${MONGODB_3_4} PYMONGO=${PYMONGO_3_10} matrix: - # Finish the build as soon as one job fails fast_finish: true @@ -52,7 +55,10 @@ matrix: env: MONGODB=${MONGODB_3_6} PYMONGO=${PYMONGO_3_6} - python: 3.7 env: MONGODB=${MONGODB_3_6} PYMONGO=${PYMONGO_3_9} - + - python: 3.7 + env: MONGODB=${MONGODB_3_6} PYMONGO=${PYMONGO_3_10} + - python: 3.8 + env: MONGODB=${MONGODB_4_0} PYMONGO=${PYMONGO_3_10} install: # Install Mongo @@ -109,5 +115,5 @@ deploy: on: tags: true repo: MongoEngine/mongoengine - condition: ($PYMONGO = ${PYMONGO_3_6}) && ($MONGODB = ${MONGODB_3_4}) + condition: ($PYMONGO = ${PYMONGO_3_10}) && ($MONGODB = ${MONGODB_3_4}) python: 2.7 diff --git a/AUTHORS b/AUTHORS index aa044bd2..7d3000ce 100644 --- a/AUTHORS +++ b/AUTHORS @@ -253,3 +253,6 @@ that much better: * Gaurav Dadhania (https://github.com/GVRV) * Yurii Andrieiev (https://github.com/yandrieiev) * Filip Kucharczyk (https://github.com/Pacu2) + * Eric Timmons (https://github.com/daewok) + * Matthew Simpson (https://github.com/mcsimps2) + * Leonardo Domingues (https://github.com/leodmgs) diff --git a/README.rst b/README.rst index 853d8fbe..b5c95888 100644 --- a/README.rst +++ b/README.rst @@ -26,10 +26,10 @@ an `API reference `_. Supported MongoDB Versions ========================== -MongoEngine is currently tested against MongoDB v3.4 and v3.6. Future versions +MongoEngine is currently tested against MongoDB v3.4, v3.6 and v4.0. Future versions should be supported as well, but aren't actively tested at the moment. Make sure to open an issue or submit a pull request if you experience any problems -with MongoDB version > 3.6. +with MongoDB version > 4.0. Installation ============ @@ -91,12 +91,11 @@ Some simple examples of what MongoEngine code looks like: # Iterate over all posts using the BlogPost superclass >>> for post in BlogPost.objects: - ... print '===', post.title, '===' + ... print('===', post.title, '===') ... if isinstance(post, TextPost): - ... print post.content + ... print(post.content) ... elif isinstance(post, LinkPost): - ... print 'Link:', post.url - ... print + ... print('Link:', post.url) ... # Count all blog posts and its subtypes diff --git a/docs/changelog.rst b/docs/changelog.rst index bac5ee17..865802bb 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -6,9 +6,18 @@ Changelog Development =========== - (Fill this out as you fix issues and develop your features). -- Codebase is now compatible with both Python2 and Python3 (no more relying on 2to3 during installation) #2087 -- Documentation improvements: - - Documented how `pymongo.monitoring` can be used to log all queries issued by MongoEngine to the driver. +- ATTENTION: Drop support for Python2 +- Add Mongo 4.0 to Travis +- Fixed a bug causing inaccurate query results, while combining ``__raw__`` and regular filters for the same field #2264 + +Changes in 0.19.1 +================= +- Requires Pillow < 7.0.0 as it dropped Python2 support +- DEPRECATION: The interface of ``QuerySet.aggregate`` method was changed, it no longer takes an unpacked list of + pipeline steps (*pipeline) but simply takes the pipeline list just like ``pymongo.Collection.aggregate`` does. #2079 + +Changes in 0.19.0 +================= - BREAKING CHANGE: ``class_check`` and ``read_preference`` keyword arguments are no longer available when filtering a ``QuerySet``. #2112 - Instead of ``Doc.objects(foo=bar, read_preference=...)`` use ``Doc.objects(foo=bar).read_preference(...)``. - Instead of ``Doc.objects(foo=bar, class_check=False)`` use ``Doc.objects(foo=bar).clear_cls_query(...)``. @@ -18,17 +27,23 @@ Development - If you catch/use ``MongoEngineConnectionError`` in your code, you'll have to rename it. - BREAKING CHANGE: Positional arguments when instantiating a document are no longer supported. #2103 - From now on keyword arguments (e.g. ``Doc(field_name=value)``) are required. +- BREAKING CHANGE: A ``LazyReferenceField`` is now stored in the ``_data`` field of its parent as a ``DBRef``, ``Document``, or ``EmbeddedDocument`` (``ObjectId`` is no longer allowed). #2182 - DEPRECATION: ``Q.empty`` & ``QNode.empty`` are marked as deprecated and will be removed in a next version of MongoEngine. #2210 - Added ability to check if Q or QNode are empty by parsing them to bool. - Instead of ``Q(name="John").empty`` use ``not Q(name="John")``. -- Improve error message related to InvalidDocumentError #2180 - Fix updating/modifying/deleting/reloading a document that's sharded by a field with ``db_field`` specified. #2125 +- Only set no_cursor_timeout when requested (fixes an incompatibility with MongoDB 4.2) #2148 - ``ListField`` now accepts an optional ``max_length`` parameter. #2110 -- Switch from nosetest to pytest as test runner #2114 -- The codebase is now formatted using ``black``. #2109 -- In bulk write insert, the detailed error message would raise in exception. +- Improve error message related to InvalidDocumentError #2180 +- Added BulkWriteError to replace NotUniqueError which was misleading in bulk write insert #2152 - Added ability to compare Q and Q operations #2204 - Added ability to use a db alias on query_counter #2194 +- Added ability to specify collations for querysets with ``Doc.objects.collation`` #2024 +- Fix updates of a list field by negative index #2094 +- Switch from nosetest to pytest as test runner #2114 +- The codebase is now formatted using ``black``. #2109 +- Documentation improvements: + - Documented how `pymongo.monitoring` can be used to log all queries issued by MongoEngine to the driver. Changes in 0.18.2 ================= diff --git a/docs/guide/defining-documents.rst b/docs/guide/defining-documents.rst index 9dcca88c..bd2b43e2 100644 --- a/docs/guide/defining-documents.rst +++ b/docs/guide/defining-documents.rst @@ -352,7 +352,7 @@ Its value can take any of the following constants: Deletion is denied if there still exist references to the object being deleted. :const:`mongoengine.NULLIFY` - Any object's fields still referring to the object being deleted are removed + Any object's fields still referring to the object being deleted are set to None (using MongoDB's "unset" operation), effectively nullifying the relationship. :const:`mongoengine.CASCADE` Any object containing fields that are referring to the object being deleted @@ -744,7 +744,7 @@ Document inheritance To create a specialised type of a :class:`~mongoengine.Document` you have defined, you may subclass it and add any extra fields or methods you may need. -As this is new class is not a direct subclass of +As this new class is not a direct subclass of :class:`~mongoengine.Document`, it will not be stored in its own collection; it will use the same collection as its superclass uses. This allows for more convenient and efficient retrieval of related documents -- all you need do is @@ -767,6 +767,27 @@ document.:: Setting :attr:`allow_inheritance` to True should also be used in :class:`~mongoengine.EmbeddedDocument` class in case you need to subclass it +When it comes to querying using :attr:`.objects()`, querying `Page.objects()` will query +both `Page` and `DatedPage` whereas querying `DatedPage` will only query the `DatedPage` documents. +Behind the scenes, MongoEngine deals with inheritance by adding a :attr:`_cls` attribute that contains +the class name in every documents. When a document is loaded, MongoEngine checks +it's :attr:`_cls` attribute and use that class to construct the instance.:: + + Page(title='a funky title').save() + DatedPage(title='another title', date=datetime.utcnow()).save() + + print(Page.objects().count()) # 2 + print(DatedPage.objects().count()) # 1 + + # print documents in their native form + # we remove 'id' to avoid polluting the output with unnecessary detail + qs = Page.objects.exclude('id').as_pymongo() + print(list(qs)) + # [ + # {'_cls': u 'Page', 'title': 'a funky title'}, + # {'_cls': u 'Page.DatedPage', 'title': u 'another title', 'date': datetime.datetime(2019, 12, 13, 20, 16, 59, 993000)} + # ] + Working with existing data -------------------------- As MongoEngine no longer defaults to needing :attr:`_cls`, you can quickly and diff --git a/docs/guide/gridfs.rst b/docs/guide/gridfs.rst index f7380e89..0baf88e0 100644 --- a/docs/guide/gridfs.rst +++ b/docs/guide/gridfs.rst @@ -10,8 +10,9 @@ Writing GridFS support comes in the form of the :class:`~mongoengine.fields.FileField` field object. This field acts as a file-like object and provides a couple of different ways of inserting and retrieving data. Arbitrary metadata such as -content type can also be stored alongside the files. In the following example, -a document is created to store details about animals, including a photo:: +content type can also be stored alongside the files. The object returned when accessing a +FileField is a proxy to `Pymongo's GridFS `_ +In the following example, a document is created to store details about animals, including a photo:: class Animal(Document): genus = StringField() @@ -20,8 +21,8 @@ a document is created to store details about animals, including a photo:: marmot = Animal(genus='Marmota', family='Sciuridae') - marmot_photo = open('marmot.jpg', 'rb') - marmot.photo.put(marmot_photo, content_type = 'image/jpeg') + with open('marmot.jpg', 'rb') as fd: + marmot.photo.put(fd, content_type = 'image/jpeg') marmot.save() Retrieval @@ -34,6 +35,20 @@ field. The file can also be retrieved just as easily:: photo = marmot.photo.read() content_type = marmot.photo.content_type +.. note:: If you need to read() the content of a file multiple times, you'll need to "rewind" + the file-like object using `seek`:: + + marmot = Animal.objects(genus='Marmota').first() + content1 = marmot.photo.read() + assert content1 != "" + + content2 = marmot.photo.read() # will be empty + assert content2 == "" + + marmot.photo.seek(0) # rewind the file by setting the current position of the cursor in the file to 0 + content3 = marmot.photo.read() + assert content3 == content1 + Streaming --------- diff --git a/docs/guide/mongomock.rst b/docs/guide/mongomock.rst index d70ee6a6..141d7b69 100644 --- a/docs/guide/mongomock.rst +++ b/docs/guide/mongomock.rst @@ -2,10 +2,10 @@ Use mongomock for testing ============================== -`mongomock `_ is a package to do just +`mongomock `_ is a package to do just what the name implies, mocking a mongo database. -To use with mongoengine, simply specify mongomock when connecting with +To use with mongoengine, simply specify mongomock when connecting with mongoengine: .. code-block:: python @@ -21,7 +21,7 @@ or with an alias: conn = get_connection('testdb') Example of test file: --------- +--------------------- .. code-block:: python import unittest @@ -45,4 +45,4 @@ Example of test file: pers.save() fresh_pers = Person.objects().first() - self.assertEqual(fresh_pers.name, 'John') + assert fresh_pers.name == 'John' diff --git a/docs/guide/querying.rst b/docs/guide/querying.rst index d64c169c..07de0378 100644 --- a/docs/guide/querying.rst +++ b/docs/guide/querying.rst @@ -222,6 +222,18 @@ keyword argument:: .. versionadded:: 0.4 +Sorting/Ordering results +======================== +It is possible to order the results by 1 or more keys using :meth:`~mongoengine.queryset.QuerySet.order_by`. +The order may be specified by prepending each of the keys by "+" or "-". Ascending order is assumed if there's no prefix.:: + + # Order by ascending date + blogs = BlogPost.objects().order_by('date') # equivalent to .order_by('+date') + + # Order by ascending date first, then descending title + blogs = BlogPost.objects().order_by('+date', '-title') + + Limiting and skipping results ============================= Just as with traditional ORMs, you may limit the number of results returned or @@ -388,7 +400,7 @@ would be generating "tag-clouds":: MongoDB aggregation API ----------------------- -If you need to run aggregation pipelines, MongoEngine provides an entry point `Pymongo's aggregation framework `_ +If you need to run aggregation pipelines, MongoEngine provides an entry point to `Pymongo's aggregation framework `_ through :meth:`~mongoengine.queryset.QuerySet.aggregate`. Check out Pymongo's documentation for the syntax and pipeline. An example of its use would be:: @@ -402,7 +414,7 @@ An example of its use would be:: {"$sort" : {"name" : -1}}, {"$project": {"_id": 0, "name": {"$toUpper": "$name"}}} ] - data = Person.objects().aggregate(*pipeline) + data = Person.objects().aggregate(pipeline) assert data == [{'name': 'BOB'}, {'name': 'JOHN'}] Query efficiency and performance @@ -585,7 +597,8 @@ cannot use the `$` syntax in keyword arguments it has been mapped to `S`:: ['database', 'mongodb'] From MongoDB version 2.6, push operator supports $position value which allows -to push values with index. +to push values with index:: + >>> post = BlogPost(title="Test", tags=["mongo"]) >>> post.save() >>> post.update(push__tags__0=["database", "code"]) diff --git a/docs/upgrade.rst b/docs/upgrade.rst index 082dbadc..250347bf 100644 --- a/docs/upgrade.rst +++ b/docs/upgrade.rst @@ -52,7 +52,7 @@ rename its occurrences. This release includes a major rehaul of MongoEngine's code quality and introduces a few breaking changes. It also touches many different parts of the package and although all the changes have been tested and scrutinized, -you're encouraged to thorougly test the upgrade. +you're encouraged to thoroughly test the upgrade. First breaking change involves renaming `ConnectionError` to `MongoEngineConnectionError`. If you import or catch this exception, you'll need to rename it in your code. diff --git a/mongoengine/__init__.py b/mongoengine/__init__.py index d7093d28..e45dfc2b 100644 --- a/mongoengine/__init__.py +++ b/mongoengine/__init__.py @@ -28,7 +28,7 @@ __all__ = ( ) -VERSION = (0, 18, 2) +VERSION = (0, 19, 1) def get_version(): diff --git a/mongoengine/base/datastructures.py b/mongoengine/base/datastructures.py index 8c93f596..9f78fec0 100644 --- a/mongoengine/base/datastructures.py +++ b/mongoengine/base/datastructures.py @@ -121,6 +121,9 @@ class BaseList(list): super(BaseList, self).__init__(list_items) def __getitem__(self, key): + # change index to positive value because MongoDB does not support negative one + if isinstance(key, int) and key < 0: + key = len(self) + key value = super(BaseList, self).__getitem__(key) if isinstance(key, slice): diff --git a/mongoengine/document.py b/mongoengine/document.py index fc6b9f16..6a7ced46 100644 --- a/mongoengine/document.py +++ b/mongoengine/document.py @@ -330,7 +330,7 @@ class Document(six.with_metaclass(TopLevelDocumentMetaclass, BaseDocument)): ): """Save the :class:`~mongoengine.Document` to the database. If the document already exists, it will be updated, otherwise it will be - created. + created. Returns the saved object instance. :param force_insert: only try to create a new document, don't allow updates of existing documents. diff --git a/mongoengine/fields.py b/mongoengine/fields.py index 562cc906..9d7d1d04 100644 --- a/mongoengine/fields.py +++ b/mongoengine/fields.py @@ -38,6 +38,7 @@ from mongoengine.common import _import_class from mongoengine.connection import DEFAULT_CONNECTION_NAME, get_db from mongoengine.document import Document, EmbeddedDocument from mongoengine.errors import DoesNotExist, InvalidQueryError, ValidationError +from mongoengine.mongodb_support import MONGODB_36, get_mongodb_version from mongoengine.python_support import StringIO from mongoengine.queryset import DO_NOTHING from mongoengine.queryset.base import BaseQuerySet @@ -1043,6 +1044,15 @@ def key_has_dot_or_dollar(d): return True +def key_starts_with_dollar(d): + """Helper function to recursively determine if any key in a + dictionary starts with a dollar + """ + for k, v in d.items(): + if (k.startswith("$")) or (isinstance(v, dict) and key_starts_with_dollar(v)): + return True + + class DictField(ComplexBaseField): """A dictionary field that wraps a standard Python dictionary. This is similar to an embedded document, but the structure is not defined. @@ -1069,11 +1079,18 @@ class DictField(ComplexBaseField): if key_not_string(value): msg = "Invalid dictionary key - documents must have only string keys" self.error(msg) - if key_has_dot_or_dollar(value): + + curr_mongo_ver = get_mongodb_version() + + if curr_mongo_ver < MONGODB_36 and key_has_dot_or_dollar(value): self.error( 'Invalid dictionary key name - keys may not contain "."' ' or startswith "$" characters' ) + elif curr_mongo_ver >= MONGODB_36 and key_starts_with_dollar(value): + self.error( + 'Invalid dictionary key name - keys may not startswith "$" characters' + ) super(DictField, self).validate(value) def lookup_member(self, member_name): @@ -2494,6 +2511,13 @@ class LazyReferenceField(BaseField): else: return pk + def to_python(self, value): + """Convert a MongoDB-compatible type to a Python type.""" + if not isinstance(value, (DBRef, Document, EmbeddedDocument)): + collection = self.document_type._get_collection_name() + value = DBRef(collection, self.document_type.id.to_python(value)) + return value + def validate(self, value): if isinstance(value, LazyReference): if value.collection != self.document_type._get_collection_name(): diff --git a/mongoengine/queryset/base.py b/mongoengine/queryset/base.py index d3176050..9f76d92d 100644 --- a/mongoengine/queryset/base.py +++ b/mongoengine/queryset/base.py @@ -302,7 +302,7 @@ class BaseQuerySet(object): ``insert(..., {w: 2, fsync: True})`` will wait until at least two servers have recorded the write and will force an fsync on each server being written to. - :parm signal_kwargs: (optional) kwargs dictionary to be passed to + :param signal_kwargs: (optional) kwargs dictionary to be passed to the signal calls. By default returns document instances, set ``load_bulk`` to False to @@ -1255,16 +1255,27 @@ class BaseQuerySet(object): for data in son_data ] - def aggregate(self, *pipeline, **kwargs): - """ - Perform a aggregate function based in your queryset params + def aggregate(self, pipeline, *suppl_pipeline, **kwargs): + """Perform a aggregate function based in your queryset params + :param pipeline: list of aggregation commands,\ see: http://docs.mongodb.org/manual/core/aggregation-pipeline/ - + :param suppl_pipeline: unpacked list of pipeline (added to support deprecation of the old interface) + parameter will be removed shortly + :param kwargs: (optional) kwargs dictionary to be passed to pymongo's aggregate call + See https://api.mongodb.com/python/current/api/pymongo/collection.html#pymongo.collection.Collection.aggregate .. versionadded:: 0.9 """ - initial_pipeline = [] + using_deprecated_interface = isinstance(pipeline, dict) or bool(suppl_pipeline) + user_pipeline = [pipeline] if isinstance(pipeline, dict) else list(pipeline) + if using_deprecated_interface: + msg = "Calling .aggregate() with un unpacked list (*pipeline) is deprecated, it will soon change and will expect a list (similar to pymongo.Collection.aggregate interface), see documentation" + warnings.warn(msg, DeprecationWarning) + + user_pipeline += suppl_pipeline + + initial_pipeline = [] if self._query: initial_pipeline.append({"$match": self._query}) @@ -1281,14 +1292,14 @@ class BaseQuerySet(object): if self._skip is not None: initial_pipeline.append({"$skip": self._skip}) - pipeline = initial_pipeline + list(pipeline) + final_pipeline = initial_pipeline + user_pipeline + collection = self._collection if self._read_preference is not None: - return self._collection.with_options( + collection = self._collection.with_options( read_preference=self._read_preference - ).aggregate(pipeline, cursor={}, **kwargs) - - return self._collection.aggregate(pipeline, cursor={}, **kwargs) + ) + return collection.aggregate(final_pipeline, cursor={}, **kwargs) # JS functionality def map_reduce( diff --git a/mongoengine/queryset/transform.py b/mongoengine/queryset/transform.py index 0b73e99b..659a97e2 100644 --- a/mongoengine/queryset/transform.py +++ b/mongoengine/queryset/transform.py @@ -169,9 +169,9 @@ def query(_doc_cls=None, **kwargs): key = ".".join(parts) - if op is None or key not in mongo_query: + if key not in mongo_query: mongo_query[key] = value - elif key in mongo_query: + else: if isinstance(mongo_query[key], dict) and isinstance(value, dict): mongo_query[key].update(value) # $max/minDistance needs to come last - convert to SON diff --git a/setup.py b/setup.py index 19cd4be7..418ff7ad 100644 --- a/setup.py +++ b/setup.py @@ -108,6 +108,10 @@ CLASSIFIERS = [ "Topic :: Software Development :: Libraries :: Python Modules", ] +PYTHON_VERSION = sys.version_info[0] +PY3 = PYTHON_VERSION == 3 +PY2 = PYTHON_VERSION == 2 + extra_opts = { "packages": find_packages(exclude=["tests", "tests.*"]), "tests_require": [ @@ -115,11 +119,12 @@ extra_opts = { "pytest-cov", "coverage<5.0", # recent coverage switched to sqlite format for the .coverage file which isn't handled properly by coveralls "blinker", - "Pillow>=2.0.0", + "Pillow>=2.0.0, <7.0.0", # 7.0.0 dropped Python2 support + "zipp<2.0.0", # (dependency of pytest) dropped python2 support ], } -if sys.version_info[0] == 3: +if PY3: if "test" in sys.argv: extra_opts["packages"] = find_packages() extra_opts["package_data"] = { @@ -143,7 +148,7 @@ setup( long_description=LONG_DESCRIPTION, platforms=["any"], classifiers=CLASSIFIERS, - install_requires=["pymongo>=3.4", "six", "future"], + install_requires=["pymongo>=3.4, <4.0", "six>=1.10.0", "future"], cmdclass={"test": PyTest}, **extra_opts ) diff --git a/tests/document/test_instance.py b/tests/document/test_instance.py index 173e02f2..a5c21323 100644 --- a/tests/document/test_instance.py +++ b/tests/document/test_instance.py @@ -41,7 +41,7 @@ from tests.utils import MongoDBTestCase, get_as_pymongo TEST_IMAGE_PATH = os.path.join(os.path.dirname(__file__), "../fields/mongoengine.png") -class TestInstance(MongoDBTestCase): +class TestDocumentInstance(MongoDBTestCase): def setUp(self): class Job(EmbeddedDocument): name = StringField() @@ -3319,6 +3319,39 @@ class TestInstance(MongoDBTestCase): f1.ref # Dereferences lazily assert f1 == f2 + def test_embedded_document_equality_with_lazy_ref(self): + class Job(EmbeddedDocument): + boss = LazyReferenceField("Person") + boss_dbref = LazyReferenceField("Person", dbref=True) + + class Person(Document): + job = EmbeddedDocumentField(Job) + + Person.drop_collection() + + boss = Person() + worker = Person(job=Job(boss=boss, boss_dbref=boss)) + boss.save() + worker.save() + + worker1 = Person.objects.get(id=worker.id) + + # worker1.job should be equal to the job used originally to create the + # document. + assert worker1.job == worker.job + + # worker1.job should be equal to a newly created Job EmbeddedDocument + # using either the Boss object or his ID. + assert worker1.job == Job(boss=boss, boss_dbref=boss) + assert worker1.job == Job(boss=boss.id, boss_dbref=boss.id) + + # The above equalities should also hold after worker1.job.boss has been + # fetch()ed. + worker1.job.boss.fetch() + assert worker1.job == worker.job + assert worker1.job == Job(boss=boss, boss_dbref=boss) + assert worker1.job == Job(boss=boss.id, boss_dbref=boss.id) + def test_dbref_equality(self): class Test2(Document): name = StringField() @@ -3584,6 +3617,51 @@ class TestInstance(MongoDBTestCase): assert b._instance == a assert idx == 2 + def test_updating_listfield_manipulate_list(self): + class Company(Document): + name = StringField() + employees = ListField(field=DictField()) + + Company.drop_collection() + + comp = Company(name="BigBank", employees=[{"name": "John"}]) + comp.save() + comp.employees.append({"name": "Bill"}) + comp.save() + + stored_comp = get_as_pymongo(comp) + self.assertEqual( + stored_comp, + { + "_id": comp.id, + "employees": [{"name": "John"}, {"name": "Bill"}], + "name": "BigBank", + }, + ) + + comp = comp.reload() + comp.employees[0]["color"] = "red" + comp.employees[-1]["color"] = "blue" + comp.employees[-1].update({"size": "xl"}) + comp.save() + + assert len(comp.employees) == 2 + assert comp.employees[0] == {"name": "John", "color": "red"} + assert comp.employees[1] == {"name": "Bill", "size": "xl", "color": "blue"} + + stored_comp = get_as_pymongo(comp) + self.assertEqual( + stored_comp, + { + "_id": comp.id, + "employees": [ + {"name": "John", "color": "red"}, + {"size": "xl", "color": "blue", "name": "Bill"}, + ], + "name": "BigBank", + }, + ) + def test_falsey_pk(self): """Ensure that we can create and update a document with Falsey PK.""" @@ -3660,13 +3738,13 @@ class TestInstance(MongoDBTestCase): value = u"I_should_be_a_dict" coll.insert_one({"light_saber": value}) - with self.assertRaises(InvalidDocumentError) as cm: + with pytest.raises(InvalidDocumentError) as exc_info: list(Jedi.objects) - self.assertEqual( - str(cm.exception), - "Invalid data to create a `Jedi` instance.\nField 'light_saber' - The source SON object needs to be of type 'dict' but a '%s' was found" - % type(value), + assert str( + exc_info.value + ) == "Invalid data to create a `Jedi` instance.\nField 'light_saber' - The source SON object needs to be of type 'dict' but a '%s' was found" % type( + value ) diff --git a/tests/fields/test_dict_field.py b/tests/fields/test_dict_field.py index e88128f9..44e628f6 100644 --- a/tests/fields/test_dict_field.py +++ b/tests/fields/test_dict_field.py @@ -3,6 +3,7 @@ import pytest from mongoengine import * from mongoengine.base import BaseDict +from mongoengine.mongodb_support import MONGODB_36, get_mongodb_version from tests.utils import MongoDBTestCase, get_as_pymongo @@ -43,11 +44,7 @@ class TestDictField(MongoDBTestCase): with pytest.raises(ValidationError): post.validate() - post.info = {"the.title": "test"} - with pytest.raises(ValidationError): - post.validate() - - post.info = {"nested": {"the.title": "test"}} + post.info = {"$title.test": "test"} with pytest.raises(ValidationError): post.validate() @@ -55,6 +52,20 @@ class TestDictField(MongoDBTestCase): with pytest.raises(ValidationError): post.validate() + post.info = {"nested": {"the.title": "test"}} + if get_mongodb_version() < MONGODB_36: + with pytest.raises(ValidationError): + post.validate() + else: + post.validate() + + post.info = {"dollar_and_dot": {"te$st.test": "test"}} + if get_mongodb_version() < MONGODB_36: + with pytest.raises(ValidationError): + post.validate() + else: + post.validate() + post.info = {"title": "test"} post.save() diff --git a/tests/fields/test_file_field.py b/tests/fields/test_file_field.py index bfc86511..b8ece1a9 100644 --- a/tests/fields/test_file_field.py +++ b/tests/fields/test_file_field.py @@ -151,7 +151,7 @@ class TestFileField(MongoDBTestCase): result = StreamFile.objects.first() assert streamfile == result assert result.the_file.read() == text + more_text - # self.assertEqual(result.the_file.content_type, content_type) + # assert result.the_file.content_type == content_type result.the_file.seek(0) assert result.the_file.tell() == 0 assert result.the_file.read(len(text)) == text diff --git a/tests/queryset/test_queryset.py b/tests/queryset/test_queryset.py index 49dab169..17bf7405 100644 --- a/tests/queryset/test_queryset.py +++ b/tests/queryset/test_queryset.py @@ -14,7 +14,7 @@ import six from six import iteritems from mongoengine import * -from mongoengine.connection import get_connection, get_db +from mongoengine.connection import get_db from mongoengine.context_managers import query_counter, switch_db from mongoengine.errors import InvalidQueryError from mongoengine.mongodb_support import MONGODB_36, get_mongodb_version @@ -4658,21 +4658,6 @@ class TestQueryset(unittest.TestCase): ) assert_read_pref(bars, ReadPreference.SECONDARY_PREFERRED) - def test_read_preference_aggregation_framework(self): - class Bar(Document): - txt = StringField() - - meta = {"indexes": ["txt"]} - - # Aggregates with read_preference - bars = Bar.objects.read_preference( - ReadPreference.SECONDARY_PREFERRED - ).aggregate() - assert ( - bars._CommandCursor__collection.read_preference - == ReadPreference.SECONDARY_PREFERRED - ) - def test_json_simple(self): class Embedded(EmbeddedDocument): string = StringField() @@ -5399,225 +5384,6 @@ class TestQueryset(unittest.TestCase): assert Person.objects.first().name == "A" assert Person.objects._has_data(), "Cursor has data and returned False" - def test_queryset_aggregation_framework(self): - class Person(Document): - name = StringField() - age = IntField() - - Person.drop_collection() - - p1 = Person(name="Isabella Luanna", age=16) - p2 = Person(name="Wilson Junior", age=21) - p3 = Person(name="Sandra Mara", age=37) - Person.objects.insert([p1, p2, p3]) - - data = Person.objects(age__lte=22).aggregate( - {"$project": {"name": {"$toUpper": "$name"}}} - ) - - assert list(data) == [ - {"_id": p1.pk, "name": "ISABELLA LUANNA"}, - {"_id": p2.pk, "name": "WILSON JUNIOR"}, - ] - - data = ( - Person.objects(age__lte=22) - .order_by("-name") - .aggregate({"$project": {"name": {"$toUpper": "$name"}}}) - ) - - assert list(data) == [ - {"_id": p2.pk, "name": "WILSON JUNIOR"}, - {"_id": p1.pk, "name": "ISABELLA LUANNA"}, - ] - - data = ( - Person.objects(age__gte=17, age__lte=40) - .order_by("-age") - .aggregate( - {"$group": {"_id": None, "total": {"$sum": 1}, "avg": {"$avg": "$age"}}} - ) - ) - assert list(data) == [{"_id": None, "avg": 29, "total": 2}] - - data = Person.objects().aggregate({"$match": {"name": "Isabella Luanna"}}) - assert list(data) == [{u"_id": p1.pk, u"age": 16, u"name": u"Isabella Luanna"}] - - def test_queryset_aggregation_with_skip(self): - class Person(Document): - name = StringField() - age = IntField() - - Person.drop_collection() - - p1 = Person(name="Isabella Luanna", age=16) - p2 = Person(name="Wilson Junior", age=21) - p3 = Person(name="Sandra Mara", age=37) - Person.objects.insert([p1, p2, p3]) - - data = Person.objects.skip(1).aggregate( - {"$project": {"name": {"$toUpper": "$name"}}} - ) - - assert list(data) == [ - {"_id": p2.pk, "name": "WILSON JUNIOR"}, - {"_id": p3.pk, "name": "SANDRA MARA"}, - ] - - def test_queryset_aggregation_with_limit(self): - class Person(Document): - name = StringField() - age = IntField() - - Person.drop_collection() - - p1 = Person(name="Isabella Luanna", age=16) - p2 = Person(name="Wilson Junior", age=21) - p3 = Person(name="Sandra Mara", age=37) - Person.objects.insert([p1, p2, p3]) - - data = Person.objects.limit(1).aggregate( - {"$project": {"name": {"$toUpper": "$name"}}} - ) - - assert list(data) == [{"_id": p1.pk, "name": "ISABELLA LUANNA"}] - - def test_queryset_aggregation_with_sort(self): - class Person(Document): - name = StringField() - age = IntField() - - Person.drop_collection() - - p1 = Person(name="Isabella Luanna", age=16) - p2 = Person(name="Wilson Junior", age=21) - p3 = Person(name="Sandra Mara", age=37) - Person.objects.insert([p1, p2, p3]) - - data = Person.objects.order_by("name").aggregate( - {"$project": {"name": {"$toUpper": "$name"}}} - ) - - assert list(data) == [ - {"_id": p1.pk, "name": "ISABELLA LUANNA"}, - {"_id": p3.pk, "name": "SANDRA MARA"}, - {"_id": p2.pk, "name": "WILSON JUNIOR"}, - ] - - def test_queryset_aggregation_with_skip_with_limit(self): - class Person(Document): - name = StringField() - age = IntField() - - Person.drop_collection() - - p1 = Person(name="Isabella Luanna", age=16) - p2 = Person(name="Wilson Junior", age=21) - p3 = Person(name="Sandra Mara", age=37) - Person.objects.insert([p1, p2, p3]) - - data = list( - Person.objects.skip(1) - .limit(1) - .aggregate({"$project": {"name": {"$toUpper": "$name"}}}) - ) - - assert list(data) == [{"_id": p2.pk, "name": "WILSON JUNIOR"}] - - # Make sure limit/skip chaining order has no impact - data2 = ( - Person.objects.limit(1) - .skip(1) - .aggregate({"$project": {"name": {"$toUpper": "$name"}}}) - ) - - assert data == list(data2) - - def test_queryset_aggregation_with_sort_with_limit(self): - class Person(Document): - name = StringField() - age = IntField() - - Person.drop_collection() - - p1 = Person(name="Isabella Luanna", age=16) - p2 = Person(name="Wilson Junior", age=21) - p3 = Person(name="Sandra Mara", age=37) - Person.objects.insert([p1, p2, p3]) - - data = ( - Person.objects.order_by("name") - .limit(2) - .aggregate({"$project": {"name": {"$toUpper": "$name"}}}) - ) - - assert list(data) == [ - {"_id": p1.pk, "name": "ISABELLA LUANNA"}, - {"_id": p3.pk, "name": "SANDRA MARA"}, - ] - - # Verify adding limit/skip steps works as expected - data = ( - Person.objects.order_by("name") - .limit(2) - .aggregate({"$project": {"name": {"$toUpper": "$name"}}}, {"$limit": 1}) - ) - - assert list(data) == [{"_id": p1.pk, "name": "ISABELLA LUANNA"}] - - data = ( - Person.objects.order_by("name") - .limit(2) - .aggregate( - {"$project": {"name": {"$toUpper": "$name"}}}, - {"$skip": 1}, - {"$limit": 1}, - ) - ) - - assert list(data) == [{"_id": p3.pk, "name": "SANDRA MARA"}] - - def test_queryset_aggregation_with_sort_with_skip(self): - class Person(Document): - name = StringField() - age = IntField() - - Person.drop_collection() - - p1 = Person(name="Isabella Luanna", age=16) - p2 = Person(name="Wilson Junior", age=21) - p3 = Person(name="Sandra Mara", age=37) - Person.objects.insert([p1, p2, p3]) - - data = ( - Person.objects.order_by("name") - .skip(2) - .aggregate({"$project": {"name": {"$toUpper": "$name"}}}) - ) - - assert list(data) == [{"_id": p2.pk, "name": "WILSON JUNIOR"}] - - def test_queryset_aggregation_with_sort_with_skip_with_limit(self): - class Person(Document): - name = StringField() - age = IntField() - - Person.drop_collection() - - p1 = Person(name="Isabella Luanna", age=16) - p2 = Person(name="Wilson Junior", age=21) - p3 = Person(name="Sandra Mara", age=37) - Person.objects.insert([p1, p2, p3]) - - data = ( - Person.objects.order_by("name") - .skip(1) - .limit(1) - .aggregate({"$project": {"name": {"$toUpper": "$name"}}}) - ) - - assert list(data) == [{"_id": p3.pk, "name": "SANDRA MARA"}] - def test_delete_count(self): [self.Person(name="User {0}".format(i), age=i * 10).save() for i in range(1, 4)] assert ( diff --git a/tests/queryset/test_queryset_aggregation.py b/tests/queryset/test_queryset_aggregation.py new file mode 100644 index 00000000..00e04a36 --- /dev/null +++ b/tests/queryset/test_queryset_aggregation.py @@ -0,0 +1,255 @@ +# -*- coding: utf-8 -*- + +import unittest +import warnings + +from pymongo.read_preferences import ReadPreference + +from mongoengine import * +from tests.utils import MongoDBTestCase + + +class TestQuerysetAggregate(MongoDBTestCase): + def test_read_preference_aggregation_framework(self): + class Bar(Document): + txt = StringField() + + meta = {"indexes": ["txt"]} + + # Aggregates with read_preference + pipeline = [] + bars = Bar.objects.read_preference( + ReadPreference.SECONDARY_PREFERRED + ).aggregate(pipeline) + assert ( + bars._CommandCursor__collection.read_preference + == ReadPreference.SECONDARY_PREFERRED + ) + + def test_queryset_aggregation_framework(self): + class Person(Document): + name = StringField() + age = IntField() + + Person.drop_collection() + + p1 = Person(name="Isabella Luanna", age=16) + p2 = Person(name="Wilson Junior", age=21) + p3 = Person(name="Sandra Mara", age=37) + Person.objects.insert([p1, p2, p3]) + + pipeline = [{"$project": {"name": {"$toUpper": "$name"}}}] + data = Person.objects(age__lte=22).aggregate(pipeline) + + assert list(data) == [ + {"_id": p1.pk, "name": "ISABELLA LUANNA"}, + {"_id": p2.pk, "name": "WILSON JUNIOR"}, + ] + + pipeline = [{"$project": {"name": {"$toUpper": "$name"}}}] + data = Person.objects(age__lte=22).order_by("-name").aggregate(pipeline) + + assert list(data) == [ + {"_id": p2.pk, "name": "WILSON JUNIOR"}, + {"_id": p1.pk, "name": "ISABELLA LUANNA"}, + ] + + pipeline = [ + {"$group": {"_id": None, "total": {"$sum": 1}, "avg": {"$avg": "$age"}}} + ] + data = ( + Person.objects(age__gte=17, age__lte=40) + .order_by("-age") + .aggregate(pipeline) + ) + assert list(data) == [{"_id": None, "avg": 29, "total": 2}] + + pipeline = [{"$match": {"name": "Isabella Luanna"}}] + data = Person.objects().aggregate(pipeline) + assert list(data) == [{u"_id": p1.pk, u"age": 16, u"name": u"Isabella Luanna"}] + + def test_queryset_aggregation_with_skip(self): + class Person(Document): + name = StringField() + age = IntField() + + Person.drop_collection() + + p1 = Person(name="Isabella Luanna", age=16) + p2 = Person(name="Wilson Junior", age=21) + p3 = Person(name="Sandra Mara", age=37) + Person.objects.insert([p1, p2, p3]) + + pipeline = [{"$project": {"name": {"$toUpper": "$name"}}}] + data = Person.objects.skip(1).aggregate(pipeline) + + assert list(data) == [ + {"_id": p2.pk, "name": "WILSON JUNIOR"}, + {"_id": p3.pk, "name": "SANDRA MARA"}, + ] + + def test_queryset_aggregation_with_limit(self): + class Person(Document): + name = StringField() + age = IntField() + + Person.drop_collection() + + p1 = Person(name="Isabella Luanna", age=16) + p2 = Person(name="Wilson Junior", age=21) + p3 = Person(name="Sandra Mara", age=37) + Person.objects.insert([p1, p2, p3]) + + pipeline = [{"$project": {"name": {"$toUpper": "$name"}}}] + data = Person.objects.limit(1).aggregate(pipeline) + + assert list(data) == [{"_id": p1.pk, "name": "ISABELLA LUANNA"}] + + def test_queryset_aggregation_with_sort(self): + class Person(Document): + name = StringField() + age = IntField() + + Person.drop_collection() + + p1 = Person(name="Isabella Luanna", age=16) + p2 = Person(name="Wilson Junior", age=21) + p3 = Person(name="Sandra Mara", age=37) + Person.objects.insert([p1, p2, p3]) + + pipeline = [{"$project": {"name": {"$toUpper": "$name"}}}] + data = Person.objects.order_by("name").aggregate(pipeline) + + assert list(data) == [ + {"_id": p1.pk, "name": "ISABELLA LUANNA"}, + {"_id": p3.pk, "name": "SANDRA MARA"}, + {"_id": p2.pk, "name": "WILSON JUNIOR"}, + ] + + def test_queryset_aggregation_with_skip_with_limit(self): + class Person(Document): + name = StringField() + age = IntField() + + Person.drop_collection() + + p1 = Person(name="Isabella Luanna", age=16) + p2 = Person(name="Wilson Junior", age=21) + p3 = Person(name="Sandra Mara", age=37) + Person.objects.insert([p1, p2, p3]) + + pipeline = [{"$project": {"name": {"$toUpper": "$name"}}}] + data = list(Person.objects.skip(1).limit(1).aggregate(pipeline)) + + assert list(data) == [{"_id": p2.pk, "name": "WILSON JUNIOR"}] + + # Make sure limit/skip chaining order has no impact + data2 = Person.objects.limit(1).skip(1).aggregate(pipeline) + + assert data == list(data2) + + def test_queryset_aggregation_with_sort_with_limit(self): + class Person(Document): + name = StringField() + age = IntField() + + Person.drop_collection() + + p1 = Person(name="Isabella Luanna", age=16) + p2 = Person(name="Wilson Junior", age=21) + p3 = Person(name="Sandra Mara", age=37) + Person.objects.insert([p1, p2, p3]) + + pipeline = [{"$project": {"name": {"$toUpper": "$name"}}}] + data = Person.objects.order_by("name").limit(2).aggregate(pipeline) + + assert list(data) == [ + {"_id": p1.pk, "name": "ISABELLA LUANNA"}, + {"_id": p3.pk, "name": "SANDRA MARA"}, + ] + + # Verify adding limit/skip steps works as expected + pipeline = [{"$project": {"name": {"$toUpper": "$name"}}}, {"$limit": 1}] + data = Person.objects.order_by("name").limit(2).aggregate(pipeline) + + assert list(data) == [{"_id": p1.pk, "name": "ISABELLA LUANNA"}] + + pipeline = [ + {"$project": {"name": {"$toUpper": "$name"}}}, + {"$skip": 1}, + {"$limit": 1}, + ] + data = Person.objects.order_by("name").limit(2).aggregate(pipeline) + + assert list(data) == [{"_id": p3.pk, "name": "SANDRA MARA"}] + + def test_queryset_aggregation_with_sort_with_skip(self): + class Person(Document): + name = StringField() + age = IntField() + + Person.drop_collection() + + p1 = Person(name="Isabella Luanna", age=16) + p2 = Person(name="Wilson Junior", age=21) + p3 = Person(name="Sandra Mara", age=37) + Person.objects.insert([p1, p2, p3]) + + pipeline = [{"$project": {"name": {"$toUpper": "$name"}}}] + data = Person.objects.order_by("name").skip(2).aggregate(pipeline) + + assert list(data) == [{"_id": p2.pk, "name": "WILSON JUNIOR"}] + + def test_queryset_aggregation_with_sort_with_skip_with_limit(self): + class Person(Document): + name = StringField() + age = IntField() + + Person.drop_collection() + + p1 = Person(name="Isabella Luanna", age=16) + p2 = Person(name="Wilson Junior", age=21) + p3 = Person(name="Sandra Mara", age=37) + Person.objects.insert([p1, p2, p3]) + + pipeline = [{"$project": {"name": {"$toUpper": "$name"}}}] + data = Person.objects.order_by("name").skip(1).limit(1).aggregate(pipeline) + + assert list(data) == [{"_id": p3.pk, "name": "SANDRA MARA"}] + + def test_queryset_aggregation_deprecated_interface(self): + class Person(Document): + name = StringField() + + Person.drop_collection() + + p1 = Person(name="Isabella Luanna") + p2 = Person(name="Wilson Junior") + p3 = Person(name="Sandra Mara") + Person.objects.insert([p1, p2, p3]) + + pipeline = [{"$project": {"name": {"$toUpper": "$name"}}}] + + # Make sure a warning is emitted + with warnings.catch_warnings(): + warnings.simplefilter("error", DeprecationWarning) + with self.assertRaises(DeprecationWarning): + Person.objects.order_by("name").limit(2).aggregate(*pipeline) + + # Make sure old interface works as expected with a 1-step pipeline + data = Person.objects.order_by("name").limit(2).aggregate(*pipeline) + + assert list(data) == [ + {"_id": p1.pk, "name": "ISABELLA LUANNA"}, + {"_id": p3.pk, "name": "SANDRA MARA"}, + ] + + # Make sure old interface works as expected with a 2-steps pipeline + pipeline = [{"$project": {"name": {"$toUpper": "$name"}}}, {"$limit": 1}] + data = Person.objects.order_by("name").limit(2).aggregate(*pipeline) + + assert list(data) == [{"_id": p1.pk, "name": "ISABELLA LUANNA"}] + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/queryset/test_transform.py b/tests/queryset/test_transform.py index 3898809e..8d6c2d06 100644 --- a/tests/queryset/test_transform.py +++ b/tests/queryset/test_transform.py @@ -24,6 +24,12 @@ class TestTransform(unittest.TestCase): } assert transform.query(friend__age__gte=30) == {"friend.age": {"$gte": 30}} assert transform.query(name__exists=True) == {"name": {"$exists": True}} + assert transform.query(name=["Mark"], __raw__={"name": {"$in": "Tom"}}) == { + "$and": [{"name": ["Mark"]}, {"name": {"$in": "Tom"}}] + } + assert transform.query(name__in=["Tom"], __raw__={"name": "Mark"}) == { + "$and": [{"name": {"$in": ["Tom"]}}, {"name": "Mark"}] + } def test_transform_update(self): class LisDoc(Document): diff --git a/tox.ini b/tox.ini index 349b5577..396817ca 100644 --- a/tox.ini +++ b/tox.ini @@ -1,5 +1,5 @@ [tox] -envlist = {py27,py35,pypy,pypy3}-{mg34,mg36} +envlist = {py27,py35,pypy,pypy3}-{mg34,mg36,mg39,mg310} [testenv] commands = @@ -7,6 +7,7 @@ commands = deps = mg34: pymongo>=3.4,<3.5 mg36: pymongo>=3.6,<3.7 - mg39: pymongo>=3.9,<4.0 + mg39: pymongo>=3.9,<3.10 + mg310: pymongo>=3.10,<3.11 setenv = PYTHON_EGG_CACHE = {envdir}/python-eggs