From 500eb920e4498c0573ceadd22f65117fd55fd075 Mon Sep 17 00:00:00 2001 From: Manuel Hermann Date: Tue, 7 Aug 2012 17:04:03 +0200 Subject: [PATCH 001/189] Use info from getlasterror whether a document has been updated or created. --- mongoengine/document.py | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/mongoengine/document.py b/mongoengine/document.py index f8bf769d..bb5a60fc 100644 --- a/mongoengine/document.py +++ b/mongoengine/document.py @@ -208,11 +208,20 @@ class Document(BaseDocument): actual_key = self._db_field_map.get(k, k) select_dict[actual_key] = doc[actual_key] + def is_new_object(last_error): + if last_error is not None: + updated = last_error.get("updatedExisting") + if updated is not None: + return not updated + return created + upsert = self._created if updates: - collection.update(select_dict, {"$set": updates}, upsert=upsert, safe=safe, **write_options) + last_error = collection.update(select_dict, {"$set": updates}, upsert=upsert, safe=safe, **write_options) + created = is_new_object(last_error) if removals: - collection.update(select_dict, {"$unset": removals}, upsert=upsert, safe=safe, **write_options) + last_error = collection.update(select_dict, {"$unset": removals}, upsert=upsert, safe=safe, **write_options) + created = created or is_new_object(last_error) cascade = self._meta.get('cascade', True) if cascade is None else cascade if cascade: From 6a31736644452ec6598dff8851f7847aef63da11 Mon Sep 17 00:00:00 2001 From: Luis Araujo Date: Wed, 26 Sep 2012 14:43:59 -0300 Subject: [PATCH 002/189] Initial support to Group and Permission. The /admin can't be exec login in MongoDB yet. Only SQLsDB (SQLite,...) This code work with django-mongoadmin pluggin. --- mongoengine/django/auth.py | 187 +++++++++++++++++++++++++++++++++++++ 1 file changed, 187 insertions(+) diff --git a/mongoengine/django/auth.py b/mongoengine/django/auth.py index a30fc57c..d9f05841 100644 --- a/mongoengine/django/auth.py +++ b/mongoengine/django/auth.py @@ -3,6 +3,8 @@ import datetime from mongoengine import * from django.utils.encoding import smart_str +from django.db import models +from django.contrib.contenttypes.models import ContentTypeManager from django.contrib.auth.models import AnonymousUser from django.utils.translation import ugettext_lazy as _ @@ -34,6 +36,191 @@ except ImportError: REDIRECT_FIELD_NAME = 'next' +class ContentType(Document): + name = StringField(max_length=100) + app_label = StringField(max_length=100) + model = StringField(max_length=100, verbose_name=_('python model class name'), + unique_with='app_label') + objects = ContentTypeManager() + + class Meta: + verbose_name = _('content type') + verbose_name_plural = _('content types') + # db_table = 'django_content_type' + # ordering = ('name',) + # unique_together = (('app_label', 'model'),) + + def __unicode__(self): + return self.name + + def model_class(self): + "Returns the Python model class for this type of content." + from django.db import models + return models.get_model(self.app_label, self.model) + + def get_object_for_this_type(self, **kwargs): + """ + Returns an object of this type for the keyword arguments given. + Basically, this is a proxy around this object_type's get_object() model + method. The ObjectNotExist exception, if thrown, will not be caught, + so code that calls this method should catch it. + """ + return self.model_class()._default_manager.using(self._state.db).get(**kwargs) + + def natural_key(self): + return (self.app_label, self.model) + +class SiteProfileNotAvailable(Exception): + pass + +class PermissionManager(models.Manager): + def get_by_natural_key(self, codename, app_label, model): + return self.get( + codename=codename, + content_type=ContentType.objects.get_by_natural_key(app_label, model) + ) + +class Permission(Document): + """The permissions system provides a way to assign permissions to specific users and groups of users. + + The permission system is used by the Django admin site, but may also be useful in your own code. The Django admin site uses permissions as follows: + + - The "add" permission limits the user's ability to view the "add" form and add an object. + - The "change" permission limits a user's ability to view the change list, view the "change" form and change an object. + - The "delete" permission limits the ability to delete an object. + + Permissions are set globally per type of object, not per specific object instance. It is possible to say "Mary may change news stories," but it's not currently possible to say "Mary may change news stories, but only the ones she created herself" or "Mary may only change news stories that have a certain status or publication date." + + Three basic permissions -- add, change and delete -- are automatically created for each Django model. + """ + name = StringField(max_length=50, verbose_name=_('username')) + content_type = ReferenceField(ContentType) + codename = StringField(max_length=100, verbose_name=_('codename')) + # FIXME: don't access field of the other class + # unique_with=['content_type__app_label', 'content_type__model']) + + objects = PermissionManager() + + class Meta: + verbose_name = _('permission') + verbose_name_plural = _('permissions') + # unique_together = (('content_type', 'codename'),) + # ordering = ('content_type__app_label', 'content_type__model', 'codename') + + def __unicode__(self): + return u"%s | %s | %s" % ( + unicode(self.content_type.app_label), + unicode(self.content_type), + unicode(self.name)) + + def natural_key(self): + return (self.codename,) + self.content_type.natural_key() + natural_key.dependencies = ['contenttypes.contenttype'] + +class Group(Document): + """Groups are a generic way of categorizing users to apply permissions, or some other label, to those users. A user can belong to any number of groups. + + A user in a group automatically has all the permissions granted to that group. For example, if the group Site editors has the permission can_edit_home_page, any user in that group will have that permission. + + Beyond permissions, groups are a convenient way to categorize users to apply some label, or extended functionality, to them. For example, you could create a group 'Special users', and you could write code that would do special things to those users -- such as giving them access to a members-only portion of your site, or sending them members-only e-mail messages. + """ + name = StringField(max_length=80, unique=True, verbose_name=_('name')) + # permissions = models.ManyToManyField(Permission, verbose_name=_('permissions'), blank=True) + permissions = ListField(ReferenceField(Permission, verbose_name=_('permissions'), required=False)) + + class Meta: + verbose_name = _('group') + verbose_name_plural = _('groups') + + def __unicode__(self): + return self.name + +class UserManager(models.Manager): + def create_user(self, username, email, password=None): + """ + Creates and saves a User with the given username, e-mail and password. + """ + now = datetime.datetime.now() + + # Normalize the address by lowercasing the domain part of the email + # address. + try: + email_name, domain_part = email.strip().split('@', 1) + except ValueError: + pass + else: + email = '@'.join([email_name, domain_part.lower()]) + + user = self.model(username=username, email=email, is_staff=False, + is_active=True, is_superuser=False, last_login=now, + date_joined=now) + + user.set_password(password) + user.save(using=self._db) + return user + + def create_superuser(self, username, email, password): + u = self.create_user(username, email, password) + u.is_staff = True + u.is_active = True + u.is_superuser = True + u.save(using=self._db) + return u + + def make_random_password(self, length=10, allowed_chars='abcdefghjkmnpqrstuvwxyzABCDEFGHJKLMNPQRSTUVWXYZ23456789'): + "Generates a random password with the given length and given allowed_chars" + # Note that default value of allowed_chars does not have "I" or letters + # that look like it -- just to avoid confusion. + from random import choice + return ''.join([choice(allowed_chars) for i in range(length)]) + + +# A few helper functions for common logic between User and AnonymousUser. +def _user_get_all_permissions(user, obj): + permissions = set() + anon = user.is_anonymous() + for backend in auth.get_backends(): + if not anon or backend.supports_anonymous_user: + if hasattr(backend, "get_all_permissions"): + if obj is not None: + if backend.supports_object_permissions: + permissions.update( + backend.get_all_permissions(user, obj) + ) + else: + permissions.update(backend.get_all_permissions(user)) + return permissions + + +def _user_has_perm(user, perm, obj): + anon = user.is_anonymous() + active = user.is_active + for backend in auth.get_backends(): + if (not active and not anon and backend.supports_inactive_user) or \ + (not anon or backend.supports_anonymous_user): + if hasattr(backend, "has_perm"): + if obj is not None: + if (backend.supports_object_permissions and + backend.has_perm(user, perm, obj)): + return True + else: + if backend.has_perm(user, perm): + return True + return False + + +def _user_has_module_perms(user, app_label): + anon = user.is_anonymous() + active = user.is_active + for backend in auth.get_backends(): + if (not active and not anon and backend.supports_inactive_user) or \ + (not anon or backend.supports_anonymous_user): + if hasattr(backend, "has_module_perms"): + if backend.has_module_perms(user, app_label): + return True + return False + + class User(Document): """A User document that aims to mirror most of the API specified by Django at http://docs.djangoproject.com/en/dev/topics/auth/#users From 3425574ddcd63d126bc96556e182cc0e7cf18bcb Mon Sep 17 00:00:00 2001 From: Luis Araujo Date: Thu, 27 Sep 2012 14:30:59 -0300 Subject: [PATCH 003/189] Adding, adjust and transplant more methods to auth.User model --- mongoengine/django/auth.py | 150 ++++++++++++++++++++++++++----------- 1 file changed, 105 insertions(+), 45 deletions(-) diff --git a/mongoengine/django/auth.py b/mongoengine/django/auth.py index d9f05841..0b6ffb98 100644 --- a/mongoengine/django/auth.py +++ b/mongoengine/django/auth.py @@ -5,6 +5,7 @@ from mongoengine import * from django.utils.encoding import smart_str from django.db import models from django.contrib.contenttypes.models import ContentTypeManager +from django.contrib import auth from django.contrib.auth.models import AnonymousUser from django.utils.translation import ugettext_lazy as _ @@ -175,51 +176,6 @@ class UserManager(models.Manager): return ''.join([choice(allowed_chars) for i in range(length)]) -# A few helper functions for common logic between User and AnonymousUser. -def _user_get_all_permissions(user, obj): - permissions = set() - anon = user.is_anonymous() - for backend in auth.get_backends(): - if not anon or backend.supports_anonymous_user: - if hasattr(backend, "get_all_permissions"): - if obj is not None: - if backend.supports_object_permissions: - permissions.update( - backend.get_all_permissions(user, obj) - ) - else: - permissions.update(backend.get_all_permissions(user)) - return permissions - - -def _user_has_perm(user, perm, obj): - anon = user.is_anonymous() - active = user.is_active - for backend in auth.get_backends(): - if (not active and not anon and backend.supports_inactive_user) or \ - (not anon or backend.supports_anonymous_user): - if hasattr(backend, "has_perm"): - if obj is not None: - if (backend.supports_object_permissions and - backend.has_perm(user, perm, obj)): - return True - else: - if backend.has_perm(user, perm): - return True - return False - - -def _user_has_module_perms(user, app_label): - anon = user.is_anonymous() - active = user.is_active - for backend in auth.get_backends(): - if (not active and not anon and backend.supports_inactive_user) or \ - (not anon or backend.supports_anonymous_user): - if hasattr(backend, "has_module_perms"): - if backend.has_module_perms(user, app_label): - return True - return False - class User(Document): """A User document that aims to mirror most of the API specified by Django @@ -313,9 +269,111 @@ class User(Document): user.save() return user + def get_all_permissions(self, obj=None): + permissions = set() + anon = self.is_anonymous() + for backend in auth.get_backends(): + if not anon or backend.supports_anonymous_user: + if hasattr(backend, "get_all_permissions"): + if obj is not None: + if backend.supports_object_permissions: + permissions.update( + backend.get_all_permissions(user, obj) + ) + else: + permissions.update(backend.get_all_permissions(self)) + return permissions + def get_and_delete_messages(self): return [] + def has_perm(self, perm, obj=None): + anon = self.is_anonymous() + active = self.is_active + for backend in auth.get_backends(): + if (not active and not anon and backend.supports_inactive_user) or \ + (not anon or backend.supports_anonymous_user): + if hasattr(backend, "has_perm"): + if obj is not None: + if (backend.supports_object_permissions and + backend.has_perm(self, perm, obj)): + return True + else: + if backend.has_perm(self, perm): + return True + return False + + def has_perms(self, perm_list, obj=None): + """ + Returns True if the user has each of the specified permissions. + If object is passed, it checks if the user has all required perms + for this object. + """ + for perm in perm_list: + if not self.has_perm(perm, obj): + return False + return True + + def has_module_perms(self, app_label): + anon = self.is_anonymous() + active = self.is_active + for backend in auth.get_backends(): + if (not active and not anon and backend.supports_inactive_user) or \ + (not anon or backend.supports_anonymous_user): + if hasattr(backend, "has_module_perms"): + if backend.has_module_perms(self, app_label): + return True + return False + + def get_and_delete_messages(self): + messages = [] + for m in self.message_set.all(): + messages.append(m.message) + m.delete() + return messages + + def email_user(self, subject, message, from_email=None): + "Sends an e-mail to this User." + from django.core.mail import send_mail + send_mail(subject, message, from_email, [self.email]) + + def get_profile(self): + """ + Returns site-specific profile for this user. Raises + SiteProfileNotAvailable if this site does not allow profiles. + """ + if not hasattr(self, '_profile_cache'): + from django.conf import settings + if not getattr(settings, 'AUTH_PROFILE_MODULE', False): + raise SiteProfileNotAvailable('You need to set AUTH_PROFILE_MO' + 'DULE in your project settings') + try: + app_label, model_name = settings.AUTH_PROFILE_MODULE.split('.') + except ValueError: + raise SiteProfileNotAvailable('app_label and model_name should' + ' be separated by a dot in the AUTH_PROFILE_MODULE set' + 'ting') + + try: + model = models.get_model(app_label, model_name) + if model is None: + raise SiteProfileNotAvailable('Unable to load the profile ' + 'model, check AUTH_PROFILE_MODULE in your project sett' + 'ings') + self._profile_cache = model._default_manager.using(self._state.db).get(user__id__exact=self.id) + self._profile_cache.user = self + except (ImportError, ImproperlyConfigured): + raise SiteProfileNotAvailable + return self._profile_cache + + def _get_message_set(self): + import warnings + warnings.warn('The user messaging API is deprecated. Please update' + ' your code to use the new messages framework.', + category=DeprecationWarning) + return self._message_set + message_set = property(_get_message_set) + class MongoEngineBackend(object): """Authenticate using MongoEngine and mongoengine.django.auth.User. @@ -329,6 +387,8 @@ class MongoEngineBackend(object): user = User.objects(username=username).first() if user: if password and user.check_password(password): + backend = auth.get_backends()[0] + user.backend = "%s.%s" % (backend.__module__, backend.__class__.__name__) return user return None From 0bfc96e459d19d188397db73fa557abbc8ecdc6d Mon Sep 17 00:00:00 2001 From: Luis Araujo Date: Thu, 27 Sep 2012 14:32:50 -0300 Subject: [PATCH 004/189] exposing mongoengine.django module --- mongoengine/__init__.py | 1 + 1 file changed, 1 insertion(+) diff --git a/mongoengine/__init__.py b/mongoengine/__init__.py index 9044e617..a4c56b85 100644 --- a/mongoengine/__init__.py +++ b/mongoengine/__init__.py @@ -8,6 +8,7 @@ import queryset from queryset import * import signals from signals import * +import django __all__ = (document.__all__ + fields.__all__ + connection.__all__ + queryset.__all__ + signals.__all__) From e4af0e361ab5b91fad7ca9dbc8f5ed631f0b2c77 Mon Sep 17 00:00:00 2001 From: Aleksey Porfirov Date: Mon, 15 Oct 2012 02:11:01 +0400 Subject: [PATCH 005/189] Add session expiration test (with django timezone support activated) --- tests/test_django.py | 45 +++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 44 insertions(+), 1 deletion(-) diff --git a/tests/test_django.py b/tests/test_django.py index 398fd3e0..3b0b04f8 100644 --- a/tests/test_django.py +++ b/tests/test_django.py @@ -12,7 +12,7 @@ try: from django.conf import settings from django.core.paginator import Paginator - settings.configure() + settings.configure(USE_TZ=True) from django.contrib.sessions.tests import SessionTestsMixin from mongoengine.django.sessions import SessionStore, MongoSession @@ -24,6 +24,37 @@ except Exception, err: raise err +from datetime import tzinfo, timedelta +ZERO = timedelta(0) + +class FixedOffset(tzinfo): + """Fixed offset in minutes east from UTC.""" + + def __init__(self, offset, name): + self.__offset = timedelta(minutes = offset) + self.__name = name + + def utcoffset(self, dt): + return self.__offset + + def tzname(self, dt): + return self.__name + + def dst(self, dt): + return ZERO + + +def activate_timezone(tz): + """Activate Django timezone support if it is available. + """ + try: + from django.utils import timezone + timezone.deactivate() + timezone.activate(tz) + except ImportError: + pass + + class QuerySetTest(unittest.TestCase): def setUp(self): @@ -120,3 +151,15 @@ class MongoDBSessionTest(SessionTestsMixin, unittest.TestCase): session['test'] = True session.save() self.assertTrue('test' in session) + + def test_session_expiration_tz(self): + activate_timezone(FixedOffset(60, 'UTC+1')) + # create and save new session + session = SessionStore() + session.set_expiry(600) # expire in 600 seconds + session['test_expire'] = True + session.save() + # reload session with key + key = session.session_key + session = SessionStore(key) + self.assertTrue('test_expire' in session, 'Session has expired before it is expected') From 0a89899ad088e712b6bca54e32f30aae64ab2bf7 Mon Sep 17 00:00:00 2001 From: Aleksey Porfirov Date: Mon, 15 Oct 2012 02:13:52 +0400 Subject: [PATCH 006/189] Fix django timezone support --- mongoengine/django/auth.py | 9 ++++----- mongoengine/django/sessions.py | 6 +++--- mongoengine/django/utils.py | 6 ++++++ 3 files changed, 13 insertions(+), 8 deletions(-) create mode 100644 mongoengine/django/utils.py diff --git a/mongoengine/django/auth.py b/mongoengine/django/auth.py index 65afacfd..3776d546 100644 --- a/mongoengine/django/auth.py +++ b/mongoengine/django/auth.py @@ -1,5 +1,3 @@ -import datetime - from mongoengine import * from django.utils.encoding import smart_str @@ -33,6 +31,7 @@ except ImportError: hash = get_hexdigest(algo, salt, raw_password) return '%s$%s$%s' % (algo, salt, hash) +from .utils import datetime_now REDIRECT_FIELD_NAME = 'next' @@ -62,9 +61,9 @@ class User(Document): is_superuser = BooleanField(default=False, verbose_name=_('superuser status'), help_text=_("Designates that this user has all permissions without explicitly assigning them.")) - last_login = DateTimeField(default=datetime.datetime.now, + last_login = DateTimeField(default=datetime_now, verbose_name=_('last login')) - date_joined = DateTimeField(default=datetime.datetime.now, + date_joined = DateTimeField(default=datetime_now, verbose_name=_('date joined')) meta = { @@ -130,7 +129,7 @@ class User(Document): """Create (and save) a new user with the given username, password and email address. """ - now = datetime.datetime.now() + now = datetime_now() # Normalize the address by lowercasing the domain part of the email # address. diff --git a/mongoengine/django/sessions.py b/mongoengine/django/sessions.py index f1783429..6e964a70 100644 --- a/mongoengine/django/sessions.py +++ b/mongoengine/django/sessions.py @@ -1,5 +1,3 @@ -from datetime import datetime - from django.conf import settings from django.contrib.sessions.backends.base import SessionBase, CreateError from django.core.exceptions import SuspiciousOperation @@ -10,6 +8,8 @@ from mongoengine import fields from mongoengine.queryset import OperationError from mongoengine.connection import DEFAULT_CONNECTION_NAME +from .utils import datetime_now + MONGOENGINE_SESSION_DB_ALIAS = getattr( settings, 'MONGOENGINE_SESSION_DB_ALIAS', @@ -33,7 +33,7 @@ class SessionStore(SessionBase): def load(self): try: s = MongoSession.objects(session_key=self.session_key, - expire_date__gt=datetime.now())[0] + expire_date__gt=datetime_now())[0] return self.decode(force_unicode(s.session_data)) except (IndexError, SuspiciousOperation): self.create() diff --git a/mongoengine/django/utils.py b/mongoengine/django/utils.py new file mode 100644 index 00000000..d3ef8a4b --- /dev/null +++ b/mongoengine/django/utils.py @@ -0,0 +1,6 @@ +try: + # django >= 1.4 + from django.utils.timezone import now as datetime_now +except ImportError: + from datetime import datetime + datetime_now = datetime.now From 6f29d1238642d395b197645f6770daeab42495d3 Mon Sep 17 00:00:00 2001 From: Ross Lawley Date: Tue, 9 Oct 2012 15:02:13 +0000 Subject: [PATCH 007/189] Changed the inheritance model to remove types The inheritance model has changed, we no longer need to store an array of `types` with the model we can just use the classname in `_cls`. See the upgrade docs for information on how to upgrade MongoEngine/mongoengine#148 --- docs/changelog.rst | 5 + docs/guide/defining-documents.rst | 23 +- docs/upgrade.rst | 39 + mongoengine/__init__.py | 6 +- mongoengine/base.py | 1523 -------------- mongoengine/base/__init__.py | 5 + mongoengine/base/common.py | 25 + mongoengine/base/datastructures.py | 124 ++ mongoengine/base/document.py | 644 ++++++ mongoengine/base/fields.py | 371 ++++ mongoengine/base/metaclasses.py | 388 ++++ mongoengine/common.py | 35 + mongoengine/dereference.py | 2 +- mongoengine/django/shortcuts.py | 2 +- mongoengine/document.py | 28 +- mongoengine/errors.py | 124 ++ mongoengine/fields.py | 6 +- mongoengine/queryset/__init__.py | 11 + mongoengine/queryset/field_list.py | 51 + mongoengine/queryset/manager.py | 61 + mongoengine/{ => queryset}/queryset.py | 818 +------- mongoengine/queryset/transform.py | 237 +++ mongoengine/queryset/visitor.py | 237 +++ setup.cfg | 2 +- tests/__init__.py | 2 + .../__init__.py} | 12 +- tests/document/__init__.py | 11 + tests/document/class_methods.py | 183 ++ tests/document/delta.py | 688 +++++++ tests/document/dynamic.py | 270 +++ tests/document/indexes.py | 637 ++++++ tests/document/inheritance.py | 395 ++++ .../instance.py} | 1752 +---------------- tests/{ => document}/mongoengine.png | Bin tests/migration/__init__.py | 4 + .../test_convert_to_new_inheritance_model.py | 51 + tests/migration/turn_off_inheritance.py | 62 + tests/test_dynamic_document.py | 533 ----- tests/test_fields.py | 5 +- tests/test_queryset.py | 106 +- 40 files changed, 4856 insertions(+), 4622 deletions(-) delete mode 100644 mongoengine/base.py create mode 100644 mongoengine/base/__init__.py create mode 100644 mongoengine/base/common.py create mode 100644 mongoengine/base/datastructures.py create mode 100644 mongoengine/base/document.py create mode 100644 mongoengine/base/fields.py create mode 100644 mongoengine/base/metaclasses.py create mode 100644 mongoengine/common.py create mode 100644 mongoengine/errors.py create mode 100644 mongoengine/queryset/__init__.py create mode 100644 mongoengine/queryset/field_list.py create mode 100644 mongoengine/queryset/manager.py rename mongoengine/{ => queryset}/queryset.py (59%) create mode 100644 mongoengine/queryset/transform.py create mode 100644 mongoengine/queryset/visitor.py rename tests/{test_all_warnings.py => all_warnings/__init__.py} (91%) create mode 100644 tests/document/__init__.py create mode 100644 tests/document/class_methods.py create mode 100644 tests/document/delta.py create mode 100644 tests/document/dynamic.py create mode 100644 tests/document/indexes.py create mode 100644 tests/document/inheritance.py rename tests/{test_document.py => document/instance.py} (50%) rename tests/{ => document}/mongoengine.png (100%) create mode 100644 tests/migration/__init__.py create mode 100644 tests/migration/test_convert_to_new_inheritance_model.py create mode 100644 tests/migration/turn_off_inheritance.py delete mode 100644 tests/test_dynamic_document.py diff --git a/docs/changelog.rst b/docs/changelog.rst index b2a855d5..8388b05a 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -2,6 +2,11 @@ Changelog ========= +Changes in 0.8 +============== +- Remove _types and just use _cls for inheritance (MongoEngine/mongoengine#148) + + Changes in 0.7.X ================ - Unicode fix for repr (MongoEngine/mongoengine#133) diff --git a/docs/guide/defining-documents.rst b/docs/guide/defining-documents.rst index 3ee77961..cf3b5a6f 100644 --- a/docs/guide/defining-documents.rst +++ b/docs/guide/defining-documents.rst @@ -461,9 +461,10 @@ If a dictionary is passed then the following options are available: :attr:`fields` (Default: None) The fields to index. Specified in the same format as described above. -:attr:`types` (Default: True) - Whether the index should have the :attr:`_types` field added automatically - to the start of the index. +:attr:`cls` (Default: True) + If you have polymorphic models that inherit and have `allow_inheritance` + turned on, you can configure whether the index should have the + :attr:`_cls` field added automatically to the start of the index. :attr:`sparse` (Default: False) Whether the index should be sparse. @@ -590,14 +591,14 @@ convenient and efficient retrieval of related documents:: Working with existing data -------------------------- To enable correct retrieval of documents involved in this kind of heirarchy, -two extra attributes are stored on each document in the database: :attr:`_cls` -and :attr:`_types`. These are hidden from the user through the MongoEngine -interface, but may not be present if you are trying to use MongoEngine with -an existing database. For this reason, you may disable this inheritance -mechansim, removing the dependency of :attr:`_cls` and :attr:`_types`, enabling -you to work with existing databases. To disable inheritance on a document -class, set :attr:`allow_inheritance` to ``False`` in the :attr:`meta` -dictionary:: +an extra attribute is stored on each document in the database: :attr:`_cls`. +These are hidden from the user through the MongoEngine interface, but may not +be present if you are trying to use MongoEngine with an existing database. + +For this reason, you may disable this inheritance mechansim, removing the +dependency of :attr:`_cls`, enabling you to work with existing databases. +To disable inheritance on a document class, set :attr:`allow_inheritance` to +``False`` in the :attr:`meta` dictionary:: # Will work with data in an existing collection named 'cmsPage' class Page(Document): diff --git a/docs/upgrade.rst b/docs/upgrade.rst index 82ac7cac..99e3078c 100644 --- a/docs/upgrade.rst +++ b/docs/upgrade.rst @@ -2,6 +2,45 @@ Upgrading ========= +0.7 to 0.8 +========== + +Inheritance +----------- + +The inheritance model has changed, we no longer need to store an array of +`types` with the model we can just use the classname in `_cls`. This means +that you will have to update your indexes for each of your inherited classes +like so: + + # 1. Declaration of the class + class Animal(Document): + name = StringField() + meta = { + 'allow_inheritance': True, + 'indexes': ['name'] + } + + # 2. Remove _types + collection = Animal._get_collection() + collection.update({}, {"$unset": {"_types": 1}}, multi=True) + + # 3. Confirm extra data is removed + count = collection.find({'_types': {"$exists": True}}).count() + assert count == 0 + + # 4. Remove indexes + info = collection.index_information() + indexes_to_drop = [key for key, value in info.iteritems() + if '_types' in dict(value['key'])] + for index in indexes_to_drop: + collection.drop_index(index) + + # 5. Recreate indexes + Animal.objects._ensure_indexes() + + + 0.6 to 0.7 ========== diff --git a/mongoengine/__init__.py b/mongoengine/__init__.py index 9044e617..d92165c5 100644 --- a/mongoengine/__init__.py +++ b/mongoengine/__init__.py @@ -9,10 +9,10 @@ from queryset import * import signals from signals import * -__all__ = (document.__all__ + fields.__all__ + connection.__all__ + - queryset.__all__ + signals.__all__) +__all__ = (list(document.__all__) + fields.__all__ + connection.__all__ + + list(queryset.__all__) + signals.__all__) -VERSION = (0, 7, 5) +VERSION = (0, 8, 0, '+') def get_version(): diff --git a/mongoengine/base.py b/mongoengine/base.py deleted file mode 100644 index fa12e35d..00000000 --- a/mongoengine/base.py +++ /dev/null @@ -1,1523 +0,0 @@ -import operator -import sys -import warnings -import weakref - -from collections import defaultdict -from functools import partial - -from queryset import QuerySet, QuerySetManager -from queryset import DoesNotExist, MultipleObjectsReturned -from queryset import DO_NOTHING - -from mongoengine import signals -from mongoengine.python_support import (PY3, UNICODE_KWARGS, txt_type, - to_str_keys_recursive) - -import pymongo -from bson import ObjectId -from bson.dbref import DBRef - -ALLOW_INHERITANCE = True - -_document_registry = {} -_class_registry = {} - - -class NotRegistered(Exception): - pass - - -class InvalidDocumentError(Exception): - pass - - -class ValidationError(AssertionError): - """Validation exception. - - May represent an error validating a field or a - document containing fields with validation errors. - - :ivar errors: A dictionary of errors for fields within this - document or list, or None if the error is for an - individual field. - """ - - errors = {} - field_name = None - _message = None - - def __init__(self, message="", **kwargs): - self.errors = kwargs.get('errors', {}) - self.field_name = kwargs.get('field_name') - self.message = message - - def __str__(self): - return txt_type(self.message) - - def __repr__(self): - return '%s(%s,)' % (self.__class__.__name__, self.message) - - def __getattribute__(self, name): - message = super(ValidationError, self).__getattribute__(name) - if name == 'message': - if self.field_name: - message = '%s' % message - if self.errors: - message = '%s(%s)' % (message, self._format_errors()) - return message - - def _get_message(self): - return self._message - - def _set_message(self, message): - self._message = message - - message = property(_get_message, _set_message) - - def to_dict(self): - """Returns a dictionary of all errors within a document - - Keys are field names or list indices and values are the - validation error messages, or a nested dictionary of - errors for an embedded document or list. - """ - - def build_dict(source): - errors_dict = {} - if not source: - return errors_dict - if isinstance(source, dict): - for field_name, error in source.iteritems(): - errors_dict[field_name] = build_dict(error) - elif isinstance(source, ValidationError) and source.errors: - return build_dict(source.errors) - else: - return unicode(source) - return errors_dict - if not self.errors: - return {} - return build_dict(self.errors) - - def _format_errors(self): - """Returns a string listing all errors within a document""" - - def generate_key(value, prefix=''): - if isinstance(value, list): - value = ' '.join([generate_key(k) for k in value]) - if isinstance(value, dict): - value = ' '.join( - [generate_key(v, k) for k, v in value.iteritems()]) - - results = "%s.%s" % (prefix, value) if prefix else value - return results - - error_dict = defaultdict(list) - for k, v in self.to_dict().iteritems(): - error_dict[generate_key(v)].append(k) - return ' '.join(["%s: %s" % (k, v) for k, v in error_dict.iteritems()]) - - -def get_document(name): - doc = _document_registry.get(name, None) - if not doc: - # Possible old style names - end = ".%s" % name - possible_match = [k for k in _document_registry.keys() - if k.endswith(end)] - if len(possible_match) == 1: - doc = _document_registry.get(possible_match.pop(), None) - if not doc: - raise NotRegistered(""" - `%s` has not been registered in the document registry. - Importing the document class automatically registers it, has it - been imported? - """.strip() % name) - return doc - - -class BaseField(object): - """A base class for fields in a MongoDB document. Instances of this class - may be added to subclasses of `Document` to define a document's schema. - - .. versionchanged:: 0.5 - added verbose and help text - """ - - name = None - - # Fields may have _types inserted into indexes by default - _index_with_types = True - _geo_index = False - - # These track each time a Field instance is created. Used to retain order. - # The auto_creation_counter is used for fields that MongoEngine implicitly - # creates, creation_counter is used for all user-specified fields. - creation_counter = 0 - auto_creation_counter = -1 - - def __init__(self, db_field=None, name=None, required=False, default=None, - unique=False, unique_with=None, primary_key=False, - validation=None, choices=None, verbose_name=None, - help_text=None): - self.db_field = (db_field or name) if not primary_key else '_id' - if name: - msg = "Fields' 'name' attribute deprecated in favour of 'db_field'" - warnings.warn(msg, DeprecationWarning) - self.name = None - self.required = required or primary_key - self.default = default - self.unique = bool(unique or unique_with) - self.unique_with = unique_with - self.primary_key = primary_key - self.validation = validation - self.choices = choices - self.verbose_name = verbose_name - self.help_text = help_text - - # Adjust the appropriate creation counter, and save our local copy. - if self.db_field == '_id': - self.creation_counter = BaseField.auto_creation_counter - BaseField.auto_creation_counter -= 1 - else: - self.creation_counter = BaseField.creation_counter - BaseField.creation_counter += 1 - - def __get__(self, instance, owner): - """Descriptor for retrieving a value from a field in a document. Do - any necessary conversion between Python and MongoDB types. - """ - if instance is None: - # Document class being used rather than a document object - return self - - # Get value from document instance if available, if not use default - value = instance._data.get(self.name) - - if value is None: - value = self.default - # Allow callable default values - if callable(value): - value = value() - - return value - - def __set__(self, instance, value): - """Descriptor for assigning a value to a field in a document. - """ - instance._data[self.name] = value - if instance._initialised: - instance._mark_as_changed(self.name) - - def error(self, message="", errors=None, field_name=None): - """Raises a ValidationError. - """ - field_name = field_name if field_name else self.name - raise ValidationError(message, errors=errors, field_name=field_name) - - def to_python(self, value): - """Convert a MongoDB-compatible type to a Python type. - """ - return value - - def to_mongo(self, value): - """Convert a Python type to a MongoDB-compatible type. - """ - return self.to_python(value) - - def prepare_query_value(self, op, value): - """Prepare a value that is being used in a query for PyMongo. - """ - return value - - def validate(self, value): - """Perform validation on a value. - """ - pass - - def _validate(self, value): - Document = _import_class('Document') - EmbeddedDocument = _import_class('EmbeddedDocument') - # check choices - if self.choices: - is_cls = isinstance(value, (Document, EmbeddedDocument)) - value_to_check = value.__class__ if is_cls else value - err_msg = 'an instance' if is_cls else 'one' - if isinstance(self.choices[0], (list, tuple)): - option_keys = [k for k, v in self.choices] - if value_to_check not in option_keys: - msg = ('Value must be %s of %s' % - (err_msg, unicode(option_keys))) - self.error(msg) - elif value_to_check not in self.choices: - msg = ('Value must be %s of %s' % - (err_msg, unicode(self.choices))) - self.error() - - # check validation argument - if self.validation is not None: - if callable(self.validation): - if not self.validation(value): - self.error('Value does not match custom validation method') - else: - raise ValueError('validation argument for "%s" must be a ' - 'callable.' % self.name) - - self.validate(value) - - -class ComplexBaseField(BaseField): - """Handles complex fields, such as lists / dictionaries. - - Allows for nesting of embedded documents inside complex types. - Handles the lazy dereferencing of a queryset by lazily dereferencing all - items in a list / dict rather than one at a time. - - .. versionadded:: 0.5 - """ - - field = None - __dereference = False - - def __get__(self, instance, owner): - """Descriptor to automatically dereference references. - """ - if instance is None: - # Document class being used rather than a document object - return self - - ReferenceField = _import_class('ReferenceField') - GenericReferenceField = _import_class('GenericReferenceField') - dereference = self.field is None or isinstance(self.field, - (GenericReferenceField, ReferenceField)) - if not self._dereference and instance._initialised and dereference: - instance._data[self.name] = self._dereference( - instance._data.get(self.name), max_depth=1, instance=instance, - name=self.name - ) - - value = super(ComplexBaseField, self).__get__(instance, owner) - - # Convert lists / values so we can watch for any changes on them - if (isinstance(value, (list, tuple)) and - not isinstance(value, BaseList)): - value = BaseList(value, instance, self.name) - instance._data[self.name] = value - elif isinstance(value, dict) and not isinstance(value, BaseDict): - value = BaseDict(value, instance, self.name) - instance._data[self.name] = value - - if (instance._initialised and isinstance(value, (BaseList, BaseDict)) - and not value._dereferenced): - value = self._dereference( - value, max_depth=1, instance=instance, name=self.name - ) - value._dereferenced = True - instance._data[self.name] = value - - return value - - def __set__(self, instance, value): - """Descriptor for assigning a value to a field in a document. - """ - instance._data[self.name] = value - instance._mark_as_changed(self.name) - - def to_python(self, value): - """Convert a MongoDB-compatible type to a Python type. - """ - Document = _import_class('Document') - - if isinstance(value, basestring): - return value - - if hasattr(value, 'to_python'): - return value.to_python() - - is_list = False - if not hasattr(value, 'items'): - try: - is_list = True - value = dict([(k, v) for k, v in enumerate(value)]) - except TypeError: # Not iterable return the value - return value - - if self.field: - value_dict = dict([(key, self.field.to_python(item)) - for key, item in value.items()]) - else: - value_dict = {} - for k, v in value.items(): - if isinstance(v, Document): - # We need the id from the saved object to create the DBRef - if v.pk is None: - self.error('You can only reference documents once they' - ' have been saved to the database') - collection = v._get_collection_name() - value_dict[k] = DBRef(collection, v.pk) - elif hasattr(v, 'to_python'): - value_dict[k] = v.to_python() - else: - value_dict[k] = self.to_python(v) - - if is_list: # Convert back to a list - return [v for k, v in sorted(value_dict.items(), - key=operator.itemgetter(0))] - return value_dict - - def to_mongo(self, value): - """Convert a Python type to a MongoDB-compatible type. - """ - Document = _import_class("Document") - - if isinstance(value, basestring): - return value - - if hasattr(value, 'to_mongo'): - return value.to_mongo() - - is_list = False - if not hasattr(value, 'items'): - try: - is_list = True - value = dict([(k, v) for k, v in enumerate(value)]) - except TypeError: # Not iterable return the value - return value - - if self.field: - value_dict = dict([(key, self.field.to_mongo(item)) - for key, item in value.items()]) - else: - value_dict = {} - for k, v in value.items(): - if isinstance(v, Document): - # We need the id from the saved object to create the DBRef - if v.pk is None: - self.error('You can only reference documents once they' - ' have been saved to the database') - - # If its a document that is not inheritable it won't have - # _types / _cls data so make it a generic reference allows - # us to dereference - meta = getattr(v, '_meta', {}) - allow_inheritance = ( - meta.get('allow_inheritance', ALLOW_INHERITANCE) - == False) - if allow_inheritance and not self.field: - GenericReferenceField = _import_class("GenericReferenceField") - value_dict[k] = GenericReferenceField().to_mongo(v) - else: - collection = v._get_collection_name() - value_dict[k] = DBRef(collection, v.pk) - elif hasattr(v, 'to_mongo'): - value_dict[k] = v.to_mongo() - else: - value_dict[k] = self.to_mongo(v) - - if is_list: # Convert back to a list - return [v for k, v in sorted(value_dict.items(), - key=operator.itemgetter(0))] - return value_dict - - def validate(self, value): - """If field is provided ensure the value is valid. - """ - errors = {} - if self.field: - if hasattr(value, 'iteritems') or hasattr(value, 'items'): - sequence = value.iteritems() - else: - sequence = enumerate(value) - for k, v in sequence: - try: - self.field._validate(v) - except ValidationError, error: - errors[k] = error.errors or error - except (ValueError, AssertionError), error: - errors[k] = error - - if errors: - field_class = self.field.__class__.__name__ - self.error('Invalid %s item (%s)' % (field_class, value), - errors=errors) - # Don't allow empty values if required - if self.required and not value: - self.error('Field is required and cannot be empty') - - def prepare_query_value(self, op, value): - return self.to_mongo(value) - - def lookup_member(self, member_name): - if self.field: - return self.field.lookup_member(member_name) - return None - - def _set_owner_document(self, owner_document): - if self.field: - self.field.owner_document = owner_document - self._owner_document = owner_document - - def _get_owner_document(self, owner_document): - self._owner_document = owner_document - - owner_document = property(_get_owner_document, _set_owner_document) - - @property - def _dereference(self,): - if not self.__dereference: - DeReference = _import_class("DeReference") - self.__dereference = DeReference() # Cached - return self.__dereference - - -class ObjectIdField(BaseField): - """An field wrapper around MongoDB's ObjectIds. - """ - - def to_python(self, value): - if not isinstance(value, ObjectId): - value = ObjectId(value) - return value - - def to_mongo(self, value): - if not isinstance(value, ObjectId): - try: - return ObjectId(unicode(value)) - except Exception, e: - # e.message attribute has been deprecated since Python 2.6 - self.error(unicode(e)) - return value - - def prepare_query_value(self, op, value): - return self.to_mongo(value) - - def validate(self, value): - try: - ObjectId(unicode(value)) - except: - self.error('Invalid Object ID') - - -class DocumentMetaclass(type): - """Metaclass for all documents. - """ - - def __new__(cls, name, bases, attrs): - flattened_bases = cls._get_bases(bases) - super_new = super(DocumentMetaclass, cls).__new__ - - # If a base class just call super - metaclass = attrs.get('my_metaclass') - if metaclass and issubclass(metaclass, DocumentMetaclass): - return super_new(cls, name, bases, attrs) - - attrs['_is_document'] = attrs.get('_is_document', False) - - # EmbeddedDocuments could have meta data for inheritance - if 'meta' in attrs: - attrs['_meta'] = attrs.pop('meta') - - # Handle document Fields - - # Merge all fields from subclasses - doc_fields = {} - for base in flattened_bases[::-1]: - if hasattr(base, '_fields'): - doc_fields.update(base._fields) - - # Standard object mixin - merge in any Fields - if not hasattr(base, '_meta'): - base_fields = {} - for attr_name, attr_value in base.__dict__.iteritems(): - if not isinstance(attr_value, BaseField): - continue - attr_value.name = attr_name - if not attr_value.db_field: - attr_value.db_field = attr_name - base_fields[attr_name] = attr_value - doc_fields.update(base_fields) - - # Discover any document fields - field_names = {} - for attr_name, attr_value in attrs.iteritems(): - if not isinstance(attr_value, BaseField): - continue - attr_value.name = attr_name - if not attr_value.db_field: - attr_value.db_field = attr_name - doc_fields[attr_name] = attr_value - - # Count names to ensure no db_field redefinitions - field_names[attr_value.db_field] = field_names.get( - attr_value.db_field, 0) + 1 - - # Ensure no duplicate db_fields - duplicate_db_fields = [k for k, v in field_names.items() if v > 1] - if duplicate_db_fields: - msg = ("Multiple db_fields defined for: %s " % - ", ".join(duplicate_db_fields)) - raise InvalidDocumentError(msg) - - # Set _fields and db_field maps - attrs['_fields'] = doc_fields - attrs['_db_field_map'] = dict([(k, getattr(v, 'db_field', k)) - for k, v in doc_fields.iteritems()]) - attrs['_reverse_db_field_map'] = dict( - (v, k) for k, v in attrs['_db_field_map'].iteritems()) - - # - # Set document hierarchy - # - superclasses = {} - class_name = [name] - for base in flattened_bases: - if (not getattr(base, '_is_base_cls', True) and - not getattr(base, '_meta', {}).get('abstract', True)): - # Collate heirarchy for _cls and _types - class_name.append(base.__name__) - - # Get superclasses from superclass - superclasses[base._class_name] = base - superclasses.update(base._superclasses) - - if hasattr(base, '_meta'): - # Warn if allow_inheritance isn't set and prevent - # inheritance of classes where inheritance is set to False - allow_inheritance = base._meta.get('allow_inheritance', - ALLOW_INHERITANCE) - if (not getattr(base, '_is_base_cls', True) - and allow_inheritance is None): - warnings.warn( - "%s uses inheritance, the default for " - "allow_inheritance is changing to off by default. " - "Please add it to the document meta." % name, - FutureWarning - ) - elif (allow_inheritance == False and - not base._meta.get('abstract')): - raise ValueError('Document %s may not be subclassed' % - base.__name__) - - attrs['_class_name'] = '.'.join(reversed(class_name)) - attrs['_superclasses'] = superclasses - - # Create the new_class - new_class = super_new(cls, name, bases, attrs) - - # Handle delete rules - Document, EmbeddedDocument, DictField = cls._import_classes() - for field in new_class._fields.itervalues(): - f = field - f.owner_document = new_class - delete_rule = getattr(f, 'reverse_delete_rule', DO_NOTHING) - if isinstance(f, ComplexBaseField) and hasattr(f, 'field'): - delete_rule = getattr(f.field, - 'reverse_delete_rule', - DO_NOTHING) - if isinstance(f, DictField) and delete_rule != DO_NOTHING: - msg = ("Reverse delete rules are not supported " - "for %s (field: %s)" % - (field.__class__.__name__, field.name)) - raise InvalidDocumentError(msg) - - f = field.field - - if delete_rule != DO_NOTHING: - if issubclass(new_class, EmbeddedDocument): - msg = ("Reverse delete rules are not supported for " - "EmbeddedDocuments (field: %s)" % field.name) - raise InvalidDocumentError(msg) - f.document_type.register_delete_rule(new_class, - field.name, delete_rule) - - if (field.name and hasattr(Document, field.name) and - EmbeddedDocument not in new_class.mro()): - msg = ("%s is a document method and not a valid " - "field name" % field.name) - raise InvalidDocumentError(msg) - - # Add class to the _document_registry - _document_registry[new_class._class_name] = new_class - - # In Python 2, User-defined methods objects have special read-only - # attributes 'im_func' and 'im_self' which contain the function obj - # and class instance object respectively. With Python 3 these special - # attributes have been replaced by __func__ and __self__. The Blinker - # module continues to use im_func and im_self, so the code below - # copies __func__ into im_func and __self__ into im_self for - # classmethod objects in Document derived classes. - if PY3: - for key, val in new_class.__dict__.items(): - if isinstance(val, classmethod): - f = val.__get__(new_class) - if hasattr(f, '__func__') and not hasattr(f, 'im_func'): - f.__dict__.update({'im_func': getattr(f, '__func__')}) - if hasattr(f, '__self__') and not hasattr(f, 'im_self'): - f.__dict__.update({'im_self': getattr(f, '__self__')}) - - return new_class - - def add_to_class(self, name, value): - setattr(self, name, value) - - @classmethod - def _get_bases(cls, bases): - if isinstance(bases, BasesTuple): - return bases - seen = [] - bases = cls.__get_bases(bases) - unique_bases = (b for b in bases if not (b in seen or seen.append(b))) - return BasesTuple(unique_bases) - - @classmethod - def __get_bases(cls, bases): - for base in bases: - if base is object: - continue - yield base - for child_base in cls.__get_bases(base.__bases__): - yield child_base - - @classmethod - def _import_classes(cls): - Document = _import_class('Document') - EmbeddedDocument = _import_class('EmbeddedDocument') - DictField = _import_class('DictField') - return (Document, EmbeddedDocument, DictField) - - -class TopLevelDocumentMetaclass(DocumentMetaclass): - """Metaclass for top-level documents (i.e. documents that have their own - collection in the database. - """ - - def __new__(cls, name, bases, attrs): - flattened_bases = cls._get_bases(bases) - super_new = super(TopLevelDocumentMetaclass, cls).__new__ - - # Set default _meta data if base class, otherwise get user defined meta - if (attrs.get('my_metaclass') == TopLevelDocumentMetaclass): - # defaults - attrs['_meta'] = { - 'abstract': True, - 'max_documents': None, - 'max_size': None, - 'ordering': [], # default ordering applied at runtime - 'indexes': [], # indexes to be ensured at runtime - 'id_field': None, - 'index_background': False, - 'index_drop_dups': False, - 'index_opts': None, - 'delete_rules': None, - 'allow_inheritance': None, - } - attrs['_is_base_cls'] = True - attrs['_meta'].update(attrs.get('meta', {})) - else: - attrs['_meta'] = attrs.get('meta', {}) - # Explictly set abstract to false unless set - attrs['_meta']['abstract'] = attrs['_meta'].get('abstract', False) - attrs['_is_base_cls'] = False - - # Set flag marking as document class - as opposed to an object mixin - attrs['_is_document'] = True - - # Ensure queryset_class is inherited - if 'objects' in attrs: - manager = attrs['objects'] - if hasattr(manager, 'queryset_class'): - attrs['_meta']['queryset_class'] = manager.queryset_class - - # Clean up top level meta - if 'meta' in attrs: - del(attrs['meta']) - - # Find the parent document class - parent_doc_cls = [b for b in flattened_bases - if b.__class__ == TopLevelDocumentMetaclass] - parent_doc_cls = None if not parent_doc_cls else parent_doc_cls[0] - - # Prevent classes setting collection different to their parents - # If parent wasn't an abstract class - if (parent_doc_cls and 'collection' in attrs.get('_meta', {}) - and not parent_doc_cls._meta.get('abstract', True)): - msg = "Trying to set a collection on a subclass (%s)" % name - warnings.warn(msg, SyntaxWarning) - del(attrs['_meta']['collection']) - - # Ensure abstract documents have abstract bases - if attrs.get('_is_base_cls') or attrs['_meta'].get('abstract'): - if (parent_doc_cls and - not parent_doc_cls._meta.get('abstract', False)): - msg = "Abstract document cannot have non-abstract base" - raise ValueError(msg) - return super_new(cls, name, bases, attrs) - - # Merge base class metas. - # Uses a special MetaDict that handles various merging rules - meta = MetaDict() - for base in flattened_bases[::-1]: - # Add any mixin metadata from plain objects - if hasattr(base, 'meta'): - meta.merge(base.meta) - elif hasattr(base, '_meta'): - meta.merge(base._meta) - - # Set collection in the meta if its callable - if (getattr(base, '_is_document', False) and - not base._meta.get('abstract')): - collection = meta.get('collection', None) - if callable(collection): - meta['collection'] = collection(base) - - meta.merge(attrs.get('_meta', {})) # Top level meta - - # Only simple classes (direct subclasses of Document) - # may set allow_inheritance to False - simple_class = all([b._meta.get('abstract') - for b in flattened_bases if hasattr(b, '_meta')]) - if (not simple_class and meta['allow_inheritance'] == False and - not meta['abstract']): - raise ValueError('Only direct subclasses of Document may set ' - '"allow_inheritance" to False') - - # Set default collection name - if 'collection' not in meta: - meta['collection'] = ''.join('_%s' % c if c.isupper() else c - for c in name).strip('_').lower() - attrs['_meta'] = meta - - # Call super and get the new class - new_class = super_new(cls, name, bases, attrs) - - meta = new_class._meta - - # Set index specifications - meta['index_specs'] = [QuerySet._build_index_spec(new_class, spec) - for spec in meta['indexes']] - unique_indexes = cls._unique_with_indexes(new_class) - new_class._meta['unique_indexes'] = unique_indexes - - # If collection is a callable - call it and set the value - collection = meta.get('collection') - if callable(collection): - new_class._meta['collection'] = collection(new_class) - - # Provide a default queryset unless one has been set - manager = attrs.get('objects', QuerySetManager()) - new_class.objects = manager - - # Validate the fields and set primary key if needed - for field_name, field in new_class._fields.iteritems(): - if field.primary_key: - # Ensure only one primary key is set - current_pk = new_class._meta.get('id_field') - if current_pk and current_pk != field_name: - raise ValueError('Cannot override primary key field') - - # Set primary key - if not current_pk: - new_class._meta['id_field'] = field_name - new_class.id = field - - # Set primary key if not defined by the document - if not new_class._meta.get('id_field'): - new_class._meta['id_field'] = 'id' - new_class._fields['id'] = ObjectIdField(db_field='_id') - new_class.id = new_class._fields['id'] - - # Merge in exceptions with parent hierarchy - exceptions_to_merge = (DoesNotExist, MultipleObjectsReturned) - module = attrs.get('__module__') - for exc in exceptions_to_merge: - name = exc.__name__ - parents = tuple(getattr(base, name) for base in flattened_bases - if hasattr(base, name)) or (exc,) - # Create new exception and set to new_class - exception = type(name, parents, {'__module__': module}) - setattr(new_class, name, exception) - - return new_class - - @classmethod - def _unique_with_indexes(cls, new_class, namespace=""): - """ - Find and set unique indexes - """ - unique_indexes = [] - for field_name, field in new_class._fields.items(): - # Generate a list of indexes needed by uniqueness constraints - if field.unique: - field.required = True - unique_fields = [field.db_field] - - # Add any unique_with fields to the back of the index spec - if field.unique_with: - if isinstance(field.unique_with, basestring): - field.unique_with = [field.unique_with] - - # Convert unique_with field names to real field names - unique_with = [] - for other_name in field.unique_with: - parts = other_name.split('.') - # Lookup real name - parts = QuerySet._lookup_field(new_class, parts) - name_parts = [part.db_field for part in parts] - unique_with.append('.'.join(name_parts)) - # Unique field should be required - parts[-1].required = True - unique_fields += unique_with - - # Add the new index to the list - index = [("%s%s" % (namespace, f), pymongo.ASCENDING) - for f in unique_fields] - unique_indexes.append(index) - - # Grab any embedded document field unique indexes - if (field.__class__.__name__ == "EmbeddedDocumentField" and - field.document_type != new_class): - field_namespace = "%s." % field_name - unique_indexes += cls._unique_with_indexes(field.document_type, - field_namespace) - - return unique_indexes - - -class MetaDict(dict): - """Custom dictionary for meta classes. - Handles the merging of set indexes - """ - _merge_options = ('indexes',) - - def merge(self, new_options): - for k, v in new_options.iteritems(): - if k in self._merge_options: - self[k] = self.get(k, []) + v - else: - self[k] = v - - -class BaseDocument(object): - - _dynamic = False - _created = True - _dynamic_lock = True - _initialised = False - - def __init__(self, **values): - signals.pre_init.send(self.__class__, document=self, values=values) - - self._data = {} - - # Assign default values to instance - for key, field in self._fields.iteritems(): - if self._db_field_map.get(key, key) in values: - continue - value = getattr(self, key, None) - setattr(self, key, value) - - # Set passed values after initialisation - if self._dynamic: - self._dynamic_fields = {} - dynamic_data = {} - for key, value in values.iteritems(): - if key in self._fields or key == '_id': - setattr(self, key, value) - elif self._dynamic: - dynamic_data[key] = value - else: - for key, value in values.iteritems(): - key = self._reverse_db_field_map.get(key, key) - setattr(self, key, value) - - # Set any get_fieldname_display methods - self.__set_field_display() - - if self._dynamic: - self._dynamic_lock = False - for key, value in dynamic_data.iteritems(): - setattr(self, key, value) - - # Flag initialised - self._initialised = True - signals.post_init.send(self.__class__, document=self) - - def __setattr__(self, name, value): - # Handle dynamic data only if an initialised dynamic document - if self._dynamic and not self._dynamic_lock: - - field = None - if not hasattr(self, name) and not name.startswith('_'): - DynamicField = _import_class("DynamicField") - field = DynamicField(db_field=name) - field.name = name - self._dynamic_fields[name] = field - - if not name.startswith('_'): - value = self.__expand_dynamic_values(name, value) - - # Handle marking data as changed - if name in self._dynamic_fields: - self._data[name] = value - if hasattr(self, '_changed_fields'): - self._mark_as_changed(name) - - if (self._is_document and not self._created and - name in self._meta.get('shard_key', tuple()) and - self._data.get(name) != value): - OperationError = _import_class('OperationError') - msg = "Shard Keys are immutable. Tried to update %s" % name - raise OperationError(msg) - - super(BaseDocument, self).__setattr__(name, value) - - def __expand_dynamic_values(self, name, value): - """expand any dynamic values to their correct types / values""" - if not isinstance(value, (dict, list, tuple)): - return value - - is_list = False - if not hasattr(value, 'items'): - is_list = True - value = dict([(k, v) for k, v in enumerate(value)]) - - if not is_list and '_cls' in value: - cls = get_document(value['_cls']) - return cls(**value) - - data = {} - for k, v in value.items(): - key = name if is_list else k - data[k] = self.__expand_dynamic_values(key, v) - - if is_list: # Convert back to a list - data_items = sorted(data.items(), key=operator.itemgetter(0)) - value = [v for k, v in data_items] - else: - value = data - - # Convert lists / values so we can watch for any changes on them - if (isinstance(value, (list, tuple)) and - not isinstance(value, BaseList)): - value = BaseList(value, self, name) - elif isinstance(value, dict) and not isinstance(value, BaseDict): - value = BaseDict(value, self, name) - - return value - - def validate(self): - """Ensure that all fields' values are valid and that required fields - are present. - """ - # Get a list of tuples of field names and their current values - fields = [(field, getattr(self, name)) - for name, field in self._fields.items()] - - # Ensure that each field is matched to a valid value - errors = {} - for field, value in fields: - if value is not None: - try: - field._validate(value) - except ValidationError, error: - errors[field.name] = error.errors or error - except (ValueError, AttributeError, AssertionError), error: - errors[field.name] = error - elif field.required: - errors[field.name] = ValidationError('Field is required', - field_name=field.name) - if errors: - raise ValidationError('ValidationError', errors=errors) - - def to_mongo(self): - """Return data dictionary ready for use with MongoDB. - """ - data = {} - for field_name, field in self._fields.items(): - value = getattr(self, field_name, None) - if value is not None: - data[field.db_field] = field.to_mongo(value) - # Only add _cls and _types if allow_inheritance is not False - if not (hasattr(self, '_meta') and - self._meta.get('allow_inheritance', ALLOW_INHERITANCE) == False): - data['_cls'] = self._class_name - data['_types'] = self._superclasses.keys() + [self._class_name] - if '_id' in data and data['_id'] is None: - del data['_id'] - - if not self._dynamic: - return data - - for name, field in self._dynamic_fields.items(): - data[name] = field.to_mongo(self._data.get(name, None)) - return data - - @classmethod - def _get_collection_name(cls): - """Returns the collection name for this class. - """ - return cls._meta.get('collection', None) - - @classmethod - def _from_son(cls, son): - """Create an instance of a Document (subclass) from a PyMongo SON. - """ - # get the class name from the document, falling back to the given - # class if unavailable - class_name = son.get('_cls', cls._class_name) - data = dict(("%s" % key, value) for key, value in son.items()) - if not UNICODE_KWARGS: - # python 2.6.4 and lower cannot handle unicode keys - # passed to class constructor example: cls(**data) - to_str_keys_recursive(data) - - if '_types' in data: - del data['_types'] - - if '_cls' in data: - del data['_cls'] - - # Return correct subclass for document type - if class_name != cls._class_name: - cls = get_document(class_name) - - changed_fields = [] - errors_dict = {} - - for field_name, field in cls._fields.items(): - if field.db_field in data: - value = data[field.db_field] - try: - data[field_name] = (value if value is None - else field.to_python(value)) - if field_name != field.db_field: - del data[field.db_field] - except (AttributeError, ValueError), e: - errors_dict[field_name] = e - elif field.default: - default = field.default - if callable(default): - default = default() - if isinstance(default, BaseDocument): - changed_fields.append(field_name) - - if errors_dict: - errors = "\n".join(["%s - %s" % (k, v) - for k, v in errors_dict.items()]) - msg = ("Invalid data to create a `%s` instance.\n%s" - % (cls._class_name, errors)) - raise InvalidDocumentError(msg) - - obj = cls(**data) - obj._changed_fields = changed_fields - obj._created = False - return obj - - def _mark_as_changed(self, key): - """Marks a key as explicitly changed by the user - """ - if not key: - return - key = self._db_field_map.get(key, key) - if (hasattr(self, '_changed_fields') and - key not in self._changed_fields): - self._changed_fields.append(key) - - def _get_changed_fields(self, key='', inspected=None): - """Returns a list of all fields that have explicitly been changed. - """ - EmbeddedDocument = _import_class("EmbeddedDocument") - DynamicEmbeddedDocument = _import_class("DynamicEmbeddedDocument") - _changed_fields = [] - _changed_fields += getattr(self, '_changed_fields', []) - - inspected = inspected or set() - if hasattr(self, 'id'): - if self.id in inspected: - return _changed_fields - inspected.add(self.id) - - field_list = self._fields.copy() - if self._dynamic: - field_list.update(self._dynamic_fields) - - for field_name in field_list: - - db_field_name = self._db_field_map.get(field_name, field_name) - key = '%s.' % db_field_name - field = self._data.get(field_name, None) - if hasattr(field, 'id'): - if field.id in inspected: - continue - inspected.add(field.id) - - if (isinstance(field, (EmbeddedDocument, DynamicEmbeddedDocument)) - and db_field_name not in _changed_fields): - # Find all embedded fields that have been changed - changed = field._get_changed_fields(key, inspected) - _changed_fields += ["%s%s" % (key, k) for k in changed if k] - elif (isinstance(field, (list, tuple, dict)) and - db_field_name not in _changed_fields): - # Loop list / dict fields as they contain documents - # Determine the iterator to use - if not hasattr(field, 'items'): - iterator = enumerate(field) - else: - iterator = field.iteritems() - for index, value in iterator: - if not hasattr(value, '_get_changed_fields'): - continue - list_key = "%s%s." % (key, index) - changed = value._get_changed_fields(list_key, inspected) - _changed_fields += ["%s%s" % (list_key, k) - for k in changed if k] - return _changed_fields - - def _delta(self): - """Returns the delta (set, unset) of the changes for a document. - Gets any values that have been explicitly changed. - """ - # Handles cases where not loaded from_son but has _id - doc = self.to_mongo() - set_fields = self._get_changed_fields() - set_data = {} - unset_data = {} - parts = [] - if hasattr(self, '_changed_fields'): - set_data = {} - # Fetch each set item from its path - for path in set_fields: - parts = path.split('.') - d = doc - new_path = [] - for p in parts: - if isinstance(d, DBRef): - break - elif p.isdigit(): - d = d[int(p)] - elif hasattr(d, 'get'): - d = d.get(p) - new_path.append(p) - path = '.'.join(new_path) - set_data[path] = d - else: - set_data = doc - if '_id' in set_data: - del(set_data['_id']) - - # Determine if any changed items were actually unset. - for path, value in set_data.items(): - if value or isinstance(value, bool): - continue - - # If we've set a value that ain't the default value dont unset it. - default = None - if (self._dynamic and len(parts) and - parts[0] in self._dynamic_fields): - del(set_data[path]) - unset_data[path] = 1 - continue - elif path in self._fields: - default = self._fields[path].default - else: # Perform a full lookup for lists / embedded lookups - d = self - parts = path.split('.') - db_field_name = parts.pop() - for p in parts: - if p.isdigit(): - d = d[int(p)] - elif (hasattr(d, '__getattribute__') and - not isinstance(d, dict)): - real_path = d._reverse_db_field_map.get(p, p) - d = getattr(d, real_path) - else: - d = d.get(p) - - if hasattr(d, '_fields'): - field_name = d._reverse_db_field_map.get(db_field_name, - db_field_name) - - if field_name in d._fields: - default = d._fields.get(field_name).default - else: - default = None - - if default is not None: - if callable(default): - default = default() - if default != value: - continue - - del(set_data[path]) - unset_data[path] = 1 - return set_data, unset_data - - @classmethod - def _geo_indices(cls, inspected=None): - inspected = inspected or [] - geo_indices = [] - inspected.append(cls) - - EmbeddedDocumentField = _import_class("EmbeddedDocumentField") - GeoPointField = _import_class("GeoPointField") - - for field in cls._fields.values(): - if not isinstance(field, (EmbeddedDocumentField, GeoPointField)): - continue - if hasattr(field, 'document_type'): - field_cls = field.document_type - if field_cls in inspected: - continue - if hasattr(field_cls, '_geo_indices'): - geo_indices += field_cls._geo_indices(inspected) - elif field._geo_index: - geo_indices.append(field) - return geo_indices - - def __getstate__(self): - removals = ("get_%s_display" % k - for k, v in self._fields.items() if v.choices) - for k in removals: - if hasattr(self, k): - delattr(self, k) - return self.__dict__ - - def __setstate__(self, __dict__): - self.__dict__ = __dict__ - self.__set_field_display() - - def __set_field_display(self): - """Dynamically set the display value for a field with choices""" - for attr_name, field in self._fields.items(): - if field.choices: - setattr(self, - 'get_%s_display' % attr_name, - partial(self.__get_field_display, field=field)) - - def __get_field_display(self, field): - """Returns the display value for a choice field""" - value = getattr(self, field.name) - if field.choices and isinstance(field.choices[0], (list, tuple)): - return dict(field.choices).get(value, value) - return value - - def __iter__(self): - return iter(self._fields) - - def __getitem__(self, name): - """Dictionary-style field access, return a field's value if present. - """ - try: - if name in self._fields: - return getattr(self, name) - except AttributeError: - pass - raise KeyError(name) - - def __setitem__(self, name, value): - """Dictionary-style field access, set a field's value. - """ - # Ensure that the field exists before settings its value - if name not in self._fields: - raise KeyError(name) - return setattr(self, name, value) - - def __contains__(self, name): - try: - val = getattr(self, name) - return val is not None - except AttributeError: - return False - - def __len__(self): - return len(self._data) - - def __repr__(self): - try: - u = self.__str__() - except (UnicodeEncodeError, UnicodeDecodeError): - u = '[Bad Unicode data]' - repr_type = type(u) - return repr_type('<%s: %s>' % (self.__class__.__name__, u)) - - def __str__(self): - if hasattr(self, '__unicode__'): - if PY3: - return self.__unicode__() - else: - return unicode(self).encode('utf-8') - return txt_type('%s object' % self.__class__.__name__) - - def __eq__(self, other): - if isinstance(other, self.__class__) and hasattr(other, 'id'): - if self.id == other.id: - return True - return False - - def __ne__(self, other): - return not self.__eq__(other) - - def __hash__(self): - if self.pk is None: - # For new object - return super(BaseDocument, self).__hash__() - else: - return hash(self.pk) - - -class BasesTuple(tuple): - """Special class to handle introspection of bases tuple in __new__""" - pass - - -class BaseList(list): - """A special list so we can watch any changes - """ - - _dereferenced = False - _instance = None - _name = None - - def __init__(self, list_items, instance, name): - self._instance = weakref.proxy(instance) - self._name = name - return super(BaseList, self).__init__(list_items) - - def __setitem__(self, *args, **kwargs): - self._mark_as_changed() - return super(BaseList, self).__setitem__(*args, **kwargs) - - def __delitem__(self, *args, **kwargs): - self._mark_as_changed() - return super(BaseList, self).__delitem__(*args, **kwargs) - - def __getstate__(self): - self.observer = None - return self - - def __setstate__(self, state): - self = state - return self - - def append(self, *args, **kwargs): - self._mark_as_changed() - return super(BaseList, self).append(*args, **kwargs) - - def extend(self, *args, **kwargs): - self._mark_as_changed() - return super(BaseList, self).extend(*args, **kwargs) - - def insert(self, *args, **kwargs): - self._mark_as_changed() - return super(BaseList, self).insert(*args, **kwargs) - - def pop(self, *args, **kwargs): - self._mark_as_changed() - return super(BaseList, self).pop(*args, **kwargs) - - def remove(self, *args, **kwargs): - self._mark_as_changed() - return super(BaseList, self).remove(*args, **kwargs) - - def reverse(self, *args, **kwargs): - self._mark_as_changed() - return super(BaseList, self).reverse(*args, **kwargs) - - def sort(self, *args, **kwargs): - self._mark_as_changed() - return super(BaseList, self).sort(*args, **kwargs) - - def _mark_as_changed(self): - if hasattr(self._instance, '_mark_as_changed'): - self._instance._mark_as_changed(self._name) - - -class BaseDict(dict): - """A special dict so we can watch any changes - """ - - _dereferenced = False - _instance = None - _name = None - - def __init__(self, dict_items, instance, name): - self._instance = weakref.proxy(instance) - self._name = name - return super(BaseDict, self).__init__(dict_items) - - def __setitem__(self, *args, **kwargs): - self._mark_as_changed() - return super(BaseDict, self).__setitem__(*args, **kwargs) - - def __delete__(self, *args, **kwargs): - self._mark_as_changed() - return super(BaseDict, self).__delete__(*args, **kwargs) - - def __delitem__(self, *args, **kwargs): - self._mark_as_changed() - return super(BaseDict, self).__delitem__(*args, **kwargs) - - def __delattr__(self, *args, **kwargs): - self._mark_as_changed() - return super(BaseDict, self).__delattr__(*args, **kwargs) - - def __getstate__(self): - self.instance = None - self._dereferenced = False - return self - - def __setstate__(self, state): - self = state - return self - - def clear(self, *args, **kwargs): - self._mark_as_changed() - return super(BaseDict, self).clear(*args, **kwargs) - - def pop(self, *args, **kwargs): - self._mark_as_changed() - return super(BaseDict, self).pop(*args, **kwargs) - - def popitem(self, *args, **kwargs): - self._mark_as_changed() - return super(BaseDict, self).popitem(*args, **kwargs) - - def update(self, *args, **kwargs): - self._mark_as_changed() - return super(BaseDict, self).update(*args, **kwargs) - - def _mark_as_changed(self): - if hasattr(self._instance, '_mark_as_changed'): - self._instance._mark_as_changed(self._name) - - -def _import_class(cls_name): - """Cached mechanism for imports""" - if cls_name in _class_registry: - return _class_registry.get(cls_name) - - doc_classes = ['Document', 'DynamicEmbeddedDocument', 'EmbeddedDocument'] - field_classes = ['DictField', 'DynamicField', 'EmbeddedDocumentField', - 'GenericReferenceField', 'GeoPointField', - 'ReferenceField'] - queryset_classes = ['OperationError'] - deref_classes = ['DeReference'] - - if cls_name in doc_classes: - from mongoengine import document as module - import_classes = doc_classes - elif cls_name in field_classes: - from mongoengine import fields as module - import_classes = field_classes - elif cls_name in queryset_classes: - from mongoengine import queryset as module - import_classes = queryset_classes - elif cls_name in deref_classes: - from mongoengine import dereference as module - import_classes = deref_classes - else: - raise ValueError('No import set for: ' % cls_name) - - for cls in import_classes: - _class_registry[cls] = getattr(module, cls) - - return _class_registry.get(cls_name) diff --git a/mongoengine/base/__init__.py b/mongoengine/base/__init__.py new file mode 100644 index 00000000..1d4a6ebe --- /dev/null +++ b/mongoengine/base/__init__.py @@ -0,0 +1,5 @@ +from .common import * +from .datastructures import * +from .document import * +from .fields import * +from .metaclasses import * diff --git a/mongoengine/base/common.py b/mongoengine/base/common.py new file mode 100644 index 00000000..648561be --- /dev/null +++ b/mongoengine/base/common.py @@ -0,0 +1,25 @@ +from mongoengine.errors import NotRegistered + +__all__ = ('ALLOW_INHERITANCE', 'get_document', '_document_registry') + +ALLOW_INHERITANCE = True + +_document_registry = {} + + +def get_document(name): + doc = _document_registry.get(name, None) + if not doc: + # Possible old style names + end = ".%s" % name + possible_match = [k for k in _document_registry.keys() + if k.endswith(end)] + if len(possible_match) == 1: + doc = _document_registry.get(possible_match.pop(), None) + if not doc: + raise NotRegistered(""" + `%s` has not been registered in the document registry. + Importing the document class automatically registers it, has it + been imported? + """.strip() % name) + return doc diff --git a/mongoengine/base/datastructures.py b/mongoengine/base/datastructures.py new file mode 100644 index 00000000..9a7620e6 --- /dev/null +++ b/mongoengine/base/datastructures.py @@ -0,0 +1,124 @@ +import weakref + +__all__ = ("BaseDict", "BaseList") + + +class BaseDict(dict): + """A special dict so we can watch any changes + """ + + _dereferenced = False + _instance = None + _name = None + + def __init__(self, dict_items, instance, name): + self._instance = weakref.proxy(instance) + self._name = name + return super(BaseDict, self).__init__(dict_items) + + def __setitem__(self, *args, **kwargs): + self._mark_as_changed() + return super(BaseDict, self).__setitem__(*args, **kwargs) + + def __delete__(self, *args, **kwargs): + self._mark_as_changed() + return super(BaseDict, self).__delete__(*args, **kwargs) + + def __delitem__(self, *args, **kwargs): + self._mark_as_changed() + return super(BaseDict, self).__delitem__(*args, **kwargs) + + def __delattr__(self, *args, **kwargs): + self._mark_as_changed() + return super(BaseDict, self).__delattr__(*args, **kwargs) + + def __getstate__(self): + self.instance = None + self._dereferenced = False + return self + + def __setstate__(self, state): + self = state + return self + + def clear(self, *args, **kwargs): + self._mark_as_changed() + return super(BaseDict, self).clear(*args, **kwargs) + + def pop(self, *args, **kwargs): + self._mark_as_changed() + return super(BaseDict, self).pop(*args, **kwargs) + + def popitem(self, *args, **kwargs): + self._mark_as_changed() + return super(BaseDict, self).popitem(*args, **kwargs) + + def update(self, *args, **kwargs): + self._mark_as_changed() + return super(BaseDict, self).update(*args, **kwargs) + + def _mark_as_changed(self): + if hasattr(self._instance, '_mark_as_changed'): + self._instance._mark_as_changed(self._name) + + +class BaseList(list): + """A special list so we can watch any changes + """ + + _dereferenced = False + _instance = None + _name = None + + def __init__(self, list_items, instance, name): + self._instance = weakref.proxy(instance) + self._name = name + return super(BaseList, self).__init__(list_items) + + def __setitem__(self, *args, **kwargs): + self._mark_as_changed() + return super(BaseList, self).__setitem__(*args, **kwargs) + + def __delitem__(self, *args, **kwargs): + self._mark_as_changed() + return super(BaseList, self).__delitem__(*args, **kwargs) + + def __getstate__(self): + self.observer = None + return self + + def __setstate__(self, state): + self = state + return self + + def append(self, *args, **kwargs): + self._mark_as_changed() + return super(BaseList, self).append(*args, **kwargs) + + def extend(self, *args, **kwargs): + self._mark_as_changed() + return super(BaseList, self).extend(*args, **kwargs) + + def insert(self, *args, **kwargs): + self._mark_as_changed() + return super(BaseList, self).insert(*args, **kwargs) + + def pop(self, *args, **kwargs): + self._mark_as_changed() + return super(BaseList, self).pop(*args, **kwargs) + + def remove(self, *args, **kwargs): + self._mark_as_changed() + return super(BaseList, self).remove(*args, **kwargs) + + def reverse(self, *args, **kwargs): + self._mark_as_changed() + return super(BaseList, self).reverse(*args, **kwargs) + + def sort(self, *args, **kwargs): + self._mark_as_changed() + return super(BaseList, self).sort(*args, **kwargs) + + def _mark_as_changed(self): + if hasattr(self._instance, '_mark_as_changed'): + self._instance._mark_as_changed(self._name) diff --git a/mongoengine/base/document.py b/mongoengine/base/document.py new file mode 100644 index 00000000..af97e1f2 --- /dev/null +++ b/mongoengine/base/document.py @@ -0,0 +1,644 @@ +import operator +from functools import partial + +import pymongo +from bson.dbref import DBRef + +from mongoengine import signals +from mongoengine.common import _import_class +from mongoengine.errors import (ValidationError, InvalidDocumentError, + LookUpError) +from mongoengine.python_support import (PY3, UNICODE_KWARGS, txt_type, + to_str_keys_recursive) + +from .common import get_document, ALLOW_INHERITANCE +from .datastructures import BaseDict, BaseList +from .fields import ComplexBaseField + +__all__ = ('BaseDocument', ) + + +class BaseDocument(object): + + _dynamic = False + _created = True + _dynamic_lock = True + _initialised = False + + def __init__(self, **values): + signals.pre_init.send(self.__class__, document=self, values=values) + + self._data = {} + + # Assign default values to instance + for key, field in self._fields.iteritems(): + if self._db_field_map.get(key, key) in values: + continue + value = getattr(self, key, None) + setattr(self, key, value) + + # Set passed values after initialisation + if self._dynamic: + self._dynamic_fields = {} + dynamic_data = {} + for key, value in values.iteritems(): + if key in self._fields or key == '_id': + setattr(self, key, value) + elif self._dynamic: + dynamic_data[key] = value + else: + for key, value in values.iteritems(): + key = self._reverse_db_field_map.get(key, key) + setattr(self, key, value) + + # Set any get_fieldname_display methods + self.__set_field_display() + + if self._dynamic: + self._dynamic_lock = False + for key, value in dynamic_data.iteritems(): + setattr(self, key, value) + + # Flag initialised + self._initialised = True + signals.post_init.send(self.__class__, document=self) + + def __setattr__(self, name, value): + # Handle dynamic data only if an initialised dynamic document + if self._dynamic and not self._dynamic_lock: + + field = None + if not hasattr(self, name) and not name.startswith('_'): + DynamicField = _import_class("DynamicField") + field = DynamicField(db_field=name) + field.name = name + self._dynamic_fields[name] = field + + if not name.startswith('_'): + value = self.__expand_dynamic_values(name, value) + + # Handle marking data as changed + if name in self._dynamic_fields: + self._data[name] = value + if hasattr(self, '_changed_fields'): + self._mark_as_changed(name) + + if (self._is_document and not self._created and + name in self._meta.get('shard_key', tuple()) and + self._data.get(name) != value): + OperationError = _import_class('OperationError') + msg = "Shard Keys are immutable. Tried to update %s" % name + raise OperationError(msg) + + super(BaseDocument, self).__setattr__(name, value) + + def __getstate__(self): + removals = ("get_%s_display" % k + for k, v in self._fields.items() if v.choices) + for k in removals: + if hasattr(self, k): + delattr(self, k) + return self.__dict__ + + def __setstate__(self, __dict__): + self.__dict__ = __dict__ + self.__set_field_display() + + def __iter__(self): + return iter(self._fields) + + def __getitem__(self, name): + """Dictionary-style field access, return a field's value if present. + """ + try: + if name in self._fields: + return getattr(self, name) + except AttributeError: + pass + raise KeyError(name) + + def __setitem__(self, name, value): + """Dictionary-style field access, set a field's value. + """ + # Ensure that the field exists before settings its value + if name not in self._fields: + raise KeyError(name) + return setattr(self, name, value) + + def __contains__(self, name): + try: + val = getattr(self, name) + return val is not None + except AttributeError: + return False + + def __len__(self): + return len(self._data) + + def __repr__(self): + try: + u = self.__str__() + except (UnicodeEncodeError, UnicodeDecodeError): + u = '[Bad Unicode data]' + repr_type = type(u) + return repr_type('<%s: %s>' % (self.__class__.__name__, u)) + + def __str__(self): + if hasattr(self, '__unicode__'): + if PY3: + return self.__unicode__() + else: + return unicode(self).encode('utf-8') + return txt_type('%s object' % self.__class__.__name__) + + def __eq__(self, other): + if isinstance(other, self.__class__) and hasattr(other, 'id'): + if self.id == other.id: + return True + return False + + def __ne__(self, other): + return not self.__eq__(other) + + def __hash__(self): + if self.pk is None: + # For new object + return super(BaseDocument, self).__hash__() + else: + return hash(self.pk) + + def to_mongo(self): + """Return data dictionary ready for use with MongoDB. + """ + data = {} + for field_name, field in self._fields.items(): + value = getattr(self, field_name, None) + if value is not None: + data[field.db_field] = field.to_mongo(value) + # Only add _cls if allow_inheritance is not False + if not (hasattr(self, '_meta') and + self._meta.get('allow_inheritance', ALLOW_INHERITANCE) == False): + data['_cls'] = self._class_name + if '_id' in data and data['_id'] is None: + del data['_id'] + + if not self._dynamic: + return data + + for name, field in self._dynamic_fields.items(): + data[name] = field.to_mongo(self._data.get(name, None)) + return data + + def validate(self): + """Ensure that all fields' values are valid and that required fields + are present. + """ + # Get a list of tuples of field names and their current values + fields = [(field, getattr(self, name)) + for name, field in self._fields.items()] + + # Ensure that each field is matched to a valid value + errors = {} + for field, value in fields: + if value is not None: + try: + field._validate(value) + except ValidationError, error: + errors[field.name] = error.errors or error + except (ValueError, AttributeError, AssertionError), error: + errors[field.name] = error + elif field.required: + errors[field.name] = ValidationError('Field is required', + field_name=field.name) + if errors: + raise ValidationError('ValidationError', errors=errors) + + def __expand_dynamic_values(self, name, value): + """expand any dynamic values to their correct types / values""" + if not isinstance(value, (dict, list, tuple)): + return value + + is_list = False + if not hasattr(value, 'items'): + is_list = True + value = dict([(k, v) for k, v in enumerate(value)]) + + if not is_list and '_cls' in value: + cls = get_document(value['_cls']) + return cls(**value) + + data = {} + for k, v in value.items(): + key = name if is_list else k + data[k] = self.__expand_dynamic_values(key, v) + + if is_list: # Convert back to a list + data_items = sorted(data.items(), key=operator.itemgetter(0)) + value = [v for k, v in data_items] + else: + value = data + + # Convert lists / values so we can watch for any changes on them + if (isinstance(value, (list, tuple)) and + not isinstance(value, BaseList)): + value = BaseList(value, self, name) + elif isinstance(value, dict) and not isinstance(value, BaseDict): + value = BaseDict(value, self, name) + + return value + + def _mark_as_changed(self, key): + """Marks a key as explicitly changed by the user + """ + if not key: + return + key = self._db_field_map.get(key, key) + if (hasattr(self, '_changed_fields') and + key not in self._changed_fields): + self._changed_fields.append(key) + + def _get_changed_fields(self, key='', inspected=None): + """Returns a list of all fields that have explicitly been changed. + """ + EmbeddedDocument = _import_class("EmbeddedDocument") + DynamicEmbeddedDocument = _import_class("DynamicEmbeddedDocument") + _changed_fields = [] + _changed_fields += getattr(self, '_changed_fields', []) + + inspected = inspected or set() + if hasattr(self, 'id'): + if self.id in inspected: + return _changed_fields + inspected.add(self.id) + + field_list = self._fields.copy() + if self._dynamic: + field_list.update(self._dynamic_fields) + + for field_name in field_list: + + db_field_name = self._db_field_map.get(field_name, field_name) + key = '%s.' % db_field_name + field = self._data.get(field_name, None) + if hasattr(field, 'id'): + if field.id in inspected: + continue + inspected.add(field.id) + + if (isinstance(field, (EmbeddedDocument, DynamicEmbeddedDocument)) + and db_field_name not in _changed_fields): + # Find all embedded fields that have been changed + changed = field._get_changed_fields(key, inspected) + _changed_fields += ["%s%s" % (key, k) for k in changed if k] + elif (isinstance(field, (list, tuple, dict)) and + db_field_name not in _changed_fields): + # Loop list / dict fields as they contain documents + # Determine the iterator to use + if not hasattr(field, 'items'): + iterator = enumerate(field) + else: + iterator = field.iteritems() + for index, value in iterator: + if not hasattr(value, '_get_changed_fields'): + continue + list_key = "%s%s." % (key, index) + changed = value._get_changed_fields(list_key, inspected) + _changed_fields += ["%s%s" % (list_key, k) + for k in changed if k] + return _changed_fields + + def _delta(self): + """Returns the delta (set, unset) of the changes for a document. + Gets any values that have been explicitly changed. + """ + # Handles cases where not loaded from_son but has _id + doc = self.to_mongo() + set_fields = self._get_changed_fields() + set_data = {} + unset_data = {} + parts = [] + if hasattr(self, '_changed_fields'): + set_data = {} + # Fetch each set item from its path + for path in set_fields: + parts = path.split('.') + d = doc + new_path = [] + for p in parts: + if isinstance(d, DBRef): + break + elif p.isdigit(): + d = d[int(p)] + elif hasattr(d, 'get'): + d = d.get(p) + new_path.append(p) + path = '.'.join(new_path) + set_data[path] = d + else: + set_data = doc + if '_id' in set_data: + del(set_data['_id']) + + # Determine if any changed items were actually unset. + for path, value in set_data.items(): + if value or isinstance(value, bool): + continue + + # If we've set a value that ain't the default value dont unset it. + default = None + if (self._dynamic and len(parts) and + parts[0] in self._dynamic_fields): + del(set_data[path]) + unset_data[path] = 1 + continue + elif path in self._fields: + default = self._fields[path].default + else: # Perform a full lookup for lists / embedded lookups + d = self + parts = path.split('.') + db_field_name = parts.pop() + for p in parts: + if p.isdigit(): + d = d[int(p)] + elif (hasattr(d, '__getattribute__') and + not isinstance(d, dict)): + real_path = d._reverse_db_field_map.get(p, p) + d = getattr(d, real_path) + else: + d = d.get(p) + + if hasattr(d, '_fields'): + field_name = d._reverse_db_field_map.get(db_field_name, + db_field_name) + + if field_name in d._fields: + default = d._fields.get(field_name).default + else: + default = None + + if default is not None: + if callable(default): + default = default() + if default != value: + continue + + del(set_data[path]) + unset_data[path] = 1 + return set_data, unset_data + + @classmethod + def _get_collection_name(cls): + """Returns the collection name for this class. + """ + return cls._meta.get('collection', None) + + @classmethod + def _from_son(cls, son): + """Create an instance of a Document (subclass) from a PyMongo SON. + """ + # get the class name from the document, falling back to the given + # class if unavailable + class_name = son.get('_cls', cls._class_name) + data = dict(("%s" % key, value) for key, value in son.items()) + if not UNICODE_KWARGS: + # python 2.6.4 and lower cannot handle unicode keys + # passed to class constructor example: cls(**data) + to_str_keys_recursive(data) + + if '_cls' in data: + del data['_cls'] + + # Return correct subclass for document type + if class_name != cls._class_name: + cls = get_document(class_name) + + changed_fields = [] + errors_dict = {} + + for field_name, field in cls._fields.items(): + if field.db_field in data: + value = data[field.db_field] + try: + data[field_name] = (value if value is None + else field.to_python(value)) + if field_name != field.db_field: + del data[field.db_field] + except (AttributeError, ValueError), e: + errors_dict[field_name] = e + elif field.default: + default = field.default + if callable(default): + default = default() + if isinstance(default, BaseDocument): + changed_fields.append(field_name) + + if errors_dict: + errors = "\n".join(["%s - %s" % (k, v) + for k, v in errors_dict.items()]) + msg = ("Invalid data to create a `%s` instance.\n%s" + % (cls._class_name, errors)) + raise InvalidDocumentError(msg) + + obj = cls(**data) + obj._changed_fields = changed_fields + obj._created = False + return obj + + @classmethod + def _build_index_spec(cls, spec): + """Build a PyMongo index spec from a MongoEngine index spec. + """ + if isinstance(spec, basestring): + spec = {'fields': [spec]} + elif isinstance(spec, (list, tuple)): + spec = {'fields': list(spec)} + elif isinstance(spec, dict): + spec = dict(spec) + + index_list = [] + direction = None + + # Check to see if we need to include _cls + allow_inheritance = cls._meta.get('allow_inheritance', + ALLOW_INHERITANCE) != False + include_cls = allow_inheritance and not spec.get('sparse', False) + + for key in spec['fields']: + # If inherited spec continue + if isinstance(key, (list, tuple)): + continue + + # ASCENDING from +, + # DESCENDING from - + # GEO2D from * + direction = pymongo.ASCENDING + if key.startswith("-"): + direction = pymongo.DESCENDING + elif key.startswith("*"): + direction = pymongo.GEO2D + if key.startswith(("+", "-", "*")): + key = key[1:] + + # Use real field name, do it manually because we need field + # objects for the next part (list field checking) + parts = key.split('.') + if parts in (['pk'], ['id'], ['_id']): + key = '_id' + fields = [] + else: + fields = cls._lookup_field(parts) + parts = [field if field == '_id' else field.db_field + for field in fields] + key = '.'.join(parts) + index_list.append((key, direction)) + + # Don't add cls to a geo index + if include_cls and direction is not pymongo.GEO2D: + index_list.insert(0, ('_cls', 1)) + + spec['fields'] = index_list + if spec.get('sparse', False) and len(spec['fields']) > 1: + raise ValueError( + 'Sparse indexes can only have one field in them. ' + 'See https://jira.mongodb.org/browse/SERVER-2193') + + return spec + + @classmethod + def _unique_with_indexes(cls, namespace=""): + """ + Find and set unique indexes + """ + unique_indexes = [] + for field_name, field in cls._fields.items(): + # Generate a list of indexes needed by uniqueness constraints + if field.unique: + field.required = True + unique_fields = [field.db_field] + + # Add any unique_with fields to the back of the index spec + if field.unique_with: + if isinstance(field.unique_with, basestring): + field.unique_with = [field.unique_with] + + # Convert unique_with field names to real field names + unique_with = [] + for other_name in field.unique_with: + parts = other_name.split('.') + # Lookup real name + parts = cls._lookup_field(parts) + name_parts = [part.db_field for part in parts] + unique_with.append('.'.join(name_parts)) + # Unique field should be required + parts[-1].required = True + unique_fields += unique_with + + # Add the new index to the list + index = [("%s%s" % (namespace, f), pymongo.ASCENDING) + for f in unique_fields] + unique_indexes.append(index) + + # Grab any embedded document field unique indexes + if (field.__class__.__name__ == "EmbeddedDocumentField" and + field.document_type != cls): + field_namespace = "%s." % field_name + doc_cls = field.document_type + unique_indexes += doc_cls._unique_with_indexes(field_namespace) + + return unique_indexes + + @classmethod + def _lookup_field(cls, parts): + """Lookup a field based on its attribute and return a list containing + the field's parents and the field. + """ + if not isinstance(parts, (list, tuple)): + parts = [parts] + fields = [] + field = None + + for field_name in parts: + # Handle ListField indexing: + if field_name.isdigit(): + new_field = field.field + fields.append(field_name) + continue + + if field is None: + # Look up first field from the document + if field_name == 'pk': + # Deal with "primary key" alias + field_name = cls._meta['id_field'] + if field_name in cls._fields: + field = cls._fields[field_name] + elif cls._dynamic: + DynamicField = _import_class('DynamicField') + field = DynamicField(db_field=field_name) + else: + raise LookUpError('Cannot resolve field "%s"' + % field_name) + else: + ReferenceField = _import_class('ReferenceField') + GenericReferenceField = _import_class('GenericReferenceField') + if isinstance(field, (ReferenceField, GenericReferenceField)): + raise LookUpError('Cannot perform join in mongoDB: %s' % + '__'.join(parts)) + if hasattr(getattr(field, 'field', None), 'lookup_member'): + new_field = field.field.lookup_member(field_name) + else: + # Look up subfield on the previous field + new_field = field.lookup_member(field_name) + if not new_field and isinstance(field, ComplexBaseField): + fields.append(field_name) + continue + elif not new_field: + raise LookUpError('Cannot resolve field "%s"' + % field_name) + field = new_field # update field to the new field type + fields.append(field) + return fields + + @classmethod + def _translate_field_name(cls, field, sep='.'): + """Translate a field attribute name to a database field name. + """ + parts = field.split(sep) + parts = [f.db_field for f in cls._lookup_field(parts)] + return '.'.join(parts) + + @classmethod + def _geo_indices(cls, inspected=None): + inspected = inspected or [] + geo_indices = [] + inspected.append(cls) + + EmbeddedDocumentField = _import_class("EmbeddedDocumentField") + GeoPointField = _import_class("GeoPointField") + + for field in cls._fields.values(): + if not isinstance(field, (EmbeddedDocumentField, GeoPointField)): + continue + if hasattr(field, 'document_type'): + field_cls = field.document_type + if field_cls in inspected: + continue + if hasattr(field_cls, '_geo_indices'): + geo_indices += field_cls._geo_indices(inspected) + elif field._geo_index: + geo_indices.append(field) + return geo_indices + + def __set_field_display(self): + """Dynamically set the display value for a field with choices""" + for attr_name, field in self._fields.items(): + if field.choices: + setattr(self, + 'get_%s_display' % attr_name, + partial(self.__get_field_display, field=field)) + + def __get_field_display(self, field): + """Returns the display value for a choice field""" + value = getattr(self, field.name) + if field.choices and isinstance(field.choices[0], (list, tuple)): + return dict(field.choices).get(value, value) + return value diff --git a/mongoengine/base/fields.py b/mongoengine/base/fields.py new file mode 100644 index 00000000..44f5e131 --- /dev/null +++ b/mongoengine/base/fields.py @@ -0,0 +1,371 @@ +import operator +import warnings + +from bson import DBRef, ObjectId + +from mongoengine.common import _import_class +from mongoengine.errors import ValidationError + +from .common import ALLOW_INHERITANCE +from .datastructures import BaseDict, BaseList + +__all__ = ("BaseField", "ComplexBaseField", "ObjectIdField") + + +class BaseField(object): + """A base class for fields in a MongoDB document. Instances of this class + may be added to subclasses of `Document` to define a document's schema. + + .. versionchanged:: 0.5 - added verbose and help text + """ + + name = None + _geo_index = False + + # These track each time a Field instance is created. Used to retain order. + # The auto_creation_counter is used for fields that MongoEngine implicitly + # creates, creation_counter is used for all user-specified fields. + creation_counter = 0 + auto_creation_counter = -1 + + def __init__(self, db_field=None, name=None, required=False, default=None, + unique=False, unique_with=None, primary_key=False, + validation=None, choices=None, verbose_name=None, + help_text=None): + self.db_field = (db_field or name) if not primary_key else '_id' + if name: + msg = "Fields' 'name' attribute deprecated in favour of 'db_field'" + warnings.warn(msg, DeprecationWarning) + self.name = None + self.required = required or primary_key + self.default = default + self.unique = bool(unique or unique_with) + self.unique_with = unique_with + self.primary_key = primary_key + self.validation = validation + self.choices = choices + self.verbose_name = verbose_name + self.help_text = help_text + + # Adjust the appropriate creation counter, and save our local copy. + if self.db_field == '_id': + self.creation_counter = BaseField.auto_creation_counter + BaseField.auto_creation_counter -= 1 + else: + self.creation_counter = BaseField.creation_counter + BaseField.creation_counter += 1 + + def __get__(self, instance, owner): + """Descriptor for retrieving a value from a field in a document. Do + any necessary conversion between Python and MongoDB types. + """ + if instance is None: + # Document class being used rather than a document object + return self + + # Get value from document instance if available, if not use default + value = instance._data.get(self.name) + + if value is None: + value = self.default + # Allow callable default values + if callable(value): + value = value() + + return value + + def __set__(self, instance, value): + """Descriptor for assigning a value to a field in a document. + """ + instance._data[self.name] = value + if instance._initialised: + instance._mark_as_changed(self.name) + + def error(self, message="", errors=None, field_name=None): + """Raises a ValidationError. + """ + field_name = field_name if field_name else self.name + raise ValidationError(message, errors=errors, field_name=field_name) + + def to_python(self, value): + """Convert a MongoDB-compatible type to a Python type. + """ + return value + + def to_mongo(self, value): + """Convert a Python type to a MongoDB-compatible type. + """ + return self.to_python(value) + + def prepare_query_value(self, op, value): + """Prepare a value that is being used in a query for PyMongo. + """ + return value + + def validate(self, value): + """Perform validation on a value. + """ + pass + + def _validate(self, value): + Document = _import_class('Document') + EmbeddedDocument = _import_class('EmbeddedDocument') + # check choices + if self.choices: + is_cls = isinstance(value, (Document, EmbeddedDocument)) + value_to_check = value.__class__ if is_cls else value + err_msg = 'an instance' if is_cls else 'one' + if isinstance(self.choices[0], (list, tuple)): + option_keys = [k for k, v in self.choices] + if value_to_check not in option_keys: + msg = ('Value must be %s of %s' % + (err_msg, unicode(option_keys))) + self.error(msg) + elif value_to_check not in self.choices: + msg = ('Value must be %s of %s' % + (err_msg, unicode(self.choices))) + self.error() + + # check validation argument + if self.validation is not None: + if callable(self.validation): + if not self.validation(value): + self.error('Value does not match custom validation method') + else: + raise ValueError('validation argument for "%s" must be a ' + 'callable.' % self.name) + + self.validate(value) + + +class ComplexBaseField(BaseField): + """Handles complex fields, such as lists / dictionaries. + + Allows for nesting of embedded documents inside complex types. + Handles the lazy dereferencing of a queryset by lazily dereferencing all + items in a list / dict rather than one at a time. + + .. versionadded:: 0.5 + """ + + field = None + __dereference = False + + def __get__(self, instance, owner): + """Descriptor to automatically dereference references. + """ + if instance is None: + # Document class being used rather than a document object + return self + + ReferenceField = _import_class('ReferenceField') + GenericReferenceField = _import_class('GenericReferenceField') + dereference = self.field is None or isinstance(self.field, + (GenericReferenceField, ReferenceField)) + if not self._dereference and instance._initialised and dereference: + instance._data[self.name] = self._dereference( + instance._data.get(self.name), max_depth=1, instance=instance, + name=self.name + ) + + value = super(ComplexBaseField, self).__get__(instance, owner) + + # Convert lists / values so we can watch for any changes on them + if (isinstance(value, (list, tuple)) and + not isinstance(value, BaseList)): + value = BaseList(value, instance, self.name) + instance._data[self.name] = value + elif isinstance(value, dict) and not isinstance(value, BaseDict): + value = BaseDict(value, instance, self.name) + instance._data[self.name] = value + + if (instance._initialised and isinstance(value, (BaseList, BaseDict)) + and not value._dereferenced): + value = self._dereference( + value, max_depth=1, instance=instance, name=self.name + ) + value._dereferenced = True + instance._data[self.name] = value + + return value + + def __set__(self, instance, value): + """Descriptor for assigning a value to a field in a document. + """ + instance._data[self.name] = value + instance._mark_as_changed(self.name) + + def to_python(self, value): + """Convert a MongoDB-compatible type to a Python type. + """ + Document = _import_class('Document') + + if isinstance(value, basestring): + return value + + if hasattr(value, 'to_python'): + return value.to_python() + + is_list = False + if not hasattr(value, 'items'): + try: + is_list = True + value = dict([(k, v) for k, v in enumerate(value)]) + except TypeError: # Not iterable return the value + return value + + if self.field: + value_dict = dict([(key, self.field.to_python(item)) + for key, item in value.items()]) + else: + value_dict = {} + for k, v in value.items(): + if isinstance(v, Document): + # We need the id from the saved object to create the DBRef + if v.pk is None: + self.error('You can only reference documents once they' + ' have been saved to the database') + collection = v._get_collection_name() + value_dict[k] = DBRef(collection, v.pk) + elif hasattr(v, 'to_python'): + value_dict[k] = v.to_python() + else: + value_dict[k] = self.to_python(v) + + if is_list: # Convert back to a list + return [v for k, v in sorted(value_dict.items(), + key=operator.itemgetter(0))] + return value_dict + + def to_mongo(self, value): + """Convert a Python type to a MongoDB-compatible type. + """ + Document = _import_class("Document") + + if isinstance(value, basestring): + return value + + if hasattr(value, 'to_mongo'): + return value.to_mongo() + + is_list = False + if not hasattr(value, 'items'): + try: + is_list = True + value = dict([(k, v) for k, v in enumerate(value)]) + except TypeError: # Not iterable return the value + return value + + if self.field: + value_dict = dict([(key, self.field.to_mongo(item)) + for key, item in value.items()]) + else: + value_dict = {} + for k, v in value.items(): + if isinstance(v, Document): + # We need the id from the saved object to create the DBRef + if v.pk is None: + self.error('You can only reference documents once they' + ' have been saved to the database') + + # If its a document that is not inheritable it won't have + # any _cls data so make it a generic reference allows + # us to dereference + meta = getattr(v, '_meta', {}) + allow_inheritance = ( + meta.get('allow_inheritance', ALLOW_INHERITANCE) + == False) + if allow_inheritance and not self.field: + GenericReferenceField = _import_class( + "GenericReferenceField") + value_dict[k] = GenericReferenceField().to_mongo(v) + else: + collection = v._get_collection_name() + value_dict[k] = DBRef(collection, v.pk) + elif hasattr(v, 'to_mongo'): + value_dict[k] = v.to_mongo() + else: + value_dict[k] = self.to_mongo(v) + + if is_list: # Convert back to a list + return [v for k, v in sorted(value_dict.items(), + key=operator.itemgetter(0))] + return value_dict + + def validate(self, value): + """If field is provided ensure the value is valid. + """ + errors = {} + if self.field: + if hasattr(value, 'iteritems') or hasattr(value, 'items'): + sequence = value.iteritems() + else: + sequence = enumerate(value) + for k, v in sequence: + try: + self.field._validate(v) + except ValidationError, error: + errors[k] = error.errors or error + except (ValueError, AssertionError), error: + errors[k] = error + + if errors: + field_class = self.field.__class__.__name__ + self.error('Invalid %s item (%s)' % (field_class, value), + errors=errors) + # Don't allow empty values if required + if self.required and not value: + self.error('Field is required and cannot be empty') + + def prepare_query_value(self, op, value): + return self.to_mongo(value) + + def lookup_member(self, member_name): + if self.field: + return self.field.lookup_member(member_name) + return None + + def _set_owner_document(self, owner_document): + if self.field: + self.field.owner_document = owner_document + self._owner_document = owner_document + + def _get_owner_document(self, owner_document): + self._owner_document = owner_document + + owner_document = property(_get_owner_document, _set_owner_document) + + @property + def _dereference(self,): + if not self.__dereference: + DeReference = _import_class("DeReference") + self.__dereference = DeReference() # Cached + return self.__dereference + + +class ObjectIdField(BaseField): + """A field wrapper around MongoDB's ObjectIds. + """ + + def to_python(self, value): + if not isinstance(value, ObjectId): + value = ObjectId(value) + return value + + def to_mongo(self, value): + if not isinstance(value, ObjectId): + try: + return ObjectId(unicode(value)) + except Exception, e: + # e.message attribute has been deprecated since Python 2.6 + self.error(unicode(e)) + return value + + def prepare_query_value(self, op, value): + return self.to_mongo(value) + + def validate(self, value): + try: + ObjectId(unicode(value)) + except: + self.error('Invalid Object ID') diff --git a/mongoengine/base/metaclasses.py b/mongoengine/base/metaclasses.py new file mode 100644 index 00000000..f87b03e4 --- /dev/null +++ b/mongoengine/base/metaclasses.py @@ -0,0 +1,388 @@ +import warnings + +import pymongo + +from mongoengine.common import _import_class +from mongoengine.errors import InvalidDocumentError +from mongoengine.python_support import PY3 +from mongoengine.queryset import (DO_NOTHING, DoesNotExist, + MultipleObjectsReturned, + QuerySet, QuerySetManager) + +from .common import _document_registry, ALLOW_INHERITANCE +from .fields import BaseField, ComplexBaseField, ObjectIdField + +__all__ = ('DocumentMetaclass', 'TopLevelDocumentMetaclass') + + +class DocumentMetaclass(type): + """Metaclass for all documents. + """ + + def __new__(cls, name, bases, attrs): + flattened_bases = cls._get_bases(bases) + super_new = super(DocumentMetaclass, cls).__new__ + + # If a base class just call super + metaclass = attrs.get('my_metaclass') + if metaclass and issubclass(metaclass, DocumentMetaclass): + return super_new(cls, name, bases, attrs) + + attrs['_is_document'] = attrs.get('_is_document', False) + + # EmbeddedDocuments could have meta data for inheritance + if 'meta' in attrs: + attrs['_meta'] = attrs.pop('meta') + + # Handle document Fields + + # Merge all fields from subclasses + doc_fields = {} + for base in flattened_bases[::-1]: + if hasattr(base, '_fields'): + doc_fields.update(base._fields) + + # Standard object mixin - merge in any Fields + if not hasattr(base, '_meta'): + base_fields = {} + for attr_name, attr_value in base.__dict__.iteritems(): + if not isinstance(attr_value, BaseField): + continue + attr_value.name = attr_name + if not attr_value.db_field: + attr_value.db_field = attr_name + base_fields[attr_name] = attr_value + doc_fields.update(base_fields) + + # Discover any document fields + field_names = {} + for attr_name, attr_value in attrs.iteritems(): + if not isinstance(attr_value, BaseField): + continue + attr_value.name = attr_name + if not attr_value.db_field: + attr_value.db_field = attr_name + doc_fields[attr_name] = attr_value + + # Count names to ensure no db_field redefinitions + field_names[attr_value.db_field] = field_names.get( + attr_value.db_field, 0) + 1 + + # Ensure no duplicate db_fields + duplicate_db_fields = [k for k, v in field_names.items() if v > 1] + if duplicate_db_fields: + msg = ("Multiple db_fields defined for: %s " % + ", ".join(duplicate_db_fields)) + raise InvalidDocumentError(msg) + + # Set _fields and db_field maps + attrs['_fields'] = doc_fields + attrs['_db_field_map'] = dict([(k, getattr(v, 'db_field', k)) + for k, v in doc_fields.iteritems()]) + attrs['_reverse_db_field_map'] = dict( + (v, k) for k, v in attrs['_db_field_map'].iteritems()) + + # + # Set document hierarchy + # + superclasses = () + class_name = [name] + for base in flattened_bases: + if (not getattr(base, '_is_base_cls', True) and + not getattr(base, '_meta', {}).get('abstract', True)): + # Collate heirarchy for _cls and _subclasses + class_name.append(base.__name__) + + if hasattr(base, '_meta'): + # Warn if allow_inheritance isn't set and prevent + # inheritance of classes where inheritance is set to False + allow_inheritance = base._meta.get('allow_inheritance', + ALLOW_INHERITANCE) + if (not getattr(base, '_is_base_cls', True) + and allow_inheritance is None): + warnings.warn( + "%s uses inheritance, the default for " + "allow_inheritance is changing to off by default. " + "Please add it to the document meta." % name, + FutureWarning + ) + elif (allow_inheritance == False and + not base._meta.get('abstract')): + raise ValueError('Document %s may not be subclassed' % + base.__name__) + + # Get superclasses from last base superclass + document_bases = [b for b in flattened_bases + if hasattr(b, '_class_name')] + if document_bases: + superclasses = document_bases[0]._superclasses + superclasses += (document_bases[0]._class_name, ) + + _cls = '.'.join(reversed(class_name)) + attrs['_class_name'] = _cls + attrs['_superclasses'] = superclasses + attrs['_subclasses'] = (_cls, ) + attrs['_types'] = attrs['_subclasses'] # TODO depreciate _types + + # Create the new_class + new_class = super_new(cls, name, bases, attrs) + + # Set _subclasses + for base in document_bases: + if _cls not in base._subclasses: + base._subclasses += (_cls,) + base._types = base._subclasses # TODO depreciate _types + + # Handle delete rules + Document, EmbeddedDocument, DictField = cls._import_classes() + for field in new_class._fields.itervalues(): + f = field + f.owner_document = new_class + delete_rule = getattr(f, 'reverse_delete_rule', DO_NOTHING) + if isinstance(f, ComplexBaseField) and hasattr(f, 'field'): + delete_rule = getattr(f.field, + 'reverse_delete_rule', + DO_NOTHING) + if isinstance(f, DictField) and delete_rule != DO_NOTHING: + msg = ("Reverse delete rules are not supported " + "for %s (field: %s)" % + (field.__class__.__name__, field.name)) + raise InvalidDocumentError(msg) + + f = field.field + + if delete_rule != DO_NOTHING: + if issubclass(new_class, EmbeddedDocument): + msg = ("Reverse delete rules are not supported for " + "EmbeddedDocuments (field: %s)" % field.name) + raise InvalidDocumentError(msg) + f.document_type.register_delete_rule(new_class, + field.name, delete_rule) + + if (field.name and hasattr(Document, field.name) and + EmbeddedDocument not in new_class.mro()): + msg = ("%s is a document method and not a valid " + "field name" % field.name) + raise InvalidDocumentError(msg) + + # Add class to the _document_registry + _document_registry[new_class._class_name] = new_class + + # In Python 2, User-defined methods objects have special read-only + # attributes 'im_func' and 'im_self' which contain the function obj + # and class instance object respectively. With Python 3 these special + # attributes have been replaced by __func__ and __self__. The Blinker + # module continues to use im_func and im_self, so the code below + # copies __func__ into im_func and __self__ into im_self for + # classmethod objects in Document derived classes. + if PY3: + for key, val in new_class.__dict__.items(): + if isinstance(val, classmethod): + f = val.__get__(new_class) + if hasattr(f, '__func__') and not hasattr(f, 'im_func'): + f.__dict__.update({'im_func': getattr(f, '__func__')}) + if hasattr(f, '__self__') and not hasattr(f, 'im_self'): + f.__dict__.update({'im_self': getattr(f, '__self__')}) + + return new_class + + def add_to_class(self, name, value): + setattr(self, name, value) + + @classmethod + def _get_bases(cls, bases): + if isinstance(bases, BasesTuple): + return bases + seen = [] + bases = cls.__get_bases(bases) + unique_bases = (b for b in bases if not (b in seen or seen.append(b))) + return BasesTuple(unique_bases) + + @classmethod + def __get_bases(cls, bases): + for base in bases: + if base is object: + continue + yield base + for child_base in cls.__get_bases(base.__bases__): + yield child_base + + @classmethod + def _import_classes(cls): + Document = _import_class('Document') + EmbeddedDocument = _import_class('EmbeddedDocument') + DictField = _import_class('DictField') + return (Document, EmbeddedDocument, DictField) + + +class TopLevelDocumentMetaclass(DocumentMetaclass): + """Metaclass for top-level documents (i.e. documents that have their own + collection in the database. + """ + + def __new__(cls, name, bases, attrs): + flattened_bases = cls._get_bases(bases) + super_new = super(TopLevelDocumentMetaclass, cls).__new__ + + # Set default _meta data if base class, otherwise get user defined meta + if (attrs.get('my_metaclass') == TopLevelDocumentMetaclass): + # defaults + attrs['_meta'] = { + 'abstract': True, + 'max_documents': None, + 'max_size': None, + 'ordering': [], # default ordering applied at runtime + 'indexes': [], # indexes to be ensured at runtime + 'id_field': None, + 'index_background': False, + 'index_drop_dups': False, + 'index_opts': None, + 'delete_rules': None, + 'allow_inheritance': None, + } + attrs['_is_base_cls'] = True + attrs['_meta'].update(attrs.get('meta', {})) + else: + attrs['_meta'] = attrs.get('meta', {}) + # Explictly set abstract to false unless set + attrs['_meta']['abstract'] = attrs['_meta'].get('abstract', False) + attrs['_is_base_cls'] = False + + # Set flag marking as document class - as opposed to an object mixin + attrs['_is_document'] = True + + # Ensure queryset_class is inherited + if 'objects' in attrs: + manager = attrs['objects'] + if hasattr(manager, 'queryset_class'): + attrs['_meta']['queryset_class'] = manager.queryset_class + + # Clean up top level meta + if 'meta' in attrs: + del(attrs['meta']) + + # Find the parent document class + parent_doc_cls = [b for b in flattened_bases + if b.__class__ == TopLevelDocumentMetaclass] + parent_doc_cls = None if not parent_doc_cls else parent_doc_cls[0] + + # Prevent classes setting collection different to their parents + # If parent wasn't an abstract class + if (parent_doc_cls and 'collection' in attrs.get('_meta', {}) + and not parent_doc_cls._meta.get('abstract', True)): + msg = "Trying to set a collection on a subclass (%s)" % name + warnings.warn(msg, SyntaxWarning) + del(attrs['_meta']['collection']) + + # Ensure abstract documents have abstract bases + if attrs.get('_is_base_cls') or attrs['_meta'].get('abstract'): + if (parent_doc_cls and + not parent_doc_cls._meta.get('abstract', False)): + msg = "Abstract document cannot have non-abstract base" + raise ValueError(msg) + return super_new(cls, name, bases, attrs) + + # Merge base class metas. + # Uses a special MetaDict that handles various merging rules + meta = MetaDict() + for base in flattened_bases[::-1]: + # Add any mixin metadata from plain objects + if hasattr(base, 'meta'): + meta.merge(base.meta) + elif hasattr(base, '_meta'): + meta.merge(base._meta) + + # Set collection in the meta if its callable + if (getattr(base, '_is_document', False) and + not base._meta.get('abstract')): + collection = meta.get('collection', None) + if callable(collection): + meta['collection'] = collection(base) + + meta.merge(attrs.get('_meta', {})) # Top level meta + + # Only simple classes (direct subclasses of Document) + # may set allow_inheritance to False + simple_class = all([b._meta.get('abstract') + for b in flattened_bases if hasattr(b, '_meta')]) + if (not simple_class and meta['allow_inheritance'] == False and + not meta['abstract']): + raise ValueError('Only direct subclasses of Document may set ' + '"allow_inheritance" to False') + + # Set default collection name + if 'collection' not in meta: + meta['collection'] = ''.join('_%s' % c if c.isupper() else c + for c in name).strip('_').lower() + attrs['_meta'] = meta + + # Call super and get the new class + new_class = super_new(cls, name, bases, attrs) + + meta = new_class._meta + + # Set index specifications + meta['index_specs'] = [new_class._build_index_spec(spec) + for spec in meta['indexes']] + unique_indexes = new_class._unique_with_indexes() + new_class._meta['unique_indexes'] = unique_indexes + + # If collection is a callable - call it and set the value + collection = meta.get('collection') + if callable(collection): + new_class._meta['collection'] = collection(new_class) + + # Provide a default queryset unless one has been set + manager = attrs.get('objects', QuerySetManager()) + new_class.objects = manager + + # Validate the fields and set primary key if needed + for field_name, field in new_class._fields.iteritems(): + if field.primary_key: + # Ensure only one primary key is set + current_pk = new_class._meta.get('id_field') + if current_pk and current_pk != field_name: + raise ValueError('Cannot override primary key field') + + # Set primary key + if not current_pk: + new_class._meta['id_field'] = field_name + new_class.id = field + + # Set primary key if not defined by the document + if not new_class._meta.get('id_field'): + new_class._meta['id_field'] = 'id' + new_class._fields['id'] = ObjectIdField(db_field='_id') + new_class.id = new_class._fields['id'] + + # Merge in exceptions with parent hierarchy + exceptions_to_merge = (DoesNotExist, MultipleObjectsReturned) + module = attrs.get('__module__') + for exc in exceptions_to_merge: + name = exc.__name__ + parents = tuple(getattr(base, name) for base in flattened_bases + if hasattr(base, name)) or (exc,) + # Create new exception and set to new_class + exception = type(name, parents, {'__module__': module}) + setattr(new_class, name, exception) + + return new_class + + +class MetaDict(dict): + """Custom dictionary for meta classes. + Handles the merging of set indexes + """ + _merge_options = ('indexes',) + + def merge(self, new_options): + for k, v in new_options.iteritems(): + if k in self._merge_options: + self[k] = self.get(k, []) + v + else: + self[k] = v + + +class BasesTuple(tuple): + """Special class to handle introspection of bases tuple in __new__""" + pass diff --git a/mongoengine/common.py b/mongoengine/common.py new file mode 100644 index 00000000..c284777e --- /dev/null +++ b/mongoengine/common.py @@ -0,0 +1,35 @@ +_class_registry_cache = {} + + +def _import_class(cls_name): + """Cached mechanism for imports""" + if cls_name in _class_registry_cache: + return _class_registry_cache.get(cls_name) + + doc_classes = ('Document', 'DynamicEmbeddedDocument', 'EmbeddedDocument', + 'MapReduceDocument') + field_classes = ('DictField', 'DynamicField', 'EmbeddedDocumentField', + 'GenericReferenceField', 'GeoPointField', + 'ReferenceField', 'StringField') + queryset_classes = ('OperationError',) + deref_classes = ('DeReference',) + + if cls_name in doc_classes: + from mongoengine import document as module + import_classes = doc_classes + elif cls_name in field_classes: + from mongoengine import fields as module + import_classes = field_classes + elif cls_name in queryset_classes: + from mongoengine import queryset as module + import_classes = queryset_classes + elif cls_name in deref_classes: + from mongoengine import dereference as module + import_classes = deref_classes + else: + raise ValueError('No import set for: ' % cls_name) + + for cls in import_classes: + _class_registry_cache[cls] = getattr(module, cls) + + return _class_registry_cache.get(cls_name) \ No newline at end of file diff --git a/mongoengine/dereference.py b/mongoengine/dereference.py index 386dbf4b..59cc0a58 100644 --- a/mongoengine/dereference.py +++ b/mongoengine/dereference.py @@ -164,7 +164,7 @@ class DeReference(object): if isinstance(items, (dict, SON)): if '_ref' in items: return self.object_map.get(items['_ref'].id, items) - elif '_types' in items and '_cls' in items: + elif '_cls' in items: doc = get_document(items['_cls'])._from_son(items) doc._data = self._attach_objects(doc._data, depth, doc, None) return doc diff --git a/mongoengine/django/shortcuts.py b/mongoengine/django/shortcuts.py index 637cee15..9cc8370b 100644 --- a/mongoengine/django/shortcuts.py +++ b/mongoengine/django/shortcuts.py @@ -1,6 +1,6 @@ from mongoengine.queryset import QuerySet from mongoengine.base import BaseDocument -from mongoengine.base import ValidationError +from mongoengine.errors import ValidationError def _get_queryset(cls): """Inspired by django.shortcuts.*""" diff --git a/mongoengine/document.py b/mongoengine/document.py index 7b3afafb..b1ce13ad 100644 --- a/mongoengine/document.py +++ b/mongoengine/document.py @@ -11,9 +11,9 @@ from base import (DocumentMetaclass, TopLevelDocumentMetaclass, BaseDocument, from queryset import OperationError, NotUniqueError from connection import get_db, DEFAULT_CONNECTION_NAME -__all__ = ['Document', 'EmbeddedDocument', 'DynamicDocument', +__all__ = ('Document', 'EmbeddedDocument', 'DynamicDocument', 'DynamicEmbeddedDocument', 'OperationError', - 'InvalidCollectionError', 'NotUniqueError'] + 'InvalidCollectionError', 'NotUniqueError', 'MapReduceDocument') class InvalidCollectionError(Exception): @@ -28,11 +28,11 @@ class EmbeddedDocument(BaseDocument): A :class:`~mongoengine.EmbeddedDocument` subclass may be itself subclassed, to create a specialised version of the embedded document that will be - stored in the same collection. To facilitate this behaviour, `_cls` and - `_types` fields are added to documents (hidden though the MongoEngine - interface though). To disable this behaviour and remove the dependence on - the presence of `_cls` and `_types`, set :attr:`allow_inheritance` to - ``False`` in the :attr:`meta` dictionary. + stored in the same collection. To facilitate this behaviour a `_cls` + field is added to documents (hidden though the MongoEngine interface). + To disable this behaviour and remove the dependence on the presence of + `_cls` set :attr:`allow_inheritance` to ``False`` in the :attr:`meta` + dictionary. """ # The __metaclass__ attribute is removed by 2to3 when running with Python3 @@ -76,11 +76,11 @@ class Document(BaseDocument): A :class:`~mongoengine.Document` subclass may be itself subclassed, to create a specialised version of the document that will be stored in the - same collection. To facilitate this behaviour, `_cls` and `_types` - fields are added to documents (hidden though the MongoEngine interface - though). To disable this behaviour and remove the dependence on the - presence of `_cls` and `_types`, set :attr:`allow_inheritance` to - ``False`` in the :attr:`meta` dictionary. + same collection. To facilitate this behaviour a `_cls` + field is added to documents (hidden though the MongoEngine interface). + To disable this behaviour and remove the dependence on the presence of + `_cls` set :attr:`allow_inheritance` to ``False`` in the :attr:`meta` + dictionary. A :class:`~mongoengine.Document` may use a **Capped Collection** by specifying :attr:`max_documents` and :attr:`max_size` in the :attr:`meta` @@ -101,10 +101,10 @@ class Document(BaseDocument): production systems where index creation is performed as part of a deployment system. - By default, _types will be added to the start of every index (that + By default, _cls will be added to the start of every index (that doesn't contain a list) if allow_inheritance is True. This can be disabled by either setting types to False on the specific index or - by setting index_types to False on the meta dictionary for the document. + by setting index_cls to False on the meta dictionary for the document. """ # The __metaclass__ attribute is removed by 2to3 when running with Python3 diff --git a/mongoengine/errors.py b/mongoengine/errors.py new file mode 100644 index 00000000..eb72503d --- /dev/null +++ b/mongoengine/errors.py @@ -0,0 +1,124 @@ +from collections import defaultdict + +from .python_support import txt_type + + +__all__ = ('NotRegistered', 'InvalidDocumentError', 'ValidationError') + + +class NotRegistered(Exception): + pass + + +class InvalidDocumentError(Exception): + pass + + +class LookUpError(AttributeError): + pass + + +class DoesNotExist(Exception): + pass + + +class MultipleObjectsReturned(Exception): + pass + + +class InvalidQueryError(Exception): + pass + + +class OperationError(Exception): + pass + + +class NotUniqueError(OperationError): + pass + + +class ValidationError(AssertionError): + """Validation exception. + + May represent an error validating a field or a + document containing fields with validation errors. + + :ivar errors: A dictionary of errors for fields within this + document or list, or None if the error is for an + individual field. + """ + + errors = {} + field_name = None + _message = None + + def __init__(self, message="", **kwargs): + self.errors = kwargs.get('errors', {}) + self.field_name = kwargs.get('field_name') + self.message = message + + def __str__(self): + return txt_type(self.message) + + def __repr__(self): + return '%s(%s,)' % (self.__class__.__name__, self.message) + + def __getattribute__(self, name): + message = super(ValidationError, self).__getattribute__(name) + if name == 'message': + if self.field_name: + message = '%s' % message + if self.errors: + message = '%s(%s)' % (message, self._format_errors()) + return message + + def _get_message(self): + return self._message + + def _set_message(self, message): + self._message = message + + message = property(_get_message, _set_message) + + def to_dict(self): + """Returns a dictionary of all errors within a document + + Keys are field names or list indices and values are the + validation error messages, or a nested dictionary of + errors for an embedded document or list. + """ + + def build_dict(source): + errors_dict = {} + if not source: + return errors_dict + if isinstance(source, dict): + for field_name, error in source.iteritems(): + errors_dict[field_name] = build_dict(error) + elif isinstance(source, ValidationError) and source.errors: + return build_dict(source.errors) + else: + return unicode(source) + return errors_dict + if not self.errors: + return {} + return build_dict(self.errors) + + def _format_errors(self): + """Returns a string listing all errors within a document""" + + def generate_key(value, prefix=''): + if isinstance(value, list): + value = ' '.join([generate_key(k) for k in value]) + if isinstance(value, dict): + value = ' '.join( + [generate_key(v, k) for k, v in value.iteritems()]) + + results = "%s.%s" % (prefix, value) if prefix else value + return results + + error_dict = defaultdict(list) + for k, v in self.to_dict().iteritems(): + error_dict[generate_key(v)].append(k) + return ' '.join(["%s: %s" % (k, v) for k, v in error_dict.iteritems()]) diff --git a/mongoengine/fields.py b/mongoengine/fields.py index 01d3fc63..9bcba9f1 100644 --- a/mongoengine/fields.py +++ b/mongoengine/fields.py @@ -12,10 +12,11 @@ from operator import itemgetter import gridfs from bson import Binary, DBRef, SON, ObjectId +from mongoengine.errors import ValidationError from mongoengine.python_support import (PY3, bin_type, txt_type, str_types, StringIO) from base import (BaseField, ComplexBaseField, ObjectIdField, - ValidationError, get_document, BaseDocument) + get_document, BaseDocument) from queryset import DO_NOTHING, QuerySet from document import Document, EmbeddedDocument from connection import get_db, DEFAULT_CONNECTION_NAME @@ -568,9 +569,6 @@ class ListField(ComplexBaseField): Required means it cannot be empty - as the default for ListFields is [] """ - # ListFields cannot be indexed with _types - MongoDB doesn't support this - _index_with_types = False - def __init__(self, field=None, **kwargs): self.field = field kwargs.setdefault('default', lambda: []) diff --git a/mongoengine/queryset/__init__.py b/mongoengine/queryset/__init__.py new file mode 100644 index 00000000..f6feeab7 --- /dev/null +++ b/mongoengine/queryset/__init__.py @@ -0,0 +1,11 @@ +from mongoengine.errors import (DoesNotExist, MultipleObjectsReturned, + InvalidQueryError, OperationError, + NotUniqueError) +from .field_list import * +from .manager import * +from .queryset import * +from .transform import * +from .visitor import * + +__all__ = (field_list.__all__ + manager.__all__ + queryset.__all__ + + transform.__all__ + visitor.__all__) diff --git a/mongoengine/queryset/field_list.py b/mongoengine/queryset/field_list.py new file mode 100644 index 00000000..1c825fa9 --- /dev/null +++ b/mongoengine/queryset/field_list.py @@ -0,0 +1,51 @@ + +__all__ = ('QueryFieldList',) + + +class QueryFieldList(object): + """Object that handles combinations of .only() and .exclude() calls""" + ONLY = 1 + EXCLUDE = 0 + + def __init__(self, fields=[], value=ONLY, always_include=[]): + self.value = value + self.fields = set(fields) + self.always_include = set(always_include) + self._id = None + + def __add__(self, f): + if not self.fields: + self.fields = f.fields + self.value = f.value + elif self.value is self.ONLY and f.value is self.ONLY: + self.fields = self.fields.intersection(f.fields) + elif self.value is self.EXCLUDE and f.value is self.EXCLUDE: + self.fields = self.fields.union(f.fields) + elif self.value is self.ONLY and f.value is self.EXCLUDE: + self.fields -= f.fields + elif self.value is self.EXCLUDE and f.value is self.ONLY: + self.value = self.ONLY + self.fields = f.fields - self.fields + + if '_id' in f.fields: + self._id = f.value + + if self.always_include: + if self.value is self.ONLY and self.fields: + self.fields = self.fields.union(self.always_include) + else: + self.fields -= self.always_include + return self + + def __nonzero__(self): + return bool(self.fields) + + def as_dict(self): + field_list = dict((field, self.value) for field in self.fields) + if self._id is not None: + field_list['_id'] = self._id + return field_list + + def reset(self): + self.fields = set([]) + self.value = self.ONLY diff --git a/mongoengine/queryset/manager.py b/mongoengine/queryset/manager.py new file mode 100644 index 00000000..7376e3c6 --- /dev/null +++ b/mongoengine/queryset/manager.py @@ -0,0 +1,61 @@ +from functools import partial +from .queryset import QuerySet + +__all__ = ('queryset_manager', 'QuerySetManager') + + +class QuerySetManager(object): + """ + The default QuerySet Manager. + + Custom QuerySet Manager functions can extend this class and users can + add extra queryset functionality. Any custom manager methods must accept a + :class:`~mongoengine.Document` class as its first argument, and a + :class:`~mongoengine.queryset.QuerySet` as its second argument. + + The method function should return a :class:`~mongoengine.queryset.QuerySet` + , probably the same one that was passed in, but modified in some way. + """ + + get_queryset = None + + def __init__(self, queryset_func=None): + if queryset_func: + self.get_queryset = queryset_func + self._collections = {} + + def __get__(self, instance, owner): + """Descriptor for instantiating a new QuerySet object when + Document.objects is accessed. + """ + if instance is not None: + # Document class being used rather than a document object + return self + + # owner is the document that contains the QuerySetManager + queryset_class = owner._meta.get('queryset_class') or QuerySet + queryset = queryset_class(owner, owner._get_collection()) + if self.get_queryset: + arg_count = self.get_queryset.func_code.co_argcount + if arg_count == 1: + queryset = self.get_queryset(queryset) + elif arg_count == 2: + queryset = self.get_queryset(owner, queryset) + else: + queryset = partial(self.get_queryset, owner, queryset) + return queryset + + +def queryset_manager(func): + """Decorator that allows you to define custom QuerySet managers on + :class:`~mongoengine.Document` classes. The manager must be a function that + accepts a :class:`~mongoengine.Document` class as its first argument, and a + :class:`~mongoengine.queryset.QuerySet` as its second argument. The method + function should return a :class:`~mongoengine.queryset.QuerySet`, probably + the same one that was passed in, but modified in some way. + """ + if func.func_code.co_argcount == 1: + import warnings + msg = 'Methods decorated with queryset_manager should take 2 arguments' + warnings.warn(msg, DeprecationWarning) + return QuerySetManager(func) diff --git a/mongoengine/queryset.py b/mongoengine/queryset/queryset.py similarity index 59% rename from mongoengine/queryset.py rename to mongoengine/queryset/queryset.py index c774322e..51080663 100644 --- a/mongoengine/queryset.py +++ b/mongoengine/queryset/queryset.py @@ -4,20 +4,21 @@ import copy import itertools import operator -from collections import defaultdict -from functools import partial - -from mongoengine.python_support import product, reduce - import pymongo from bson.code import Code from mongoengine import signals +from mongoengine.common import _import_class +from mongoengine.errors import (OperationError, NotUniqueError, + InvalidQueryError) -__all__ = ['queryset_manager', 'Q', 'InvalidQueryError', - 'DO_NOTHING', 'NULLIFY', 'CASCADE', 'DENY', 'PULL'] +from . import transform +from .field_list import QueryFieldList +from .visitor import Q +__all__ = ('QuerySet', 'DO_NOTHING', 'NULLIFY', 'CASCADE', 'DENY', 'PULL') + # The maximum number of items to display in a QuerySet.__repr__ REPR_OUTPUT_SIZE = 20 @@ -28,308 +29,9 @@ CASCADE = 2 DENY = 3 PULL = 4 - -class DoesNotExist(Exception): - pass - - -class MultipleObjectsReturned(Exception): - pass - - -class InvalidQueryError(Exception): - pass - - -class OperationError(Exception): - pass - - -class NotUniqueError(OperationError): - pass - - RE_TYPE = type(re.compile('')) -class QNodeVisitor(object): - """Base visitor class for visiting Q-object nodes in a query tree. - """ - - def visit_combination(self, combination): - """Called by QCombination objects. - """ - return combination - - def visit_query(self, query): - """Called by (New)Q objects. - """ - return query - - -class SimplificationVisitor(QNodeVisitor): - """Simplifies query trees by combinging unnecessary 'and' connection nodes - into a single Q-object. - """ - - def visit_combination(self, combination): - if combination.operation == combination.AND: - # The simplification only applies to 'simple' queries - if all(isinstance(node, Q) for node in combination.children): - queries = [node.query for node in combination.children] - return Q(**self._query_conjunction(queries)) - return combination - - def _query_conjunction(self, queries): - """Merges query dicts - effectively &ing them together. - """ - query_ops = set() - combined_query = {} - for query in queries: - ops = set(query.keys()) - # Make sure that the same operation isn't applied more than once - # to a single field - intersection = ops.intersection(query_ops) - if intersection: - msg = 'Duplicate query conditions: ' - raise InvalidQueryError(msg + ', '.join(intersection)) - - query_ops.update(ops) - combined_query.update(copy.deepcopy(query)) - return combined_query - - -class QueryTreeTransformerVisitor(QNodeVisitor): - """Transforms the query tree in to a form that may be used with MongoDB. - """ - - def visit_combination(self, combination): - if combination.operation == combination.AND: - # MongoDB doesn't allow us to have too many $or operations in our - # queries, so the aim is to move the ORs up the tree to one - # 'master' $or. Firstly, we must find all the necessary parts (part - # of an AND combination or just standard Q object), and store them - # separately from the OR parts. - or_groups = [] - and_parts = [] - for node in combination.children: - if isinstance(node, QCombination): - if node.operation == node.OR: - # Any of the children in an $or component may cause - # the query to succeed - or_groups.append(node.children) - elif node.operation == node.AND: - and_parts.append(node) - elif isinstance(node, Q): - and_parts.append(node) - - # Now we combine the parts into a usable query. AND together all of - # the necessary parts. Then for each $or part, create a new query - # that ANDs the necessary part with the $or part. - clauses = [] - for or_group in product(*or_groups): - q_object = reduce(lambda a, b: a & b, and_parts, Q()) - q_object = reduce(lambda a, b: a & b, or_group, q_object) - clauses.append(q_object) - # Finally, $or the generated clauses in to one query. Each of the - # clauses is sufficient for the query to succeed. - return reduce(lambda a, b: a | b, clauses, Q()) - - if combination.operation == combination.OR: - children = [] - # Crush any nested ORs in to this combination as MongoDB doesn't - # support nested $or operations - for node in combination.children: - if (isinstance(node, QCombination) and - node.operation == combination.OR): - children += node.children - else: - children.append(node) - combination.children = children - - return combination - - -class QueryCompilerVisitor(QNodeVisitor): - """Compiles the nodes in a query tree to a PyMongo-compatible query - dictionary. - """ - - def __init__(self, document): - self.document = document - - def visit_combination(self, combination): - if combination.operation == combination.OR: - return {'$or': combination.children} - elif combination.operation == combination.AND: - return self._mongo_query_conjunction(combination.children) - return combination - - def visit_query(self, query): - return QuerySet._transform_query(self.document, **query.query) - - def _mongo_query_conjunction(self, queries): - """Merges Mongo query dicts - effectively &ing them together. - """ - combined_query = {} - for query in queries: - for field, ops in query.items(): - if field not in combined_query: - combined_query[field] = ops - else: - # The field is already present in the query the only way - # we can merge is if both the existing value and the new - # value are operation dicts, reject anything else - if (not isinstance(combined_query[field], dict) or - not isinstance(ops, dict)): - message = 'Conflicting values for ' + field - raise InvalidQueryError(message) - - current_ops = set(combined_query[field].keys()) - new_ops = set(ops.keys()) - # Make sure that the same operation isn't applied more than - # once to a single field - intersection = current_ops.intersection(new_ops) - if intersection: - msg = 'Duplicate query conditions: ' - raise InvalidQueryError(msg + ', '.join(intersection)) - - # Right! We've got two non-overlapping dicts of operations! - combined_query[field].update(copy.deepcopy(ops)) - return combined_query - - -class QNode(object): - """Base class for nodes in query trees. - """ - - AND = 0 - OR = 1 - - def to_query(self, document): - query = self.accept(SimplificationVisitor()) - query = query.accept(QueryTreeTransformerVisitor()) - query = query.accept(QueryCompilerVisitor(document)) - return query - - def accept(self, visitor): - raise NotImplementedError - - def _combine(self, other, operation): - """Combine this node with another node into a QCombination object. - """ - if getattr(other, 'empty', True): - return self - - if self.empty: - return other - - return QCombination(operation, [self, other]) - - @property - def empty(self): - return False - - def __or__(self, other): - return self._combine(other, self.OR) - - def __and__(self, other): - return self._combine(other, self.AND) - - -class QCombination(QNode): - """Represents the combination of several conditions by a given logical - operator. - """ - - def __init__(self, operation, children): - self.operation = operation - self.children = [] - for node in children: - # If the child is a combination of the same type, we can merge its - # children directly into this combinations children - if isinstance(node, QCombination) and node.operation == operation: - self.children += node.children - else: - self.children.append(node) - - def accept(self, visitor): - for i in range(len(self.children)): - if isinstance(self.children[i], QNode): - self.children[i] = self.children[i].accept(visitor) - - return visitor.visit_combination(self) - - @property - def empty(self): - return not bool(self.children) - - -class Q(QNode): - """A simple query object, used in a query tree to build up more complex - query structures. - """ - - def __init__(self, **query): - self.query = query - - def accept(self, visitor): - return visitor.visit_query(self) - - @property - def empty(self): - return not bool(self.query) - - -class QueryFieldList(object): - """Object that handles combinations of .only() and .exclude() calls""" - ONLY = 1 - EXCLUDE = 0 - - def __init__(self, fields=[], value=ONLY, always_include=[]): - self.value = value - self.fields = set(fields) - self.always_include = set(always_include) - self._id = None - - def as_dict(self): - field_list = dict((field, self.value) for field in self.fields) - if self._id is not None: - field_list['_id'] = self._id - return field_list - - def __add__(self, f): - if not self.fields: - self.fields = f.fields - self.value = f.value - elif self.value is self.ONLY and f.value is self.ONLY: - self.fields = self.fields.intersection(f.fields) - elif self.value is self.EXCLUDE and f.value is self.EXCLUDE: - self.fields = self.fields.union(f.fields) - elif self.value is self.ONLY and f.value is self.EXCLUDE: - self.fields -= f.fields - elif self.value is self.EXCLUDE and f.value is self.ONLY: - self.value = self.ONLY - self.fields = f.fields - self.fields - - if '_id' in f.fields: - self._id = f.value - - if self.always_include: - if self.value is self.ONLY and self.fields: - self.fields = self.fields.union(self.always_include) - else: - self.fields -= self.always_include - return self - - def reset(self): - self.fields = set([]) - self.value = self.ONLY - - def __nonzero__(self): - return bool(self.fields) - - class QuerySet(object): """A set of results returned from a query. Wraps a MongoDB cursor, providing :class:`~mongoengine.Document` objects as the results. @@ -357,7 +59,7 @@ class QuerySet(object): # If inheritance is allowed, only return instances and instances of # subclasses of the class being used if document._meta.get('allow_inheritance') != False: - self._initial_query = {'_types': self._document._class_name} + self._initial_query = {"_cls": {"$in": self._document._subclasses}} self._loaded_fields = QueryFieldList(always_include=['_cls']) self._cursor_obj = None self._limit = None @@ -397,7 +99,7 @@ class QuerySet(object): construct a multi-field index); keys may be prefixed with a **+** or a **-** to determine the index ordering """ - index_spec = QuerySet._build_index_spec(self._document, key_or_list) + index_spec = self._document._build_index_spec(key_or_list) index_spec = index_spec.copy() fields = index_spec.pop('fields') index_spec['drop_dups'] = drop_dups @@ -448,26 +150,26 @@ class QuerySet(object): background = self._document._meta.get('index_background', False) drop_dups = self._document._meta.get('index_drop_dups', False) index_opts = self._document._meta.get('index_opts') or {} - index_types = self._document._meta.get('index_types', True) + index_cls = self._document._meta.get('index_cls', True) # determine if an index which we are creating includes - # _type as its first field; if so, we can avoid creating - # an extra index on _type, as mongodb will use the existing - # index to service queries against _type - types_indexed = False + # _cls as its first field; if so, we can avoid creating + # an extra index on _cls, as mongodb will use the existing + # index to service queries against _cls + cls_indexed = False - def includes_types(fields): + def includes_cls(fields): first_field = None if len(fields): if isinstance(fields[0], basestring): first_field = fields[0] elif isinstance(fields[0], (list, tuple)) and len(fields[0]): first_field = fields[0][0] - return first_field == '_types' + return first_field == '_cls' # Ensure indexes created by uniqueness constraints for index in self._document._meta['unique_indexes']: - types_indexed = types_indexed or includes_types(index) + cls_indexed = cls_indexed or includes_cls(index) self._collection.ensure_index(index, unique=True, background=background, drop_dups=drop_dups, **index_opts) @@ -477,16 +179,16 @@ class QuerySet(object): for spec in index_spec: spec = spec.copy() fields = spec.pop('fields') - types_indexed = types_indexed or includes_types(fields) + cls_indexed = cls_indexed or includes_cls(fields) opts = index_opts.copy() opts.update(spec) self._collection.ensure_index(fields, background=background, **opts) - # If _types is being used (for polymorphism), it needs an index, - # only if another index doesn't begin with _types - if index_types and '_types' in self._query and not types_indexed: - self._collection.ensure_index('_types', + # If _cls is being used (for polymorphism), it needs an index, + # only if another index doesn't begin with _cls + if index_cls and '_cls' in self._query and not cls_indexed: + self._collection.ensure_index('_cls', background=background, **index_opts) # Add geo indicies @@ -495,79 +197,14 @@ class QuerySet(object): self._collection.ensure_index(index_spec, background=background, **index_opts) - @classmethod - def _build_index_spec(cls, doc_cls, spec): - """Build a PyMongo index spec from a MongoEngine index spec. - """ - if isinstance(spec, basestring): - spec = {'fields': [spec]} - elif isinstance(spec, (list, tuple)): - spec = {'fields': list(spec)} - elif isinstance(spec, dict): - spec = dict(spec) - - index_list = [] - direction = None - - allow_inheritance = doc_cls._meta.get('allow_inheritance') != False - - # If sparse - dont include types - use_types = allow_inheritance and not spec.get('sparse', False) - - for key in spec['fields']: - # If inherited spec continue - if isinstance(key, (list, tuple)): - continue - - # Get ASCENDING direction from +, DESCENDING from -, and GEO2D from * - direction = pymongo.ASCENDING - if key.startswith("-"): - direction = pymongo.DESCENDING - elif key.startswith("*"): - direction = pymongo.GEO2D - if key.startswith(("+", "-", "*")): - key = key[1:] - - # Use real field name, do it manually because we need field - # objects for the next part (list field checking) - parts = key.split('.') - if parts in (['pk'], ['id'], ['_id']): - key = '_id' - fields = [] - else: - fields = QuerySet._lookup_field(doc_cls, parts) - parts = [field if field == '_id' else field.db_field - for field in fields] - key = '.'.join(parts) - index_list.append((key, direction)) - - # Check if a list field is being used, don't use _types if it is - if use_types and not all(f._index_with_types for f in fields): - use_types = False - - # If _types is being used, prepend it to every specified index - index_types = doc_cls._meta.get('index_types', True) - - if (spec.get('types', index_types) and use_types - and direction is not pymongo.GEO2D): - index_list.insert(0, ('_types', 1)) - - spec['fields'] = index_list - if spec.get('sparse', False) and len(spec['fields']) > 1: - raise ValueError( - 'Sparse indexes can only have one field in them. ' - 'See https://jira.mongodb.org/browse/SERVER-2193') - - return spec - @classmethod def _reset_already_indexed(cls, document=None): - """Helper to reset already indexed, can be useful for testing purposes""" + """Helper to reset already indexed, can be useful for testing purposes + """ if document: cls.__already_indexed.discard(document) cls.__already_indexed.clear() - @property def _collection(self): """Property that returns the collection object. This allows us to @@ -624,195 +261,12 @@ class QuerySet(object): self._cursor_obj.hint(self._hint) return self._cursor_obj - @classmethod - def _lookup_field(cls, document, parts): - """Lookup a field based on its attribute and return a list containing - the field's parents and the field. - """ - if not isinstance(parts, (list, tuple)): - parts = [parts] - fields = [] - field = None - - for field_name in parts: - # Handle ListField indexing: - if field_name.isdigit(): - try: - new_field = field.field - except AttributeError, err: - raise InvalidQueryError( - "Can't use index on unsubscriptable field (%s)" % err) - fields.append(field_name) - continue - - if field is None: - # Look up first field from the document - if field_name == 'pk': - # Deal with "primary key" alias - field_name = document._meta['id_field'] - if field_name in document._fields: - field = document._fields[field_name] - elif document._dynamic: - from fields import DynamicField - field = DynamicField(db_field=field_name) - else: - raise InvalidQueryError('Cannot resolve field "%s"' - % field_name) - else: - from mongoengine.fields import ReferenceField, GenericReferenceField - if isinstance(field, (ReferenceField, GenericReferenceField)): - raise InvalidQueryError('Cannot perform join in mongoDB: %s' % '__'.join(parts)) - if hasattr(getattr(field, 'field', None), 'lookup_member'): - new_field = field.field.lookup_member(field_name) - else: - # Look up subfield on the previous field - new_field = field.lookup_member(field_name) - from base import ComplexBaseField - if not new_field and isinstance(field, ComplexBaseField): - fields.append(field_name) - continue - elif not new_field: - raise InvalidQueryError('Cannot resolve field "%s"' - % field_name) - field = new_field # update field to the new field type - fields.append(field) - return fields - - @classmethod - def _translate_field_name(cls, doc_cls, field, sep='.'): - """Translate a field attribute name to a database field name. - """ - parts = field.split(sep) - parts = [f.db_field for f in QuerySet._lookup_field(doc_cls, parts)] - return '.'.join(parts) - - @classmethod - def _transform_query(cls, _doc_cls=None, _field_operation=False, **query): - """Transform a query from Django-style format to Mongo format. - """ - operators = ['ne', 'gt', 'gte', 'lt', 'lte', 'in', 'nin', 'mod', - 'all', 'size', 'exists', 'not'] - geo_operators = ['within_distance', 'within_spherical_distance', 'within_box', 'within_polygon', 'near', 'near_sphere'] - match_operators = ['contains', 'icontains', 'startswith', - 'istartswith', 'endswith', 'iendswith', - 'exact', 'iexact'] - custom_operators = ['match'] - - mongo_query = {} - merge_query = defaultdict(list) - for key, value in query.items(): - if key == "__raw__": - mongo_query.update(value) - continue - - parts = key.split('__') - indices = [(i, p) for i, p in enumerate(parts) if p.isdigit()] - parts = [part for part in parts if not part.isdigit()] - # Check for an operator and transform to mongo-style if there is - op = None - if parts[-1] in operators + match_operators + geo_operators + custom_operators: - op = parts.pop() - - negate = False - if parts[-1] == 'not': - parts.pop() - negate = True - - if _doc_cls: - # Switch field names to proper names [set in Field(name='foo')] - fields = QuerySet._lookup_field(_doc_cls, parts) - parts = [] - - cleaned_fields = [] - for field in fields: - append_field = True - if isinstance(field, basestring): - parts.append(field) - append_field = False - else: - parts.append(field.db_field) - if append_field: - cleaned_fields.append(field) - - # Convert value to proper value - field = cleaned_fields[-1] - - singular_ops = [None, 'ne', 'gt', 'gte', 'lt', 'lte', 'not'] - singular_ops += match_operators - if op in singular_ops: - if isinstance(field, basestring): - if op in match_operators and isinstance(value, basestring): - from mongoengine import StringField - value = StringField.prepare_query_value(op, value) - else: - value = field - else: - value = field.prepare_query_value(op, value) - elif op in ('in', 'nin', 'all', 'near'): - # 'in', 'nin' and 'all' require a list of values - value = [field.prepare_query_value(op, v) for v in value] - - # if op and op not in match_operators: - if op: - if op in geo_operators: - if op == "within_distance": - value = {'$within': {'$center': value}} - elif op == "within_spherical_distance": - value = {'$within': {'$centerSphere': value}} - elif op == "within_polygon": - value = {'$within': {'$polygon': value}} - elif op == "near": - value = {'$near': value} - elif op == "near_sphere": - value = {'$nearSphere': value} - elif op == 'within_box': - value = {'$within': {'$box': value}} - else: - raise NotImplementedError("Geo method '%s' has not " - "been implemented" % op) - elif op in custom_operators: - if op == 'match': - value = {"$elemMatch": value} - else: - NotImplementedError("Custom method '%s' has not " - "been implemented" % op) - elif op not in match_operators: - value = {'$' + op: value} - - if negate: - value = {'$not': value} - - for i, part in indices: - parts.insert(i, part) - key = '.'.join(parts) - if op is None or key not in mongo_query: - mongo_query[key] = value - elif key in mongo_query: - if key in mongo_query and isinstance(mongo_query[key], dict): - mongo_query[key].update(value) - else: - # Store for manually merging later - merge_query[key].append(value) - - # The queryset has been filter in such a way we must manually merge - for k, v in merge_query.items(): - merge_query[k].append(mongo_query[k]) - del mongo_query[k] - if isinstance(v, list): - value = [{k:val} for val in v] - if '$and' in mongo_query.keys(): - mongo_query['$and'].append(value) - else: - mongo_query['$and'] = value - - return mongo_query - def get(self, *q_objs, **query): """Retrieve the the matching object raising :class:`~mongoengine.queryset.MultipleObjectsReturned` or - `DocumentName.MultipleObjectsReturned` exception if multiple results and - :class:`~mongoengine.queryset.DoesNotExist` or `DocumentName.DoesNotExist` - if no results are found. + `DocumentName.MultipleObjectsReturned` exception if multiple results + and :class:`~mongoengine.queryset.DoesNotExist` or + `DocumentName.DoesNotExist` if no results are found. .. versionadded:: 0.3 """ @@ -910,7 +364,7 @@ class QuerySet(object): .. versionadded:: 0.5 """ - from document import Document + Document = _import_class('Document') if not write_options: write_options = {} @@ -1064,7 +518,7 @@ class QuerySet(object): .. versionadded:: 0.3 """ - from document import MapReduceDocument + MapReduceDocument = _import_class('MapReduceDocument') if not hasattr(self._collection, "map_reduce"): raise NotImplementedError("Requires MongoDB >= 1.7.1") @@ -1267,14 +721,16 @@ class QuerySet(object): .. versionadded:: 0.5 """ - self._loaded_fields = QueryFieldList(always_include=self._loaded_fields.always_include) + self._loaded_fields = QueryFieldList( + always_include=self._loaded_fields.always_include) return self def _fields_to_dbfields(self, fields): """Translate fields paths to its db equivalents""" ret = [] for field in fields: - field = ".".join(f.db_field for f in QuerySet._lookup_field(self._document, field.split('.'))) + field = ".".join(f.db_field for f in + self._document._lookup_field(field.split('.'))) ret.append(field) return ret @@ -1288,7 +744,8 @@ class QuerySet(object): """ key_list = [] for key in keys: - if not key: continue + if not key: + continue direction = pymongo.ASCENDING if key[0] == '-': direction = pymongo.DESCENDING @@ -1296,7 +753,7 @@ class QuerySet(object): key = key[1:] key = key.replace('__', '.') try: - key = QuerySet._translate_field_name(self._document, key) + key = self._document._translate_field_name(key) except: pass key_list.append((key, direction)) @@ -1389,107 +846,6 @@ class QuerySet(object): self._collection.remove(self._query, safe=safe) - @classmethod - def _transform_update(cls, _doc_cls=None, **update): - """Transform an update spec from Django-style format to Mongo format. - """ - operators = ['set', 'unset', 'inc', 'dec', 'pop', 'push', 'push_all', - 'pull', 'pull_all', 'add_to_set'] - match_operators = ['ne', 'gt', 'gte', 'lt', 'lte', 'in', 'nin', 'mod', - 'all', 'size', 'exists', 'not'] - - mongo_update = {} - for key, value in update.items(): - if key == "__raw__": - mongo_update.update(value) - continue - parts = key.split('__') - # Check for an operator and transform to mongo-style if there is - op = None - if parts[0] in operators: - op = parts.pop(0) - # Convert Pythonic names to Mongo equivalents - if op in ('push_all', 'pull_all'): - op = op.replace('_all', 'All') - elif op == 'dec': - # Support decrement by flipping a positive value's sign - # and using 'inc' - op = 'inc' - if value > 0: - value = -value - elif op == 'add_to_set': - op = op.replace('_to_set', 'ToSet') - - match = None - if parts[-1] in match_operators: - match = parts.pop() - - if _doc_cls: - # Switch field names to proper names [set in Field(name='foo')] - fields = QuerySet._lookup_field(_doc_cls, parts) - parts = [] - - cleaned_fields = [] - for field in fields: - append_field = True - if isinstance(field, basestring): - # Convert the S operator to $ - if field == 'S': - field = '$' - parts.append(field) - append_field = False - else: - parts.append(field.db_field) - if append_field: - cleaned_fields.append(field) - - # Convert value to proper value - field = cleaned_fields[-1] - - if op in (None, 'set', 'push', 'pull'): - if field.required or value is not None: - value = field.prepare_query_value(op, value) - elif op in ('pushAll', 'pullAll'): - value = [field.prepare_query_value(op, v) for v in value] - elif op == 'addToSet': - if isinstance(value, (list, tuple, set)): - value = [field.prepare_query_value(op, v) for v in value] - elif field.required or value is not None: - value = field.prepare_query_value(op, value) - - if match: - match = '$' + match - value = {match: value} - - key = '.'.join(parts) - - if not op: - raise InvalidQueryError("Updates must supply an operation " - "eg: set__FIELD=value") - - if 'pull' in op and '.' in key: - # Dot operators don't work on pull operations - # it uses nested dict syntax - if op == 'pullAll': - raise InvalidQueryError("pullAll operations only support " - "a single field depth") - - parts.reverse() - for key in parts: - value = {key: value} - elif op == 'addToSet' and isinstance(value, list): - value = {key: {"$each": value}} - else: - value = {key: value} - key = '$' + op - - if key not in mongo_update: - mongo_update[key] = value - elif key in mongo_update and isinstance(mongo_update[key], dict): - mongo_update[key].update(value) - - return mongo_update - def update(self, safe_update=True, upsert=False, multi=True, write_options=None, **update): """Perform an atomic update on the fields matched by the query. When ``safe_update`` is used, the number of affected documents is returned. @@ -1506,14 +862,9 @@ class QuerySet(object): if not write_options: write_options = {} - update = QuerySet._transform_update(self._document, **update) + update = transform.update(self._document, **update) query = self._query - # SERVER-5247 hack - remove_types = "_types" in query and ".$." in unicode(update) - if remove_types: - del query["_types"] - try: ret = self._collection.update(query, update, multi=multi, upsert=upsert, safe=safe_update, @@ -1537,30 +888,8 @@ class QuerySet(object): .. versionadded:: 0.2 """ - if not update: - raise OperationError("No update parameters, would remove data") - - if not write_options: - write_options = {} - update = QuerySet._transform_update(self._document, **update) - query = self._query - - # SERVER-5247 hack - remove_types = "_types" in query and ".$." in unicode(update) - if remove_types: - del query["_types"] - - try: - # Explicitly provide 'multi=False' to newer versions of PyMongo - # as the default may change to 'True' - ret = self._collection.update(query, update, multi=False, - upsert=upsert, safe=safe_update, - **write_options) - - if ret is not None and 'n' in ret: - return ret['n'] - except pymongo.errors.OperationFailure, e: - raise OperationError(u'Update failed [%s]' % unicode(e)) + return self.update(safe_update=True, upsert=False, multi=False, + write_options=None, **update) def __iter__(self): self.rewind() @@ -1611,14 +940,14 @@ class QuerySet(object): def field_sub(match): # Extract just the field name, and look up the field objects field_name = match.group(1).split('.') - fields = QuerySet._lookup_field(self._document, field_name) + fields = self._document._lookup_field(field_name) # Substitute the correct name for the field into the javascript return u'["%s"]' % fields[-1].db_field def field_path_sub(match): # Extract just the field name, and look up the field objects field_name = match.group(1).split('.') - fields = QuerySet._lookup_field(self._document, field_name) + fields = self._document._lookup_field(field_name) # Substitute the correct name for the field into the javascript return ".".join([f.db_field for f in fields]) @@ -1650,8 +979,7 @@ class QuerySet(object): """ code = self._sub_js_fields(code) - fields = [QuerySet._translate_field_name(self._document, f) - for f in fields] + fields = [self._document._translate_field_name(f) for f in fields] collection = self._document._get_collection_name() scope = { @@ -1925,63 +1253,5 @@ class QuerySet(object): @property def _dereference(self): if not self.__dereference: - from dereference import DeReference - self.__dereference = DeReference() # Cached + self.__dereference = _import_class('DeReference')() return self.__dereference - - -class QuerySetManager(object): - """ - The default QuerySet Manager. - - Custom QuerySet Manager functions can extend this class and users can - add extra queryset functionality. Any custom manager methods must accept a - :class:`~mongoengine.Document` class as its first argument, and a - :class:`~mongoengine.queryset.QuerySet` as its second argument. - - The method function should return a :class:`~mongoengine.queryset.QuerySet` - , probably the same one that was passed in, but modified in some way. - """ - - get_queryset = None - - def __init__(self, queryset_func=None): - if queryset_func: - self.get_queryset = queryset_func - self._collections = {} - - def __get__(self, instance, owner): - """Descriptor for instantiating a new QuerySet object when - Document.objects is accessed. - """ - if instance is not None: - # Document class being used rather than a document object - return self - - # owner is the document that contains the QuerySetManager - queryset_class = owner._meta.get('queryset_class') or QuerySet - queryset = queryset_class(owner, owner._get_collection()) - if self.get_queryset: - arg_count = self.get_queryset.func_code.co_argcount - if arg_count == 1: - queryset = self.get_queryset(queryset) - elif arg_count == 2: - queryset = self.get_queryset(owner, queryset) - else: - queryset = partial(self.get_queryset, owner, queryset) - return queryset - - -def queryset_manager(func): - """Decorator that allows you to define custom QuerySet managers on - :class:`~mongoengine.Document` classes. The manager must be a function that - accepts a :class:`~mongoengine.Document` class as its first argument, and a - :class:`~mongoengine.queryset.QuerySet` as its second argument. The method - function should return a :class:`~mongoengine.queryset.QuerySet`, probably - the same one that was passed in, but modified in some way. - """ - if func.func_code.co_argcount == 1: - import warnings - msg = 'Methods decorated with queryset_manager should take 2 arguments' - warnings.warn(msg, DeprecationWarning) - return QuerySetManager(func) diff --git a/mongoengine/queryset/transform.py b/mongoengine/queryset/transform.py new file mode 100644 index 00000000..8ee84eed --- /dev/null +++ b/mongoengine/queryset/transform.py @@ -0,0 +1,237 @@ +from collections import defaultdict + +from mongoengine.common import _import_class +from mongoengine.errors import InvalidQueryError, LookUpError + +__all__ = ('query', 'update') + + +COMPARISON_OPERATORS = ('ne', 'gt', 'gte', 'lt', 'lte', 'in', 'nin', 'mod', + 'all', 'size', 'exists', 'not') +GEO_OPERATORS = ('within_distance', 'within_spherical_distance', + 'within_box', 'within_polygon', 'near', 'near_sphere') +STRING_OPERATORS = ('contains', 'icontains', 'startswith', + 'istartswith', 'endswith', 'iendswith', + 'exact', 'iexact') +CUSTOM_OPERATORS = ('match',) +MATCH_OPERATORS = (COMPARISON_OPERATORS + GEO_OPERATORS + + STRING_OPERATORS + CUSTOM_OPERATORS) + +UPDATE_OPERATORS = ('set', 'unset', 'inc', 'dec', 'pop', 'push', + 'push_all', 'pull', 'pull_all', 'add_to_set') + + +def query(_doc_cls=None, _field_operation=False, **query): + """Transform a query from Django-style format to Mongo format. + """ + mongo_query = {} + merge_query = defaultdict(list) + for key, value in query.items(): + if key == "__raw__": + mongo_query.update(value) + continue + + parts = key.split('__') + indices = [(i, p) for i, p in enumerate(parts) if p.isdigit()] + parts = [part for part in parts if not part.isdigit()] + # Check for an operator and transform to mongo-style if there is + op = None + if parts[-1] in MATCH_OPERATORS: + op = parts.pop() + + negate = False + if parts[-1] == 'not': + parts.pop() + negate = True + + if _doc_cls: + # Switch field names to proper names [set in Field(name='foo')] + try: + fields = _doc_cls._lookup_field(parts) + except Exception, e: + raise InvalidQueryError(e) + parts = [] + + cleaned_fields = [] + for field in fields: + append_field = True + if isinstance(field, basestring): + parts.append(field) + append_field = False + else: + parts.append(field.db_field) + if append_field: + cleaned_fields.append(field) + + # Convert value to proper value + field = cleaned_fields[-1] + + singular_ops = [None, 'ne', 'gt', 'gte', 'lt', 'lte', 'not'] + singular_ops += STRING_OPERATORS + if op in singular_ops: + if isinstance(field, basestring): + if (op in STRING_OPERATORS and + isinstance(value, basestring)): + StringField = _import_class('StringField') + value = StringField.prepare_query_value(op, value) + else: + value = field + else: + value = field.prepare_query_value(op, value) + elif op in ('in', 'nin', 'all', 'near'): + # 'in', 'nin' and 'all' require a list of values + value = [field.prepare_query_value(op, v) for v in value] + + # if op and op not in COMPARISON_OPERATORS: + if op: + if op in GEO_OPERATORS: + if op == "within_distance": + value = {'$within': {'$center': value}} + elif op == "within_spherical_distance": + value = {'$within': {'$centerSphere': value}} + elif op == "within_polygon": + value = {'$within': {'$polygon': value}} + elif op == "near": + value = {'$near': value} + elif op == "near_sphere": + value = {'$nearSphere': value} + elif op == 'within_box': + value = {'$within': {'$box': value}} + else: + raise NotImplementedError("Geo method '%s' has not " + "been implemented" % op) + elif op in CUSTOM_OPERATORS: + if op == 'match': + value = {"$elemMatch": value} + else: + NotImplementedError("Custom method '%s' has not " + "been implemented" % op) + elif op not in STRING_OPERATORS: + value = {'$' + op: value} + + if negate: + value = {'$not': value} + + for i, part in indices: + parts.insert(i, part) + key = '.'.join(parts) + if op is None or key not in mongo_query: + mongo_query[key] = value + elif key in mongo_query: + if key in mongo_query and isinstance(mongo_query[key], dict): + mongo_query[key].update(value) + else: + # Store for manually merging later + merge_query[key].append(value) + + # The queryset has been filter in such a way we must manually merge + for k, v in merge_query.items(): + merge_query[k].append(mongo_query[k]) + del mongo_query[k] + if isinstance(v, list): + value = [{k:val} for val in v] + if '$and' in mongo_query.keys(): + mongo_query['$and'].append(value) + else: + mongo_query['$and'] = value + + return mongo_query + + +def update(_doc_cls=None, **update): + """Transform an update spec from Django-style format to Mongo format. + """ + mongo_update = {} + for key, value in update.items(): + if key == "__raw__": + mongo_update.update(value) + continue + parts = key.split('__') + # Check for an operator and transform to mongo-style if there is + op = None + if parts[0] in UPDATE_OPERATORS: + op = parts.pop(0) + # Convert Pythonic names to Mongo equivalents + if op in ('push_all', 'pull_all'): + op = op.replace('_all', 'All') + elif op == 'dec': + # Support decrement by flipping a positive value's sign + # and using 'inc' + op = 'inc' + if value > 0: + value = -value + elif op == 'add_to_set': + op = op.replace('_to_set', 'ToSet') + + match = None + if parts[-1] in COMPARISON_OPERATORS: + match = parts.pop() + + if _doc_cls: + # Switch field names to proper names [set in Field(name='foo')] + try: + fields = _doc_cls._lookup_field(parts) + except Exception, e: + raise InvalidQueryError(e) + parts = [] + + cleaned_fields = [] + for field in fields: + append_field = True + if isinstance(field, basestring): + # Convert the S operator to $ + if field == 'S': + field = '$' + parts.append(field) + append_field = False + else: + parts.append(field.db_field) + if append_field: + cleaned_fields.append(field) + + # Convert value to proper value + field = cleaned_fields[-1] + + if op in (None, 'set', 'push', 'pull'): + if field.required or value is not None: + value = field.prepare_query_value(op, value) + elif op in ('pushAll', 'pullAll'): + value = [field.prepare_query_value(op, v) for v in value] + elif op == 'addToSet': + if isinstance(value, (list, tuple, set)): + value = [field.prepare_query_value(op, v) for v in value] + elif field.required or value is not None: + value = field.prepare_query_value(op, value) + + if match: + match = '$' + match + value = {match: value} + + key = '.'.join(parts) + + if not op: + raise InvalidQueryError("Updates must supply an operation " + "eg: set__FIELD=value") + + if 'pull' in op and '.' in key: + # Dot operators don't work on pull operations + # it uses nested dict syntax + if op == 'pullAll': + raise InvalidQueryError("pullAll operations only support " + "a single field depth") + + parts.reverse() + for key in parts: + value = {key: value} + elif op == 'addToSet' and isinstance(value, list): + value = {key: {"$each": value}} + else: + value = {key: value} + key = '$' + op + + if key not in mongo_update: + mongo_update[key] = value + elif key in mongo_update and isinstance(mongo_update[key], dict): + mongo_update[key].update(value) + + return mongo_update diff --git a/mongoengine/queryset/visitor.py b/mongoengine/queryset/visitor.py new file mode 100644 index 00000000..94d6a5e1 --- /dev/null +++ b/mongoengine/queryset/visitor.py @@ -0,0 +1,237 @@ +import copy + +from mongoengine.errors import InvalidQueryError +from mongoengine.python_support import product, reduce + +from mongoengine.queryset import transform + +__all__ = ('Q',) + + +class QNodeVisitor(object): + """Base visitor class for visiting Q-object nodes in a query tree. + """ + + def visit_combination(self, combination): + """Called by QCombination objects. + """ + return combination + + def visit_query(self, query): + """Called by (New)Q objects. + """ + return query + + +class SimplificationVisitor(QNodeVisitor): + """Simplifies query trees by combinging unnecessary 'and' connection nodes + into a single Q-object. + """ + + def visit_combination(self, combination): + if combination.operation == combination.AND: + # The simplification only applies to 'simple' queries + if all(isinstance(node, Q) for node in combination.children): + queries = [node.query for node in combination.children] + return Q(**self._query_conjunction(queries)) + return combination + + def _query_conjunction(self, queries): + """Merges query dicts - effectively &ing them together. + """ + query_ops = set() + combined_query = {} + for query in queries: + ops = set(query.keys()) + # Make sure that the same operation isn't applied more than once + # to a single field + intersection = ops.intersection(query_ops) + if intersection: + msg = 'Duplicate query conditions: ' + raise InvalidQueryError(msg + ', '.join(intersection)) + + query_ops.update(ops) + combined_query.update(copy.deepcopy(query)) + return combined_query + + +class QueryTreeTransformerVisitor(QNodeVisitor): + """Transforms the query tree in to a form that may be used with MongoDB. + """ + + def visit_combination(self, combination): + if combination.operation == combination.AND: + # MongoDB doesn't allow us to have too many $or operations in our + # queries, so the aim is to move the ORs up the tree to one + # 'master' $or. Firstly, we must find all the necessary parts (part + # of an AND combination or just standard Q object), and store them + # separately from the OR parts. + or_groups = [] + and_parts = [] + for node in combination.children: + if isinstance(node, QCombination): + if node.operation == node.OR: + # Any of the children in an $or component may cause + # the query to succeed + or_groups.append(node.children) + elif node.operation == node.AND: + and_parts.append(node) + elif isinstance(node, Q): + and_parts.append(node) + + # Now we combine the parts into a usable query. AND together all of + # the necessary parts. Then for each $or part, create a new query + # that ANDs the necessary part with the $or part. + clauses = [] + for or_group in product(*or_groups): + q_object = reduce(lambda a, b: a & b, and_parts, Q()) + q_object = reduce(lambda a, b: a & b, or_group, q_object) + clauses.append(q_object) + # Finally, $or the generated clauses in to one query. Each of the + # clauses is sufficient for the query to succeed. + return reduce(lambda a, b: a | b, clauses, Q()) + + if combination.operation == combination.OR: + children = [] + # Crush any nested ORs in to this combination as MongoDB doesn't + # support nested $or operations + for node in combination.children: + if (isinstance(node, QCombination) and + node.operation == combination.OR): + children += node.children + else: + children.append(node) + combination.children = children + + return combination + + +class QueryCompilerVisitor(QNodeVisitor): + """Compiles the nodes in a query tree to a PyMongo-compatible query + dictionary. + """ + + def __init__(self, document): + self.document = document + + def visit_combination(self, combination): + if combination.operation == combination.OR: + return {'$or': combination.children} + elif combination.operation == combination.AND: + return self._mongo_query_conjunction(combination.children) + return combination + + def visit_query(self, query): + return transform.query(self.document, **query.query) + + def _mongo_query_conjunction(self, queries): + """Merges Mongo query dicts - effectively &ing them together. + """ + combined_query = {} + for query in queries: + for field, ops in query.items(): + if field not in combined_query: + combined_query[field] = ops + else: + # The field is already present in the query the only way + # we can merge is if both the existing value and the new + # value are operation dicts, reject anything else + if (not isinstance(combined_query[field], dict) or + not isinstance(ops, dict)): + message = 'Conflicting values for ' + field + raise InvalidQueryError(message) + + current_ops = set(combined_query[field].keys()) + new_ops = set(ops.keys()) + # Make sure that the same operation isn't applied more than + # once to a single field + intersection = current_ops.intersection(new_ops) + if intersection: + msg = 'Duplicate query conditions: ' + raise InvalidQueryError(msg + ', '.join(intersection)) + + # Right! We've got two non-overlapping dicts of operations! + combined_query[field].update(copy.deepcopy(ops)) + return combined_query + + +class QNode(object): + """Base class for nodes in query trees. + """ + + AND = 0 + OR = 1 + + def to_query(self, document): + query = self.accept(SimplificationVisitor()) + query = query.accept(QueryTreeTransformerVisitor()) + query = query.accept(QueryCompilerVisitor(document)) + return query + + def accept(self, visitor): + raise NotImplementedError + + def _combine(self, other, operation): + """Combine this node with another node into a QCombination object. + """ + if getattr(other, 'empty', True): + return self + + if self.empty: + return other + + return QCombination(operation, [self, other]) + + @property + def empty(self): + return False + + def __or__(self, other): + return self._combine(other, self.OR) + + def __and__(self, other): + return self._combine(other, self.AND) + + +class QCombination(QNode): + """Represents the combination of several conditions by a given logical + operator. + """ + + def __init__(self, operation, children): + self.operation = operation + self.children = [] + for node in children: + # If the child is a combination of the same type, we can merge its + # children directly into this combinations children + if isinstance(node, QCombination) and node.operation == operation: + self.children += node.children + else: + self.children.append(node) + + def accept(self, visitor): + for i in range(len(self.children)): + if isinstance(self.children[i], QNode): + self.children[i] = self.children[i].accept(visitor) + + return visitor.visit_combination(self) + + @property + def empty(self): + return not bool(self.children) + + +class Q(QNode): + """A simple query object, used in a query tree to build up more complex + query structures. + """ + + def __init__(self, **query): + self.query = query + + def accept(self, visitor): + return visitor.visit_query(self) + + @property + def empty(self): + return not bool(self.query) diff --git a/setup.cfg b/setup.cfg index d95a9176..3f3faa8c 100644 --- a/setup.cfg +++ b/setup.cfg @@ -8,4 +8,4 @@ detailed-errors = 1 #cover-package = mongoengine py3where = build where = tests -#tests = test_bugfix.py \ No newline at end of file +#tests = document/__init__.py \ No newline at end of file diff --git a/tests/__init__.py b/tests/__init__.py index e69de29b..f2a43b05 100644 --- a/tests/__init__.py +++ b/tests/__init__.py @@ -0,0 +1,2 @@ +from .all_warnings import AllWarnings +from .document import * \ No newline at end of file diff --git a/tests/test_all_warnings.py b/tests/all_warnings/__init__.py similarity index 91% rename from tests/test_all_warnings.py rename to tests/all_warnings/__init__.py index 9b38fa61..72de8222 100644 --- a/tests/test_all_warnings.py +++ b/tests/all_warnings/__init__.py @@ -1,11 +1,19 @@ +""" +This test has been put into a module. This is because it tests warnings that +only get triggered on first hit. This way we can ensure its imported into the +top level and called first by the test suite. +""" + import unittest import warnings from mongoengine import * -from mongoengine.tests import query_counter -class TestWarnings(unittest.TestCase): +__all__ = ('AllWarnings', ) + + +class AllWarnings(unittest.TestCase): def setUp(self): conn = connect(db='mongoenginetest') diff --git a/tests/document/__init__.py b/tests/document/__init__.py new file mode 100644 index 00000000..1ef25201 --- /dev/null +++ b/tests/document/__init__.py @@ -0,0 +1,11 @@ +# TODO EXPLICT IMPORTS + +from class_methods import * +from delta import * +from dynamic import * +from indexes import * +from inheritance import * +from instance import * + +if __name__ == '__main__': + unittest.main() diff --git a/tests/document/class_methods.py b/tests/document/class_methods.py new file mode 100644 index 00000000..8050998c --- /dev/null +++ b/tests/document/class_methods.py @@ -0,0 +1,183 @@ +# -*- coding: utf-8 -*- +from __future__ import with_statement +import unittest + +from mongoengine import * + +from mongoengine.queryset import NULLIFY +from mongoengine.connection import get_db + +__all__ = ("ClassMethodsTest", ) + + +class ClassMethodsTest(unittest.TestCase): + + def setUp(self): + connect(db='mongoenginetest') + self.db = get_db() + + class Person(Document): + name = StringField() + age = IntField() + + non_field = True + + meta = {"allow_inheritance": True} + + self.Person = Person + + def tearDown(self): + for collection in self.db.collection_names(): + if 'system.' in collection: + continue + self.db.drop_collection(collection) + + def test_definition(self): + """Ensure that document may be defined using fields. + """ + self.assertEqual(['age', 'name', 'id'], self.Person._fields.keys()) + self.assertEqual([IntField, StringField, ObjectIdField], + [x.__class__ for x in self.Person._fields.values()]) + + def test_get_db(self): + """Ensure that get_db returns the expected db. + """ + db = self.Person._get_db() + self.assertEqual(self.db, db) + + def test_get_collection_name(self): + """Ensure that get_collection_name returns the expected collection + name. + """ + collection_name = 'person' + self.assertEqual(collection_name, self.Person._get_collection_name()) + + def test_get_collection(self): + """Ensure that get_collection returns the expected collection. + """ + collection_name = 'person' + collection = self.Person._get_collection() + self.assertEqual(self.db[collection_name], collection) + + def test_drop_collection(self): + """Ensure that the collection may be dropped from the database. + """ + collection_name = 'person' + self.Person(name='Test').save() + self.assertTrue(collection_name in self.db.collection_names()) + + self.Person.drop_collection() + self.assertFalse(collection_name in self.db.collection_names()) + + def test_register_delete_rule(self): + """Ensure that register delete rule adds a delete rule to the document + meta. + """ + class Job(Document): + employee = ReferenceField(self.Person) + + self.assertEqual(self.Person._meta.get('delete_rules'), None) + + self.Person.register_delete_rule(Job, 'employee', NULLIFY) + self.assertEqual(self.Person._meta['delete_rules'], + {(Job, 'employee'): NULLIFY}) + + def test_collection_naming(self): + """Ensure that a collection with a specified name may be used. + """ + + class DefaultNamingTest(Document): + pass + self.assertEqual('default_naming_test', + DefaultNamingTest._get_collection_name()) + + class CustomNamingTest(Document): + meta = {'collection': 'pimp_my_collection'} + + self.assertEqual('pimp_my_collection', + CustomNamingTest._get_collection_name()) + + class DynamicNamingTest(Document): + meta = {'collection': lambda c: "DYNAMO"} + self.assertEqual('DYNAMO', DynamicNamingTest._get_collection_name()) + + # Use Abstract class to handle backwards compatibility + class BaseDocument(Document): + meta = { + 'abstract': True, + 'collection': lambda c: c.__name__.lower() + } + + class OldNamingConvention(BaseDocument): + pass + self.assertEqual('oldnamingconvention', + OldNamingConvention._get_collection_name()) + + class InheritedAbstractNamingTest(BaseDocument): + meta = {'collection': 'wibble'} + self.assertEqual('wibble', + InheritedAbstractNamingTest._get_collection_name()) + + # Mixin tests + class BaseMixin(object): + meta = { + 'collection': lambda c: c.__name__.lower() + } + + class OldMixinNamingConvention(Document, BaseMixin): + pass + self.assertEqual('oldmixinnamingconvention', + OldMixinNamingConvention._get_collection_name()) + + class BaseMixin(object): + meta = { + 'collection': lambda c: c.__name__.lower() + } + + class BaseDocument(Document, BaseMixin): + meta = {'allow_inheritance': True} + + class MyDocument(BaseDocument): + pass + + self.assertEqual('basedocument', MyDocument._get_collection_name()) + + def test_custom_collection_name_operations(self): + """Ensure that a collection with a specified name is used as expected. + """ + collection_name = 'personCollTest' + + class Person(Document): + name = StringField() + meta = {'collection': collection_name} + + Person(name="Test User").save() + self.assertTrue(collection_name in self.db.collection_names()) + + user_obj = self.db[collection_name].find_one() + self.assertEqual(user_obj['name'], "Test User") + + user_obj = Person.objects[0] + self.assertEqual(user_obj.name, "Test User") + + Person.drop_collection() + self.assertFalse(collection_name in self.db.collection_names()) + + def test_collection_name_and_primary(self): + """Ensure that a collection with a specified name may be used. + """ + + class Person(Document): + name = StringField(primary_key=True) + meta = {'collection': 'app'} + + Person(name="Test User").save() + + user_obj = Person.objects.first() + self.assertEqual(user_obj.name, "Test User") + + Person.drop_collection() + + +if __name__ == '__main__': + unittest.main() diff --git a/tests/document/delta.py b/tests/document/delta.py new file mode 100644 index 00000000..f8a071d6 --- /dev/null +++ b/tests/document/delta.py @@ -0,0 +1,688 @@ +# -*- coding: utf-8 -*- +import unittest + +from mongoengine import * +from mongoengine.connection import get_db + +__all__ = ("DeltaTest",) + + +class DeltaTest(unittest.TestCase): + + def setUp(self): + connect(db='mongoenginetest') + self.db = get_db() + + class Person(Document): + name = StringField() + age = IntField() + + non_field = True + + meta = {"allow_inheritance": True} + + self.Person = Person + + def tearDown(self): + for collection in self.db.collection_names(): + if 'system.' in collection: + continue + self.db.drop_collection(collection) + + def test_delta(self): + self.delta(Document) + self.delta(DynamicDocument) + + def delta(self, DocClass): + + class Doc(DocClass): + string_field = StringField() + int_field = IntField() + dict_field = DictField() + list_field = ListField() + + Doc.drop_collection() + doc = Doc() + doc.save() + + doc = Doc.objects.first() + self.assertEqual(doc._get_changed_fields(), []) + self.assertEqual(doc._delta(), ({}, {})) + + doc.string_field = 'hello' + self.assertEqual(doc._get_changed_fields(), ['string_field']) + self.assertEqual(doc._delta(), ({'string_field': 'hello'}, {})) + + doc._changed_fields = [] + doc.int_field = 1 + self.assertEqual(doc._get_changed_fields(), ['int_field']) + self.assertEqual(doc._delta(), ({'int_field': 1}, {})) + + doc._changed_fields = [] + dict_value = {'hello': 'world', 'ping': 'pong'} + doc.dict_field = dict_value + self.assertEqual(doc._get_changed_fields(), ['dict_field']) + self.assertEqual(doc._delta(), ({'dict_field': dict_value}, {})) + + doc._changed_fields = [] + list_value = ['1', 2, {'hello': 'world'}] + doc.list_field = list_value + self.assertEqual(doc._get_changed_fields(), ['list_field']) + self.assertEqual(doc._delta(), ({'list_field': list_value}, {})) + + # Test unsetting + doc._changed_fields = [] + doc.dict_field = {} + self.assertEqual(doc._get_changed_fields(), ['dict_field']) + self.assertEqual(doc._delta(), ({}, {'dict_field': 1})) + + doc._changed_fields = [] + doc.list_field = [] + self.assertEqual(doc._get_changed_fields(), ['list_field']) + self.assertEqual(doc._delta(), ({}, {'list_field': 1})) + + def test_delta_recursive(self): + self.delta_recursive(Document, EmbeddedDocument) + self.delta_recursive(DynamicDocument, EmbeddedDocument) + self.delta_recursive(Document, DynamicEmbeddedDocument) + self.delta_recursive(DynamicDocument, DynamicEmbeddedDocument) + + def delta_recursive(self, DocClass, EmbeddedClass): + + class Embedded(EmbeddedClass): + string_field = StringField() + int_field = IntField() + dict_field = DictField() + list_field = ListField() + + class Doc(DocClass): + string_field = StringField() + int_field = IntField() + dict_field = DictField() + list_field = ListField() + embedded_field = EmbeddedDocumentField(Embedded) + + Doc.drop_collection() + doc = Doc() + doc.save() + + doc = Doc.objects.first() + self.assertEqual(doc._get_changed_fields(), []) + self.assertEqual(doc._delta(), ({}, {})) + + embedded_1 = Embedded() + embedded_1.string_field = 'hello' + embedded_1.int_field = 1 + embedded_1.dict_field = {'hello': 'world'} + embedded_1.list_field = ['1', 2, {'hello': 'world'}] + doc.embedded_field = embedded_1 + + self.assertEqual(doc._get_changed_fields(), ['embedded_field']) + + embedded_delta = { + 'string_field': 'hello', + 'int_field': 1, + 'dict_field': {'hello': 'world'}, + 'list_field': ['1', 2, {'hello': 'world'}] + } + self.assertEqual(doc.embedded_field._delta(), (embedded_delta, {})) + embedded_delta.update({ + '_cls': 'Embedded', + }) + self.assertEqual(doc._delta(), + ({'embedded_field': embedded_delta}, {})) + + doc.save() + doc = doc.reload(10) + + doc.embedded_field.dict_field = {} + self.assertEqual(doc._get_changed_fields(), + ['embedded_field.dict_field']) + self.assertEqual(doc.embedded_field._delta(), ({}, {'dict_field': 1})) + self.assertEqual(doc._delta(), ({}, {'embedded_field.dict_field': 1})) + doc.save() + doc = doc.reload(10) + self.assertEqual(doc.embedded_field.dict_field, {}) + + doc.embedded_field.list_field = [] + self.assertEqual(doc._get_changed_fields(), + ['embedded_field.list_field']) + self.assertEqual(doc.embedded_field._delta(), ({}, {'list_field': 1})) + self.assertEqual(doc._delta(), ({}, {'embedded_field.list_field': 1})) + doc.save() + doc = doc.reload(10) + self.assertEqual(doc.embedded_field.list_field, []) + + embedded_2 = Embedded() + embedded_2.string_field = 'hello' + embedded_2.int_field = 1 + embedded_2.dict_field = {'hello': 'world'} + embedded_2.list_field = ['1', 2, {'hello': 'world'}] + + doc.embedded_field.list_field = ['1', 2, embedded_2] + self.assertEqual(doc._get_changed_fields(), + ['embedded_field.list_field']) + self.assertEqual(doc.embedded_field._delta(), ({ + 'list_field': ['1', 2, { + '_cls': 'Embedded', + 'string_field': 'hello', + 'dict_field': {'hello': 'world'}, + 'int_field': 1, + 'list_field': ['1', 2, {'hello': 'world'}], + }] + }, {})) + + self.assertEqual(doc._delta(), ({ + 'embedded_field.list_field': ['1', 2, { + '_cls': 'Embedded', + 'string_field': 'hello', + 'dict_field': {'hello': 'world'}, + 'int_field': 1, + 'list_field': ['1', 2, {'hello': 'world'}], + }] + }, {})) + doc.save() + doc = doc.reload(10) + + self.assertEqual(doc.embedded_field.list_field[0], '1') + self.assertEqual(doc.embedded_field.list_field[1], 2) + for k in doc.embedded_field.list_field[2]._fields: + self.assertEqual(doc.embedded_field.list_field[2][k], + embedded_2[k]) + + doc.embedded_field.list_field[2].string_field = 'world' + self.assertEqual(doc._get_changed_fields(), + ['embedded_field.list_field.2.string_field']) + self.assertEqual(doc.embedded_field._delta(), + ({'list_field.2.string_field': 'world'}, {})) + self.assertEqual(doc._delta(), + ({'embedded_field.list_field.2.string_field': 'world'}, {})) + doc.save() + doc = doc.reload(10) + self.assertEqual(doc.embedded_field.list_field[2].string_field, + 'world') + + # Test multiple assignments + doc.embedded_field.list_field[2].string_field = 'hello world' + doc.embedded_field.list_field[2] = doc.embedded_field.list_field[2] + self.assertEqual(doc._get_changed_fields(), + ['embedded_field.list_field']) + self.assertEqual(doc.embedded_field._delta(), ({ + 'list_field': ['1', 2, { + '_cls': 'Embedded', + 'string_field': 'hello world', + 'int_field': 1, + 'list_field': ['1', 2, {'hello': 'world'}], + 'dict_field': {'hello': 'world'}}]}, {})) + self.assertEqual(doc._delta(), ({ + 'embedded_field.list_field': ['1', 2, { + '_cls': 'Embedded', + 'string_field': 'hello world', + 'int_field': 1, + 'list_field': ['1', 2, {'hello': 'world'}], + 'dict_field': {'hello': 'world'}} + ]}, {})) + doc.save() + doc = doc.reload(10) + self.assertEqual(doc.embedded_field.list_field[2].string_field, + 'hello world') + + # Test list native methods + doc.embedded_field.list_field[2].list_field.pop(0) + self.assertEqual(doc._delta(), + ({'embedded_field.list_field.2.list_field': + [2, {'hello': 'world'}]}, {})) + doc.save() + doc = doc.reload(10) + + doc.embedded_field.list_field[2].list_field.append(1) + self.assertEqual(doc._delta(), + ({'embedded_field.list_field.2.list_field': + [2, {'hello': 'world'}, 1]}, {})) + doc.save() + doc = doc.reload(10) + self.assertEqual(doc.embedded_field.list_field[2].list_field, + [2, {'hello': 'world'}, 1]) + + doc.embedded_field.list_field[2].list_field.sort(key=str) + doc.save() + doc = doc.reload(10) + self.assertEqual(doc.embedded_field.list_field[2].list_field, + [1, 2, {'hello': 'world'}]) + + del(doc.embedded_field.list_field[2].list_field[2]['hello']) + self.assertEqual(doc._delta(), + ({'embedded_field.list_field.2.list_field': [1, 2, {}]}, {})) + doc.save() + doc = doc.reload(10) + + del(doc.embedded_field.list_field[2].list_field) + self.assertEqual(doc._delta(), + ({}, {'embedded_field.list_field.2.list_field': 1})) + + doc.save() + doc = doc.reload(10) + + doc.dict_field['Embedded'] = embedded_1 + doc.save() + doc = doc.reload(10) + + doc.dict_field['Embedded'].string_field = 'Hello World' + self.assertEqual(doc._get_changed_fields(), + ['dict_field.Embedded.string_field']) + self.assertEqual(doc._delta(), + ({'dict_field.Embedded.string_field': 'Hello World'}, {})) + + def test_circular_reference_deltas(self): + self.circular_reference_deltas(Document, Document) + self.circular_reference_deltas(Document, DynamicDocument) + self.circular_reference_deltas(DynamicDocument, Document) + self.circular_reference_deltas(DynamicDocument, DynamicDocument) + + def circular_reference_deltas(self, DocClass1, DocClass2): + + class Person(DocClass1): + name = StringField() + owns = ListField(ReferenceField('Organization')) + + class Organization(DocClass2): + name = StringField() + owner = ReferenceField('Person') + + person = Person(name="owner") + person.save() + organization = Organization(name="company") + organization.save() + + person.owns.append(organization) + organization.owner = person + + person.save() + organization.save() + + p = Person.objects[0].select_related() + o = Organization.objects.first() + self.assertEqual(p.owns[0], o) + self.assertEqual(o.owner, p) + + def test_circular_reference_deltas_2(self): + self.circular_reference_deltas_2(Document, Document) + self.circular_reference_deltas_2(Document, DynamicDocument) + self.circular_reference_deltas_2(DynamicDocument, Document) + self.circular_reference_deltas_2(DynamicDocument, DynamicDocument) + + def circular_reference_deltas_2(self, DocClass1, DocClass2): + + class Person(DocClass1): + name = StringField() + owns = ListField(ReferenceField('Organization')) + employer = ReferenceField('Organization') + + class Organization(DocClass2): + name = StringField() + owner = ReferenceField('Person') + employees = ListField(ReferenceField('Person')) + + Person.drop_collection() + Organization.drop_collection() + + person = Person(name="owner") + person.save() + + employee = Person(name="employee") + employee.save() + + organization = Organization(name="company") + organization.save() + + person.owns.append(organization) + organization.owner = person + + organization.employees.append(employee) + employee.employer = organization + + person.save() + organization.save() + employee.save() + + p = Person.objects.get(name="owner") + e = Person.objects.get(name="employee") + o = Organization.objects.first() + + self.assertEqual(p.owns[0], o) + self.assertEqual(o.owner, p) + self.assertEqual(e.employer, o) + + def test_delta_db_field(self): + self.delta_db_field(Document) + self.delta_db_field(DynamicDocument) + + def delta_db_field(self, DocClass): + + class Doc(DocClass): + string_field = StringField(db_field='db_string_field') + int_field = IntField(db_field='db_int_field') + dict_field = DictField(db_field='db_dict_field') + list_field = ListField(db_field='db_list_field') + + Doc.drop_collection() + doc = Doc() + doc.save() + + doc = Doc.objects.first() + self.assertEqual(doc._get_changed_fields(), []) + self.assertEqual(doc._delta(), ({}, {})) + + doc.string_field = 'hello' + self.assertEqual(doc._get_changed_fields(), ['db_string_field']) + self.assertEqual(doc._delta(), ({'db_string_field': 'hello'}, {})) + + doc._changed_fields = [] + doc.int_field = 1 + self.assertEqual(doc._get_changed_fields(), ['db_int_field']) + self.assertEqual(doc._delta(), ({'db_int_field': 1}, {})) + + doc._changed_fields = [] + dict_value = {'hello': 'world', 'ping': 'pong'} + doc.dict_field = dict_value + self.assertEqual(doc._get_changed_fields(), ['db_dict_field']) + self.assertEqual(doc._delta(), ({'db_dict_field': dict_value}, {})) + + doc._changed_fields = [] + list_value = ['1', 2, {'hello': 'world'}] + doc.list_field = list_value + self.assertEqual(doc._get_changed_fields(), ['db_list_field']) + self.assertEqual(doc._delta(), ({'db_list_field': list_value}, {})) + + # Test unsetting + doc._changed_fields = [] + doc.dict_field = {} + self.assertEqual(doc._get_changed_fields(), ['db_dict_field']) + self.assertEqual(doc._delta(), ({}, {'db_dict_field': 1})) + + doc._changed_fields = [] + doc.list_field = [] + self.assertEqual(doc._get_changed_fields(), ['db_list_field']) + self.assertEqual(doc._delta(), ({}, {'db_list_field': 1})) + + # Test it saves that data + doc = Doc() + doc.save() + + doc.string_field = 'hello' + doc.int_field = 1 + doc.dict_field = {'hello': 'world'} + doc.list_field = ['1', 2, {'hello': 'world'}] + doc.save() + doc = doc.reload(10) + + self.assertEqual(doc.string_field, 'hello') + self.assertEqual(doc.int_field, 1) + self.assertEqual(doc.dict_field, {'hello': 'world'}) + self.assertEqual(doc.list_field, ['1', 2, {'hello': 'world'}]) + + def test_delta_recursive_db_field(self): + self.delta_recursive_db_field(Document, EmbeddedDocument) + self.delta_recursive_db_field(Document, DynamicEmbeddedDocument) + self.delta_recursive_db_field(DynamicDocument, EmbeddedDocument) + self.delta_recursive_db_field(DynamicDocument, DynamicEmbeddedDocument) + + def delta_recursive_db_field(self, DocClass, EmbeddedClass): + + class Embedded(EmbeddedClass): + string_field = StringField(db_field='db_string_field') + int_field = IntField(db_field='db_int_field') + dict_field = DictField(db_field='db_dict_field') + list_field = ListField(db_field='db_list_field') + + class Doc(DocClass): + string_field = StringField(db_field='db_string_field') + int_field = IntField(db_field='db_int_field') + dict_field = DictField(db_field='db_dict_field') + list_field = ListField(db_field='db_list_field') + embedded_field = EmbeddedDocumentField(Embedded, + db_field='db_embedded_field') + + Doc.drop_collection() + doc = Doc() + doc.save() + + doc = Doc.objects.first() + self.assertEqual(doc._get_changed_fields(), []) + self.assertEqual(doc._delta(), ({}, {})) + + embedded_1 = Embedded() + embedded_1.string_field = 'hello' + embedded_1.int_field = 1 + embedded_1.dict_field = {'hello': 'world'} + embedded_1.list_field = ['1', 2, {'hello': 'world'}] + doc.embedded_field = embedded_1 + + self.assertEqual(doc._get_changed_fields(), ['db_embedded_field']) + + embedded_delta = { + 'db_string_field': 'hello', + 'db_int_field': 1, + 'db_dict_field': {'hello': 'world'}, + 'db_list_field': ['1', 2, {'hello': 'world'}] + } + self.assertEqual(doc.embedded_field._delta(), (embedded_delta, {})) + embedded_delta.update({ + '_cls': 'Embedded', + }) + self.assertEqual(doc._delta(), + ({'db_embedded_field': embedded_delta}, {})) + + doc.save() + doc = doc.reload(10) + + doc.embedded_field.dict_field = {} + self.assertEqual(doc._get_changed_fields(), + ['db_embedded_field.db_dict_field']) + self.assertEqual(doc.embedded_field._delta(), + ({}, {'db_dict_field': 1})) + self.assertEqual(doc._delta(), + ({}, {'db_embedded_field.db_dict_field': 1})) + doc.save() + doc = doc.reload(10) + self.assertEqual(doc.embedded_field.dict_field, {}) + + doc.embedded_field.list_field = [] + self.assertEqual(doc._get_changed_fields(), + ['db_embedded_field.db_list_field']) + self.assertEqual(doc.embedded_field._delta(), + ({}, {'db_list_field': 1})) + self.assertEqual(doc._delta(), + ({}, {'db_embedded_field.db_list_field': 1})) + doc.save() + doc = doc.reload(10) + self.assertEqual(doc.embedded_field.list_field, []) + + embedded_2 = Embedded() + embedded_2.string_field = 'hello' + embedded_2.int_field = 1 + embedded_2.dict_field = {'hello': 'world'} + embedded_2.list_field = ['1', 2, {'hello': 'world'}] + + doc.embedded_field.list_field = ['1', 2, embedded_2] + self.assertEqual(doc._get_changed_fields(), + ['db_embedded_field.db_list_field']) + self.assertEqual(doc.embedded_field._delta(), ({ + 'db_list_field': ['1', 2, { + '_cls': 'Embedded', + 'db_string_field': 'hello', + 'db_dict_field': {'hello': 'world'}, + 'db_int_field': 1, + 'db_list_field': ['1', 2, {'hello': 'world'}], + }] + }, {})) + + self.assertEqual(doc._delta(), ({ + 'db_embedded_field.db_list_field': ['1', 2, { + '_cls': 'Embedded', + 'db_string_field': 'hello', + 'db_dict_field': {'hello': 'world'}, + 'db_int_field': 1, + 'db_list_field': ['1', 2, {'hello': 'world'}], + }] + }, {})) + doc.save() + doc = doc.reload(10) + + self.assertEqual(doc.embedded_field.list_field[0], '1') + self.assertEqual(doc.embedded_field.list_field[1], 2) + for k in doc.embedded_field.list_field[2]._fields: + self.assertEqual(doc.embedded_field.list_field[2][k], + embedded_2[k]) + + doc.embedded_field.list_field[2].string_field = 'world' + self.assertEqual(doc._get_changed_fields(), + ['db_embedded_field.db_list_field.2.db_string_field']) + self.assertEqual(doc.embedded_field._delta(), + ({'db_list_field.2.db_string_field': 'world'}, {})) + self.assertEqual(doc._delta(), + ({'db_embedded_field.db_list_field.2.db_string_field': 'world'}, + {})) + doc.save() + doc = doc.reload(10) + self.assertEqual(doc.embedded_field.list_field[2].string_field, + 'world') + + # Test multiple assignments + doc.embedded_field.list_field[2].string_field = 'hello world' + doc.embedded_field.list_field[2] = doc.embedded_field.list_field[2] + self.assertEqual(doc._get_changed_fields(), + ['db_embedded_field.db_list_field']) + self.assertEqual(doc.embedded_field._delta(), ({ + 'db_list_field': ['1', 2, { + '_cls': 'Embedded', + 'db_string_field': 'hello world', + 'db_int_field': 1, + 'db_list_field': ['1', 2, {'hello': 'world'}], + 'db_dict_field': {'hello': 'world'}}]}, {})) + self.assertEqual(doc._delta(), ({ + 'db_embedded_field.db_list_field': ['1', 2, { + '_cls': 'Embedded', + 'db_string_field': 'hello world', + 'db_int_field': 1, + 'db_list_field': ['1', 2, {'hello': 'world'}], + 'db_dict_field': {'hello': 'world'}} + ]}, {})) + doc.save() + doc = doc.reload(10) + self.assertEqual(doc.embedded_field.list_field[2].string_field, + 'hello world') + + # Test list native methods + doc.embedded_field.list_field[2].list_field.pop(0) + self.assertEqual(doc._delta(), + ({'db_embedded_field.db_list_field.2.db_list_field': + [2, {'hello': 'world'}]}, {})) + doc.save() + doc = doc.reload(10) + + doc.embedded_field.list_field[2].list_field.append(1) + self.assertEqual(doc._delta(), + ({'db_embedded_field.db_list_field.2.db_list_field': + [2, {'hello': 'world'}, 1]}, {})) + doc.save() + doc = doc.reload(10) + self.assertEqual(doc.embedded_field.list_field[2].list_field, + [2, {'hello': 'world'}, 1]) + + doc.embedded_field.list_field[2].list_field.sort(key=str) + doc.save() + doc = doc.reload(10) + self.assertEqual(doc.embedded_field.list_field[2].list_field, + [1, 2, {'hello': 'world'}]) + + del(doc.embedded_field.list_field[2].list_field[2]['hello']) + self.assertEqual(doc._delta(), + ({'db_embedded_field.db_list_field.2.db_list_field': + [1, 2, {}]}, {})) + doc.save() + doc = doc.reload(10) + + del(doc.embedded_field.list_field[2].list_field) + self.assertEqual(doc._delta(), ({}, + {'db_embedded_field.db_list_field.2.db_list_field': 1})) + + def test_delta_for_dynamic_documents(self): + class Person(DynamicDocument): + name = StringField() + meta = {'allow_inheritance': True} + + Person.drop_collection() + + p = Person(name="James", age=34) + self.assertEqual(p._delta(), ({'age': 34, 'name': 'James', + '_cls': 'Person'}, {})) + + p.doc = 123 + del(p.doc) + self.assertEqual(p._delta(), ({'age': 34, 'name': 'James', + '_cls': 'Person'}, {'doc': 1})) + + p = Person() + p.name = "Dean" + p.age = 22 + p.save() + + p.age = 24 + self.assertEqual(p.age, 24) + self.assertEqual(p._get_changed_fields(), ['age']) + self.assertEqual(p._delta(), ({'age': 24}, {})) + + p = self.Person.objects(age=22).get() + p.age = 24 + self.assertEqual(p.age, 24) + self.assertEqual(p._get_changed_fields(), ['age']) + self.assertEqual(p._delta(), ({'age': 24}, {})) + + p.save() + self.assertEqual(1, self.Person.objects(age=24).count()) + + def test_dynamic_delta(self): + + class Doc(DynamicDocument): + pass + + Doc.drop_collection() + doc = Doc() + doc.save() + + doc = Doc.objects.first() + self.assertEqual(doc._get_changed_fields(), []) + self.assertEqual(doc._delta(), ({}, {})) + + doc.string_field = 'hello' + self.assertEqual(doc._get_changed_fields(), ['string_field']) + self.assertEqual(doc._delta(), ({'string_field': 'hello'}, {})) + + doc._changed_fields = [] + doc.int_field = 1 + self.assertEqual(doc._get_changed_fields(), ['int_field']) + self.assertEqual(doc._delta(), ({'int_field': 1}, {})) + + doc._changed_fields = [] + dict_value = {'hello': 'world', 'ping': 'pong'} + doc.dict_field = dict_value + self.assertEqual(doc._get_changed_fields(), ['dict_field']) + self.assertEqual(doc._delta(), ({'dict_field': dict_value}, {})) + + doc._changed_fields = [] + list_value = ['1', 2, {'hello': 'world'}] + doc.list_field = list_value + self.assertEqual(doc._get_changed_fields(), ['list_field']) + self.assertEqual(doc._delta(), ({'list_field': list_value}, {})) + + # Test unsetting + doc._changed_fields = [] + doc.dict_field = {} + self.assertEqual(doc._get_changed_fields(), ['dict_field']) + self.assertEqual(doc._delta(), ({}, {'dict_field': 1})) + + doc._changed_fields = [] + doc.list_field = [] + self.assertEqual(doc._get_changed_fields(), ['list_field']) + self.assertEqual(doc._delta(), ({}, {'list_field': 1})) diff --git a/tests/document/dynamic.py b/tests/document/dynamic.py new file mode 100644 index 00000000..ef279179 --- /dev/null +++ b/tests/document/dynamic.py @@ -0,0 +1,270 @@ +import unittest + +from mongoengine import * +from mongoengine.connection import get_db + +__all__ = ("DynamicTest", ) + + +class DynamicTest(unittest.TestCase): + + def setUp(self): + connect(db='mongoenginetest') + self.db = get_db() + + class Person(DynamicDocument): + name = StringField() + meta = {'allow_inheritance': True} + + Person.drop_collection() + + self.Person = Person + + def test_simple_dynamic_document(self): + """Ensures simple dynamic documents are saved correctly""" + + p = self.Person() + p.name = "James" + p.age = 34 + + self.assertEqual(p.to_mongo(), {"_cls": "Person", "name": "James", + "age": 34}) + + p.save() + + self.assertEqual(self.Person.objects.first().age, 34) + + # Confirm no changes to self.Person + self.assertFalse(hasattr(self.Person, 'age')) + + def test_change_scope_of_variable(self): + """Test changing the scope of a dynamic field has no adverse effects""" + p = self.Person() + p.name = "Dean" + p.misc = 22 + p.save() + + p = self.Person.objects.get() + p.misc = {'hello': 'world'} + p.save() + + p = self.Person.objects.get() + self.assertEqual(p.misc, {'hello': 'world'}) + + def test_delete_dynamic_field(self): + """Test deleting a dynamic field works""" + self.Person.drop_collection() + p = self.Person() + p.name = "Dean" + p.misc = 22 + p.save() + + p = self.Person.objects.get() + p.misc = {'hello': 'world'} + p.save() + + p = self.Person.objects.get() + self.assertEqual(p.misc, {'hello': 'world'}) + collection = self.db[self.Person._get_collection_name()] + obj = collection.find_one() + self.assertEqual(sorted(obj.keys()), ['_cls', '_id', 'misc', 'name']) + + del(p.misc) + p.save() + + p = self.Person.objects.get() + self.assertFalse(hasattr(p, 'misc')) + + obj = collection.find_one() + self.assertEqual(sorted(obj.keys()), ['_cls', '_id', 'name']) + + def test_dynamic_document_queries(self): + """Ensure we can query dynamic fields""" + p = self.Person() + p.name = "Dean" + p.age = 22 + p.save() + + self.assertEqual(1, self.Person.objects(age=22).count()) + p = self.Person.objects(age=22) + p = p.get() + self.assertEqual(22, p.age) + + def test_complex_dynamic_document_queries(self): + class Person(DynamicDocument): + name = StringField() + + Person.drop_collection() + + p = Person(name="test") + p.age = "ten" + p.save() + + p1 = Person(name="test1") + p1.age = "less then ten and a half" + p1.save() + + p2 = Person(name="test2") + p2.age = 10 + p2.save() + + self.assertEqual(Person.objects(age__icontains='ten').count(), 2) + self.assertEqual(Person.objects(age__gte=10).count(), 1) + + def test_complex_data_lookups(self): + """Ensure you can query dynamic document dynamic fields""" + p = self.Person() + p.misc = {'hello': 'world'} + p.save() + + self.assertEqual(1, self.Person.objects(misc__hello='world').count()) + + def test_inheritance(self): + """Ensure that dynamic document plays nice with inheritance""" + class Employee(self.Person): + salary = IntField() + + Employee.drop_collection() + + self.assertTrue('name' in Employee._fields) + self.assertTrue('salary' in Employee._fields) + self.assertEqual(Employee._get_collection_name(), + self.Person._get_collection_name()) + + joe_bloggs = Employee() + joe_bloggs.name = "Joe Bloggs" + joe_bloggs.salary = 10 + joe_bloggs.age = 20 + joe_bloggs.save() + + self.assertEqual(1, self.Person.objects(age=20).count()) + self.assertEqual(1, Employee.objects(age=20).count()) + + joe_bloggs = self.Person.objects.first() + self.assertTrue(isinstance(joe_bloggs, Employee)) + + def test_embedded_dynamic_document(self): + """Test dynamic embedded documents""" + class Embedded(DynamicEmbeddedDocument): + pass + + class Doc(DynamicDocument): + pass + + Doc.drop_collection() + doc = Doc() + + embedded_1 = Embedded() + embedded_1.string_field = 'hello' + embedded_1.int_field = 1 + embedded_1.dict_field = {'hello': 'world'} + embedded_1.list_field = ['1', 2, {'hello': 'world'}] + doc.embedded_field = embedded_1 + + self.assertEqual(doc.to_mongo(), {"_cls": "Doc", + "embedded_field": { + "_cls": "Embedded", + "string_field": "hello", + "int_field": 1, + "dict_field": {"hello": "world"}, + "list_field": ['1', 2, {'hello': 'world'}] + } + }) + doc.save() + + doc = Doc.objects.first() + self.assertEqual(doc.embedded_field.__class__, Embedded) + self.assertEqual(doc.embedded_field.string_field, "hello") + self.assertEqual(doc.embedded_field.int_field, 1) + self.assertEqual(doc.embedded_field.dict_field, {'hello': 'world'}) + self.assertEqual(doc.embedded_field.list_field, + ['1', 2, {'hello': 'world'}]) + + def test_complex_embedded_documents(self): + """Test complex dynamic embedded documents setups""" + class Embedded(DynamicEmbeddedDocument): + pass + + class Doc(DynamicDocument): + pass + + Doc.drop_collection() + doc = Doc() + + embedded_1 = Embedded() + embedded_1.string_field = 'hello' + embedded_1.int_field = 1 + embedded_1.dict_field = {'hello': 'world'} + + embedded_2 = Embedded() + embedded_2.string_field = 'hello' + embedded_2.int_field = 1 + embedded_2.dict_field = {'hello': 'world'} + embedded_2.list_field = ['1', 2, {'hello': 'world'}] + + embedded_1.list_field = ['1', 2, embedded_2] + doc.embedded_field = embedded_1 + + self.assertEqual(doc.to_mongo(), {"_cls": "Doc", + "embedded_field": { + "_cls": "Embedded", + "string_field": "hello", + "int_field": 1, + "dict_field": {"hello": "world"}, + "list_field": ['1', 2, + {"_cls": "Embedded", + "string_field": "hello", + "int_field": 1, + "dict_field": {"hello": "world"}, + "list_field": ['1', 2, {'hello': 'world'}]} + ] + } + }) + doc.save() + doc = Doc.objects.first() + self.assertEqual(doc.embedded_field.__class__, Embedded) + self.assertEqual(doc.embedded_field.string_field, "hello") + self.assertEqual(doc.embedded_field.int_field, 1) + self.assertEqual(doc.embedded_field.dict_field, {'hello': 'world'}) + self.assertEqual(doc.embedded_field.list_field[0], '1') + self.assertEqual(doc.embedded_field.list_field[1], 2) + + embedded_field = doc.embedded_field.list_field[2] + + self.assertEqual(embedded_field.__class__, Embedded) + self.assertEqual(embedded_field.string_field, "hello") + self.assertEqual(embedded_field.int_field, 1) + self.assertEqual(embedded_field.dict_field, {'hello': 'world'}) + self.assertEqual(embedded_field.list_field, ['1', 2, + {'hello': 'world'}]) + + def test_dynamic_and_embedded(self): + """Ensure embedded documents play nicely""" + + class Address(EmbeddedDocument): + city = StringField() + + class Person(DynamicDocument): + name = StringField() + meta = {'allow_inheritance': True} + + Person.drop_collection() + + Person(name="Ross", address=Address(city="London")).save() + + person = Person.objects.first() + person.address.city = "Lundenne" + person.save() + + self.assertEqual(Person.objects.first().address.city, "Lundenne") + + person = Person.objects.first() + person.address = Address(city="Londinium") + person.save() + + self.assertEqual(Person.objects.first().address.city, "Londinium") + + person = Person.objects.first() + person.age = 35 + person.save() + self.assertEqual(Person.objects.first().age, 35) diff --git a/tests/document/indexes.py b/tests/document/indexes.py new file mode 100644 index 00000000..a6b74cd0 --- /dev/null +++ b/tests/document/indexes.py @@ -0,0 +1,637 @@ +# -*- coding: utf-8 -*- +from __future__ import with_statement +import bson +import os +import pickle +import pymongo +import sys +import unittest +import uuid +import warnings + +from nose.plugins.skip import SkipTest +from datetime import datetime + +from tests.fixtures import Base, Mixin, PickleEmbedded, PickleTest + +from mongoengine import * +from mongoengine.errors import (NotRegistered, InvalidDocumentError, + InvalidQueryError) +from mongoengine.queryset import NULLIFY, Q +from mongoengine.connection import get_db, get_connection + +TEST_IMAGE_PATH = os.path.join(os.path.dirname(__file__), 'mongoengine.png') + +__all__ = ("InstanceTest", ) + + +class InstanceTest(unittest.TestCase): + + def setUp(self): + connect(db='mongoenginetest') + self.db = get_db() + + class Person(Document): + name = StringField() + age = IntField() + + non_field = True + + meta = {"allow_inheritance": True} + + self.Person = Person + + def tearDown(self): + for collection in self.db.collection_names(): + if 'system.' in collection: + continue + self.db.drop_collection(collection) + + def test_indexes_document(self, ): + """Ensure that indexes are used when meta[indexes] is specified for + Documents + """ + index_test(Document) + + def test_indexes_dynamic_document(self, ): + """Ensure that indexes are used when meta[indexes] is specified for + Dynamic Documents + """ + index_test(DynamicDocument) + + def index_test(self, InheritFrom): + + class BlogPost(InheritFrom): + date = DateTimeField(db_field='addDate', default=datetime.now) + category = StringField() + tags = ListField(StringField()) + meta = { + 'indexes': [ + '-date', + 'tags', + ('category', '-date') + ], + 'allow_inheritance': True + } + + expected_specs = [{'fields': [('_cls', 1), ('addDate', -1)]}, + {'fields': [('_cls', 1), ('tags', 1)]}, + {'fields': [('_cls', 1), ('category', 1), + ('addDate', -1)]}] + self.assertEqual(expected_specs, BlogPost._meta['index_specs']) + + BlogPost.objects._ensure_indexes() + info = BlogPost.objects._collection.index_information() + # _id, '-date', 'tags', ('cat', 'date') + # NB: there is no index on _cls by itself, since + # the indices on -date and tags will both contain + # _cls as first element in the key + self.assertEqual(len(info), 4) + info = [value['key'] for key, value in info.iteritems()] + for expected in expected_specs: + self.assertTrue(expected['fields'] in info) + + class ExtendedBlogPost(BlogPost): + title = StringField() + meta = {'indexes': ['title']} + + expected_specs.append({'fields': [('_cls', 1), ('title', 1)]}) + self.assertEqual(expected_specs, ExtendedBlogPost._meta['index_specs']) + + BlogPost.drop_collection() + + ExtendedBlogPost.objects._ensure_indexes() + info = ExtendedBlogPost.objects._collection.index_information() + info = [value['key'] for key, value in info.iteritems()] + for expected in expected_specs: + self.assertTrue(expected['fields'] in info) + + def test_inherited_index(self): + """Ensure index specs are inhertited correctly""" + + class A(Document): + title = StringField() + meta = { + 'indexes': [ + { + 'fields': ('title',), + }, + ], + 'allow_inheritance': True, + } + + class B(A): + description = StringField() + + self.assertEqual(A._meta['index_specs'], B._meta['index_specs']) + self.assertEqual([{'fields': [('_cls', 1), ('title', 1)]}], + A._meta['index_specs']) + + def test_build_index_spec_is_not_destructive(self): + + class MyDoc(Document): + keywords = StringField() + + meta = { + 'indexes': ['keywords'], + 'allow_inheritance': False + } + + self.assertEqual(MyDoc._meta['index_specs'], + [{'fields': [('keywords', 1)]}]) + + # Force index creation + MyDoc.objects._ensure_indexes() + + self.assertEqual(MyDoc._meta['index_specs'], + [{'fields': [('keywords', 1)]}]) + + def test_embedded_document_index_meta(self): + """Ensure that embedded document indexes are created explicitly + """ + class Rank(EmbeddedDocument): + title = StringField(required=True) + + class Person(Document): + name = StringField(required=True) + rank = EmbeddedDocumentField(Rank, required=False) + + meta = { + 'indexes': [ + 'rank.title', + ], + 'allow_inheritance': False + } + + self.assertEqual([{'fields': [('rank.title', 1)]}], + Person._meta['index_specs']) + + Person.drop_collection() + + # Indexes are lazy so use list() to perform query + list(Person.objects) + info = Person.objects._collection.index_information() + info = [value['key'] for key, value in info.iteritems()] + self.assertTrue([('rank.title', 1)] in info) + + def test_explicit_geo2d_index(self): + """Ensure that geo2d indexes work when created via meta[indexes] + """ + class Place(Document): + location = DictField() + meta = { + 'allow_inheritance': True, + 'indexes': [ + '*location.point', + ] + } + + self.assertEqual([{'fields': [('location.point', '2d')]}], + Place._meta['index_specs']) + + Place.objects()._ensure_indexes() + info = Place._get_collection().index_information() + info = [value['key'] for key, value in info.iteritems()] + self.assertTrue([('location.point', '2d')] in info) + + def test_dictionary_indexes(self): + """Ensure that indexes are used when meta[indexes] contains + dictionaries instead of lists. + """ + class BlogPost(Document): + date = DateTimeField(db_field='addDate', default=datetime.now) + category = StringField() + tags = ListField(StringField()) + meta = { + 'indexes': [ + {'fields': ['-date'], 'unique': True, + 'sparse': True, 'types': False}, + ], + } + + self.assertEqual([{'fields': [('addDate', -1)], 'unique': True, + 'sparse': True, 'types': False}], + BlogPost._meta['index_specs']) + + BlogPost.drop_collection() + + info = BlogPost.objects._collection.index_information() + # _id, '-date' + self.assertEqual(len(info), 3) + + # Indexes are lazy so use list() to perform query + list(BlogPost.objects) + info = BlogPost.objects._collection.index_information() + info = [(value['key'], + value.get('unique', False), + value.get('sparse', False)) + for key, value in info.iteritems()] + self.assertTrue(([('addDate', -1)], True, True) in info) + + BlogPost.drop_collection() + + def test_abstract_index_inheritance(self): + + class UserBase(Document): + user_guid = StringField(required=True) + meta = { + 'abstract': True, + 'indexes': ['user_guid'], + 'allow_inheritance': True + } + + class Person(UserBase): + name = StringField() + + meta = { + 'indexes': ['name'], + } + + Person(name="test", user_guid='123').save() + + self.assertEqual(1, Person.objects.count()) + info = Person.objects._collection.index_information() + self.assertEqual(info.keys(), ['_cls_1_name_1', '_cls_1_user_guid_1', + '_id_']) + + def test_disable_index_creation(self): + """Tests setting auto_create_index to False on the connection will + disable any index generation. + """ + class User(Document): + meta = { + 'indexes': ['user_guid'], + 'auto_create_index': False + } + user_guid = StringField(required=True) + + + User.drop_collection() + + u = User(user_guid='123') + u.save() + + self.assertEqual(1, User.objects.count()) + info = User.objects._collection.index_information() + self.assertEqual(info.keys(), ['_id_']) + User.drop_collection() + + def test_embedded_document_index(self): + """Tests settings an index on an embedded document + """ + class Date(EmbeddedDocument): + year = IntField(db_field='yr') + + class BlogPost(Document): + title = StringField() + date = EmbeddedDocumentField(Date) + + meta = { + 'indexes': [ + '-date.year' + ], + } + + BlogPost.drop_collection() + + info = BlogPost.objects._collection.index_information() + self.assertEqual(info.keys(), ['_cls_1_date.yr_-1', '_id_']) + BlogPost.drop_collection() + + def test_list_embedded_document_index(self): + """Ensure list embedded documents can be indexed + """ + class Tag(EmbeddedDocument): + name = StringField(db_field='tag') + + class BlogPost(Document): + title = StringField() + tags = ListField(EmbeddedDocumentField(Tag)) + + meta = { + 'indexes': [ + 'tags.name' + ] + } + + BlogPost.drop_collection() + + info = BlogPost.objects._collection.index_information() + # we don't use _cls in with list fields by default + self.assertEqual(info.keys(), ['_id_', '_cls_1_tags.tag_1']) + + post1 = BlogPost(title="Embedded Indexes tests in place", + tags=[Tag(name="about"), Tag(name="time")] + ) + post1.save() + BlogPost.drop_collection() + + def test_recursive_embedded_objects_dont_break_indexes(self): + + class RecursiveObject(EmbeddedDocument): + obj = EmbeddedDocumentField('self') + + class RecursiveDocument(Document): + recursive_obj = EmbeddedDocumentField(RecursiveObject) + meta = {'allow_inheritance': True} + + RecursiveDocument.objects._ensure_indexes() + info = RecursiveDocument._get_collection().index_information() + self.assertEqual(info.keys(), ['_id_', '_cls_1']) + + def test_geo_indexes_recursion(self): + + class Location(Document): + name = StringField() + location = GeoPointField() + + class Parent(Document): + name = StringField() + location = ReferenceField(Location) + + Location.drop_collection() + Parent.drop_collection() + + list(Parent.objects) + + collection = Parent._get_collection() + info = collection.index_information() + + self.assertFalse('location_2d' in info) + + self.assertEqual(len(Parent._geo_indices()), 0) + self.assertEqual(len(Location._geo_indices()), 1) + + def test_covered_index(self): + """Ensure that covered indexes can be used + """ + + class Test(Document): + a = IntField() + + meta = { + 'indexes': ['a'], + 'allow_inheritance': False + } + + Test.drop_collection() + + obj = Test(a=1) + obj.save() + + # Need to be explicit about covered indexes as mongoDB doesn't know if + # the documents returned might have more keys in that here. + query_plan = Test.objects(id=obj.id).exclude('a').explain() + self.assertFalse(query_plan['indexOnly']) + + query_plan = Test.objects(id=obj.id).only('id').explain() + self.assertTrue(query_plan['indexOnly']) + + query_plan = Test.objects(a=1).only('a').exclude('id').explain() + self.assertTrue(query_plan['indexOnly']) + + def test_index_on_id(self): + + class BlogPost(Document): + meta = { + 'indexes': [ + ['categories', 'id'] + ], + 'allow_inheritance': False + } + + title = StringField(required=True) + description = StringField(required=True) + categories = ListField() + + BlogPost.drop_collection() + + indexes = BlogPost.objects._collection.index_information() + self.assertEqual(indexes['categories_1__id_1']['key'], + [('categories', 1), ('_id', 1)]) + + def test_hint(self): + + class BlogPost(Document): + tags = ListField(StringField()) + meta = { + 'indexes': [ + 'tags', + ], + } + + BlogPost.drop_collection() + + for i in xrange(0, 10): + tags = [("tag %i" % n) for n in xrange(0, i % 2)] + BlogPost(tags=tags).save() + + self.assertEqual(BlogPost.objects.count(), 10) + self.assertEqual(BlogPost.objects.hint().count(), 10) + self.assertEqual(BlogPost.objects.hint([('tags', 1)]).count(), 10) + + self.assertEqual(BlogPost.objects.hint([('ZZ', 1)]).count(), 10) + + def invalid_index(): + BlogPost.objects.hint('tags') + self.assertRaises(TypeError, invalid_index) + + def invalid_index_2(): + return BlogPost.objects.hint(('tags', 1)) + self.assertRaises(TypeError, invalid_index_2) + + def test_unique(self): + """Ensure that uniqueness constraints are applied to fields. + """ + class BlogPost(Document): + title = StringField() + slug = StringField(unique=True) + + BlogPost.drop_collection() + + post1 = BlogPost(title='test1', slug='test') + post1.save() + + # Two posts with the same slug is not allowed + post2 = BlogPost(title='test2', slug='test') + self.assertRaises(NotUniqueError, post2.save) + + # Ensure backwards compatibilty for errors + self.assertRaises(OperationError, post2.save) + + def test_unique_with(self): + """Ensure that unique_with constraints are applied to fields. + """ + class Date(EmbeddedDocument): + year = IntField(db_field='yr') + + class BlogPost(Document): + title = StringField() + date = EmbeddedDocumentField(Date) + slug = StringField(unique_with='date.year') + + BlogPost.drop_collection() + + post1 = BlogPost(title='test1', date=Date(year=2009), slug='test') + post1.save() + + # day is different so won't raise exception + post2 = BlogPost(title='test2', date=Date(year=2010), slug='test') + post2.save() + + # Now there will be two docs with the same slug and the same day: fail + post3 = BlogPost(title='test3', date=Date(year=2010), slug='test') + self.assertRaises(OperationError, post3.save) + + BlogPost.drop_collection() + + def test_unique_embedded_document(self): + """Ensure that uniqueness constraints are applied to fields on embedded documents. + """ + class SubDocument(EmbeddedDocument): + year = IntField(db_field='yr') + slug = StringField(unique=True) + + class BlogPost(Document): + title = StringField() + sub = EmbeddedDocumentField(SubDocument) + + BlogPost.drop_collection() + + post1 = BlogPost(title='test1', sub=SubDocument(year=2009, slug="test")) + post1.save() + + # sub.slug is different so won't raise exception + post2 = BlogPost(title='test2', sub=SubDocument(year=2010, slug='another-slug')) + post2.save() + + # Now there will be two docs with the same sub.slug + post3 = BlogPost(title='test3', sub=SubDocument(year=2010, slug='test')) + self.assertRaises(NotUniqueError, post3.save) + + BlogPost.drop_collection() + + def test_unique_with_embedded_document_and_embedded_unique(self): + """Ensure that uniqueness constraints are applied to fields on + embedded documents. And work with unique_with as well. + """ + class SubDocument(EmbeddedDocument): + year = IntField(db_field='yr') + slug = StringField(unique=True) + + class BlogPost(Document): + title = StringField(unique_with='sub.year') + sub = EmbeddedDocumentField(SubDocument) + + BlogPost.drop_collection() + + post1 = BlogPost(title='test1', sub=SubDocument(year=2009, slug="test")) + post1.save() + + # sub.slug is different so won't raise exception + post2 = BlogPost(title='test2', sub=SubDocument(year=2010, slug='another-slug')) + post2.save() + + # Now there will be two docs with the same sub.slug + post3 = BlogPost(title='test3', sub=SubDocument(year=2010, slug='test')) + self.assertRaises(NotUniqueError, post3.save) + + # Now there will be two docs with the same title and year + post3 = BlogPost(title='test1', sub=SubDocument(year=2009, slug='test-1')) + self.assertRaises(NotUniqueError, post3.save) + + BlogPost.drop_collection() + + def test_ttl_indexes(self): + + class Log(Document): + created = DateTimeField(default=datetime.now) + meta = { + 'indexes': [ + {'fields': ['created'], 'expireAfterSeconds': 3600} + ] + } + + Log.drop_collection() + + if pymongo.version_tuple[0] < 2 and pymongo.version_tuple[1] < 3: + raise SkipTest('pymongo needs to be 2.3 or higher for this test') + + connection = get_connection() + version_array = connection.server_info()['versionArray'] + if version_array[0] < 2 and version_array[1] < 2: + raise SkipTest('MongoDB needs to be 2.2 or higher for this test') + + # Indexes are lazy so use list() to perform query + list(Log.objects) + info = Log.objects._collection.index_information() + self.assertEqual(3600, + info['_cls_1_created_1']['expireAfterSeconds']) + + def test_unique_and_indexes(self): + """Ensure that 'unique' constraints aren't overridden by + meta.indexes. + """ + class Customer(Document): + cust_id = IntField(unique=True, required=True) + meta = { + 'indexes': ['cust_id'], + 'allow_inheritance': False, + } + + Customer.drop_collection() + cust = Customer(cust_id=1) + cust.save() + + cust_dupe = Customer(cust_id=1) + try: + cust_dupe.save() + raise AssertionError, "We saved a dupe!" + except NotUniqueError: + pass + Customer.drop_collection() + + def test_unique_and_primary(self): + """If you set a field as primary, then unexpected behaviour can occur. + You won't create a duplicate but you will update an existing document. + """ + + class User(Document): + name = StringField(primary_key=True, unique=True) + password = StringField() + + User.drop_collection() + + user = User(name='huangz', password='secret') + user.save() + + user = User(name='huangz', password='secret2') + user.save() + + self.assertEqual(User.objects.count(), 1) + self.assertEqual(User.objects.get().password, 'secret2') + + User.drop_collection() + + def test_types_index_with_pk(self): + """Ensure you can use `pk` as part of a query""" + + class Comment(EmbeddedDocument): + comment_id = IntField(required=True) + + try: + class BlogPost(Document): + comments = EmbeddedDocumentField(Comment) + meta = {'indexes': [ + {'fields': ['pk', 'comments.comment_id'], + 'unique': True}]} + except UnboundLocalError: + self.fail('Unbound local error at types index + pk definition') + + info = BlogPost.objects._collection.index_information() + info = [value['key'] for key, value in info.iteritems()] + index_item = [('_cls', 1), ('_id', 1), ('comments.comment_id', 1)] + self.assertTrue(index_item in info) + +if __name__ == '__main__': + unittest.main() diff --git a/tests/document/inheritance.py b/tests/document/inheritance.py new file mode 100644 index 00000000..d269ac0e --- /dev/null +++ b/tests/document/inheritance.py @@ -0,0 +1,395 @@ +# -*- coding: utf-8 -*- +import unittest +import warnings + +from datetime import datetime + +from tests.fixtures import Base + +from mongoengine import Document, EmbeddedDocument, connect +from mongoengine.connection import get_db +from mongoengine.fields import (BooleanField, GenericReferenceField, + IntField, StringField) + +__all__ = ('InheritanceTest', ) + + +class InheritanceTest(unittest.TestCase): + + def setUp(self): + connect(db='mongoenginetest') + self.db = get_db() + + def tearDown(self): + for collection in self.db.collection_names(): + if 'system.' in collection: + continue + self.db.drop_collection(collection) + + def test_superclasses(self): + """Ensure that the correct list of superclasses is assembled. + """ + class Animal(Document): + meta = {'allow_inheritance': True} + class Fish(Animal): pass + class Guppy(Fish): pass + class Mammal(Animal): pass + class Dog(Mammal): pass + class Human(Mammal): pass + + self.assertEqual(Animal._superclasses, ()) + self.assertEqual(Fish._superclasses, ('Animal',)) + self.assertEqual(Guppy._superclasses, ('Animal', 'Animal.Fish')) + self.assertEqual(Mammal._superclasses, ('Animal',)) + self.assertEqual(Dog._superclasses, ('Animal', 'Animal.Mammal')) + self.assertEqual(Human._superclasses, ('Animal', 'Animal.Mammal')) + + def test_external_superclasses(self): + """Ensure that the correct list of super classes is assembled when + importing part of the model. + """ + class Animal(Base): pass + class Fish(Animal): pass + class Guppy(Fish): pass + class Mammal(Animal): pass + class Dog(Mammal): pass + class Human(Mammal): pass + + self.assertEqual(Animal._superclasses, ('Base', )) + self.assertEqual(Fish._superclasses, ('Base', 'Base.Animal',)) + self.assertEqual(Guppy._superclasses, ('Base', 'Base.Animal', + 'Base.Animal.Fish')) + self.assertEqual(Mammal._superclasses, ('Base', 'Base.Animal',)) + self.assertEqual(Dog._superclasses, ('Base', 'Base.Animal', + 'Base.Animal.Mammal')) + self.assertEqual(Human._superclasses, ('Base', 'Base.Animal', + 'Base.Animal.Mammal')) + + def test_subclasses(self): + """Ensure that the correct list of _subclasses (subclasses) is + assembled. + """ + class Animal(Document): + meta = {'allow_inheritance': True} + class Fish(Animal): pass + class Guppy(Fish): pass + class Mammal(Animal): pass + class Dog(Mammal): pass + class Human(Mammal): pass + + self.assertEqual(Animal._subclasses, ('Animal', + 'Animal.Fish', + 'Animal.Fish.Guppy', + 'Animal.Mammal', + 'Animal.Mammal.Dog', + 'Animal.Mammal.Human')) + self.assertEqual(Fish._subclasses, ('Animal.Fish', + 'Animal.Fish.Guppy',)) + self.assertEqual(Guppy._subclasses, ('Animal.Fish.Guppy',)) + self.assertEqual(Mammal._subclasses, ('Animal.Mammal', + 'Animal.Mammal.Dog', + 'Animal.Mammal.Human')) + self.assertEqual(Human._subclasses, ('Animal.Mammal.Human',)) + + def test_external_subclasses(self): + """Ensure that the correct list of _subclasses (subclasses) is + assembled when importing part of the model. + """ + class Animal(Base): pass + class Fish(Animal): pass + class Guppy(Fish): pass + class Mammal(Animal): pass + class Dog(Mammal): pass + class Human(Mammal): pass + + self.assertEqual(Animal._subclasses, ('Base.Animal', + 'Base.Animal.Fish', + 'Base.Animal.Fish.Guppy', + 'Base.Animal.Mammal', + 'Base.Animal.Mammal.Dog', + 'Base.Animal.Mammal.Human')) + self.assertEqual(Fish._subclasses, ('Base.Animal.Fish', + 'Base.Animal.Fish.Guppy',)) + self.assertEqual(Guppy._subclasses, ('Base.Animal.Fish.Guppy',)) + self.assertEqual(Mammal._subclasses, ('Base.Animal.Mammal', + 'Base.Animal.Mammal.Dog', + 'Base.Animal.Mammal.Human')) + self.assertEqual(Human._subclasses, ('Base.Animal.Mammal.Human',)) + + def test_dynamic_declarations(self): + """Test that declaring an extra class updates meta data""" + + class Animal(Document): + meta = {'allow_inheritance': True} + + self.assertEqual(Animal._superclasses, ()) + self.assertEqual(Animal._subclasses, ('Animal',)) + + # Test dynamically adding a class changes the meta data + class Fish(Animal): + pass + + self.assertEqual(Animal._superclasses, ()) + self.assertEqual(Animal._subclasses, ('Animal', 'Animal.Fish')) + + self.assertEqual(Fish._superclasses, ('Animal', )) + self.assertEqual(Fish._subclasses, ('Animal.Fish',)) + + # Test dynamically adding an inherited class changes the meta data + class Pike(Fish): + pass + + self.assertEqual(Animal._superclasses, ()) + self.assertEqual(Animal._subclasses, ('Animal', 'Animal.Fish', + 'Animal.Fish.Pike')) + + self.assertEqual(Fish._superclasses, ('Animal', )) + self.assertEqual(Fish._subclasses, ('Animal.Fish', 'Animal.Fish.Pike')) + + self.assertEqual(Pike._superclasses, ('Animal', 'Animal.Fish')) + self.assertEqual(Pike._subclasses, ('Animal.Fish.Pike',)) + + def test_inheritance_meta_data(self): + """Ensure that document may inherit fields from a superclass document. + """ + class Person(Document): + name = StringField() + age = IntField() + + meta = {'allow_inheritance': True} + + class Employee(Person): + salary = IntField() + + self.assertEqual(['salary', 'age', 'name', 'id'], + Employee._fields.keys()) + self.assertEqual(Employee._get_collection_name(), + Person._get_collection_name()) + + + def test_polymorphic_queries(self): + """Ensure that the correct subclasses are returned from a query + """ + + class Animal(Document): + meta = {'allow_inheritance': True} + class Fish(Animal): pass + class Mammal(Animal): pass + class Dog(Mammal): pass + class Human(Mammal): pass + + Animal.drop_collection() + + Animal().save() + Fish().save() + Mammal().save() + Dog().save() + Human().save() + + classes = [obj.__class__ for obj in Animal.objects] + self.assertEqual(classes, [Animal, Fish, Mammal, Dog, Human]) + + classes = [obj.__class__ for obj in Mammal.objects] + self.assertEqual(classes, [Mammal, Dog, Human]) + + classes = [obj.__class__ for obj in Human.objects] + self.assertEqual(classes, [Human]) + + + def test_allow_inheritance(self): + """Ensure that inheritance may be disabled on simple classes and that + _cls and _subclasses will not be used. + """ + + class Animal(Document): + name = StringField() + meta = {'allow_inheritance': False} + + def create_dog_class(): + class Dog(Animal): + pass + + self.assertRaises(ValueError, create_dog_class) + + # Check that _cls etc aren't present on simple documents + dog = Animal(name='dog') + dog.save() + + collection = self.db[Animal._get_collection_name()] + obj = collection.find_one() + self.assertFalse('_cls' in obj) + + def test_cant_turn_off_inheritance_on_subclass(self): + """Ensure if inheritance is on in a subclass you cant turn it off + """ + + class Animal(Document): + name = StringField() + meta = {'allow_inheritance': True} + + def create_mammal_class(): + class Mammal(Animal): + meta = {'allow_inheritance': False} + self.assertRaises(ValueError, create_mammal_class) + + def test_allow_inheritance_abstract_document(self): + """Ensure that abstract documents can set inheritance rules and that + _cls will not be used. + """ + class FinalDocument(Document): + meta = {'abstract': True, + 'allow_inheritance': False} + + class Animal(FinalDocument): + name = StringField() + + def create_mammal_class(): + class Mammal(Animal): + pass + self.assertRaises(ValueError, create_mammal_class) + + # Check that _cls isn't present in simple documents + doc = Animal(name='dog') + self.assertFalse('_cls' in doc.to_mongo()) + + def test_allow_inheritance_embedded_document(self): + """Ensure embedded documents respect inheritance + """ + + class Comment(EmbeddedDocument): + content = StringField() + meta = {'allow_inheritance': False} + + def create_special_comment(): + class SpecialComment(Comment): + pass + + self.assertRaises(ValueError, create_special_comment) + + doc = Comment(content='test') + self.assertFalse('_cls' in doc.to_mongo()) + + class Comment(EmbeddedDocument): + content = StringField() + meta = {'allow_inheritance': True} + + doc = Comment(content='test') + self.assertTrue('_cls' in doc.to_mongo()) + + def test_document_inheritance(self): + """Ensure mutliple inheritance of abstract documents + """ + class DateCreatedDocument(Document): + meta = { + 'allow_inheritance': True, + 'abstract': True, + } + + class DateUpdatedDocument(Document): + meta = { + 'allow_inheritance': True, + 'abstract': True, + } + + try: + class MyDocument(DateCreatedDocument, DateUpdatedDocument): + pass + except: + self.assertTrue(False, "Couldn't create MyDocument class") + + def test_abstract_documents(self): + """Ensure that a document superclass can be marked as abstract + thereby not using it as the name for the collection.""" + + defaults = {'index_background': True, + 'index_drop_dups': True, + 'index_opts': {'hello': 'world'}, + 'allow_inheritance': True, + 'queryset_class': 'QuerySet', + 'db_alias': 'myDB', + 'shard_key': ('hello', 'world')} + + meta_settings = {'abstract': True} + meta_settings.update(defaults) + + class Animal(Document): + name = StringField() + meta = meta_settings + + class Fish(Animal): pass + class Guppy(Fish): pass + + class Mammal(Animal): + meta = {'abstract': True} + class Human(Mammal): pass + + for k, v in defaults.iteritems(): + for cls in [Animal, Fish, Guppy]: + self.assertEqual(cls._meta[k], v) + + self.assertFalse('collection' in Animal._meta) + self.assertFalse('collection' in Mammal._meta) + + self.assertEqual(Animal._get_collection_name(), None) + self.assertEqual(Mammal._get_collection_name(), None) + + self.assertEqual(Fish._get_collection_name(), 'fish') + self.assertEqual(Guppy._get_collection_name(), 'fish') + self.assertEqual(Human._get_collection_name(), 'human') + + def create_bad_abstract(): + class EvilHuman(Human): + evil = BooleanField(default=True) + meta = {'abstract': True} + self.assertRaises(ValueError, create_bad_abstract) + + def test_inherited_collections(self): + """Ensure that subclassed documents don't override parents' + collections + """ + + class Drink(Document): + name = StringField() + meta = {'allow_inheritance': True} + + class Drinker(Document): + drink = GenericReferenceField() + + try: + warnings.simplefilter("error") + + class AcloholicDrink(Drink): + meta = {'collection': 'booze'} + + except SyntaxWarning: + warnings.simplefilter("ignore") + + class AlcoholicDrink(Drink): + meta = {'collection': 'booze'} + + else: + raise AssertionError("SyntaxWarning should be triggered") + + warnings.resetwarnings() + + Drink.drop_collection() + AlcoholicDrink.drop_collection() + Drinker.drop_collection() + + red_bull = Drink(name='Red Bull') + red_bull.save() + + programmer = Drinker(drink=red_bull) + programmer.save() + + beer = AlcoholicDrink(name='Beer') + beer.save() + real_person = Drinker(drink=beer) + real_person.save() + + self.assertEqual(Drinker.objects[0].drink.name, red_bull.name) + self.assertEqual(Drinker.objects[1].drink.name, beer.name) + + +if __name__ == '__main__': + unittest.main() diff --git a/tests/test_document.py b/tests/document/instance.py similarity index 50% rename from tests/test_document.py rename to tests/document/instance.py index a09aaeca..95f37d9e 100644 --- a/tests/test_document.py +++ b/tests/document/instance.py @@ -15,14 +15,17 @@ from datetime import datetime from tests.fixtures import Base, Mixin, PickleEmbedded, PickleTest from mongoengine import * -from mongoengine.base import NotRegistered, InvalidDocumentError -from mongoengine.queryset import InvalidQueryError +from mongoengine.errors import (NotRegistered, InvalidDocumentError, + InvalidQueryError) +from mongoengine.queryset import NULLIFY, Q from mongoengine.connection import get_db, get_connection TEST_IMAGE_PATH = os.path.join(os.path.dirname(__file__), 'mongoengine.png') +__all__ = ("InstanceTest",) -class DocumentTest(unittest.TestCase): + +class InstanceTest(unittest.TestCase): def setUp(self): connect(db='mongoenginetest') @@ -32,59 +35,57 @@ class DocumentTest(unittest.TestCase): name = StringField() age = IntField() - meta = {'allow_inheritance': True} + non_field = True + + meta = {"allow_inheritance": True} self.Person = Person def tearDown(self): - self.Person.drop_collection() + for collection in self.db.collection_names(): + if 'system.' in collection: + continue + self.db.drop_collection(collection) - def test_drop_collection(self): - """Ensure that the collection may be dropped from the database. + def test_capped_collection(self): + """Ensure that capped collections work properly. """ - self.Person(name='Test').save() + class Log(Document): + date = DateTimeField(default=datetime.now) + meta = { + 'max_documents': 10, + 'max_size': 90000, + } - collection = self.Person._get_collection_name() - self.assertTrue(collection in self.db.collection_names()) + Log.drop_collection() - self.Person.drop_collection() - self.assertFalse(collection in self.db.collection_names()) + # Ensure that the collection handles up to its maximum + for _ in range(10): + Log().save() - def test_queryset_resurrects_dropped_collection(self): + self.assertEqual(len(Log.objects), 10) - self.Person.objects().item_frequencies('name') - self.Person.drop_collection() + # Check that extra documents don't increase the size + Log().save() + self.assertEqual(len(Log.objects), 10) - self.assertEqual({}, self.Person.objects().item_frequencies('name')) + options = Log.objects._collection.options() + self.assertEqual(options['capped'], True) + self.assertEqual(options['max'], 10) + self.assertEqual(options['size'], 90000) - class Actor(self.Person): - pass + # Check that the document cannot be redefined with different options + def recreate_log_document(): + class Log(Document): + date = DateTimeField(default=datetime.now) + meta = { + 'max_documents': 11, + } + # Create the collection by accessing Document.objects + Log.objects + self.assertRaises(InvalidCollectionError, recreate_log_document) - # Ensure works correctly with inhertited classes - Actor.objects().item_frequencies('name') - self.Person.drop_collection() - self.assertEqual({}, Actor.objects().item_frequencies('name')) - - def test_definition(self): - """Ensure that document may be defined using fields. - """ - name_field = StringField() - age_field = IntField() - - class Person(Document): - name = name_field - age = age_field - non_field = True - - self.assertEqual(Person._fields['name'], name_field) - self.assertEqual(Person._fields['age'], age_field) - self.assertFalse('non_field' in Person._fields) - self.assertTrue('id' in Person._fields) - # Test iteration over fields - fields = list(Person()) - self.assertTrue('name' in fields and 'age' in fields) - # Ensure Document isn't treated like an actual document - self.assertFalse(hasattr(Document, '_fields')) + Log.drop_collection() def test_repr(self): """Ensure that unicode representation works @@ -95,146 +96,22 @@ class DocumentTest(unittest.TestCase): def __unicode__(self): return self.title - Article.drop_collection() + doc = Article(title=u'привет мир') - Article(title=u'привет мир').save() + self.assertEqual('', repr(doc)) - self.assertEqual('', repr(Article.objects.first())) - self.assertEqual('[]', repr(Article.objects.all())) + def test_queryset_resurrects_dropped_collection(self): + self.Person.drop_collection() - def test_collection_naming(self): - """Ensure that a collection with a specified name may be used. - """ + self.assertEqual([], list(self.Person.objects())) - class DefaultNamingTest(Document): - pass - self.assertEqual('default_naming_test', DefaultNamingTest._get_collection_name()) - - class CustomNamingTest(Document): - meta = {'collection': 'pimp_my_collection'} - - self.assertEqual('pimp_my_collection', CustomNamingTest._get_collection_name()) - - class DynamicNamingTest(Document): - meta = {'collection': lambda c: "DYNAMO"} - self.assertEqual('DYNAMO', DynamicNamingTest._get_collection_name()) - - # Use Abstract class to handle backwards compatibility - class BaseDocument(Document): - meta = { - 'abstract': True, - 'collection': lambda c: c.__name__.lower() - } - - class OldNamingConvention(BaseDocument): - pass - self.assertEqual('oldnamingconvention', OldNamingConvention._get_collection_name()) - - class InheritedAbstractNamingTest(BaseDocument): - meta = {'collection': 'wibble'} - self.assertEqual('wibble', InheritedAbstractNamingTest._get_collection_name()) - - - # Mixin tests - class BaseMixin(object): - meta = { - 'collection': lambda c: c.__name__.lower() - } - - class OldMixinNamingConvention(Document, BaseMixin): - pass - self.assertEqual('oldmixinnamingconvention', OldMixinNamingConvention._get_collection_name()) - - class BaseMixin(object): - meta = { - 'collection': lambda c: c.__name__.lower() - } - - class BaseDocument(Document, BaseMixin): - meta = {'allow_inheritance': True} - - class MyDocument(BaseDocument): + class Actor(self.Person): pass - self.assertEqual('basedocument', MyDocument._get_collection_name()) - - def test_get_superclasses(self): - """Ensure that the correct list of superclasses is assembled. - """ - class Animal(Document): - meta = {'allow_inheritance': True} - class Fish(Animal): pass - class Mammal(Animal): pass - class Human(Mammal): pass - class Dog(Mammal): pass - - mammal_superclasses = {'Animal': Animal} - self.assertEqual(Mammal._superclasses, mammal_superclasses) - - dog_superclasses = { - 'Animal': Animal, - 'Animal.Mammal': Mammal, - } - self.assertEqual(Dog._superclasses, dog_superclasses) - - def test_external_superclasses(self): - """Ensure that the correct list of sub and super classes is assembled. - when importing part of the model - """ - class Animal(Base): pass - class Fish(Animal): pass - class Mammal(Animal): pass - class Human(Mammal): pass - class Dog(Mammal): pass - - mammal_superclasses = {'Base': Base, 'Base.Animal': Animal} - self.assertEqual(Mammal._superclasses, mammal_superclasses) - - dog_superclasses = { - 'Base': Base, - 'Base.Animal': Animal, - 'Base.Animal.Mammal': Mammal, - } - self.assertEqual(Dog._superclasses, dog_superclasses) - - Base.drop_collection() - - h = Human() - h.save() - - self.assertEqual(Human.objects.count(), 1) - self.assertEqual(Mammal.objects.count(), 1) - self.assertEqual(Animal.objects.count(), 1) - self.assertEqual(Base.objects.count(), 1) - Base.drop_collection() - - def test_polymorphic_queries(self): - """Ensure that the correct subclasses are returned from a query""" - class Animal(Document): - meta = {'allow_inheritance': True} - class Fish(Animal): pass - class Mammal(Animal): pass - class Human(Mammal): pass - class Dog(Mammal): pass - - Animal.drop_collection() - - Animal().save() - Fish().save() - Mammal().save() - Human().save() - Dog().save() - - classes = [obj.__class__ for obj in Animal.objects] - self.assertEqual(classes, [Animal, Fish, Mammal, Human, Dog]) - - classes = [obj.__class__ for obj in Mammal.objects] - self.assertEqual(classes, [Mammal, Human, Dog]) - - classes = [obj.__class__ for obj in Human.objects] - self.assertEqual(classes, [Human]) - - Animal.drop_collection() + # Ensure works correctly with inhertited classes + Actor.objects() + self.Person.drop_collection() + self.assertEqual([], list(Actor.objects())) def test_polymorphic_references(self): """Ensure that the correct subclasses are returned from a query when @@ -244,8 +121,8 @@ class DocumentTest(unittest.TestCase): meta = {'allow_inheritance': True} class Fish(Animal): pass class Mammal(Animal): pass - class Human(Mammal): pass class Dog(Mammal): pass + class Human(Mammal): pass class Zoo(Document): animals = ListField(ReferenceField(Animal)) @@ -256,8 +133,8 @@ class DocumentTest(unittest.TestCase): Animal().save() Fish().save() Mammal().save() - Human().save() Dog().save() + Human().save() # Save a reference to each animal zoo = Zoo(animals=Animal.objects) @@ -265,7 +142,7 @@ class DocumentTest(unittest.TestCase): zoo.reload() classes = [a.__class__ for a in Zoo.objects.first().animals] - self.assertEqual(classes, [Animal, Fish, Mammal, Human, Dog]) + self.assertEqual(classes, [Animal, Fish, Mammal, Dog, Human]) Zoo.drop_collection() @@ -278,7 +155,7 @@ class DocumentTest(unittest.TestCase): zoo.reload() classes = [a.__class__ for a in Zoo.objects.first().animals] - self.assertEqual(classes, [Animal, Fish, Mammal, Human, Dog]) + self.assertEqual(classes, [Animal, Fish, Mammal, Dog, Human]) Zoo.drop_collection() Animal.drop_collection() @@ -308,466 +185,8 @@ class DocumentTest(unittest.TestCase): self.assertEqual(list_stats, CompareStats.objects.first().stats) - def test_inheritance(self): - """Ensure that document may inherit fields from a superclass document. - """ - class Employee(self.Person): - salary = IntField() - self.assertTrue('name' in Employee._fields) - self.assertTrue('salary' in Employee._fields) - self.assertEqual(Employee._get_collection_name(), - self.Person._get_collection_name()) - # Ensure that MRO error is not raised - class A(Document): - meta = {'allow_inheritance': True} - class B(A): pass - class C(B): pass - - def test_allow_inheritance(self): - """Ensure that inheritance may be disabled on simple classes and that - _cls and _types will not be used. - """ - - class Animal(Document): - name = StringField() - meta = {'allow_inheritance': False} - - Animal.drop_collection() - def create_dog_class(): - class Dog(Animal): - pass - self.assertRaises(ValueError, create_dog_class) - - # Check that _cls etc aren't present on simple documents - dog = Animal(name='dog') - dog.save() - collection = self.db[Animal._get_collection_name()] - obj = collection.find_one() - self.assertFalse('_cls' in obj) - self.assertFalse('_types' in obj) - - Animal.drop_collection() - - def create_employee_class(): - class Employee(self.Person): - meta = {'allow_inheritance': False} - self.assertRaises(ValueError, create_employee_class) - - def test_allow_inheritance_abstract_document(self): - """Ensure that abstract documents can set inheritance rules and that - _cls and _types will not be used. - """ - class FinalDocument(Document): - meta = {'abstract': True, - 'allow_inheritance': False} - - class Animal(FinalDocument): - name = StringField() - - Animal.drop_collection() - def create_dog_class(): - class Dog(Animal): - pass - self.assertRaises(ValueError, create_dog_class) - - # Check that _cls etc aren't present on simple documents - dog = Animal(name='dog') - dog.save() - collection = self.db[Animal._get_collection_name()] - obj = collection.find_one() - self.assertFalse('_cls' in obj) - self.assertFalse('_types' in obj) - - Animal.drop_collection() - - def test_allow_inheritance_embedded_document(self): - - # Test the same for embedded documents - class Comment(EmbeddedDocument): - content = StringField() - meta = {'allow_inheritance': False} - - def create_special_comment(): - class SpecialComment(Comment): - pass - - self.assertRaises(ValueError, create_special_comment) - - comment = Comment(content='test') - self.assertFalse('_cls' in comment.to_mongo()) - self.assertFalse('_types' in comment.to_mongo()) - - class Comment(EmbeddedDocument): - content = StringField() - meta = {'allow_inheritance': True} - - comment = Comment(content='test') - self.assertTrue('_cls' in comment.to_mongo()) - self.assertTrue('_types' in comment.to_mongo()) - - def test_document_inheritance(self): - """Ensure mutliple inheritance of abstract docs works - """ - class DateCreatedDocument(Document): - meta = { - 'allow_inheritance': True, - 'abstract': True, - } - - class DateUpdatedDocument(Document): - meta = { - 'allow_inheritance': True, - 'abstract': True, - } - - try: - class MyDocument(DateCreatedDocument, DateUpdatedDocument): - pass - except: - self.assertTrue(False, "Couldn't create MyDocument class") - - def test_how_to_turn_off_inheritance(self): - """Demonstrates migrating from allow_inheritance = True to False. - """ - class Animal(Document): - name = StringField() - meta = { - 'indexes': ['name'] - } - - self.assertEqual(Animal._meta['index_specs'], - [{'fields': [('_types', 1), ('name', 1)]}]) - - Animal.drop_collection() - - dog = Animal(name='dog') - dog.save() - - collection = self.db[Animal._get_collection_name()] - obj = collection.find_one() - self.assertTrue('_cls' in obj) - self.assertTrue('_types' in obj) - - info = collection.index_information() - info = [value['key'] for key, value in info.iteritems()] - self.assertEqual([[(u'_id', 1)], [(u'_types', 1), (u'name', 1)]], info) - - # Turn off inheritance - class Animal(Document): - name = StringField() - meta = { - 'allow_inheritance': False, - 'indexes': ['name'] - } - - self.assertEqual(Animal._meta['index_specs'], - [{'fields': [('name', 1)]}]) - collection.update({}, {"$unset": {"_types": 1, "_cls": 1}}, multi=True) - - # Confirm extra data is removed - obj = collection.find_one() - self.assertFalse('_cls' in obj) - self.assertFalse('_types' in obj) - - info = collection.index_information() - info = [value['key'] for key, value in info.iteritems()] - self.assertEqual([[(u'_id', 1)], [(u'_types', 1), (u'name', 1)]], info) - - info = collection.index_information() - indexes_to_drop = [key for key, value in info.iteritems() if '_types' in dict(value['key'])] - for index in indexes_to_drop: - collection.drop_index(index) - - info = collection.index_information() - info = [value['key'] for key, value in info.iteritems()] - self.assertEqual([[(u'_id', 1)]], info) - - # Recreate indexes - dog = Animal.objects.first() - dog.save() - info = collection.index_information() - info = [value['key'] for key, value in info.iteritems()] - self.assertEqual([[(u'_id', 1)], [(u'name', 1),]], info) - - Animal.drop_collection() - - def test_abstract_documents(self): - """Ensure that a document superclass can be marked as abstract - thereby not using it as the name for the collection.""" - - defaults = {'index_background': True, - 'index_drop_dups': True, - 'index_opts': {'hello': 'world'}, - 'allow_inheritance': True, - 'queryset_class': 'QuerySet', - 'db_alias': 'myDB', - 'shard_key': ('hello', 'world')} - - meta_settings = {'abstract': True} - meta_settings.update(defaults) - - class Animal(Document): - name = StringField() - meta = meta_settings - - class Fish(Animal): pass - class Guppy(Fish): pass - - class Mammal(Animal): - meta = {'abstract': True} - class Human(Mammal): pass - - for k, v in defaults.iteritems(): - for cls in [Animal, Fish, Guppy]: - self.assertEqual(cls._meta[k], v) - - self.assertFalse('collection' in Animal._meta) - self.assertFalse('collection' in Mammal._meta) - - self.assertEqual(Animal._get_collection_name(), None) - self.assertEqual(Mammal._get_collection_name(), None) - - self.assertEqual(Fish._get_collection_name(), 'fish') - self.assertEqual(Guppy._get_collection_name(), 'fish') - self.assertEqual(Human._get_collection_name(), 'human') - - def create_bad_abstract(): - class EvilHuman(Human): - evil = BooleanField(default=True) - meta = {'abstract': True} - self.assertRaises(ValueError, create_bad_abstract) - - def test_collection_name(self): - """Ensure that a collection with a specified name may be used. - """ - collection = 'personCollTest' - if collection in self.db.collection_names(): - self.db.drop_collection(collection) - - class Person(Document): - name = StringField() - meta = {'collection': collection} - - user = Person(name="Test User") - user.save() - self.assertTrue(collection in self.db.collection_names()) - - user_obj = self.db[collection].find_one() - self.assertEqual(user_obj['name'], "Test User") - - user_obj = Person.objects[0] - self.assertEqual(user_obj.name, "Test User") - - Person.drop_collection() - self.assertFalse(collection in self.db.collection_names()) - - def test_collection_name_and_primary(self): - """Ensure that a collection with a specified name may be used. - """ - - class Person(Document): - name = StringField(primary_key=True) - meta = {'collection': 'app'} - - user = Person(name="Test User") - user.save() - - user_obj = Person.objects[0] - self.assertEqual(user_obj.name, "Test User") - - Person.drop_collection() - - def test_inherited_collections(self): - """Ensure that subclassed documents don't override parents' collections. - """ - - class Drink(Document): - name = StringField() - meta = {'allow_inheritance': True} - - class Drinker(Document): - drink = GenericReferenceField() - - try: - warnings.simplefilter("error") - - class AcloholicDrink(Drink): - meta = {'collection': 'booze'} - - except SyntaxWarning, w: - warnings.simplefilter("ignore") - - class AlcoholicDrink(Drink): - meta = {'collection': 'booze'} - - else: - raise AssertionError("SyntaxWarning should be triggered") - - warnings.resetwarnings() - - Drink.drop_collection() - AlcoholicDrink.drop_collection() - Drinker.drop_collection() - - red_bull = Drink(name='Red Bull') - red_bull.save() - - programmer = Drinker(drink=red_bull) - programmer.save() - - beer = AlcoholicDrink(name='Beer') - beer.save() - real_person = Drinker(drink=beer) - real_person.save() - - self.assertEqual(Drinker.objects[0].drink.name, red_bull.name) - self.assertEqual(Drinker.objects[1].drink.name, beer.name) - - def test_capped_collection(self): - """Ensure that capped collections work properly. - """ - class Log(Document): - date = DateTimeField(default=datetime.now) - meta = { - 'max_documents': 10, - 'max_size': 90000, - } - - Log.drop_collection() - - # Ensure that the collection handles up to its maximum - for i in range(10): - Log().save() - - self.assertEqual(len(Log.objects), 10) - - # Check that extra documents don't increase the size - Log().save() - self.assertEqual(len(Log.objects), 10) - - options = Log.objects._collection.options() - self.assertEqual(options['capped'], True) - self.assertEqual(options['max'], 10) - self.assertEqual(options['size'], 90000) - - # Check that the document cannot be redefined with different options - def recreate_log_document(): - class Log(Document): - date = DateTimeField(default=datetime.now) - meta = { - 'max_documents': 11, - } - # Create the collection by accessing Document.objects - Log.objects - self.assertRaises(InvalidCollectionError, recreate_log_document) - - Log.drop_collection() - - def test_indexes(self): - """Ensure that indexes are used when meta[indexes] is specified. - """ - class BlogPost(Document): - date = DateTimeField(db_field='addDate', default=datetime.now) - category = StringField() - tags = ListField(StringField()) - meta = { - 'indexes': [ - '-date', - 'tags', - ('category', '-date') - ], - 'allow_inheritance': True - } - - self.assertEqual(BlogPost._meta['index_specs'], - [{'fields': [('_types', 1), ('addDate', -1)]}, - {'fields': [('tags', 1)]}, - {'fields': [('_types', 1), ('category', 1), - ('addDate', -1)]}]) - - BlogPost.drop_collection() - - info = BlogPost.objects._collection.index_information() - # _id, '-date', 'tags', ('cat', 'date') - # NB: there is no index on _types by itself, since - # the indices on -date and tags will both contain - # _types as first element in the key - self.assertEqual(len(info), 4) - - # Indexes are lazy so use list() to perform query - list(BlogPost.objects) - info = BlogPost.objects._collection.index_information() - info = [value['key'] for key, value in info.iteritems()] - self.assertTrue([('_types', 1), ('category', 1), ('addDate', -1)] - in info) - self.assertTrue([('_types', 1), ('addDate', -1)] in info) - # tags is a list field so it shouldn't have _types in the index - self.assertTrue([('tags', 1)] in info) - - class ExtendedBlogPost(BlogPost): - title = StringField() - meta = {'indexes': ['title']} - - self.assertEqual(ExtendedBlogPost._meta['index_specs'], - [{'fields': [('_types', 1), ('addDate', -1)]}, - {'fields': [('tags', 1)]}, - {'fields': [('_types', 1), ('category', 1), - ('addDate', -1)]}, - {'fields': [('_types', 1), ('title', 1)]}]) - - BlogPost.drop_collection() - - list(ExtendedBlogPost.objects) - info = ExtendedBlogPost.objects._collection.index_information() - info = [value['key'] for key, value in info.iteritems()] - self.assertTrue([('_types', 1), ('category', 1), ('addDate', -1)] - in info) - self.assertTrue([('_types', 1), ('addDate', -1)] in info) - self.assertTrue([('_types', 1), ('title', 1)] in info) - - BlogPost.drop_collection() - - def test_inherited_index(self): - """Ensure index specs are inhertited correctly""" - - class A(Document): - title = StringField() - meta = { - 'indexes': [ - { - 'fields': ('title',), - }, - ], - 'allow_inheritance': True, - } - - class B(A): - description = StringField() - - self.assertEqual(A._meta['index_specs'], B._meta['index_specs']) - self.assertEqual([{'fields': [('_types', 1), ('title', 1)]}], - A._meta['index_specs']) - - def test_build_index_spec_is_not_destructive(self): - - class MyDoc(Document): - keywords = StringField() - - meta = { - 'indexes': ['keywords'], - 'allow_inheritance': False - } - - self.assertEqual(MyDoc._meta['index_specs'], - [{'fields': [('keywords', 1)]}]) - - # Force index creation - MyDoc.objects._ensure_indexes() - - self.assertEqual(MyDoc._meta['index_specs'], - [{'fields': [('keywords', 1)]}]) def test_db_field_load(self): """Ensure we load data correctly @@ -812,477 +231,8 @@ class DocumentTest(unittest.TestCase): self.assertEqual(Person.objects.get(name="Jack").rank, "Corporal") self.assertEqual(Person.objects.get(name="Fred").rank, "Private") - def test_embedded_document_index_meta(self): - """Ensure that embedded document indexes are created explicitly - """ - class Rank(EmbeddedDocument): - title = StringField(required=True) - class Person(Document): - name = StringField(required=True) - rank = EmbeddedDocumentField(Rank, required=False) - meta = { - 'indexes': [ - 'rank.title', - ], - 'allow_inheritance': False - } - - self.assertEqual([{'fields': [('rank.title', 1)]}], - Person._meta['index_specs']) - - Person.drop_collection() - - # Indexes are lazy so use list() to perform query - list(Person.objects) - info = Person.objects._collection.index_information() - info = [value['key'] for key, value in info.iteritems()] - self.assertTrue([('rank.title', 1)] in info) - - def test_explicit_geo2d_index(self): - """Ensure that geo2d indexes work when created via meta[indexes] - """ - class Place(Document): - location = DictField() - meta = { - 'indexes': [ - '*location.point', - ], - } - - self.assertEqual([{'fields': [('location.point', '2d')]}], - Place._meta['index_specs']) - - Place.drop_collection() - - info = Place.objects._collection.index_information() - # Indexes are lazy so use list() to perform query - list(Place.objects) - info = Place.objects._collection.index_information() - info = [value['key'] for key, value in info.iteritems()] - - self.assertTrue([('location.point', '2d')] in info) - - def test_dictionary_indexes(self): - """Ensure that indexes are used when meta[indexes] contains dictionaries - instead of lists. - """ - class BlogPost(Document): - date = DateTimeField(db_field='addDate', default=datetime.now) - category = StringField() - tags = ListField(StringField()) - meta = { - 'indexes': [ - {'fields': ['-date'], 'unique': True, - 'sparse': True, 'types': False }, - ], - } - - self.assertEqual([{'fields': [('addDate', -1)], 'unique': True, - 'sparse': True, 'types': False}], - BlogPost._meta['index_specs']) - - BlogPost.drop_collection() - - info = BlogPost.objects._collection.index_information() - # _id, '-date' - self.assertEqual(len(info), 3) - - # Indexes are lazy so use list() to perform query - list(BlogPost.objects) - info = BlogPost.objects._collection.index_information() - info = [(value['key'], - value.get('unique', False), - value.get('sparse', False)) - for key, value in info.iteritems()] - self.assertTrue(([('addDate', -1)], True, True) in info) - - BlogPost.drop_collection() - - def test_abstract_index_inheritance(self): - - class UserBase(Document): - meta = { - 'abstract': True, - 'indexes': ['user_guid'] - } - - user_guid = StringField(required=True) - - class Person(UserBase): - meta = { - 'indexes': ['name'], - } - - name = StringField() - - Person.drop_collection() - - p = Person(name="test", user_guid='123') - p.save() - - self.assertEqual(1, Person.objects.count()) - info = Person.objects._collection.index_information() - self.assertEqual(info.keys(), ['_types_1_user_guid_1', '_id_', '_types_1_name_1']) - Person.drop_collection() - - def test_disable_index_creation(self): - """Tests setting auto_create_index to False on the connection will - disable any index generation. - """ - class User(Document): - meta = { - 'indexes': ['user_guid'], - 'auto_create_index': False - } - user_guid = StringField(required=True) - - - User.drop_collection() - - u = User(user_guid='123') - u.save() - - self.assertEqual(1, User.objects.count()) - info = User.objects._collection.index_information() - self.assertEqual(info.keys(), ['_id_']) - User.drop_collection() - - def test_embedded_document_index(self): - """Tests settings an index on an embedded document - """ - class Date(EmbeddedDocument): - year = IntField(db_field='yr') - - class BlogPost(Document): - title = StringField() - date = EmbeddedDocumentField(Date) - - meta = { - 'indexes': [ - '-date.year' - ], - } - - BlogPost.drop_collection() - - info = BlogPost.objects._collection.index_information() - self.assertEqual(info.keys(), ['_types_1_date.yr_-1', '_id_']) - BlogPost.drop_collection() - - def test_list_embedded_document_index(self): - """Ensure list embedded documents can be indexed - """ - class Tag(EmbeddedDocument): - name = StringField(db_field='tag') - - class BlogPost(Document): - title = StringField() - tags = ListField(EmbeddedDocumentField(Tag)) - - meta = { - 'indexes': [ - 'tags.name' - ], - } - - BlogPost.drop_collection() - - info = BlogPost.objects._collection.index_information() - # we don't use _types in with list fields by default - self.assertEqual(info.keys(), ['_id_', '_types_1', 'tags.tag_1']) - - post1 = BlogPost(title="Embedded Indexes tests in place", - tags=[Tag(name="about"), Tag(name="time")] - ) - post1.save() - BlogPost.drop_collection() - - def test_recursive_embedded_objects_dont_break_indexes(self): - - class RecursiveObject(EmbeddedDocument): - obj = EmbeddedDocumentField('self') - - class RecursiveDocument(Document): - recursive_obj = EmbeddedDocumentField(RecursiveObject) - - info = RecursiveDocument.objects._collection.index_information() - self.assertEqual(info.keys(), ['_id_', '_types_1']) - - def test_geo_indexes_recursion(self): - - class Location(Document): - name = StringField() - location = GeoPointField() - - class Parent(Document): - name = StringField() - location = ReferenceField(Location) - - Location.drop_collection() - Parent.drop_collection() - - list(Parent.objects) - - collection = Parent._get_collection() - info = collection.index_information() - - self.assertFalse('location_2d' in info) - - self.assertEqual(len(Parent._geo_indices()), 0) - self.assertEqual(len(Location._geo_indices()), 1) - - def test_covered_index(self): - """Ensure that covered indexes can be used - """ - - class Test(Document): - a = IntField() - - meta = { - 'indexes': ['a'], - 'allow_inheritance': False - } - - Test.drop_collection() - - obj = Test(a=1) - obj.save() - - # Need to be explicit about covered indexes as mongoDB doesn't know if - # the documents returned might have more keys in that here. - query_plan = Test.objects(id=obj.id).exclude('a').explain() - self.assertFalse(query_plan['indexOnly']) - - query_plan = Test.objects(id=obj.id).only('id').explain() - self.assertTrue(query_plan['indexOnly']) - - query_plan = Test.objects(a=1).only('a').exclude('id').explain() - self.assertTrue(query_plan['indexOnly']) - - def test_index_on_id(self): - - class BlogPost(Document): - meta = { - 'indexes': [ - ['categories', 'id'] - ], - 'allow_inheritance': False - } - - title = StringField(required=True) - description = StringField(required=True) - categories = ListField() - - BlogPost.drop_collection() - - indexes = BlogPost.objects._collection.index_information() - self.assertEqual(indexes['categories_1__id_1']['key'], - [('categories', 1), ('_id', 1)]) - - def test_hint(self): - - class BlogPost(Document): - tags = ListField(StringField()) - meta = { - 'indexes': [ - 'tags', - ], - } - - BlogPost.drop_collection() - - for i in xrange(0, 10): - tags = [("tag %i" % n) for n in xrange(0, i % 2)] - BlogPost(tags=tags).save() - - self.assertEqual(BlogPost.objects.count(), 10) - self.assertEqual(BlogPost.objects.hint().count(), 10) - self.assertEqual(BlogPost.objects.hint([('tags', 1)]).count(), 10) - - self.assertEqual(BlogPost.objects.hint([('ZZ', 1)]).count(), 10) - - def invalid_index(): - BlogPost.objects.hint('tags') - self.assertRaises(TypeError, invalid_index) - - def invalid_index_2(): - return BlogPost.objects.hint(('tags', 1)) - self.assertRaises(TypeError, invalid_index_2) - - def test_unique(self): - """Ensure that uniqueness constraints are applied to fields. - """ - class BlogPost(Document): - title = StringField() - slug = StringField(unique=True) - - BlogPost.drop_collection() - - post1 = BlogPost(title='test1', slug='test') - post1.save() - - # Two posts with the same slug is not allowed - post2 = BlogPost(title='test2', slug='test') - self.assertRaises(NotUniqueError, post2.save) - - # Ensure backwards compatibilty for errors - self.assertRaises(OperationError, post2.save) - - def test_unique_with(self): - """Ensure that unique_with constraints are applied to fields. - """ - class Date(EmbeddedDocument): - year = IntField(db_field='yr') - - class BlogPost(Document): - title = StringField() - date = EmbeddedDocumentField(Date) - slug = StringField(unique_with='date.year') - - BlogPost.drop_collection() - - post1 = BlogPost(title='test1', date=Date(year=2009), slug='test') - post1.save() - - # day is different so won't raise exception - post2 = BlogPost(title='test2', date=Date(year=2010), slug='test') - post2.save() - - # Now there will be two docs with the same slug and the same day: fail - post3 = BlogPost(title='test3', date=Date(year=2010), slug='test') - self.assertRaises(OperationError, post3.save) - - BlogPost.drop_collection() - - def test_unique_embedded_document(self): - """Ensure that uniqueness constraints are applied to fields on embedded documents. - """ - class SubDocument(EmbeddedDocument): - year = IntField(db_field='yr') - slug = StringField(unique=True) - - class BlogPost(Document): - title = StringField() - sub = EmbeddedDocumentField(SubDocument) - - BlogPost.drop_collection() - - post1 = BlogPost(title='test1', sub=SubDocument(year=2009, slug="test")) - post1.save() - - # sub.slug is different so won't raise exception - post2 = BlogPost(title='test2', sub=SubDocument(year=2010, slug='another-slug')) - post2.save() - - # Now there will be two docs with the same sub.slug - post3 = BlogPost(title='test3', sub=SubDocument(year=2010, slug='test')) - self.assertRaises(NotUniqueError, post3.save) - - BlogPost.drop_collection() - - def test_unique_with_embedded_document_and_embedded_unique(self): - """Ensure that uniqueness constraints are applied to fields on - embedded documents. And work with unique_with as well. - """ - class SubDocument(EmbeddedDocument): - year = IntField(db_field='yr') - slug = StringField(unique=True) - - class BlogPost(Document): - title = StringField(unique_with='sub.year') - sub = EmbeddedDocumentField(SubDocument) - - BlogPost.drop_collection() - - post1 = BlogPost(title='test1', sub=SubDocument(year=2009, slug="test")) - post1.save() - - # sub.slug is different so won't raise exception - post2 = BlogPost(title='test2', sub=SubDocument(year=2010, slug='another-slug')) - post2.save() - - # Now there will be two docs with the same sub.slug - post3 = BlogPost(title='test3', sub=SubDocument(year=2010, slug='test')) - self.assertRaises(NotUniqueError, post3.save) - - # Now there will be two docs with the same title and year - post3 = BlogPost(title='test1', sub=SubDocument(year=2009, slug='test-1')) - self.assertRaises(NotUniqueError, post3.save) - - BlogPost.drop_collection() - - def test_ttl_indexes(self): - - class Log(Document): - created = DateTimeField(default=datetime.now) - meta = { - 'indexes': [ - {'fields': ['created'], 'expireAfterSeconds': 3600} - ] - } - - Log.drop_collection() - - if pymongo.version_tuple[0] < 2 and pymongo.version_tuple[1] < 3: - raise SkipTest('pymongo needs to be 2.3 or higher for this test') - - connection = get_connection() - version_array = connection.server_info()['versionArray'] - if version_array[0] < 2 and version_array[1] < 2: - raise SkipTest('MongoDB needs to be 2.2 or higher for this test') - - # Indexes are lazy so use list() to perform query - list(Log.objects) - info = Log.objects._collection.index_information() - self.assertEqual(3600, - info['_types_1_created_1']['expireAfterSeconds']) - - def test_unique_and_indexes(self): - """Ensure that 'unique' constraints aren't overridden by - meta.indexes. - """ - class Customer(Document): - cust_id = IntField(unique=True, required=True) - meta = { - 'indexes': ['cust_id'], - 'allow_inheritance': False, - } - - Customer.drop_collection() - cust = Customer(cust_id=1) - cust.save() - - cust_dupe = Customer(cust_id=1) - try: - cust_dupe.save() - raise AssertionError, "We saved a dupe!" - except NotUniqueError: - pass - Customer.drop_collection() - - def test_unique_and_primary(self): - """If you set a field as primary, then unexpected behaviour can occur. - You won't create a duplicate but you will update an existing document. - """ - - class User(Document): - name = StringField(primary_key=True, unique=True) - password = StringField() - - User.drop_collection() - - user = User(name='huangz', password='secret') - user.save() - - user = User(name='huangz', password='secret2') - user.save() - - self.assertEqual(User.objects.count(), 1) - self.assertEqual(User.objects.get().password, 'secret2') - - User.drop_collection() def test_custom_id_field(self): """Ensure that documents may be created with custom primary keys. @@ -1876,7 +826,6 @@ class DocumentTest(unittest.TestCase): class Site(Document): page = EmbeddedDocumentField(Page) - Site.drop_collection() site = Site(page=Page(log_message="Warning: Dummy message")) site.save() @@ -1903,7 +852,6 @@ class DocumentTest(unittest.TestCase): class Site(Document): page = EmbeddedDocumentField(Page) - Site.drop_collection() site = Site(page=Page(log_message="Warning: Dummy message")) @@ -1917,519 +865,6 @@ class DocumentTest(unittest.TestCase): site = Site.objects.first() self.assertEqual(site.page.log_message, "Error: Dummy message") - def test_circular_reference_deltas(self): - - class Person(Document): - name = StringField() - owns = ListField(ReferenceField('Organization')) - - class Organization(Document): - name = StringField() - owner = ReferenceField('Person') - - Person.drop_collection() - Organization.drop_collection() - - person = Person(name="owner") - person.save() - organization = Organization(name="company") - organization.save() - - person.owns.append(organization) - organization.owner = person - - person.save() - organization.save() - - p = Person.objects[0].select_related() - o = Organization.objects.first() - self.assertEqual(p.owns[0], o) - self.assertEqual(o.owner, p) - - def test_circular_reference_deltas_2(self): - - class Person(Document): - name = StringField() - owns = ListField( ReferenceField( 'Organization' ) ) - employer = ReferenceField( 'Organization' ) - - class Organization( Document ): - name = StringField() - owner = ReferenceField( 'Person' ) - employees = ListField( ReferenceField( 'Person' ) ) - - Person.drop_collection() - Organization.drop_collection() - - person = Person( name="owner" ) - person.save() - - employee = Person( name="employee" ) - employee.save() - - organization = Organization( name="company" ) - organization.save() - - person.owns.append( organization ) - organization.owner = person - - organization.employees.append( employee ) - employee.employer = organization - - person.save() - organization.save() - employee.save() - - p = Person.objects.get(name="owner") - e = Person.objects.get(name="employee") - o = Organization.objects.first() - - self.assertEqual(p.owns[0], o) - self.assertEqual(o.owner, p) - self.assertEqual(e.employer, o) - - def test_delta(self): - - class Doc(Document): - string_field = StringField() - int_field = IntField() - dict_field = DictField() - list_field = ListField() - - Doc.drop_collection() - doc = Doc() - doc.save() - - doc = Doc.objects.first() - self.assertEqual(doc._get_changed_fields(), []) - self.assertEqual(doc._delta(), ({}, {})) - - doc.string_field = 'hello' - self.assertEqual(doc._get_changed_fields(), ['string_field']) - self.assertEqual(doc._delta(), ({'string_field': 'hello'}, {})) - - doc._changed_fields = [] - doc.int_field = 1 - self.assertEqual(doc._get_changed_fields(), ['int_field']) - self.assertEqual(doc._delta(), ({'int_field': 1}, {})) - - doc._changed_fields = [] - dict_value = {'hello': 'world', 'ping': 'pong'} - doc.dict_field = dict_value - self.assertEqual(doc._get_changed_fields(), ['dict_field']) - self.assertEqual(doc._delta(), ({'dict_field': dict_value}, {})) - - doc._changed_fields = [] - list_value = ['1', 2, {'hello': 'world'}] - doc.list_field = list_value - self.assertEqual(doc._get_changed_fields(), ['list_field']) - self.assertEqual(doc._delta(), ({'list_field': list_value}, {})) - - # Test unsetting - doc._changed_fields = [] - doc.dict_field = {} - self.assertEqual(doc._get_changed_fields(), ['dict_field']) - self.assertEqual(doc._delta(), ({}, {'dict_field': 1})) - - doc._changed_fields = [] - doc.list_field = [] - self.assertEqual(doc._get_changed_fields(), ['list_field']) - self.assertEqual(doc._delta(), ({}, {'list_field': 1})) - - def test_delta_recursive(self): - - class Embedded(EmbeddedDocument): - string_field = StringField() - int_field = IntField() - dict_field = DictField() - list_field = ListField() - - class Doc(Document): - string_field = StringField() - int_field = IntField() - dict_field = DictField() - list_field = ListField() - embedded_field = EmbeddedDocumentField(Embedded) - - Doc.drop_collection() - doc = Doc() - doc.save() - - doc = Doc.objects.first() - self.assertEqual(doc._get_changed_fields(), []) - self.assertEqual(doc._delta(), ({}, {})) - - embedded_1 = Embedded() - embedded_1.string_field = 'hello' - embedded_1.int_field = 1 - embedded_1.dict_field = {'hello': 'world'} - embedded_1.list_field = ['1', 2, {'hello': 'world'}] - doc.embedded_field = embedded_1 - - self.assertEqual(doc._get_changed_fields(), ['embedded_field']) - - embedded_delta = { - 'string_field': 'hello', - 'int_field': 1, - 'dict_field': {'hello': 'world'}, - 'list_field': ['1', 2, {'hello': 'world'}] - } - self.assertEqual(doc.embedded_field._delta(), (embedded_delta, {})) - embedded_delta.update({ - '_types': ['Embedded'], - '_cls': 'Embedded', - }) - self.assertEqual(doc._delta(), ({'embedded_field': embedded_delta}, {})) - - doc.save() - doc = doc.reload(10) - - doc.embedded_field.dict_field = {} - self.assertEqual(doc._get_changed_fields(), ['embedded_field.dict_field']) - self.assertEqual(doc.embedded_field._delta(), ({}, {'dict_field': 1})) - self.assertEqual(doc._delta(), ({}, {'embedded_field.dict_field': 1})) - doc.save() - doc = doc.reload(10) - self.assertEqual(doc.embedded_field.dict_field, {}) - - doc.embedded_field.list_field = [] - self.assertEqual(doc._get_changed_fields(), ['embedded_field.list_field']) - self.assertEqual(doc.embedded_field._delta(), ({}, {'list_field': 1})) - self.assertEqual(doc._delta(), ({}, {'embedded_field.list_field': 1})) - doc.save() - doc = doc.reload(10) - self.assertEqual(doc.embedded_field.list_field, []) - - embedded_2 = Embedded() - embedded_2.string_field = 'hello' - embedded_2.int_field = 1 - embedded_2.dict_field = {'hello': 'world'} - embedded_2.list_field = ['1', 2, {'hello': 'world'}] - - doc.embedded_field.list_field = ['1', 2, embedded_2] - self.assertEqual(doc._get_changed_fields(), ['embedded_field.list_field']) - self.assertEqual(doc.embedded_field._delta(), ({ - 'list_field': ['1', 2, { - '_cls': 'Embedded', - '_types': ['Embedded'], - 'string_field': 'hello', - 'dict_field': {'hello': 'world'}, - 'int_field': 1, - 'list_field': ['1', 2, {'hello': 'world'}], - }] - }, {})) - - self.assertEqual(doc._delta(), ({ - 'embedded_field.list_field': ['1', 2, { - '_cls': 'Embedded', - '_types': ['Embedded'], - 'string_field': 'hello', - 'dict_field': {'hello': 'world'}, - 'int_field': 1, - 'list_field': ['1', 2, {'hello': 'world'}], - }] - }, {})) - doc.save() - doc = doc.reload(10) - - self.assertEqual(doc.embedded_field.list_field[0], '1') - self.assertEqual(doc.embedded_field.list_field[1], 2) - for k in doc.embedded_field.list_field[2]._fields: - self.assertEqual(doc.embedded_field.list_field[2][k], embedded_2[k]) - - doc.embedded_field.list_field[2].string_field = 'world' - self.assertEqual(doc._get_changed_fields(), ['embedded_field.list_field.2.string_field']) - self.assertEqual(doc.embedded_field._delta(), ({'list_field.2.string_field': 'world'}, {})) - self.assertEqual(doc._delta(), ({'embedded_field.list_field.2.string_field': 'world'}, {})) - doc.save() - doc = doc.reload(10) - self.assertEqual(doc.embedded_field.list_field[2].string_field, 'world') - - # Test multiple assignments - doc.embedded_field.list_field[2].string_field = 'hello world' - doc.embedded_field.list_field[2] = doc.embedded_field.list_field[2] - self.assertEqual(doc._get_changed_fields(), ['embedded_field.list_field']) - self.assertEqual(doc.embedded_field._delta(), ({ - 'list_field': ['1', 2, { - '_types': ['Embedded'], - '_cls': 'Embedded', - 'string_field': 'hello world', - 'int_field': 1, - 'list_field': ['1', 2, {'hello': 'world'}], - 'dict_field': {'hello': 'world'}}]}, {})) - self.assertEqual(doc._delta(), ({ - 'embedded_field.list_field': ['1', 2, { - '_types': ['Embedded'], - '_cls': 'Embedded', - 'string_field': 'hello world', - 'int_field': 1, - 'list_field': ['1', 2, {'hello': 'world'}], - 'dict_field': {'hello': 'world'}} - ]}, {})) - doc.save() - doc = doc.reload(10) - self.assertEqual(doc.embedded_field.list_field[2].string_field, 'hello world') - - # Test list native methods - doc.embedded_field.list_field[2].list_field.pop(0) - self.assertEqual(doc._delta(), ({'embedded_field.list_field.2.list_field': [2, {'hello': 'world'}]}, {})) - doc.save() - doc = doc.reload(10) - - doc.embedded_field.list_field[2].list_field.append(1) - self.assertEqual(doc._delta(), ({'embedded_field.list_field.2.list_field': [2, {'hello': 'world'}, 1]}, {})) - doc.save() - doc = doc.reload(10) - self.assertEqual(doc.embedded_field.list_field[2].list_field, [2, {'hello': 'world'}, 1]) - - doc.embedded_field.list_field[2].list_field.sort(key=str) - doc.save() - doc = doc.reload(10) - self.assertEqual(doc.embedded_field.list_field[2].list_field, [1, 2, {'hello': 'world'}]) - - del(doc.embedded_field.list_field[2].list_field[2]['hello']) - self.assertEqual(doc._delta(), ({'embedded_field.list_field.2.list_field': [1, 2, {}]}, {})) - doc.save() - doc = doc.reload(10) - - del(doc.embedded_field.list_field[2].list_field) - self.assertEqual(doc._delta(), ({}, {'embedded_field.list_field.2.list_field': 1})) - - doc.save() - doc = doc.reload(10) - - doc.dict_field['Embedded'] = embedded_1 - doc.save() - doc = doc.reload(10) - - doc.dict_field['Embedded'].string_field = 'Hello World' - self.assertEqual(doc._get_changed_fields(), ['dict_field.Embedded.string_field']) - self.assertEqual(doc._delta(), ({'dict_field.Embedded.string_field': 'Hello World'}, {})) - - - def test_delta_db_field(self): - - class Doc(Document): - string_field = StringField(db_field='db_string_field') - int_field = IntField(db_field='db_int_field') - dict_field = DictField(db_field='db_dict_field') - list_field = ListField(db_field='db_list_field') - - Doc.drop_collection() - doc = Doc() - doc.save() - - doc = Doc.objects.first() - self.assertEqual(doc._get_changed_fields(), []) - self.assertEqual(doc._delta(), ({}, {})) - - doc.string_field = 'hello' - self.assertEqual(doc._get_changed_fields(), ['db_string_field']) - self.assertEqual(doc._delta(), ({'db_string_field': 'hello'}, {})) - - doc._changed_fields = [] - doc.int_field = 1 - self.assertEqual(doc._get_changed_fields(), ['db_int_field']) - self.assertEqual(doc._delta(), ({'db_int_field': 1}, {})) - - doc._changed_fields = [] - dict_value = {'hello': 'world', 'ping': 'pong'} - doc.dict_field = dict_value - self.assertEqual(doc._get_changed_fields(), ['db_dict_field']) - self.assertEqual(doc._delta(), ({'db_dict_field': dict_value}, {})) - - doc._changed_fields = [] - list_value = ['1', 2, {'hello': 'world'}] - doc.list_field = list_value - self.assertEqual(doc._get_changed_fields(), ['db_list_field']) - self.assertEqual(doc._delta(), ({'db_list_field': list_value}, {})) - - # Test unsetting - doc._changed_fields = [] - doc.dict_field = {} - self.assertEqual(doc._get_changed_fields(), ['db_dict_field']) - self.assertEqual(doc._delta(), ({}, {'db_dict_field': 1})) - - doc._changed_fields = [] - doc.list_field = [] - self.assertEqual(doc._get_changed_fields(), ['db_list_field']) - self.assertEqual(doc._delta(), ({}, {'db_list_field': 1})) - - # Test it saves that data - doc = Doc() - doc.save() - - doc.string_field = 'hello' - doc.int_field = 1 - doc.dict_field = {'hello': 'world'} - doc.list_field = ['1', 2, {'hello': 'world'}] - doc.save() - doc = doc.reload(10) - - self.assertEqual(doc.string_field, 'hello') - self.assertEqual(doc.int_field, 1) - self.assertEqual(doc.dict_field, {'hello': 'world'}) - self.assertEqual(doc.list_field, ['1', 2, {'hello': 'world'}]) - - def test_delta_recursive_db_field(self): - - class Embedded(EmbeddedDocument): - string_field = StringField(db_field='db_string_field') - int_field = IntField(db_field='db_int_field') - dict_field = DictField(db_field='db_dict_field') - list_field = ListField(db_field='db_list_field') - - class Doc(Document): - string_field = StringField(db_field='db_string_field') - int_field = IntField(db_field='db_int_field') - dict_field = DictField(db_field='db_dict_field') - list_field = ListField(db_field='db_list_field') - embedded_field = EmbeddedDocumentField(Embedded, db_field='db_embedded_field') - - Doc.drop_collection() - doc = Doc() - doc.save() - - doc = Doc.objects.first() - self.assertEqual(doc._get_changed_fields(), []) - self.assertEqual(doc._delta(), ({}, {})) - - embedded_1 = Embedded() - embedded_1.string_field = 'hello' - embedded_1.int_field = 1 - embedded_1.dict_field = {'hello': 'world'} - embedded_1.list_field = ['1', 2, {'hello': 'world'}] - doc.embedded_field = embedded_1 - - self.assertEqual(doc._get_changed_fields(), ['db_embedded_field']) - - embedded_delta = { - 'db_string_field': 'hello', - 'db_int_field': 1, - 'db_dict_field': {'hello': 'world'}, - 'db_list_field': ['1', 2, {'hello': 'world'}] - } - self.assertEqual(doc.embedded_field._delta(), (embedded_delta, {})) - embedded_delta.update({ - '_types': ['Embedded'], - '_cls': 'Embedded', - }) - self.assertEqual(doc._delta(), ({'db_embedded_field': embedded_delta}, {})) - - doc.save() - doc = doc.reload(10) - - doc.embedded_field.dict_field = {} - self.assertEqual(doc._get_changed_fields(), ['db_embedded_field.db_dict_field']) - self.assertEqual(doc.embedded_field._delta(), ({}, {'db_dict_field': 1})) - self.assertEqual(doc._delta(), ({}, {'db_embedded_field.db_dict_field': 1})) - doc.save() - doc = doc.reload(10) - self.assertEqual(doc.embedded_field.dict_field, {}) - - doc.embedded_field.list_field = [] - self.assertEqual(doc._get_changed_fields(), ['db_embedded_field.db_list_field']) - self.assertEqual(doc.embedded_field._delta(), ({}, {'db_list_field': 1})) - self.assertEqual(doc._delta(), ({}, {'db_embedded_field.db_list_field': 1})) - doc.save() - doc = doc.reload(10) - self.assertEqual(doc.embedded_field.list_field, []) - - embedded_2 = Embedded() - embedded_2.string_field = 'hello' - embedded_2.int_field = 1 - embedded_2.dict_field = {'hello': 'world'} - embedded_2.list_field = ['1', 2, {'hello': 'world'}] - - doc.embedded_field.list_field = ['1', 2, embedded_2] - self.assertEqual(doc._get_changed_fields(), ['db_embedded_field.db_list_field']) - self.assertEqual(doc.embedded_field._delta(), ({ - 'db_list_field': ['1', 2, { - '_cls': 'Embedded', - '_types': ['Embedded'], - 'db_string_field': 'hello', - 'db_dict_field': {'hello': 'world'}, - 'db_int_field': 1, - 'db_list_field': ['1', 2, {'hello': 'world'}], - }] - }, {})) - - self.assertEqual(doc._delta(), ({ - 'db_embedded_field.db_list_field': ['1', 2, { - '_cls': 'Embedded', - '_types': ['Embedded'], - 'db_string_field': 'hello', - 'db_dict_field': {'hello': 'world'}, - 'db_int_field': 1, - 'db_list_field': ['1', 2, {'hello': 'world'}], - }] - }, {})) - doc.save() - doc = doc.reload(10) - - self.assertEqual(doc.embedded_field.list_field[0], '1') - self.assertEqual(doc.embedded_field.list_field[1], 2) - for k in doc.embedded_field.list_field[2]._fields: - self.assertEqual(doc.embedded_field.list_field[2][k], embedded_2[k]) - - doc.embedded_field.list_field[2].string_field = 'world' - self.assertEqual(doc._get_changed_fields(), ['db_embedded_field.db_list_field.2.db_string_field']) - self.assertEqual(doc.embedded_field._delta(), ({'db_list_field.2.db_string_field': 'world'}, {})) - self.assertEqual(doc._delta(), ({'db_embedded_field.db_list_field.2.db_string_field': 'world'}, {})) - doc.save() - doc = doc.reload(10) - self.assertEqual(doc.embedded_field.list_field[2].string_field, 'world') - - # Test multiple assignments - doc.embedded_field.list_field[2].string_field = 'hello world' - doc.embedded_field.list_field[2] = doc.embedded_field.list_field[2] - self.assertEqual(doc._get_changed_fields(), ['db_embedded_field.db_list_field']) - self.assertEqual(doc.embedded_field._delta(), ({ - 'db_list_field': ['1', 2, { - '_types': ['Embedded'], - '_cls': 'Embedded', - 'db_string_field': 'hello world', - 'db_int_field': 1, - 'db_list_field': ['1', 2, {'hello': 'world'}], - 'db_dict_field': {'hello': 'world'}}]}, {})) - self.assertEqual(doc._delta(), ({ - 'db_embedded_field.db_list_field': ['1', 2, { - '_types': ['Embedded'], - '_cls': 'Embedded', - 'db_string_field': 'hello world', - 'db_int_field': 1, - 'db_list_field': ['1', 2, {'hello': 'world'}], - 'db_dict_field': {'hello': 'world'}} - ]}, {})) - doc.save() - doc = doc.reload(10) - self.assertEqual(doc.embedded_field.list_field[2].string_field, 'hello world') - - # Test list native methods - doc.embedded_field.list_field[2].list_field.pop(0) - self.assertEqual(doc._delta(), ({'db_embedded_field.db_list_field.2.db_list_field': [2, {'hello': 'world'}]}, {})) - doc.save() - doc = doc.reload(10) - - doc.embedded_field.list_field[2].list_field.append(1) - self.assertEqual(doc._delta(), ({'db_embedded_field.db_list_field.2.db_list_field': [2, {'hello': 'world'}, 1]}, {})) - doc.save() - doc = doc.reload(10) - self.assertEqual(doc.embedded_field.list_field[2].list_field, [2, {'hello': 'world'}, 1]) - - doc.embedded_field.list_field[2].list_field.sort(key=str) - doc.save() - doc = doc.reload(10) - self.assertEqual(doc.embedded_field.list_field[2].list_field, [1, 2, {'hello': 'world'}]) - - del(doc.embedded_field.list_field[2].list_field[2]['hello']) - self.assertEqual(doc._delta(), ({'db_embedded_field.db_list_field.2.db_list_field': [1, 2, {}]}, {})) - doc.save() - doc = doc.reload(10) - - del(doc.embedded_field.list_field[2].list_field) - self.assertEqual(doc._delta(), ({}, {'db_embedded_field.db_list_field.2.db_list_field': 1})) - def test_save_only_changed_fields(self): """Ensure save only sets / unsets changed fields """ @@ -2437,7 +872,6 @@ class DocumentTest(unittest.TestCase): class User(self.Person): active = BooleanField(default=True) - User.drop_collection() # Create person object and save it to the database @@ -2697,29 +1131,6 @@ class DocumentTest(unittest.TestCase): promoted_employee.reload() self.assertEqual(promoted_employee.details, None) - def test_mixins_dont_add_to_types(self): - - class Mixin(object): - name = StringField() - - class Person(Document, Mixin): - pass - - Person.drop_collection() - - self.assertEqual(Person._fields.keys(), ['name', 'id']) - - Person(name="Rozza").save() - - collection = self.db[Person._get_collection_name()] - obj = collection.find_one() - self.assertEqual(obj['_cls'], 'Person') - self.assertEqual(obj['_types'], ['Person']) - - self.assertEqual(Person.objects.count(), 1) - - Person.drop_collection() - def test_object_mixins(self): class NameMixin(object): @@ -2795,22 +1206,6 @@ class DocumentTest(unittest.TestCase): BlogPost.drop_collection() - def test_cannot_perform_joins_references(self): - - class BlogPost(Document): - author = ReferenceField(self.Person) - author2 = GenericReferenceField() - - def test_reference(): - list(BlogPost.objects(author__name="test")) - - self.assertRaises(InvalidQueryError, test_reference) - - def test_generic_reference(): - list(BlogPost.objects(author2__name="test")) - - self.assertRaises(InvalidQueryError, test_generic_reference) - def test_duplicate_db_fields_raise_invalid_document_error(self): """Ensure a InvalidDocumentError is thrown if duplicate fields declare the same db_field""" @@ -3082,17 +1477,17 @@ class DocumentTest(unittest.TestCase): for u in User.objects.all(): all_user_dic[u] = "OK" - self.assertEqual(all_user_dic.get(u1, False), "OK" ) - self.assertEqual(all_user_dic.get(u2, False), "OK" ) - self.assertEqual(all_user_dic.get(u3, False), "OK" ) - self.assertEqual(all_user_dic.get(u4, False), False ) # New object - self.assertEqual(all_user_dic.get(b1, False), False ) # Other object - self.assertEqual(all_user_dic.get(b2, False), False ) # Other object + self.assertEqual(all_user_dic.get(u1, False), "OK") + self.assertEqual(all_user_dic.get(u2, False), "OK") + self.assertEqual(all_user_dic.get(u3, False), "OK") + self.assertEqual(all_user_dic.get(u4, False), False) # New object + self.assertEqual(all_user_dic.get(b1, False), False) # Other object + self.assertEqual(all_user_dic.get(b2, False), False) # Other object # in Set all_user_set = set(User.objects.all()) - self.assertTrue(u1 in all_user_set ) + self.assertTrue(u1 in all_user_set) def test_picklable(self): @@ -3313,7 +1708,7 @@ class DocumentTest(unittest.TestCase): # Bob Book.objects.create(name="1", author=bob, extra={"a": bob.to_dbref(), "b": [karl.to_dbref(), susan.to_dbref()]}) - Book.objects.create(name="2", author=bob, extra={"a": bob.to_dbref(), "b": karl.to_dbref()} ) + Book.objects.create(name="2", author=bob, extra={"a": bob.to_dbref(), "b": karl.to_dbref()}) Book.objects.create(name="3", author=bob, extra={"a": bob.to_dbref(), "c": [jon.to_dbref(), peter.to_dbref()]}) Book.objects.create(name="4", author=bob) @@ -3325,20 +1720,20 @@ class DocumentTest(unittest.TestCase): Book.objects.create(name="9", author=jon, extra={"a": peter.to_dbref()}) # Checks - self.assertEqual(u",".join([str(b) for b in Book.objects.all()] ) , "1,2,3,4,5,6,7,8,9" ) + self.assertEqual(u",".join([str(b) for b in Book.objects.all()]) , "1,2,3,4,5,6,7,8,9") # bob related books self.assertEqual(u",".join([str(b) for b in Book.objects.filter( - Q(extra__a=bob ) | + Q(extra__a=bob) | Q(author=bob) | Q(extra__b=bob))]) , "1,2,3,4") # Susan & Karl related books self.assertEqual(u",".join([str(b) for b in Book.objects.filter( - Q(extra__a__all=[karl, susan] ) | - Q(author__all=[karl, susan ] ) | - Q(extra__b__all=[karl.to_dbref(), susan.to_dbref()] ) - ) ] ) , "1" ) + Q(extra__a__all=[karl, susan]) | + Q(author__all=[karl, susan ]) | + Q(extra__b__all=[karl.to_dbref(), susan.to_dbref()]) + ) ]) , "1") # $Where self.assertEqual(u",".join([str(b) for b in Book.objects.filter( @@ -3348,7 +1743,7 @@ class DocumentTest(unittest.TestCase): return this.name == '1' || this.name == '2';}""" } - ) ]), "1,2") + ) ]), "1,2") class ValidatorErrorTest(unittest.TestCase): @@ -3504,5 +1899,6 @@ class ValidatorErrorTest(unittest.TestCase): self.assertRaises(OperationError, change_shard_key) + if __name__ == '__main__': unittest.main() diff --git a/tests/mongoengine.png b/tests/document/mongoengine.png similarity index 100% rename from tests/mongoengine.png rename to tests/document/mongoengine.png diff --git a/tests/migration/__init__.py b/tests/migration/__init__.py new file mode 100644 index 00000000..882e7370 --- /dev/null +++ b/tests/migration/__init__.py @@ -0,0 +1,4 @@ +from turn_off_inheritance import * + +if __name__ == '__main__': + unittest.main() \ No newline at end of file diff --git a/tests/migration/test_convert_to_new_inheritance_model.py b/tests/migration/test_convert_to_new_inheritance_model.py new file mode 100644 index 00000000..0ef37f74 --- /dev/null +++ b/tests/migration/test_convert_to_new_inheritance_model.py @@ -0,0 +1,51 @@ +# -*- coding: utf-8 -*- +import unittest + +from mongoengine import Document, connect +from mongoengine.connection import get_db +from mongoengine.fields import StringField + +__all__ = ('ConvertToNewInheritanceModel', ) + + +class ConvertToNewInheritanceModel(unittest.TestCase): + + def setUp(self): + connect(db='mongoenginetest') + self.db = get_db() + + def tearDown(self): + for collection in self.db.collection_names(): + if 'system.' in collection: + continue + self.db.drop_collection(collection) + + def test_how_to_convert_to_the_new_inheritance_model(self): + """Demonstrates migrating from 0.7 to 0.8 + """ + + # 1. Declaration of the class + class Animal(Document): + name = StringField() + meta = { + 'allow_inheritance': True, + 'indexes': ['name'] + } + + # 2. Remove _types + collection = Animal._get_collection() + collection.update({}, {"$unset": {"_types": 1}}, multi=True) + + # 3. Confirm extra data is removed + count = collection.find({'_types': {"$exists": True}}).count() + assert count == 0 + + # 4. Remove indexes + info = collection.index_information() + indexes_to_drop = [key for key, value in info.iteritems() + if '_types' in dict(value['key'])] + for index in indexes_to_drop: + collection.drop_index(index) + + # 5. Recreate indexes + Animal.objects._ensure_indexes() diff --git a/tests/migration/turn_off_inheritance.py b/tests/migration/turn_off_inheritance.py new file mode 100644 index 00000000..5d0f7d73 --- /dev/null +++ b/tests/migration/turn_off_inheritance.py @@ -0,0 +1,62 @@ +# -*- coding: utf-8 -*- +import unittest + +from mongoengine import Document, connect +from mongoengine.connection import get_db +from mongoengine.fields import StringField + +__all__ = ('TurnOffInheritanceTest', ) + + +class TurnOffInheritanceTest(unittest.TestCase): + + def setUp(self): + connect(db='mongoenginetest') + self.db = get_db() + + def tearDown(self): + for collection in self.db.collection_names(): + if 'system.' in collection: + continue + self.db.drop_collection(collection) + + def test_how_to_turn_off_inheritance(self): + """Demonstrates migrating from allow_inheritance = True to False. + """ + + # 1. Old declaration of the class + + class Animal(Document): + name = StringField() + meta = { + 'allow_inheritance': True, + 'indexes': ['name'] + } + + # 2. Turn off inheritance + class Animal(Document): + name = StringField() + meta = { + 'allow_inheritance': False, + 'indexes': ['name'] + } + + # 3. Remove _types and _cls + collection = Animal._get_collection() + collection.update({}, {"$unset": {"_types": 1, "_cls": 1}}, multi=True) + + # 3. Confirm extra data is removed + count = collection.find({"$or": [{'_types': {"$exists": True}}, + {'_cls': {"$exists": True}}]}).count() + assert count == 0 + + # 4. Remove indexes + info = collection.index_information() + indexes_to_drop = [key for key, value in info.iteritems() + if '_types' in dict(value['key']) + or '_cls' in dict(value['key'])] + for index in indexes_to_drop: + collection.drop_index(index) + + # 5. Recreate indexes + Animal.objects._ensure_indexes() diff --git a/tests/test_dynamic_document.py b/tests/test_dynamic_document.py deleted file mode 100644 index 23762a34..00000000 --- a/tests/test_dynamic_document.py +++ /dev/null @@ -1,533 +0,0 @@ -import unittest - -from mongoengine import * -from mongoengine.connection import get_db - - -class DynamicDocTest(unittest.TestCase): - - def setUp(self): - connect(db='mongoenginetest') - self.db = get_db() - - class Person(DynamicDocument): - name = StringField() - meta = {'allow_inheritance': True} - - Person.drop_collection() - - self.Person = Person - - def test_simple_dynamic_document(self): - """Ensures simple dynamic documents are saved correctly""" - - p = self.Person() - p.name = "James" - p.age = 34 - - self.assertEqual(p.to_mongo(), - {"_types": ["Person"], "_cls": "Person", - "name": "James", "age": 34} - ) - - p.save() - - self.assertEqual(self.Person.objects.first().age, 34) - - # Confirm no changes to self.Person - self.assertFalse(hasattr(self.Person, 'age')) - - def test_dynamic_document_delta(self): - """Ensures simple dynamic documents can delta correctly""" - p = self.Person(name="James", age=34) - self.assertEqual(p._delta(), ({'_types': ['Person'], 'age': 34, 'name': 'James', '_cls': 'Person'}, {})) - - p.doc = 123 - del(p.doc) - self.assertEqual(p._delta(), ({'_types': ['Person'], 'age': 34, 'name': 'James', '_cls': 'Person'}, {'doc': 1})) - - def test_change_scope_of_variable(self): - """Test changing the scope of a dynamic field has no adverse effects""" - p = self.Person() - p.name = "Dean" - p.misc = 22 - p.save() - - p = self.Person.objects.get() - p.misc = {'hello': 'world'} - p.save() - - p = self.Person.objects.get() - self.assertEqual(p.misc, {'hello': 'world'}) - - def test_delete_dynamic_field(self): - """Test deleting a dynamic field works""" - self.Person.drop_collection() - p = self.Person() - p.name = "Dean" - p.misc = 22 - p.save() - - p = self.Person.objects.get() - p.misc = {'hello': 'world'} - p.save() - - p = self.Person.objects.get() - self.assertEqual(p.misc, {'hello': 'world'}) - collection = self.db[self.Person._get_collection_name()] - obj = collection.find_one() - self.assertEqual(sorted(obj.keys()), ['_cls', '_id', '_types', 'misc', 'name']) - - del(p.misc) - p.save() - - p = self.Person.objects.get() - self.assertFalse(hasattr(p, 'misc')) - - obj = collection.find_one() - self.assertEqual(sorted(obj.keys()), ['_cls', '_id', '_types', 'name']) - - def test_dynamic_document_queries(self): - """Ensure we can query dynamic fields""" - p = self.Person() - p.name = "Dean" - p.age = 22 - p.save() - - self.assertEqual(1, self.Person.objects(age=22).count()) - p = self.Person.objects(age=22) - p = p.get() - self.assertEqual(22, p.age) - - def test_complex_dynamic_document_queries(self): - class Person(DynamicDocument): - name = StringField() - - Person.drop_collection() - - p = Person(name="test") - p.age = "ten" - p.save() - - p1 = Person(name="test1") - p1.age = "less then ten and a half" - p1.save() - - p2 = Person(name="test2") - p2.age = 10 - p2.save() - - self.assertEqual(Person.objects(age__icontains='ten').count(), 2) - self.assertEqual(Person.objects(age__gte=10).count(), 1) - - def test_complex_data_lookups(self): - """Ensure you can query dynamic document dynamic fields""" - p = self.Person() - p.misc = {'hello': 'world'} - p.save() - - self.assertEqual(1, self.Person.objects(misc__hello='world').count()) - - def test_inheritance(self): - """Ensure that dynamic document plays nice with inheritance""" - class Employee(self.Person): - salary = IntField() - - Employee.drop_collection() - - self.assertTrue('name' in Employee._fields) - self.assertTrue('salary' in Employee._fields) - self.assertEqual(Employee._get_collection_name(), - self.Person._get_collection_name()) - - joe_bloggs = Employee() - joe_bloggs.name = "Joe Bloggs" - joe_bloggs.salary = 10 - joe_bloggs.age = 20 - joe_bloggs.save() - - self.assertEqual(1, self.Person.objects(age=20).count()) - self.assertEqual(1, Employee.objects(age=20).count()) - - joe_bloggs = self.Person.objects.first() - self.assertTrue(isinstance(joe_bloggs, Employee)) - - def test_embedded_dynamic_document(self): - """Test dynamic embedded documents""" - class Embedded(DynamicEmbeddedDocument): - pass - - class Doc(DynamicDocument): - pass - - Doc.drop_collection() - doc = Doc() - - embedded_1 = Embedded() - embedded_1.string_field = 'hello' - embedded_1.int_field = 1 - embedded_1.dict_field = {'hello': 'world'} - embedded_1.list_field = ['1', 2, {'hello': 'world'}] - doc.embedded_field = embedded_1 - - self.assertEqual(doc.to_mongo(), {"_types": ['Doc'], "_cls": "Doc", - "embedded_field": { - "_types": ['Embedded'], "_cls": "Embedded", - "string_field": "hello", - "int_field": 1, - "dict_field": {"hello": "world"}, - "list_field": ['1', 2, {'hello': 'world'}] - } - }) - doc.save() - - doc = Doc.objects.first() - self.assertEqual(doc.embedded_field.__class__, Embedded) - self.assertEqual(doc.embedded_field.string_field, "hello") - self.assertEqual(doc.embedded_field.int_field, 1) - self.assertEqual(doc.embedded_field.dict_field, {'hello': 'world'}) - self.assertEqual(doc.embedded_field.list_field, ['1', 2, {'hello': 'world'}]) - - def test_complex_embedded_documents(self): - """Test complex dynamic embedded documents setups""" - class Embedded(DynamicEmbeddedDocument): - pass - - class Doc(DynamicDocument): - pass - - Doc.drop_collection() - doc = Doc() - - embedded_1 = Embedded() - embedded_1.string_field = 'hello' - embedded_1.int_field = 1 - embedded_1.dict_field = {'hello': 'world'} - - embedded_2 = Embedded() - embedded_2.string_field = 'hello' - embedded_2.int_field = 1 - embedded_2.dict_field = {'hello': 'world'} - embedded_2.list_field = ['1', 2, {'hello': 'world'}] - - embedded_1.list_field = ['1', 2, embedded_2] - doc.embedded_field = embedded_1 - - self.assertEqual(doc.to_mongo(), {"_types": ['Doc'], "_cls": "Doc", - "embedded_field": { - "_types": ['Embedded'], "_cls": "Embedded", - "string_field": "hello", - "int_field": 1, - "dict_field": {"hello": "world"}, - "list_field": ['1', 2, - {"_types": ['Embedded'], "_cls": "Embedded", - "string_field": "hello", - "int_field": 1, - "dict_field": {"hello": "world"}, - "list_field": ['1', 2, {'hello': 'world'}]} - ] - } - }) - doc.save() - doc = Doc.objects.first() - self.assertEqual(doc.embedded_field.__class__, Embedded) - self.assertEqual(doc.embedded_field.string_field, "hello") - self.assertEqual(doc.embedded_field.int_field, 1) - self.assertEqual(doc.embedded_field.dict_field, {'hello': 'world'}) - self.assertEqual(doc.embedded_field.list_field[0], '1') - self.assertEqual(doc.embedded_field.list_field[1], 2) - - embedded_field = doc.embedded_field.list_field[2] - - self.assertEqual(embedded_field.__class__, Embedded) - self.assertEqual(embedded_field.string_field, "hello") - self.assertEqual(embedded_field.int_field, 1) - self.assertEqual(embedded_field.dict_field, {'hello': 'world'}) - self.assertEqual(embedded_field.list_field, ['1', 2, {'hello': 'world'}]) - - def test_delta_for_dynamic_documents(self): - p = self.Person() - p.name = "Dean" - p.age = 22 - p.save() - - p.age = 24 - self.assertEqual(p.age, 24) - self.assertEqual(p._get_changed_fields(), ['age']) - self.assertEqual(p._delta(), ({'age': 24}, {})) - - p = self.Person.objects(age=22).get() - p.age = 24 - self.assertEqual(p.age, 24) - self.assertEqual(p._get_changed_fields(), ['age']) - self.assertEqual(p._delta(), ({'age': 24}, {})) - - p.save() - self.assertEqual(1, self.Person.objects(age=24).count()) - - def test_delta(self): - - class Doc(DynamicDocument): - pass - - Doc.drop_collection() - doc = Doc() - doc.save() - - doc = Doc.objects.first() - self.assertEqual(doc._get_changed_fields(), []) - self.assertEqual(doc._delta(), ({}, {})) - - doc.string_field = 'hello' - self.assertEqual(doc._get_changed_fields(), ['string_field']) - self.assertEqual(doc._delta(), ({'string_field': 'hello'}, {})) - - doc._changed_fields = [] - doc.int_field = 1 - self.assertEqual(doc._get_changed_fields(), ['int_field']) - self.assertEqual(doc._delta(), ({'int_field': 1}, {})) - - doc._changed_fields = [] - dict_value = {'hello': 'world', 'ping': 'pong'} - doc.dict_field = dict_value - self.assertEqual(doc._get_changed_fields(), ['dict_field']) - self.assertEqual(doc._delta(), ({'dict_field': dict_value}, {})) - - doc._changed_fields = [] - list_value = ['1', 2, {'hello': 'world'}] - doc.list_field = list_value - self.assertEqual(doc._get_changed_fields(), ['list_field']) - self.assertEqual(doc._delta(), ({'list_field': list_value}, {})) - - # Test unsetting - doc._changed_fields = [] - doc.dict_field = {} - self.assertEqual(doc._get_changed_fields(), ['dict_field']) - self.assertEqual(doc._delta(), ({}, {'dict_field': 1})) - - doc._changed_fields = [] - doc.list_field = [] - self.assertEqual(doc._get_changed_fields(), ['list_field']) - self.assertEqual(doc._delta(), ({}, {'list_field': 1})) - - def test_delta_recursive(self): - """Testing deltaing works with dynamic documents""" - class Embedded(DynamicEmbeddedDocument): - pass - - class Doc(DynamicDocument): - pass - - Doc.drop_collection() - doc = Doc() - doc.save() - - doc = Doc.objects.first() - self.assertEqual(doc._get_changed_fields(), []) - self.assertEqual(doc._delta(), ({}, {})) - - embedded_1 = Embedded() - embedded_1.string_field = 'hello' - embedded_1.int_field = 1 - embedded_1.dict_field = {'hello': 'world'} - embedded_1.list_field = ['1', 2, {'hello': 'world'}] - doc.embedded_field = embedded_1 - - self.assertEqual(doc._get_changed_fields(), ['embedded_field']) - - embedded_delta = { - 'string_field': 'hello', - 'int_field': 1, - 'dict_field': {'hello': 'world'}, - 'list_field': ['1', 2, {'hello': 'world'}] - } - self.assertEqual(doc.embedded_field._delta(), (embedded_delta, {})) - embedded_delta.update({ - '_types': ['Embedded'], - '_cls': 'Embedded', - }) - self.assertEqual(doc._delta(), ({'embedded_field': embedded_delta}, {})) - - doc.save() - doc.reload() - - doc.embedded_field.dict_field = {} - self.assertEqual(doc._get_changed_fields(), ['embedded_field.dict_field']) - self.assertEqual(doc.embedded_field._delta(), ({}, {'dict_field': 1})) - - self.assertEqual(doc._delta(), ({}, {'embedded_field.dict_field': 1})) - doc.save() - doc.reload() - - doc.embedded_field.list_field = [] - self.assertEqual(doc._get_changed_fields(), ['embedded_field.list_field']) - self.assertEqual(doc.embedded_field._delta(), ({}, {'list_field': 1})) - self.assertEqual(doc._delta(), ({}, {'embedded_field.list_field': 1})) - doc.save() - doc.reload() - - embedded_2 = Embedded() - embedded_2.string_field = 'hello' - embedded_2.int_field = 1 - embedded_2.dict_field = {'hello': 'world'} - embedded_2.list_field = ['1', 2, {'hello': 'world'}] - - doc.embedded_field.list_field = ['1', 2, embedded_2] - self.assertEqual(doc._get_changed_fields(), ['embedded_field.list_field']) - self.assertEqual(doc.embedded_field._delta(), ({ - 'list_field': ['1', 2, { - '_cls': 'Embedded', - '_types': ['Embedded'], - 'string_field': 'hello', - 'dict_field': {'hello': 'world'}, - 'int_field': 1, - 'list_field': ['1', 2, {'hello': 'world'}], - }] - }, {})) - - self.assertEqual(doc._delta(), ({ - 'embedded_field.list_field': ['1', 2, { - '_cls': 'Embedded', - '_types': ['Embedded'], - 'string_field': 'hello', - 'dict_field': {'hello': 'world'}, - 'int_field': 1, - 'list_field': ['1', 2, {'hello': 'world'}], - }] - }, {})) - doc.save() - doc.reload() - - self.assertEqual(doc.embedded_field.list_field[2]._changed_fields, []) - self.assertEqual(doc.embedded_field.list_field[0], '1') - self.assertEqual(doc.embedded_field.list_field[1], 2) - for k in doc.embedded_field.list_field[2]._fields: - self.assertEqual(doc.embedded_field.list_field[2][k], embedded_2[k]) - - doc.embedded_field.list_field[2].string_field = 'world' - self.assertEqual(doc._get_changed_fields(), ['embedded_field.list_field.2.string_field']) - self.assertEqual(doc.embedded_field._delta(), ({'list_field.2.string_field': 'world'}, {})) - self.assertEqual(doc._delta(), ({'embedded_field.list_field.2.string_field': 'world'}, {})) - doc.save() - doc.reload() - self.assertEqual(doc.embedded_field.list_field[2].string_field, 'world') - - # Test multiple assignments - doc.embedded_field.list_field[2].string_field = 'hello world' - doc.embedded_field.list_field[2] = doc.embedded_field.list_field[2] - self.assertEqual(doc._get_changed_fields(), ['embedded_field.list_field']) - self.assertEqual(doc.embedded_field._delta(), ({ - 'list_field': ['1', 2, { - '_types': ['Embedded'], - '_cls': 'Embedded', - 'string_field': 'hello world', - 'int_field': 1, - 'list_field': ['1', 2, {'hello': 'world'}], - 'dict_field': {'hello': 'world'}}]}, {})) - self.assertEqual(doc._delta(), ({ - 'embedded_field.list_field': ['1', 2, { - '_types': ['Embedded'], - '_cls': 'Embedded', - 'string_field': 'hello world', - 'int_field': 1, - 'list_field': ['1', 2, {'hello': 'world'}], - 'dict_field': {'hello': 'world'}} - ]}, {})) - doc.save() - doc.reload() - self.assertEqual(doc.embedded_field.list_field[2].string_field, 'hello world') - - # Test list native methods - doc.embedded_field.list_field[2].list_field.pop(0) - self.assertEqual(doc._delta(), ({'embedded_field.list_field.2.list_field': [2, {'hello': 'world'}]}, {})) - doc.save() - doc.reload() - - doc.embedded_field.list_field[2].list_field.append(1) - self.assertEqual(doc._delta(), ({'embedded_field.list_field.2.list_field': [2, {'hello': 'world'}, 1]}, {})) - doc.save() - doc.reload() - self.assertEqual(doc.embedded_field.list_field[2].list_field, [2, {'hello': 'world'}, 1]) - - doc.embedded_field.list_field[2].list_field.sort(key=str)# use str as a key to allow comparing uncomperable types - doc.save() - doc.reload() - self.assertEqual(doc.embedded_field.list_field[2].list_field, [1, 2, {'hello': 'world'}]) - - del(doc.embedded_field.list_field[2].list_field[2]['hello']) - self.assertEqual(doc._delta(), ({'embedded_field.list_field.2.list_field': [1, 2, {}]}, {})) - doc.save() - doc.reload() - - del(doc.embedded_field.list_field[2].list_field) - self.assertEqual(doc._delta(), ({}, {'embedded_field.list_field.2.list_field': 1})) - - doc.save() - doc.reload() - - doc.dict_field = {'embedded': embedded_1} - doc.save() - doc.reload() - - doc.dict_field['embedded'].string_field = 'Hello World' - self.assertEqual(doc._get_changed_fields(), ['dict_field.embedded.string_field']) - self.assertEqual(doc._delta(), ({'dict_field.embedded.string_field': 'Hello World'}, {})) - - def test_indexes(self): - """Ensure that indexes are used when meta[indexes] is specified. - """ - class BlogPost(DynamicDocument): - meta = { - 'indexes': [ - '-date', - ('category', '-date') - ], - } - - BlogPost.drop_collection() - - info = BlogPost.objects._collection.index_information() - # _id, '-date', ('cat', 'date') - # NB: there is no index on _types by itself, since - # the indices on -date and tags will both contain - # _types as first element in the key - self.assertEqual(len(info), 3) - - # Indexes are lazy so use list() to perform query - list(BlogPost.objects) - info = BlogPost.objects._collection.index_information() - info = [value['key'] for key, value in info.iteritems()] - self.assertTrue([('_types', 1), ('category', 1), ('date', -1)] - in info) - self.assertTrue([('_types', 1), ('date', -1)] in info) - - def test_dynamic_and_embedded(self): - """Ensure embedded documents play nicely""" - - class Address(EmbeddedDocument): - city = StringField() - - class Person(DynamicDocument): - name = StringField() - meta = {'allow_inheritance': True} - - Person.drop_collection() - - Person(name="Ross", address=Address(city="London")).save() - - person = Person.objects.first() - person.address.city = "Lundenne" - person.save() - - self.assertEqual(Person.objects.first().address.city, "Lundenne") - - person = Person.objects.first() - person.address = Address(city="Londinium") - person.save() - - self.assertEqual(Person.objects.first().address.city, "Londinium") - - person = Person.objects.first() - person.age = 35 - person.save() - self.assertEqual(Person.objects.first().age, 35) diff --git a/tests/test_fields.py b/tests/test_fields.py index 98065501..118521fd 100644 --- a/tests/test_fields.py +++ b/tests/test_fields.py @@ -14,10 +14,11 @@ import gridfs from nose.plugins.skip import SkipTest from mongoengine import * from mongoengine.connection import get_db -from mongoengine.base import _document_registry, NotRegistered +from mongoengine.base import _document_registry +from mongoengine.errors import NotRegistered from mongoengine.python_support import PY3, b, StringIO, bin_type -TEST_IMAGE_PATH = os.path.join(os.path.dirname(__file__), 'mongoengine.png') +TEST_IMAGE_PATH = os.path.join(os.path.dirname(__file__), 'document/mongoengine.png') class FieldTest(unittest.TestCase): diff --git a/tests/test_queryset.py b/tests/test_queryset.py index 690df5eb..cdabadb4 100644 --- a/tests/test_queryset.py +++ b/tests/test_queryset.py @@ -11,9 +11,12 @@ from mongoengine import * from mongoengine.connection import get_connection from mongoengine.python_support import PY3 from mongoengine.tests import query_counter -from mongoengine.queryset import (QuerySet, QuerySetManager, +from mongoengine.queryset import (Q, QuerySet, QuerySetManager, MultipleObjectsReturned, DoesNotExist, - QueryFieldList) + QueryFieldList, queryset_manager) +from mongoengine.queryset import transform +from mongoengine.errors import InvalidQueryError + class QuerySetTest(unittest.TestCase): @@ -40,19 +43,34 @@ class QuerySetTest(unittest.TestCase): def test_transform_query(self): """Ensure that the _transform_query function operates correctly. """ - self.assertEqual(QuerySet._transform_query(name='test', age=30), + self.assertEqual(transform.query(name='test', age=30), {'name': 'test', 'age': 30}) - self.assertEqual(QuerySet._transform_query(age__lt=30), + self.assertEqual(transform.query(age__lt=30), {'age': {'$lt': 30}}) - self.assertEqual(QuerySet._transform_query(age__gt=20, age__lt=50), + self.assertEqual(transform.query(age__gt=20, age__lt=50), {'age': {'$gt': 20, '$lt': 50}}) - self.assertEqual(QuerySet._transform_query(age=20, age__gt=50), + self.assertEqual(transform.query(age=20, age__gt=50), {'age': 20}) - self.assertEqual(QuerySet._transform_query(friend__age__gte=30), + self.assertEqual(transform.query(friend__age__gte=30), {'friend.age': {'$gte': 30}}) - self.assertEqual(QuerySet._transform_query(name__exists=True), + self.assertEqual(transform.query(name__exists=True), {'name': {'$exists': True}}) + def test_cannot_perform_joins_references(self): + + class BlogPost(Document): + author = ReferenceField(self.Person) + author2 = GenericReferenceField() + + def test_reference(): + list(BlogPost.objects(author__name="test")) + + self.assertRaises(InvalidQueryError, test_reference) + + def test_generic_reference(): + list(BlogPost.objects(author2__name="test")) + + def test_find(self): """Ensure that a query returns a valid set of results. """ @@ -921,10 +939,9 @@ class QuerySetTest(unittest.TestCase): # find all published blog posts before 2010-01-07 published_posts = BlogPost.published() published_posts = published_posts.filter( - published_date__lt=datetime(2010, 1, 7, 0, 0 ,0)) + published_date__lt=datetime(2010, 1, 7, 0, 0, 0)) self.assertEqual(published_posts.count(), 2) - blog_posts = BlogPost.objects blog_posts = blog_posts.filter(blog__in=[blog_1, blog_2]) blog_posts = blog_posts.filter(blog=blog_3) @@ -935,7 +952,7 @@ class QuerySetTest(unittest.TestCase): def test_raw_and_merging(self): class Doc(Document): - pass + meta = {'allow_inheritance': False} raw_query = Doc.objects(__raw__={'deleted': False, 'scraped': 'yes', @@ -943,7 +960,7 @@ class QuerySetTest(unittest.TestCase): {'attachments.views.extracted':'no'}] })._query - expected = {'deleted': False, '_types': 'Doc', 'scraped': 'yes', + expected = {'deleted': False, 'scraped': 'yes', '$nor': [{'views.extracted': 'no'}, {'attachments.views.extracted': 'no'}]} self.assertEqual(expected, raw_query) @@ -2598,68 +2615,6 @@ class QuerySetTest(unittest.TestCase): Group.drop_collection() - def test_types_index(self): - """Ensure that and index is used when '_types' is being used in a - query. - """ - class BlogPost(Document): - date = DateTimeField() - meta = {'indexes': ['-date']} - - # Indexes are lazy so use list() to perform query - list(BlogPost.objects) - info = BlogPost.objects._collection.index_information() - info = [value['key'] for key, value in info.iteritems()] - self.assertTrue([('_types', 1)] in info) - self.assertTrue([('_types', 1), ('date', -1)] in info) - - def test_dont_index_types(self): - """Ensure that index_types will, when disabled, prevent _types - being added to all indices. - """ - class BloggPost(Document): - date = DateTimeField() - meta = {'index_types': False, - 'indexes': ['-date']} - - # Indexes are lazy so use list() to perform query - list(BloggPost.objects) - info = BloggPost.objects._collection.index_information() - info = [value['key'] for key, value in info.iteritems()] - self.assertTrue([('_types', 1)] not in info) - self.assertTrue([('date', -1)] in info) - - BloggPost.drop_collection() - - class BloggPost(Document): - title = StringField() - meta = {'allow_inheritance': False} - - # _types is not used on objects where allow_inheritance is False - list(BloggPost.objects) - info = BloggPost.objects._collection.index_information() - self.assertFalse([('_types', 1)] in info.values()) - - BloggPost.drop_collection() - - def test_types_index_with_pk(self): - - class Comment(EmbeddedDocument): - comment_id = IntField(required=True) - - try: - class BlogPost(Document): - comments = EmbeddedDocumentField(Comment) - meta = {'indexes': [{'fields': ['pk', 'comments.comment_id'], - 'unique': True}]} - except UnboundLocalError: - self.fail('Unbound local error at types index + pk definition') - - info = BlogPost.objects._collection.index_information() - info = [value['key'] for key, value in info.iteritems()] - index_item = [(u'_types', 1), (u'_id', 1), (u'comments.comment_id', 1)] - self.assertTrue(index_item in info) - def test_dict_with_custom_baseclass(self): """Ensure DictField working with custom base clases. """ @@ -3116,6 +3071,7 @@ class QuerySetTest(unittest.TestCase): """ class Comment(Document): message = StringField() + meta = {'allow_inheritance': True} Comment.objects.ensure_index('message') @@ -3124,7 +3080,7 @@ class QuerySetTest(unittest.TestCase): value.get('unique', False), value.get('sparse', False)) for key, value in info.iteritems()] - self.assertTrue(([('_types', 1), ('message', 1)], False, False) in info) + self.assertTrue(([('_cls', 1), ('message', 1)], False, False) in info) def test_where(self): """Ensure that where clauses work. From 59826c8cfd42a13bb0e4cf9f9e0f62ab1d27a543 Mon Sep 17 00:00:00 2001 From: Marcelo Anton Date: Thu, 18 Oct 2012 11:44:18 -0300 Subject: [PATCH 008/189] This change in how the variable is declared DESCRIPTION corrects problems when running the command ``python setup.py bdist_rpm`` --- setup.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/setup.py b/setup.py index 6d9b51bc..35ca5793 100644 --- a/setup.py +++ b/setup.py @@ -8,8 +8,8 @@ try: except ImportError: pass -DESCRIPTION = """MongoEngine is a Python Object-Document -Mapper for working with MongoDB.""" +DESCRIPTION = 'MongoEngine is a Python Object-Document ' + \ +'Mapper for working with MongoDB.' LONG_DESCRIPTION = None try: LONG_DESCRIPTION = open('README.rst').read() From 3d5b6ae332291821d60d09e761f5e5b02529a404 Mon Sep 17 00:00:00 2001 From: Ross Lawley Date: Wed, 17 Oct 2012 11:36:18 +0000 Subject: [PATCH 009/189] Inheritance is off by default (MongoEngine/mongoengine#122) --- docs/changelog.rst | 1 + docs/guide/defining-documents.rst | 30 +++++++------- docs/tutorial.rst | 5 ++- docs/upgrade.rst | 24 ++++++++++-- mongoengine/base/common.py | 2 +- mongoengine/base/document.py | 42 +++++++++++++------- mongoengine/base/fields.py | 31 ++++++++++----- mongoengine/base/metaclasses.py | 23 ++++++----- mongoengine/dereference.py | 5 ++- mongoengine/document.py | 19 ++++++--- mongoengine/fields.py | 51 ++++++++++++++---------- mongoengine/queryset/queryset.py | 2 +- tests/all_warnings/__init__.py | 18 +-------- tests/document/delta.py | 29 +++++++------- tests/document/dynamic.py | 12 ++++-- tests/document/inheritance.py | 2 - tests/document/instance.py | 65 +++++++++++++++++-------------- tests/test_dereference.py | 12 ++---- tests/test_fields.py | 32 +++++++++------ tests/test_queryset.py | 17 ++++---- 20 files changed, 245 insertions(+), 177 deletions(-) diff --git a/docs/changelog.rst b/docs/changelog.rst index 8388b05a..1970bf02 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -4,6 +4,7 @@ Changelog Changes in 0.8 ============== +- Inheritance is off by default (MongoEngine/mongoengine#122) - Remove _types and just use _cls for inheritance (MongoEngine/mongoengine#148) diff --git a/docs/guide/defining-documents.rst b/docs/guide/defining-documents.rst index cf3b5a6f..ea8e05b2 100644 --- a/docs/guide/defining-documents.rst +++ b/docs/guide/defining-documents.rst @@ -462,9 +462,10 @@ If a dictionary is passed then the following options are available: The fields to index. Specified in the same format as described above. :attr:`cls` (Default: True) - If you have polymorphic models that inherit and have `allow_inheritance` - turned on, you can configure whether the index should have the - :attr:`_cls` field added automatically to the start of the index. + If you have polymorphic models that inherit and have + :attr:`allow_inheritance` turned on, you can configure whether the index + should have the :attr:`_cls` field added automatically to the start of the + index. :attr:`sparse` (Default: False) Whether the index should be sparse. @@ -573,7 +574,9 @@ defined, you may subclass it and add any extra fields or methods you may need. As this is new class is not a direct subclass of :class:`~mongoengine.Document`, it will not be stored in its own collection; it will use the same collection as its superclass uses. This allows for more -convenient and efficient retrieval of related documents:: +convenient and efficient retrieval of related documents - all you need do is +set :attr:`allow_inheritance` to True in the :attr:`meta` data for a +document.:: # Stored in a collection named 'page' class Page(Document): @@ -585,25 +588,20 @@ convenient and efficient retrieval of related documents:: class DatedPage(Page): date = DateTimeField() -.. note:: From 0.7 onwards you must declare `allow_inheritance` in the document meta. +.. note:: From 0.8 onwards you must declare :attr:`allow_inheritance` defaults + to False, meaning you must set it to True to use inheritance. Working with existing data -------------------------- -To enable correct retrieval of documents involved in this kind of heirarchy, -an extra attribute is stored on each document in the database: :attr:`_cls`. -These are hidden from the user through the MongoEngine interface, but may not -be present if you are trying to use MongoEngine with an existing database. - -For this reason, you may disable this inheritance mechansim, removing the -dependency of :attr:`_cls`, enabling you to work with existing databases. -To disable inheritance on a document class, set :attr:`allow_inheritance` to -``False`` in the :attr:`meta` dictionary:: +As MongoEngine no longer defaults to needing :attr:`_cls` you can quickly and +easily get working with existing data. Just define the document to match +the expected schema in your database. If you have wildly varying schemas then +a :class:`~mongoengine.DynamicDocument` might be more appropriate. # Will work with data in an existing collection named 'cmsPage' class Page(Document): title = StringField(max_length=200, required=True) meta = { - 'collection': 'cmsPage', - 'allow_inheritance': False, + 'collection': 'cmsPage' } diff --git a/docs/tutorial.rst b/docs/tutorial.rst index a5284c8f..c2fb5b91 100644 --- a/docs/tutorial.rst +++ b/docs/tutorial.rst @@ -84,12 +84,15 @@ using* the new fields we need to support video posts. This fits with the Object-Oriented principle of *inheritance* nicely. We can think of :class:`Post` as a base class, and :class:`TextPost`, :class:`ImagePost` and :class:`LinkPost` as subclasses of :class:`Post`. In fact, MongoEngine supports -this kind of modelling out of the box:: +this kind of modelling out of the box - all you need do is turn on inheritance +by setting :attr:`allow_inheritance` to True in the :attr:`meta`:: class Post(Document): title = StringField(max_length=120, required=True) author = ReferenceField(User) + meta = {'allow_inheritance': True} + class TextPost(Post): content = StringField() diff --git a/docs/upgrade.rst b/docs/upgrade.rst index 99e3078c..bf0a8421 100644 --- a/docs/upgrade.rst +++ b/docs/upgrade.rst @@ -8,10 +8,13 @@ Upgrading Inheritance ----------- +Data Model +~~~~~~~~~~ + The inheritance model has changed, we no longer need to store an array of -`types` with the model we can just use the classname in `_cls`. This means -that you will have to update your indexes for each of your inherited classes -like so: +:attr:`types` with the model we can just use the classname in :attr:`_cls`. +This means that you will have to update your indexes for each of your +inherited classes like so: # 1. Declaration of the class class Animal(Document): @@ -40,6 +43,19 @@ like so: Animal.objects._ensure_indexes() +Document Definition +~~~~~~~~~~~~~~~~~~~ + +The default for inheritance has changed - its now off by default and +:attr:`_cls` will not be stored automatically with the class. So if you extend +your :class:`~mongoengine.Document` or :class:`~mongoengine.EmbeddedDocuments` +you will need to declare :attr:`allow_inheritance` in the meta data like so: + + class Animal(Document): + name = StringField() + + meta = {'allow_inheritance': True} + 0.6 to 0.7 ========== @@ -123,7 +139,7 @@ Document.objects.with_id - now raises an InvalidQueryError if used with a filter. FutureWarning - A future warning has been added to all inherited classes that -don't define `allow_inheritance` in their meta. +don't define :attr:`allow_inheritance` in their meta. You may need to update pyMongo to 2.0 for use with Sharding. diff --git a/mongoengine/base/common.py b/mongoengine/base/common.py index 648561be..82728d1e 100644 --- a/mongoengine/base/common.py +++ b/mongoengine/base/common.py @@ -2,7 +2,7 @@ from mongoengine.errors import NotRegistered __all__ = ('ALLOW_INHERITANCE', 'get_document', '_document_registry') -ALLOW_INHERITANCE = True +ALLOW_INHERITANCE = False _document_registry = {} diff --git a/mongoengine/base/document.py b/mongoengine/base/document.py index af97e1f2..bc509af2 100644 --- a/mongoengine/base/document.py +++ b/mongoengine/base/document.py @@ -50,7 +50,6 @@ class BaseDocument(object): for key, value in values.iteritems(): key = self._reverse_db_field_map.get(key, key) setattr(self, key, value) - # Set any get_fieldname_display methods self.__set_field_display() @@ -83,6 +82,11 @@ class BaseDocument(object): if hasattr(self, '_changed_fields'): self._mark_as_changed(name) + # Check if the user has created a new instance of a class + if (self._is_document and self._initialised + and self._created and name == self._meta['id_field']): + super(BaseDocument, self).__setattr__('_created', False) + if (self._is_document and not self._created and name in self._meta.get('shard_key', tuple()) and self._data.get(name) != value): @@ -171,14 +175,24 @@ class BaseDocument(object): """Return data dictionary ready for use with MongoDB. """ data = {} - for field_name, field in self._fields.items(): - value = getattr(self, field_name, None) + for field_name, field in self._fields.iteritems(): + value = self._data.get(field_name, None) if value is not None: - data[field.db_field] = field.to_mongo(value) - # Only add _cls if allow_inheritance is not False - if not (hasattr(self, '_meta') and - self._meta.get('allow_inheritance', ALLOW_INHERITANCE) == False): + value = field.to_mongo(value) + + # Handle self generating fields + if value is None and field._auto_gen: + value = field.generate() + self._data[field_name] = value + + if value is not None: + data[field.db_field] = value + + # Only add _cls if allow_inheritance is True + if (hasattr(self, '_meta') and + self._meta.get('allow_inheritance', ALLOW_INHERITANCE) == True): data['_cls'] = self._class_name + if '_id' in data and data['_id'] is None: del data['_id'] @@ -194,7 +208,7 @@ class BaseDocument(object): are present. """ # Get a list of tuples of field names and their current values - fields = [(field, getattr(self, name)) + fields = [(field, self._data.get(name)) for name, field in self._fields.items()] # Ensure that each field is matched to a valid value @@ -207,7 +221,7 @@ class BaseDocument(object): errors[field.name] = error.errors or error except (ValueError, AttributeError, AssertionError), error: errors[field.name] = error - elif field.required: + elif field.required and not getattr(field, '_auto_gen', False): errors[field.name] = ValidationError('Field is required', field_name=field.name) if errors: @@ -313,6 +327,7 @@ class BaseDocument(object): """ # Handles cases where not loaded from_son but has _id doc = self.to_mongo() + set_fields = self._get_changed_fields() set_data = {} unset_data = {} @@ -370,7 +385,6 @@ class BaseDocument(object): if hasattr(d, '_fields'): field_name = d._reverse_db_field_map.get(db_field_name, db_field_name) - if field_name in d._fields: default = d._fields.get(field_name).default else: @@ -379,6 +393,7 @@ class BaseDocument(object): if default is not None: if callable(default): default = default() + if default != value: continue @@ -399,15 +414,12 @@ class BaseDocument(object): # get the class name from the document, falling back to the given # class if unavailable class_name = son.get('_cls', cls._class_name) - data = dict(("%s" % key, value) for key, value in son.items()) + data = dict(("%s" % key, value) for key, value in son.iteritems()) if not UNICODE_KWARGS: # python 2.6.4 and lower cannot handle unicode keys # passed to class constructor example: cls(**data) to_str_keys_recursive(data) - if '_cls' in data: - del data['_cls'] - # Return correct subclass for document type if class_name != cls._class_name: cls = get_document(class_name) @@ -415,7 +427,7 @@ class BaseDocument(object): changed_fields = [] errors_dict = {} - for field_name, field in cls._fields.items(): + for field_name, field in cls._fields.iteritems(): if field.db_field in data: value = data[field.db_field] try: diff --git a/mongoengine/base/fields.py b/mongoengine/base/fields.py index 44f5e131..00e040ca 100644 --- a/mongoengine/base/fields.py +++ b/mongoengine/base/fields.py @@ -21,6 +21,7 @@ class BaseField(object): name = None _geo_index = False + _auto_gen = False # Call `generate` to generate a value # These track each time a Field instance is created. Used to retain order. # The auto_creation_counter is used for fields that MongoEngine implicitly @@ -36,7 +37,6 @@ class BaseField(object): if name: msg = "Fields' 'name' attribute deprecated in favour of 'db_field'" warnings.warn(msg, DeprecationWarning) - self.name = None self.required = required or primary_key self.default = default self.unique = bool(unique or unique_with) @@ -62,7 +62,6 @@ class BaseField(object): if instance is None: # Document class being used rather than a document object return self - # Get value from document instance if available, if not use default value = instance._data.get(self.name) @@ -241,12 +240,21 @@ class ComplexBaseField(BaseField): """Convert a Python type to a MongoDB-compatible type. """ Document = _import_class("Document") + EmbeddedDocument = _import_class("EmbeddedDocument") + GenericReferenceField = _import_class("GenericReferenceField") if isinstance(value, basestring): return value if hasattr(value, 'to_mongo'): - return value.to_mongo() + if isinstance(value, Document): + return GenericReferenceField().to_mongo(value) + cls = value.__class__ + val = value.to_mongo() + # If we its a document thats not inherited add _cls + if (isinstance(value, EmbeddedDocument)): + val['_cls'] = cls.__name__ + return val is_list = False if not hasattr(value, 'items'): @@ -258,10 +266,10 @@ class ComplexBaseField(BaseField): if self.field: value_dict = dict([(key, self.field.to_mongo(item)) - for key, item in value.items()]) + for key, item in value.iteritems()]) else: value_dict = {} - for k, v in value.items(): + for k, v in value.iteritems(): if isinstance(v, Document): # We need the id from the saved object to create the DBRef if v.pk is None: @@ -274,16 +282,19 @@ class ComplexBaseField(BaseField): meta = getattr(v, '_meta', {}) allow_inheritance = ( meta.get('allow_inheritance', ALLOW_INHERITANCE) - == False) - if allow_inheritance and not self.field: - GenericReferenceField = _import_class( - "GenericReferenceField") + == True) + if not allow_inheritance and not self.field: value_dict[k] = GenericReferenceField().to_mongo(v) else: collection = v._get_collection_name() value_dict[k] = DBRef(collection, v.pk) elif hasattr(v, 'to_mongo'): - value_dict[k] = v.to_mongo() + cls = v.__class__ + val = v.to_mongo() + # If we its a document thats not inherited add _cls + if (isinstance(v, (Document, EmbeddedDocument))): + val['_cls'] = cls.__name__ + value_dict[k] = val else: value_dict[k] = self.to_mongo(v) diff --git a/mongoengine/base/metaclasses.py b/mongoengine/base/metaclasses.py index f87b03e4..e68ec13d 100644 --- a/mongoengine/base/metaclasses.py +++ b/mongoengine/base/metaclasses.py @@ -34,6 +34,17 @@ class DocumentMetaclass(type): if 'meta' in attrs: attrs['_meta'] = attrs.pop('meta') + # EmbeddedDocuments should inherit meta data + if '_meta' not in attrs: + meta = MetaDict() + for base in flattened_bases[::-1]: + # Add any mixin metadata from plain objects + if hasattr(base, 'meta'): + meta.merge(base.meta) + elif hasattr(base, '_meta'): + meta.merge(base._meta) + attrs['_meta'] = meta + # Handle document Fields # Merge all fields from subclasses @@ -52,6 +63,7 @@ class DocumentMetaclass(type): if not attr_value.db_field: attr_value.db_field = attr_name base_fields[attr_name] = attr_value + doc_fields.update(base_fields) # Discover any document fields @@ -98,15 +110,7 @@ class DocumentMetaclass(type): # inheritance of classes where inheritance is set to False allow_inheritance = base._meta.get('allow_inheritance', ALLOW_INHERITANCE) - if (not getattr(base, '_is_base_cls', True) - and allow_inheritance is None): - warnings.warn( - "%s uses inheritance, the default for " - "allow_inheritance is changing to off by default. " - "Please add it to the document meta." % name, - FutureWarning - ) - elif (allow_inheritance == False and + if (allow_inheritance != True and not base._meta.get('abstract')): raise ValueError('Document %s may not be subclassed' % base.__name__) @@ -353,6 +357,7 @@ class TopLevelDocumentMetaclass(DocumentMetaclass): if not new_class._meta.get('id_field'): new_class._meta['id_field'] = 'id' new_class._fields['id'] = ObjectIdField(db_field='_id') + new_class._fields['id'].name = 'id' new_class.id = new_class._fields['id'] # Merge in exceptions with parent hierarchy diff --git a/mongoengine/dereference.py b/mongoengine/dereference.py index 59cc0a58..25d46b46 100644 --- a/mongoengine/dereference.py +++ b/mongoengine/dereference.py @@ -121,7 +121,10 @@ class DeReference(object): for key, doc in references.iteritems(): object_map[key] = doc else: # Generic reference: use the refs data to convert to document - if doc_type and not isinstance(doc_type, (ListField, DictField, MapField,) ): + if isinstance(doc_type, (ListField, DictField, MapField,)): + continue + + if doc_type: references = doc_type._get_db()[col].find({'_id': {'$in': refs}}) for ref in references: doc = doc_type._from_son(ref) diff --git a/mongoengine/document.py b/mongoengine/document.py index b1ce13ad..95dd6246 100644 --- a/mongoengine/document.py +++ b/mongoengine/document.py @@ -117,6 +117,7 @@ class Document(BaseDocument): """ def fget(self): return getattr(self, self._meta['id_field']) + def fset(self, value): return setattr(self, self._meta['id_field'], value) return property(fget, fset) @@ -125,7 +126,7 @@ class Document(BaseDocument): @classmethod def _get_db(cls): """Some Model using other db_alias""" - return get_db(cls._meta.get("db_alias", DEFAULT_CONNECTION_NAME )) + return get_db(cls._meta.get("db_alias", DEFAULT_CONNECTION_NAME)) @classmethod def _get_collection(cls): @@ -212,11 +213,11 @@ class Document(BaseDocument): doc = self.to_mongo() - created = force_insert or '_id' not in doc + find_delta = ('_id' not in doc or self._created or force_insert) try: collection = self.__class__.objects._collection - if created: + if find_delta: if force_insert: object_id = collection.insert(doc, safe=safe, **write_options) @@ -271,7 +272,8 @@ class Document(BaseDocument): self._changed_fields = [] self._created = False - signals.post_save.send(self.__class__, document=self, created=created) + signals.post_save.send(self.__class__, document=self, + created=find_delta) return self def cascade_save(self, warn_cascade=None, *args, **kwargs): @@ -373,6 +375,7 @@ class Document(BaseDocument): for name in self._dynamic_fields.keys(): setattr(self, name, self._reload(name, obj._data[name])) self._changed_fields = obj._changed_fields + self._created = False return obj def _reload(self, key, value): @@ -464,7 +467,13 @@ class DynamicEmbeddedDocument(EmbeddedDocument): """Deletes the attribute by setting to None and allowing _delta to unset it""" field_name = args[0] - setattr(self, field_name, None) + if field_name in self._fields: + default = self._fields[field_name].default + if callable(default): + default = default() + setattr(self, field_name, default) + else: + setattr(self, field_name, None) class MapReduceDocument(object): diff --git a/mongoengine/fields.py b/mongoengine/fields.py index 9bcba9f1..15e1626f 100644 --- a/mongoengine/fields.py +++ b/mongoengine/fields.py @@ -16,12 +16,11 @@ from mongoengine.errors import ValidationError from mongoengine.python_support import (PY3, bin_type, txt_type, str_types, StringIO) from base import (BaseField, ComplexBaseField, ObjectIdField, - get_document, BaseDocument) + get_document, BaseDocument, ALLOW_INHERITANCE) from queryset import DO_NOTHING, QuerySet from document import Document, EmbeddedDocument from connection import get_db, DEFAULT_CONNECTION_NAME - try: from PIL import Image, ImageOps except ImportError: @@ -314,16 +313,16 @@ class DateTimeField(BaseField): usecs = 0 kwargs = {'microsecond': usecs} try: # Seconds are optional, so try converting seconds first. - return datetime.datetime(*time.strptime(value, '%Y-%m-%d %H:%M:%S')[:6], - **kwargs) + return datetime.datetime(*time.strptime(value, + '%Y-%m-%d %H:%M:%S')[:6], **kwargs) except ValueError: try: # Try without seconds. - return datetime.datetime(*time.strptime(value, '%Y-%m-%d %H:%M')[:5], - **kwargs) + return datetime.datetime(*time.strptime(value, + '%Y-%m-%d %H:%M')[:5], **kwargs) except ValueError: # Try without hour/minutes/seconds. try: - return datetime.datetime(*time.strptime(value, '%Y-%m-%d')[:3], - **kwargs) + return datetime.datetime(*time.strptime(value, + '%Y-%m-%d')[:3], **kwargs) except ValueError: return None @@ -410,6 +409,7 @@ class ComplexDateTimeField(StringField): return super(ComplexDateTimeField, self).__set__(instance, value) def validate(self, value): + value = self.to_python(value) if not isinstance(value, datetime.datetime): self.error('Only datetime objects may used in a ' 'ComplexDateTimeField') @@ -422,6 +422,7 @@ class ComplexDateTimeField(StringField): return original_value def to_mongo(self, value): + value = self.to_python(value) return self._convert_from_datetime(value) def prepare_query_value(self, op, value): @@ -529,7 +530,12 @@ class DynamicField(BaseField): return value if hasattr(value, 'to_mongo'): - return value.to_mongo() + cls = value.__class__ + val = value.to_mongo() + # If we its a document thats not inherited add _cls + if (isinstance(value, (Document, EmbeddedDocument))): + val['_cls'] = cls.__name__ + return val if not isinstance(value, (dict, list, tuple)): return value @@ -540,13 +546,12 @@ class DynamicField(BaseField): value = dict([(k, v) for k, v in enumerate(value)]) data = {} - for k, v in value.items(): + for k, v in value.iteritems(): data[k] = self.to_mongo(v) + value = data if is_list: # Convert back to a list - value = [v for k, v in sorted(data.items(), key=itemgetter(0))] - else: - value = data + value = [v for k, v in sorted(data.iteritems(), key=itemgetter(0))] return value def lookup_member(self, member_name): @@ -666,7 +671,6 @@ class DictField(ComplexBaseField): if op in match_operators and isinstance(value, basestring): return StringField().prepare_query_value(op, value) - return super(DictField, self).prepare_query_value(op, value) @@ -1323,7 +1327,8 @@ class GeoPointField(BaseField): class SequenceField(IntField): - """Provides a sequental counter (see http://www.mongodb.org/display/DOCS/Object+IDs#ObjectIDs-SequenceNumbers) + """Provides a sequental counter see: + http://www.mongodb.org/display/DOCS/Object+IDs#ObjectIDs-SequenceNumbers .. note:: @@ -1335,17 +1340,21 @@ class SequenceField(IntField): .. versionadded:: 0.5 """ - def __init__(self, collection_name=None, db_alias = None, sequence_name = None, *args, **kwargs): + _auto_gen = True + + def __init__(self, collection_name=None, db_alias=None, + sequence_name=None, *args, **kwargs): self.collection_name = collection_name or 'mongoengine.counters' self.db_alias = db_alias or DEFAULT_CONNECTION_NAME self.sequence_name = sequence_name return super(SequenceField, self).__init__(*args, **kwargs) - def generate_new_value(self): + def generate(self): """ Generate and Increment the counter """ - sequence_name = self.sequence_name or self.owner_document._get_collection_name() + sequence_name = (self.sequence_name or + self.owner_document._get_collection_name()) sequence_id = "%s.%s" % (sequence_name, self.name) collection = get_db(alias=self.db_alias)[self.collection_name] counter = collection.find_and_modify(query={"_id": sequence_id}, @@ -1365,7 +1374,7 @@ class SequenceField(IntField): value = instance._data.get(self.name) if not value and instance._initialised: - value = self.generate_new_value() + value = self.generate() instance._data[self.name] = value instance._mark_as_changed(self.name) @@ -1374,13 +1383,13 @@ class SequenceField(IntField): def __set__(self, instance, value): if value is None and instance._initialised: - value = self.generate_new_value() + value = self.generate() return super(SequenceField, self).__set__(instance, value) def to_python(self, value): if value is None: - value = self.generate_new_value() + value = self.generate() return value diff --git a/mongoengine/queryset/queryset.py b/mongoengine/queryset/queryset.py index 51080663..dd7200b2 100644 --- a/mongoengine/queryset/queryset.py +++ b/mongoengine/queryset/queryset.py @@ -58,7 +58,7 @@ class QuerySet(object): # If inheritance is allowed, only return instances and instances of # subclasses of the class being used - if document._meta.get('allow_inheritance') != False: + if document._meta.get('allow_inheritance') == True: self._initial_query = {"_cls": {"$in": self._document._subclasses}} self._loaded_fields = QueryFieldList(always_include=['_cls']) self._cursor_obj = None diff --git a/tests/all_warnings/__init__.py b/tests/all_warnings/__init__.py index 72de8222..4609c5a5 100644 --- a/tests/all_warnings/__init__.py +++ b/tests/all_warnings/__init__.py @@ -29,22 +29,6 @@ class AllWarnings(unittest.TestCase): # restore default handling of warnings warnings.showwarning = self.showwarning_default - def test_allow_inheritance_future_warning(self): - """Add FutureWarning for future allow_inhertiance default change. - """ - - class SimpleBase(Document): - a = IntField() - - class InheritedClass(SimpleBase): - b = IntField() - - InheritedClass() - self.assertEqual(len(self.warning_list), 1) - warning = self.warning_list[0] - self.assertEqual(FutureWarning, warning["category"]) - self.assertTrue("InheritedClass" in str(warning["message"])) - def test_dbref_reference_field_future_warning(self): class Person(Document): @@ -93,7 +77,7 @@ class AllWarnings(unittest.TestCase): def test_document_collection_syntax_warning(self): class NonAbstractBase(Document): - pass + meta = {'allow_inheritance': True} class InheritedDocumentFailTest(NonAbstractBase): meta = {'collection': 'fail'} diff --git a/tests/document/delta.py b/tests/document/delta.py index f8a071d6..c6191d9b 100644 --- a/tests/document/delta.py +++ b/tests/document/delta.py @@ -1,4 +1,6 @@ # -*- coding: utf-8 -*- +import sys +sys.path[0:0] = [""] import unittest from mongoengine import * @@ -126,9 +128,6 @@ class DeltaTest(unittest.TestCase): 'list_field': ['1', 2, {'hello': 'world'}] } self.assertEqual(doc.embedded_field._delta(), (embedded_delta, {})) - embedded_delta.update({ - '_cls': 'Embedded', - }) self.assertEqual(doc._delta(), ({'embedded_field': embedded_delta}, {})) @@ -162,6 +161,7 @@ class DeltaTest(unittest.TestCase): doc.embedded_field.list_field = ['1', 2, embedded_2] self.assertEqual(doc._get_changed_fields(), ['embedded_field.list_field']) + self.assertEqual(doc.embedded_field._delta(), ({ 'list_field': ['1', 2, { '_cls': 'Embedded', @@ -175,10 +175,10 @@ class DeltaTest(unittest.TestCase): self.assertEqual(doc._delta(), ({ 'embedded_field.list_field': ['1', 2, { '_cls': 'Embedded', - 'string_field': 'hello', - 'dict_field': {'hello': 'world'}, - 'int_field': 1, - 'list_field': ['1', 2, {'hello': 'world'}], + 'string_field': 'hello', + 'dict_field': {'hello': 'world'}, + 'int_field': 1, + 'list_field': ['1', 2, {'hello': 'world'}], }] }, {})) doc.save() @@ -467,9 +467,6 @@ class DeltaTest(unittest.TestCase): 'db_list_field': ['1', 2, {'hello': 'world'}] } self.assertEqual(doc.embedded_field._delta(), (embedded_delta, {})) - embedded_delta.update({ - '_cls': 'Embedded', - }) self.assertEqual(doc._delta(), ({'db_embedded_field': embedded_delta}, {})) @@ -520,10 +517,10 @@ class DeltaTest(unittest.TestCase): self.assertEqual(doc._delta(), ({ 'db_embedded_field.db_list_field': ['1', 2, { '_cls': 'Embedded', - 'db_string_field': 'hello', - 'db_dict_field': {'hello': 'world'}, - 'db_int_field': 1, - 'db_list_field': ['1', 2, {'hello': 'world'}], + 'db_string_field': 'hello', + 'db_dict_field': {'hello': 'world'}, + 'db_int_field': 1, + 'db_list_field': ['1', 2, {'hello': 'world'}], }] }, {})) doc.save() @@ -686,3 +683,7 @@ class DeltaTest(unittest.TestCase): doc.list_field = [] self.assertEqual(doc._get_changed_fields(), ['list_field']) self.assertEqual(doc._delta(), ({}, {'list_field': 1})) + + +if __name__ == '__main__': + unittest.main() diff --git a/tests/document/dynamic.py b/tests/document/dynamic.py index ef279179..d879b54c 100644 --- a/tests/document/dynamic.py +++ b/tests/document/dynamic.py @@ -1,4 +1,7 @@ import unittest +import sys + +sys.path[0:0] = [""] from mongoengine import * from mongoengine.connection import get_db @@ -161,7 +164,7 @@ class DynamicTest(unittest.TestCase): embedded_1.list_field = ['1', 2, {'hello': 'world'}] doc.embedded_field = embedded_1 - self.assertEqual(doc.to_mongo(), {"_cls": "Doc", + self.assertEqual(doc.to_mongo(), { "embedded_field": { "_cls": "Embedded", "string_field": "hello", @@ -205,7 +208,7 @@ class DynamicTest(unittest.TestCase): embedded_1.list_field = ['1', 2, embedded_2] doc.embedded_field = embedded_1 - self.assertEqual(doc.to_mongo(), {"_cls": "Doc", + self.assertEqual(doc.to_mongo(), { "embedded_field": { "_cls": "Embedded", "string_field": "hello", @@ -246,7 +249,6 @@ class DynamicTest(unittest.TestCase): class Person(DynamicDocument): name = StringField() - meta = {'allow_inheritance': True} Person.drop_collection() @@ -268,3 +270,7 @@ class DynamicTest(unittest.TestCase): person.age = 35 person.save() self.assertEqual(Person.objects.first().age, 35) + + +if __name__ == '__main__': + unittest.main() diff --git a/tests/document/inheritance.py b/tests/document/inheritance.py index d269ac0e..08e29048 100644 --- a/tests/document/inheritance.py +++ b/tests/document/inheritance.py @@ -203,7 +203,6 @@ class InheritanceTest(unittest.TestCase): class Animal(Document): name = StringField() - meta = {'allow_inheritance': False} def create_dog_class(): class Dog(Animal): @@ -258,7 +257,6 @@ class InheritanceTest(unittest.TestCase): class Comment(EmbeddedDocument): content = StringField() - meta = {'allow_inheritance': False} def create_special_comment(): class SpecialComment(Comment): diff --git a/tests/document/instance.py b/tests/document/instance.py index 95f37d9e..fcc43bad 100644 --- a/tests/document/instance.py +++ b/tests/document/instance.py @@ -1,24 +1,22 @@ # -*- coding: utf-8 -*- from __future__ import with_statement +import sys +sys.path[0:0] = [""] + import bson import os import pickle -import pymongo -import sys import unittest import uuid -import warnings -from nose.plugins.skip import SkipTest from datetime import datetime - -from tests.fixtures import Base, Mixin, PickleEmbedded, PickleTest +from tests.fixtures import PickleEmbedded, PickleTest from mongoengine import * from mongoengine.errors import (NotRegistered, InvalidDocumentError, InvalidQueryError) from mongoengine.queryset import NULLIFY, Q -from mongoengine.connection import get_db, get_connection +from mongoengine.connection import get_db TEST_IMAGE_PATH = os.path.join(os.path.dirname(__file__), 'mongoengine.png') @@ -461,7 +459,7 @@ class InstanceTest(unittest.TestCase): doc.validate() keys = doc._data.keys() self.assertEqual(2, len(keys)) - self.assertTrue(None in keys) + self.assertTrue('id' in keys) self.assertTrue('e' in keys) def test_save(self): @@ -656,8 +654,8 @@ class InstanceTest(unittest.TestCase): self.assertEqual(p1.name, p.parent.name) def test_update(self): - """Ensure that an existing document is updated instead of be overwritten. - """ + """Ensure that an existing document is updated instead of be + overwritten.""" # Create person object and save it to the database person = self.Person(name='Test User', age=30) person.save() @@ -753,30 +751,33 @@ class InstanceTest(unittest.TestCase): float_field = FloatField(default=1.1) boolean_field = BooleanField(default=True) datetime_field = DateTimeField(default=datetime.now) - embedded_document_field = EmbeddedDocumentField(EmbeddedDoc, default=lambda: EmbeddedDoc()) + embedded_document_field = EmbeddedDocumentField(EmbeddedDoc, + default=lambda: EmbeddedDoc()) list_field = ListField(default=lambda: [1, 2, 3]) dict_field = DictField(default=lambda: {"hello": "world"}) objectid_field = ObjectIdField(default=bson.ObjectId) - reference_field = ReferenceField(Simple, default=lambda: Simple().save()) + reference_field = ReferenceField(Simple, default=lambda: + Simple().save()) map_field = MapField(IntField(), default=lambda: {"simple": 1}) decimal_field = DecimalField(default=1.0) complex_datetime_field = ComplexDateTimeField(default=datetime.now) url_field = URLField(default="http://mongoengine.org") dynamic_field = DynamicField(default=1) - generic_reference_field = GenericReferenceField(default=lambda: Simple().save()) - sorted_list_field = SortedListField(IntField(), default=lambda: [1, 2, 3]) + generic_reference_field = GenericReferenceField( + default=lambda: Simple().save()) + sorted_list_field = SortedListField(IntField(), + default=lambda: [1, 2, 3]) email_field = EmailField(default="ross@example.com") geo_point_field = GeoPointField(default=lambda: [1, 2]) sequence_field = SequenceField() uuid_field = UUIDField(default=uuid.uuid4) - generic_embedded_document_field = GenericEmbeddedDocumentField(default=lambda: EmbeddedDoc()) - + generic_embedded_document_field = GenericEmbeddedDocumentField( + default=lambda: EmbeddedDoc()) Simple.drop_collection() Doc.drop_collection() Doc().save() - my_doc = Doc.objects.only("string_field").first() my_doc.string_field = "string" my_doc.save() @@ -1707,9 +1708,12 @@ class InstanceTest(unittest.TestCase): peter = User.objects.create(name="Peter") # Bob - Book.objects.create(name="1", author=bob, extra={"a": bob.to_dbref(), "b": [karl.to_dbref(), susan.to_dbref()]}) - Book.objects.create(name="2", author=bob, extra={"a": bob.to_dbref(), "b": karl.to_dbref()}) - Book.objects.create(name="3", author=bob, extra={"a": bob.to_dbref(), "c": [jon.to_dbref(), peter.to_dbref()]}) + Book.objects.create(name="1", author=bob, extra={ + "a": bob.to_dbref(), "b": [karl.to_dbref(), susan.to_dbref()]}) + Book.objects.create(name="2", author=bob, extra={ + "a": bob.to_dbref(), "b": karl.to_dbref()}) + Book.objects.create(name="3", author=bob, extra={ + "a": bob.to_dbref(), "c": [jon.to_dbref(), peter.to_dbref()]}) Book.objects.create(name="4", author=bob) # Jon @@ -1717,23 +1721,26 @@ class InstanceTest(unittest.TestCase): Book.objects.create(name="6", author=peter) Book.objects.create(name="7", author=jon) Book.objects.create(name="8", author=jon) - Book.objects.create(name="9", author=jon, extra={"a": peter.to_dbref()}) + Book.objects.create(name="9", author=jon, + extra={"a": peter.to_dbref()}) # Checks - self.assertEqual(u",".join([str(b) for b in Book.objects.all()]) , "1,2,3,4,5,6,7,8,9") + self.assertEqual(",".join([str(b) for b in Book.objects.all()]), + "1,2,3,4,5,6,7,8,9") # bob related books - self.assertEqual(u",".join([str(b) for b in Book.objects.filter( + self.assertEqual(",".join([str(b) for b in Book.objects.filter( Q(extra__a=bob) | Q(author=bob) | - Q(extra__b=bob))]) , + Q(extra__b=bob))]), "1,2,3,4") # Susan & Karl related books - self.assertEqual(u",".join([str(b) for b in Book.objects.filter( + self.assertEqual(",".join([str(b) for b in Book.objects.filter( Q(extra__a__all=[karl, susan]) | - Q(author__all=[karl, susan ]) | - Q(extra__b__all=[karl.to_dbref(), susan.to_dbref()]) - ) ]) , "1") + Q(author__all=[karl, susan]) | + Q(extra__b__all=[ + karl.to_dbref(), susan.to_dbref()])) + ]), "1") # $Where self.assertEqual(u",".join([str(b) for b in Book.objects.filter( @@ -1743,7 +1750,7 @@ class InstanceTest(unittest.TestCase): return this.name == '1' || this.name == '2';}""" } - ) ]), "1,2") + )]), "1,2") class ValidatorErrorTest(unittest.TestCase): diff --git a/tests/test_dereference.py b/tests/test_dereference.py index 7b149dbd..c9631ebb 100644 --- a/tests/test_dereference.py +++ b/tests/test_dereference.py @@ -331,14 +331,10 @@ class FieldTest(unittest.TestCase): return "" % self.name Person.drop_collection() - paul = Person(name="Paul") - paul.save() - maria = Person(name="Maria") - maria.save() - julia = Person(name='Julia') - julia.save() - anna = Person(name='Anna') - anna.save() + paul = Person(name="Paul").save() + maria = Person(name="Maria").save() + julia = Person(name='Julia').save() + anna = Person(name='Anna').save() paul.other.friends = [maria, julia, anna] paul.other.name = "Paul's friends" diff --git a/tests/test_fields.py b/tests/test_fields.py index 118521fd..1c13a58c 100644 --- a/tests/test_fields.py +++ b/tests/test_fields.py @@ -727,7 +727,7 @@ class FieldTest(unittest.TestCase): """Ensure that the list fields can handle the complex types.""" class SettingBase(EmbeddedDocument): - pass + meta = {'allow_inheritance': True} class StringSetting(SettingBase): value = StringField() @@ -743,8 +743,9 @@ class FieldTest(unittest.TestCase): e.mapping.append(StringSetting(value='foo')) e.mapping.append(IntegerSetting(value=42)) e.mapping.append({'number': 1, 'string': 'Hi!', 'float': 1.001, - 'complex': IntegerSetting(value=42), 'list': - [IntegerSetting(value=42), StringSetting(value='foo')]}) + 'complex': IntegerSetting(value=42), + 'list': [IntegerSetting(value=42), + StringSetting(value='foo')]}) e.save() e2 = Simple.objects.get(id=e.id) @@ -844,7 +845,7 @@ class FieldTest(unittest.TestCase): """Ensure that the dict field can handle the complex types.""" class SettingBase(EmbeddedDocument): - pass + meta = {'allow_inheritance': True} class StringSetting(SettingBase): value = StringField() @@ -859,9 +860,11 @@ class FieldTest(unittest.TestCase): e = Simple() e.mapping['somestring'] = StringSetting(value='foo') e.mapping['someint'] = IntegerSetting(value=42) - e.mapping['nested_dict'] = {'number': 1, 'string': 'Hi!', 'float': 1.001, - 'complex': IntegerSetting(value=42), 'list': - [IntegerSetting(value=42), StringSetting(value='foo')]} + e.mapping['nested_dict'] = {'number': 1, 'string': 'Hi!', + 'float': 1.001, + 'complex': IntegerSetting(value=42), + 'list': [IntegerSetting(value=42), + StringSetting(value='foo')]} e.save() e2 = Simple.objects.get(id=e.id) @@ -915,7 +918,7 @@ class FieldTest(unittest.TestCase): """Ensure that the MapField can handle complex declared types.""" class SettingBase(EmbeddedDocument): - pass + meta = {"allow_inheritance": True} class StringSetting(SettingBase): value = StringField() @@ -951,7 +954,8 @@ class FieldTest(unittest.TestCase): number = IntField(default=0, db_field='i') class Test(Document): - my_map = MapField(field=EmbeddedDocumentField(Embedded), db_field='x') + my_map = MapField(field=EmbeddedDocumentField(Embedded), + db_field='x') Test.drop_collection() @@ -1038,6 +1042,8 @@ class FieldTest(unittest.TestCase): class User(EmbeddedDocument): name = StringField() + meta = {'allow_inheritance': True} + class PowerUser(User): power = IntField() @@ -1046,8 +1052,10 @@ class FieldTest(unittest.TestCase): author = EmbeddedDocumentField(User) post = BlogPost(content='What I did today...') - post.author = User(name='Test User') post.author = PowerUser(name='Test User', power=47) + post.save() + + self.assertEqual(47, BlogPost.objects.first().author.power) def test_reference_validation(self): """Ensure that invalid docment objects cannot be assigned to reference @@ -2117,12 +2125,12 @@ class FieldTest(unittest.TestCase): def test_sequence_fields_reload(self): class Animal(Document): counter = SequenceField() - type = StringField() + name = StringField() self.db['mongoengine.counters'].drop() Animal.drop_collection() - a = Animal(type="Boi") + a = Animal(name="Boi") a.save() self.assertEqual(a.counter, 1) diff --git a/tests/test_queryset.py b/tests/test_queryset.py index cdabadb4..e9e78b4f 100644 --- a/tests/test_queryset.py +++ b/tests/test_queryset.py @@ -647,7 +647,8 @@ class QuerySetTest(unittest.TestCase): self.assertRaises(NotUniqueError, throw_operation_error_not_unique) self.assertEqual(Blog.objects.count(), 2) - Blog.objects.insert([blog2, blog3], write_options={'continue_on_error': True}) + Blog.objects.insert([blog2, blog3], write_options={ + 'continue_on_error': True}) self.assertEqual(Blog.objects.count(), 3) def test_get_changed_fields_query_count(self): @@ -673,7 +674,7 @@ class QuerySetTest(unittest.TestCase): r2 = Project(name="r2").save() r3 = Project(name="r3").save() p1 = Person(name="p1", projects=[r1, r2]).save() - p2 = Person(name="p2", projects=[r2]).save() + p2 = Person(name="p2", projects=[r2, r3]).save() o1 = Organization(name="o1", employees=[p1]).save() with query_counter() as q: @@ -688,24 +689,24 @@ class QuerySetTest(unittest.TestCase): self.assertEqual(q, 0) fresh_o1 = Organization.objects.get(id=o1.id) - fresh_o1.save() + fresh_o1.save() # No changes, does nothing - self.assertEqual(q, 2) + self.assertEqual(q, 1) with query_counter() as q: self.assertEqual(q, 0) fresh_o1 = Organization.objects.get(id=o1.id) - fresh_o1.save(cascade=False) + fresh_o1.save(cascade=False) # No changes, does nothing - self.assertEqual(q, 2) + self.assertEqual(q, 1) with query_counter() as q: self.assertEqual(q, 0) fresh_o1 = Organization.objects.get(id=o1.id) - fresh_o1.employees.append(p2) - fresh_o1.save(cascade=False) + fresh_o1.employees.append(p2) # Dereferences + fresh_o1.save(cascade=False) # Saves self.assertEqual(q, 3) From 7d90aa76ff7116269dea42f2c6629ea6b868b0de Mon Sep 17 00:00:00 2001 From: Ross Lawley Date: Tue, 6 Nov 2012 16:04:23 +0000 Subject: [PATCH 010/189] Add _instance to Embedded Documents Fixes MongoEngine/mongoengine#139 --- mongoengine/base/datastructures.py | 20 +++++++++++- mongoengine/base/fields.py | 4 +++ mongoengine/document.py | 2 ++ mongoengine/fields.py | 7 +++-- tests/document/instance.py | 50 ++++++++++++++++++++++++------ 5 files changed, 70 insertions(+), 13 deletions(-) diff --git a/mongoengine/base/datastructures.py b/mongoengine/base/datastructures.py index 9a7620e6..c750b5ba 100644 --- a/mongoengine/base/datastructures.py +++ b/mongoengine/base/datastructures.py @@ -1,4 +1,5 @@ import weakref +from mongoengine.common import _import_class __all__ = ("BaseDict", "BaseList") @@ -16,6 +17,14 @@ class BaseDict(dict): self._name = name return super(BaseDict, self).__init__(dict_items) + def __getitem__(self, *args, **kwargs): + value = super(BaseDict, self).__getitem__(*args, **kwargs) + + EmbeddedDocument = _import_class('EmbeddedDocument') + if isinstance(value, EmbeddedDocument) and value._instance is None: + value._instance = self._instance + return value + def __setitem__(self, *args, **kwargs): self._mark_as_changed() return super(BaseDict, self).__setitem__(*args, **kwargs) @@ -75,6 +84,14 @@ class BaseList(list): self._name = name return super(BaseList, self).__init__(list_items) + def __getitem__(self, *args, **kwargs): + value = super(BaseList, self).__getitem__(*args, **kwargs) + + EmbeddedDocument = _import_class('EmbeddedDocument') + if isinstance(value, EmbeddedDocument) and value._instance is None: + value._instance = self._instance + return value + def __setitem__(self, *args, **kwargs): self._mark_as_changed() return super(BaseList, self).__setitem__(*args, **kwargs) @@ -84,7 +101,8 @@ class BaseList(list): return super(BaseList, self).__delitem__(*args, **kwargs) def __getstate__(self): - self.observer = None + self.instance = None + self._dereferenced = False return self def __setstate__(self, state): diff --git a/mongoengine/base/fields.py b/mongoengine/base/fields.py index 00e040ca..fc1a0767 100644 --- a/mongoengine/base/fields.py +++ b/mongoengine/base/fields.py @@ -1,5 +1,6 @@ import operator import warnings +import weakref from bson import DBRef, ObjectId @@ -71,6 +72,9 @@ class BaseField(object): if callable(value): value = value() + EmbeddedDocument = _import_class('EmbeddedDocument') + if isinstance(value, EmbeddedDocument) and value._instance is None: + value._instance = weakref.proxy(instance) return value def __set__(self, instance, value): diff --git a/mongoengine/document.py b/mongoengine/document.py index 95dd6246..adbdcca2 100644 --- a/mongoengine/document.py +++ b/mongoengine/document.py @@ -40,6 +40,8 @@ class EmbeddedDocument(BaseDocument): my_metaclass = DocumentMetaclass __metaclass__ = DocumentMetaclass + _instance = None + def __init__(self, *args, **kwargs): super(EmbeddedDocument, self).__init__(*args, **kwargs) self._changed_fields = [] diff --git a/mongoengine/fields.py b/mongoengine/fields.py index 15e1626f..94e11556 100644 --- a/mongoengine/fields.py +++ b/mongoengine/fields.py @@ -625,7 +625,8 @@ class SortedListField(ListField): def to_mongo(self, value): value = super(SortedListField, self).to_mongo(value) if self._ordering is not None: - return sorted(value, key=itemgetter(self._ordering), reverse=self._order_reverse) + return sorted(value, key=itemgetter(self._ordering), + reverse=self._order_reverse) return sorted(value, reverse=self._order_reverse) @@ -655,7 +656,9 @@ class DictField(ComplexBaseField): self.error('Only dictionaries may be used in a DictField') if any(k for k in value.keys() if not isinstance(k, basestring)): - self.error('Invalid dictionary key - documents must have only string keys') + msg = ("Invalid dictionary key - documents must " + "have only string keys") + self.error(msg) if any(('.' in k or '$' in k) for k in value.keys()): self.error('Invalid dictionary key name - keys may not contain "."' ' or "$" characters') diff --git a/tests/document/instance.py b/tests/document/instance.py index fcc43bad..48ddc10d 100644 --- a/tests/document/instance.py +++ b/tests/document/instance.py @@ -183,9 +183,6 @@ class InstanceTest(unittest.TestCase): self.assertEqual(list_stats, CompareStats.objects.first().stats) - - - def test_db_field_load(self): """Ensure we load data correctly """ @@ -214,24 +211,24 @@ class InstanceTest(unittest.TestCase): class Person(Document): name = StringField(required=True) - rank_ = EmbeddedDocumentField(Rank, required=False, db_field='rank') + rank_ = EmbeddedDocumentField(Rank, + required=False, + db_field='rank') @property def rank(self): - return self.rank_.title if self.rank_ is not None else "Private" + if self.rank_ is None: + return "Private" + return self.rank_.title Person.drop_collection() Person(name="Jack", rank_=Rank(title="Corporal")).save() - Person(name="Fred").save() self.assertEqual(Person.objects.get(name="Jack").rank, "Corporal") self.assertEqual(Person.objects.get(name="Fred").rank, "Private") - - - def test_custom_id_field(self): """Ensure that documents may be created with custom primary keys. """ @@ -247,7 +244,7 @@ class InstanceTest(unittest.TestCase): self.assertEqual(User._meta['id_field'], 'username') def create_invalid_user(): - User(name='test').save() # no primary key field + User(name='test').save() # no primary key field self.assertRaises(ValidationError, create_invalid_user) def define_invalid_user(): @@ -424,6 +421,36 @@ class InstanceTest(unittest.TestCase): self.assertTrue('content' in Comment._fields) self.assertFalse('id' in Comment._fields) + def test_embedded_document_instance(self): + """Ensure that embedded documents can reference parent instance + """ + class Embedded(EmbeddedDocument): + string = StringField() + + class Doc(Document): + embedded_field = EmbeddedDocumentField(Embedded) + + Doc.drop_collection() + Doc(embedded_field=Embedded(string="Hi")).save() + + doc = Doc.objects.get() + self.assertEqual(doc, doc.embedded_field._instance) + + def test_embedded_document_complex_instance(self): + """Ensure that embedded documents in complex fields can reference + parent instance""" + class Embedded(EmbeddedDocument): + string = StringField() + + class Doc(Document): + embedded_field = ListField(EmbeddedDocumentField(Embedded)) + + Doc.drop_collection() + Doc(embedded_field=[Embedded(string="Hi")]).save() + + doc = Doc.objects.get() + self.assertEqual(doc, doc.embedded_field[0]._instance) + def test_embedded_document_validation(self): """Ensure that embedded documents may be validated. """ @@ -442,6 +469,7 @@ class InstanceTest(unittest.TestCase): comment.date = datetime.now() comment.validate() + self.assertEqual(comment._instance, None) def test_embedded_db_field_validate(self): @@ -475,11 +503,13 @@ class InstanceTest(unittest.TestCase): self.assertEqual(person_obj['age'], 30) self.assertEqual(person_obj['_id'], person.id) # Test skipping validation on save + class Recipient(Document): email = EmailField(required=True) recipient = Recipient(email='root@localhost') self.assertRaises(ValidationError, recipient.save) + try: recipient.save(validate=False) except ValidationError: From f0f1308465773545a6b3cead3abe2c1f82b2f2e8 Mon Sep 17 00:00:00 2001 From: Ross Lawley Date: Tue, 6 Nov 2012 16:06:54 +0000 Subject: [PATCH 011/189] Updated changelog --- docs/changelog.rst | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/changelog.rst b/docs/changelog.rst index ecd487fc..33d22e19 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -4,6 +4,7 @@ Changelog Changes in 0.8 ============== +- Added _instance to EmbeddedDocuments pointing to the parent (MongoEngine/mongoengine#139) - Inheritance is off by default (MongoEngine/mongoengine#122) - Remove _types and just use _cls for inheritance (MongoEngine/mongoengine#148) From f2049e9c1896eae2e34b9a25d6bdbf82fa8375e2 Mon Sep 17 00:00:00 2001 From: Samuel Clay Date: Tue, 6 Nov 2012 18:55:13 +0000 Subject: [PATCH 012/189] Adding QuerySet(read_preference=pymongo.ReadPreference.X) and QuerySet().read_preference() method to override connection-level read_preference on a per-query basis. --- mongoengine/queryset/queryset.py | 24 +++++++++++++++++++++--- 1 file changed, 21 insertions(+), 3 deletions(-) diff --git a/mongoengine/queryset/queryset.py b/mongoengine/queryset/queryset.py index dd7200b2..0437395a 100644 --- a/mongoengine/queryset/queryset.py +++ b/mongoengine/queryset/queryset.py @@ -53,6 +53,7 @@ class QuerySet(object): self._timeout = True self._class_check = True self._slave_okay = False + self._read_preference = None self._iter = False self._scalar = [] @@ -75,7 +76,8 @@ class QuerySet(object): copy_props = ('_initial_query', '_query_obj', '_where_clause', '_loaded_fields', '_ordering', '_snapshot', - '_timeout', '_limit', '_skip', '_slave_okay', '_hint') + '_timeout', '_limit', '_skip', '_slave_okay', '_hint', + '_read_preference') for prop in copy_props: val = getattr(self, prop) @@ -109,7 +111,8 @@ class QuerySet(object): self._collection.ensure_index(fields, **index_spec) return self - def __call__(self, q_obj=None, class_check=True, slave_okay=False, **query): + def __call__(self, q_obj=None, class_check=True, slave_okay=False, read_preference=None, + **query): """Filter the selected documents by calling the :class:`~mongoengine.queryset.QuerySet` with a query. @@ -121,6 +124,8 @@ class QuerySet(object): querying collection :param slave_okay: if True, allows this query to be run against a replica secondary. + :params read_preference: if set, overrides connection-level + read_preference from `ReplicaSetConnection`. :param query: Django-style query keyword arguments """ query = Q(**query) @@ -129,6 +134,8 @@ class QuerySet(object): self._query_obj &= query self._mongo_query = None self._cursor_obj = None + if read_preference is not None: + self._read_preference = read_preference self._class_check = class_check return self @@ -229,8 +236,10 @@ class QuerySet(object): cursor_args = { 'snapshot': self._snapshot, 'timeout': self._timeout, - 'slave_okay': self._slave_okay + 'slave_okay': self._slave_okay, } + if self._read_preference is not None: + cursor_args['read_preference'] = self._read_preference if self._loaded_fields: cursor_args['fields'] = self._loaded_fields.as_dict() return cursor_args @@ -802,6 +811,15 @@ class QuerySet(object): self._slave_okay = enabled return self + def read_preference(self, read_preference): + """Change the read_preference when querying. + + :param read_preference: override ReplicaSetConnection-level + preference. + """ + self._read_preference = read_preference + return self + def delete(self, safe=False): """Delete the documents matched by the query. From 7073b9d395429a753f95ecdb26decb4344784691 Mon Sep 17 00:00:00 2001 From: Ross Lawley Date: Tue, 6 Nov 2012 18:55:14 +0000 Subject: [PATCH 013/189] Added validation and tests --- docs/changelog.rst | 1 + docs/guide/connecting.rst | 6 ++ mongoengine/queryset/queryset.py | 118 ++++++++++++++++++------------- tests/test_queryset.py | 16 +++++ 4 files changed, 93 insertions(+), 48 deletions(-) diff --git a/docs/changelog.rst b/docs/changelog.rst index 33d22e19..5ea1e4f0 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -4,6 +4,7 @@ Changelog Changes in 0.8 ============== +- Added support setting for read prefrence at a query level (MongoEngine/mongoengine#157) - Added _instance to EmbeddedDocuments pointing to the parent (MongoEngine/mongoengine#139) - Inheritance is off by default (MongoEngine/mongoengine#122) - Remove _types and just use _cls for inheritance (MongoEngine/mongoengine#148) diff --git a/docs/guide/connecting.rst b/docs/guide/connecting.rst index bc45dbfe..657c46c2 100644 --- a/docs/guide/connecting.rst +++ b/docs/guide/connecting.rst @@ -33,6 +33,12 @@ MongoEngine now supports :func:`~pymongo.replica_set_connection.ReplicaSetConnec to use them please use a URI style connection and provide the `replicaSet` name in the connection kwargs. +Read preferences are supported throught the connection or via individual +queries by passing the read_preference :: + + Bar.objects().read_preference(ReadPreference.PRIMARY) + Bar.objects(read_preference=ReadPreference.PRIMARY) + Multiple Databases ================== diff --git a/mongoengine/queryset/queryset.py b/mongoengine/queryset/queryset.py index 0437395a..cf4b4f82 100644 --- a/mongoengine/queryset/queryset.py +++ b/mongoengine/queryset/queryset.py @@ -6,6 +6,7 @@ import operator import pymongo from bson.code import Code +from pymongo.common import validate_read_preference from mongoengine import signals from mongoengine.common import _import_class @@ -68,7 +69,8 @@ class QuerySet(object): self._hint = -1 # Using -1 as None is a valid value for hint def clone(self): - """Creates a copy of the current :class:`~mongoengine.queryset.QuerySet` + """Creates a copy of the current + :class:`~mongoengine.queryset.QuerySet` .. versionadded:: 0.5 """ @@ -111,8 +113,8 @@ class QuerySet(object): self._collection.ensure_index(fields, **index_spec) return self - def __call__(self, q_obj=None, class_check=True, slave_okay=False, read_preference=None, - **query): + def __call__(self, q_obj=None, class_check=True, slave_okay=False, + read_preference=None, **query): """Filter the selected documents by calling the :class:`~mongoengine.queryset.QuerySet` with a query. @@ -124,7 +126,7 @@ class QuerySet(object): querying collection :param slave_okay: if True, allows this query to be run against a replica secondary. - :params read_preference: if set, overrides connection-level + :params read_preference: if set, overrides connection-level read_preference from `ReplicaSetConnection`. :param query: Django-style query keyword arguments """ @@ -135,7 +137,7 @@ class QuerySet(object): self._mongo_query = None self._cursor_obj = None if read_preference is not None: - self._read_preference = read_preference + self.read_preference(read_preference) self._class_check = class_check return self @@ -282,39 +284,43 @@ class QuerySet(object): self.limit(2) self.__call__(*q_objs, **query) try: - result1 = self.next() + result = self.next() except StopIteration: - raise self._document.DoesNotExist("%s matching query does not exist." - % self._document._class_name) + msg = ("%s matching query does not exist." + % self._document._class_name) + raise self._document.DoesNotExist(msg) try: - result2 = self.next() + self.next() except StopIteration: - return result1 + return result self.rewind() message = u'%d items returned, instead of 1' % self.count() raise self._document.MultipleObjectsReturned(message) - def get_or_create(self, write_options=None, auto_save=True, *q_objs, **query): - """Retrieve unique object or create, if it doesn't exist. Returns a tuple of - ``(object, created)``, where ``object`` is the retrieved or created object - and ``created`` is a boolean specifying whether a new object was created. Raises + def get_or_create(self, write_options=None, auto_save=True, + *q_objs, **query): + """Retrieve unique object or create, if it doesn't exist. Returns a + tuple of ``(object, created)``, where ``object`` is the retrieved or + created object and ``created`` is a boolean specifying whether a new + object was created. Raises :class:`~mongoengine.queryset.MultipleObjectsReturned` or `DocumentName.MultipleObjectsReturned` if multiple results are found. A new document will be created if the document doesn't exists; a dictionary of default values for the new document may be provided as a keyword argument called :attr:`defaults`. - .. note:: This requires two separate operations and therefore a - race condition exists. Because there are no transactions in mongoDB - other approaches should be investigated, to ensure you don't - accidently duplicate data when using this method. + .. warning:: This requires two separate operations and therefore a + race condition exists. Because there are no transactions in + mongoDB other approaches should be investigated, to ensure you + don't accidently duplicate data when using this method. :param write_options: optional extra keyword arguments used if we have to create a new document. Passes any write_options onto :meth:`~mongoengine.Document.save` - :param auto_save: if the object is to be saved automatically if not found. + :param auto_save: if the object is to be saved automatically if + not found. .. versionchanged:: 0.6 - added `auto_save` .. versionadded:: 0.3 @@ -352,21 +358,24 @@ class QuerySet(object): result = None return result - def insert(self, doc_or_docs, load_bulk=True, safe=False, write_options=None): + def insert(self, doc_or_docs, load_bulk=True, safe=False, + write_options=None): """bulk insert documents If ``safe=True`` and the operation is unsuccessful, an :class:`~mongoengine.OperationError` will be raised. :param docs_or_doc: a document or list of documents to be inserted - :param load_bulk (optional): If True returns the list of document instances + :param load_bulk (optional): If True returns the list of document + instances :param safe: check if the operation succeeded before returning :param write_options: Extra keyword arguments are passed down to :meth:`~pymongo.collection.Collection.insert` - which will be used as options for the resultant ``getLastError`` command. - For example, ``insert(..., {w: 2, fsync: True})`` will wait until at least two - servers have recorded the write and will force an fsync on each server being - written to. + which will be used as options for the resultant + ``getLastError`` command. For example, + ``insert(..., {w: 2, fsync: True})`` will wait until at least + two servers have recorded the write and will force an fsync on + each server being written to. By default returns document instances, set ``load_bulk`` to False to return just ``ObjectIds`` @@ -388,7 +397,8 @@ class QuerySet(object): raw = [] for doc in docs: if not isinstance(doc, self._document): - msg = "Some documents inserted aren't instances of %s" % str(self._document) + msg = ("Some documents inserted aren't instances of %s" + % str(self._document)) raise OperationError(msg) if doc.pk: msg = "Some documents have ObjectIds use doc.update() instead" @@ -429,7 +439,8 @@ class QuerySet(object): .. versionchanged:: 0.6 Raises InvalidQueryError if filter has been set """ if not self._query_obj.empty: - raise InvalidQueryError("Cannot use a filter whilst using `with_id`") + msg = "Cannot use a filter whilst using `with_id`" + raise InvalidQueryError(msg) return self.filter(pk=object_id).first() def in_bulk(self, object_ids): @@ -503,9 +514,9 @@ class QuerySet(object): :param reduce_f: reduce function, as :class:`~bson.code.Code` or string :param output: output collection name, if set to 'inline' will try to - use :class:`~pymongo.collection.Collection.inline_map_reduce` - This can also be a dictionary containing output options - see: http://docs.mongodb.org/manual/reference/commands/#mapReduce + use :class:`~pymongo.collection.Collection.inline_map_reduce` + This can also be a dictionary containing output options + see: http://docs.mongodb.org/manual/reference/commands/#mapReduce :param finalize_f: finalize function, an optional function that performs any post-reduction processing. :param scope: values to insert into map/reduce global scope. Optional. @@ -568,7 +579,8 @@ class QuerySet(object): map_reduce_function = 'map_reduce' mr_args['out'] = output - results = getattr(self._collection, map_reduce_function)(map_f, reduce_f, **mr_args) + results = getattr(self._collection, map_reduce_function)( + map_f, reduce_f, **mr_args) if map_reduce_function == 'map_reduce': results = results.find() @@ -609,9 +621,9 @@ class QuerySet(object): """Added 'hint' support, telling Mongo the proper index to use for the query. - Judicious use of hints can greatly improve query performance. When doing - a query on multiple fields (at least one of which is indexed) pass the - indexed field as a hint to the query. + Judicious use of hints can greatly improve query performance. When + doing a query on multiple fields (at least one of which is indexed) + pass the indexed field as a hint to the query. Hinting will not do anything if the corresponding index does not exist. The last hint applied to this cursor takes precedence over all others. @@ -695,9 +707,9 @@ class QuerySet(object): Retrieving a Subrange of Array Elements: You can use the $slice operator to retrieve a subrange of elements in - an array :: + an array. For example to get the first 5 comments:: - post = BlogPost.objects(...).fields(slice__comments=5) // first 5 comments + post = BlogPost.objects(...).fields(slice__comments=5) :param kwargs: A dictionary identifying what to include @@ -724,9 +736,10 @@ class QuerySet(object): return self def all_fields(self): - """Include all fields. Reset all previously calls of .only() and .exclude(). :: + """Include all fields. Reset all previously calls of .only() or + .exclude(). :: - post = BlogPost.objects(...).exclude("comments").only("title").all_fields() + post = BlogPost.objects.exclude("comments").all_fields() .. versionadded:: 0.5 """ @@ -817,6 +830,7 @@ class QuerySet(object): :param read_preference: override ReplicaSetConnection-level preference. """ + validate_read_preference('read_preference', read_preference) self._read_preference = read_preference return self @@ -839,9 +853,10 @@ class QuerySet(object): for rule_entry in delete_rules: document_cls, field_name = rule_entry rule = doc._meta['delete_rules'][rule_entry] - if rule == DENY and document_cls.objects(**{field_name + '__in': self}).count() > 0: - msg = u'Could not delete document (at least %s.%s refers to it)' % \ - (document_cls.__name__, field_name) + if rule == DENY and document_cls.objects( + **{field_name + '__in': self}).count() > 0: + msg = ("Could not delete document (%s.%s refers to it)" + % (document_cls.__name__, field_name)) raise OperationError(msg) for rule_entry in delete_rules: @@ -864,13 +879,15 @@ class QuerySet(object): self._collection.remove(self._query, safe=safe) - def update(self, safe_update=True, upsert=False, multi=True, write_options=None, **update): + def update(self, safe_update=True, upsert=False, multi=True, + write_options=None, **update): """Perform an atomic update on the fields matched by the query. When ``safe_update`` is used, the number of affected documents is returned. :param safe_update: check if the operation succeeded before returning :param upsert: Any existing document with that "_id" is overwritten. - :param write_options: extra keyword arguments for :meth:`~pymongo.collection.Collection.update` + :param write_options: extra keyword arguments for + :meth:`~pymongo.collection.Collection.update` .. versionadded:: 0.2 """ @@ -895,13 +912,15 @@ class QuerySet(object): raise OperationError(message) raise OperationError(u'Update failed (%s)' % unicode(err)) - def update_one(self, safe_update=True, upsert=False, write_options=None, **update): + def update_one(self, safe_update=True, upsert=False, write_options=None, + **update): """Perform an atomic update on first field matched by the query. When ``safe_update`` is used, the number of affected documents is returned. :param safe_update: check if the operation succeeded before returning :param upsert: Any existing document with that "_id" is overwritten. - :param write_options: extra keyword arguments for :meth:`~pymongo.collection.Collection.update` + :param write_options: extra keyword arguments for + :meth:`~pymongo.collection.Collection.update` :param update: Django-style update keyword arguments .. versionadded:: 0.2 @@ -970,7 +989,8 @@ class QuerySet(object): return ".".join([f.db_field for f in fields]) code = re.sub(u'\[\s*~([A-z_][A-z_0-9.]+?)\s*\]', field_sub, code) - code = re.sub(u'\{\{\s*~([A-z_][A-z_0-9.]+?)\s*\}\}', field_path_sub, code) + code = re.sub(u'\{\{\s*~([A-z_][A-z_0-9.]+?)\s*\}\}', field_path_sub, + code) return code def exec_js(self, code, *fields, **options): @@ -1094,7 +1114,8 @@ class QuerySet(object): } """) - for result in self.map_reduce(map_func, reduce_func, finalize_f=finalize_func, output='inline'): + for result in self.map_reduce(map_func, reduce_func, + finalize_f=finalize_func, output='inline'): return result.value else: return 0 @@ -1122,7 +1143,8 @@ class QuerySet(object): document lookups """ if map_reduce: - return self._item_frequencies_map_reduce(field, normalize=normalize) + return self._item_frequencies_map_reduce(field, + normalize=normalize) return self._item_frequencies_exec_js(field, normalize=normalize) def _item_frequencies_map_reduce(self, field, normalize=False): diff --git a/tests/test_queryset.py b/tests/test_queryset.py index e9e78b4f..dcb25241 100644 --- a/tests/test_queryset.py +++ b/tests/test_queryset.py @@ -1,9 +1,13 @@ from __future__ import with_statement +import sys +sys.path[0:0] = [""] import unittest from datetime import datetime, timedelta import pymongo +from pymongo.errors import ConfigurationError +from pymongo.read_preferences import ReadPreference from bson import ObjectId @@ -3648,6 +3652,18 @@ class QueryFieldListTest(unittest.TestCase): ak = list(Bar.objects(foo__match={'shape': "square", "color": "purple"})) self.assertEqual([b1], ak) + def test_read_preference(self): + class Bar(Document): + pass + + Bar.drop_collection() + bars = list(Bar.objects(read_preference=ReadPreference.PRIMARY)) + self.assertEqual([], bars) + + self.assertRaises(ConfigurationError, Bar.objects, + read_preference='Primary') + + if __name__ == '__main__': unittest.main() From 1986e82783ba7432728ee1a21a45dae7d8971d34 Mon Sep 17 00:00:00 2001 From: Ross Lawley Date: Wed, 7 Nov 2012 12:12:28 +0000 Subject: [PATCH 014/189] Added clean method to documents for pre validation data cleaning (MongoEngine/mongoengine#60) --- docs/changelog.rst | 1 + docs/guide/document-instances.rst | 28 ++++++++++++ mongoengine/base/document.py | 46 +++++++++++++++----- mongoengine/base/fields.py | 6 +-- mongoengine/common.py | 4 +- mongoengine/document.py | 10 +++-- mongoengine/fields.py | 8 ++-- tests/document/instance.py | 72 ++++++++++++++++++++++++++++++- 8 files changed, 150 insertions(+), 25 deletions(-) diff --git a/docs/changelog.rst b/docs/changelog.rst index 5ea1e4f0..ca18d3e9 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -4,6 +4,7 @@ Changelog Changes in 0.8 ============== +- Added clean method to documents for pre validation data cleaning (MongoEngine/mongoengine#60) - Added support setting for read prefrence at a query level (MongoEngine/mongoengine#157) - Added _instance to EmbeddedDocuments pointing to the parent (MongoEngine/mongoengine#139) - Inheritance is off by default (MongoEngine/mongoengine#122) diff --git a/docs/guide/document-instances.rst b/docs/guide/document-instances.rst index 54fa804b..b3bf687b 100644 --- a/docs/guide/document-instances.rst +++ b/docs/guide/document-instances.rst @@ -38,6 +38,34 @@ already exist, then any changes will be updated atomically. For example:: .. seealso:: :ref:`guide-atomic-updates` +Pre save data validation and cleaning +------------------------------------- +MongoEngine allows you to create custom cleaning rules for your documents when +calling :meth:`~mongoengine.Document.save`. By providing a custom +:meth:`~mongoengine.Document.clean` method you can do any pre validation / data +cleaning. + +This might be useful if you want to ensure a default value based on other +document values for example:: + + class Essay(Document): + status = StringField(choices=('Published', 'Draft'), required=True) + pub_date = DateTimeField() + + def clean(self): + """Ensures that only published essays have a `pub_date` and + automatically sets the pub_date if published and not set""" + if self.status == 'Draft' and self.pub_date is not None: + msg = 'Draft entries should not have a publication date.' + raise ValidationError(msg) + # Set the pub_date for published items if not set. + if self.status == 'Published' and self.pub_date is None: + self.pub_date = datetime.now() + +.. note:: + Cleaning is only called if validation is turned on and when calling +:meth:`~mongoengine.Document.save`. + Cascading Saves --------------- If your document contains :class:`~mongoengine.ReferenceField` or diff --git a/mongoengine/base/document.py b/mongoengine/base/document.py index bc509af2..46f53205 100644 --- a/mongoengine/base/document.py +++ b/mongoengine/base/document.py @@ -15,7 +15,9 @@ from .common import get_document, ALLOW_INHERITANCE from .datastructures import BaseDict, BaseList from .fields import ComplexBaseField -__all__ = ('BaseDocument', ) +__all__ = ('BaseDocument', 'NON_FIELD_ERRORS') + +NON_FIELD_ERRORS = '__all__' class BaseDocument(object): @@ -82,11 +84,6 @@ class BaseDocument(object): if hasattr(self, '_changed_fields'): self._mark_as_changed(name) - # Check if the user has created a new instance of a class - if (self._is_document and self._initialised - and self._created and name == self._meta['id_field']): - super(BaseDocument, self).__setattr__('_created', False) - if (self._is_document and not self._created and name in self._meta.get('shard_key', tuple()) and self._data.get(name) != value): @@ -94,6 +91,11 @@ class BaseDocument(object): msg = "Shard Keys are immutable. Tried to update %s" % name raise OperationError(msg) + # Check if the user has created a new instance of a class + if (self._is_document and self._initialised + and self._created and name == self._meta['id_field']): + super(BaseDocument, self).__setattr__('_created', False) + super(BaseDocument, self).__setattr__(name, value) def __getstate__(self): @@ -171,6 +173,16 @@ class BaseDocument(object): else: return hash(self.pk) + def clean(self): + """ + Hook for doing document level data cleaning before validation is run. + + Any ValidationError raised by this method will not be associated with + a particular field; it will have a special-case association with the + field defined by NON_FIELD_ERRORS. + """ + pass + def to_mongo(self): """Return data dictionary ready for use with MongoDB. """ @@ -203,20 +215,33 @@ class BaseDocument(object): data[name] = field.to_mongo(self._data.get(name, None)) return data - def validate(self): + def validate(self, clean=True): """Ensure that all fields' values are valid and that required fields are present. """ + # Ensure that each field is matched to a valid value + errors = {} + if clean: + try: + self.clean() + except ValidationError, error: + errors[NON_FIELD_ERRORS] = error + # Get a list of tuples of field names and their current values fields = [(field, self._data.get(name)) for name, field in self._fields.items()] - # Ensure that each field is matched to a valid value - errors = {} + EmbeddedDocumentField = _import_class("EmbeddedDocumentField") + GenericEmbeddedDocumentField = _import_class("GenericEmbeddedDocumentField") + for field, value in fields: if value is not None: try: - field._validate(value) + if isinstance(field, (EmbeddedDocumentField, + GenericEmbeddedDocumentField)): + field._validate(value, clean=clean) + else: + field._validate(value) except ValidationError, error: errors[field.name] = error.errors or error except (ValueError, AttributeError, AssertionError), error: @@ -224,6 +249,7 @@ class BaseDocument(object): elif field.required and not getattr(field, '_auto_gen', False): errors[field.name] = ValidationError('Field is required', field_name=field.name) + if errors: raise ValidationError('ValidationError', errors=errors) diff --git a/mongoengine/base/fields.py b/mongoengine/base/fields.py index fc1a0767..11719b55 100644 --- a/mongoengine/base/fields.py +++ b/mongoengine/base/fields.py @@ -105,12 +105,12 @@ class BaseField(object): """ return value - def validate(self, value): + def validate(self, value, clean=True): """Perform validation on a value. """ pass - def _validate(self, value): + def _validate(self, value, **kwargs): Document = _import_class('Document') EmbeddedDocument = _import_class('EmbeddedDocument') # check choices @@ -138,7 +138,7 @@ class BaseField(object): raise ValueError('validation argument for "%s" must be a ' 'callable.' % self.name) - self.validate(value) + self.validate(value, **kwargs) class ComplexBaseField(BaseField): diff --git a/mongoengine/common.py b/mongoengine/common.py index c284777e..c76801ce 100644 --- a/mongoengine/common.py +++ b/mongoengine/common.py @@ -9,8 +9,8 @@ def _import_class(cls_name): doc_classes = ('Document', 'DynamicEmbeddedDocument', 'EmbeddedDocument', 'MapReduceDocument') field_classes = ('DictField', 'DynamicField', 'EmbeddedDocumentField', - 'GenericReferenceField', 'GeoPointField', - 'ReferenceField', 'StringField') + 'GenericReferenceField', 'GenericEmbeddedDocumentField', + 'GeoPointField', 'ReferenceField', 'StringField') queryset_classes = ('OperationError',) deref_classes = ('DeReference',) diff --git a/mongoengine/document.py b/mongoengine/document.py index adbdcca2..fcf82563 100644 --- a/mongoengine/document.py +++ b/mongoengine/document.py @@ -100,8 +100,8 @@ class Document(BaseDocument): Automatic index creation can be disabled by specifying attr:`auto_create_index` in the :attr:`meta` dictionary. If this is set to False then indexes will not be created by MongoEngine. This is useful in - production systems where index creation is performed as part of a deployment - system. + production systems where index creation is performed as part of a + deployment system. By default, _cls will be added to the start of every index (that doesn't contain a list) if allow_inheritance is True. This can be @@ -165,7 +165,7 @@ class Document(BaseDocument): cls._collection = db[collection_name] return cls._collection - def save(self, safe=True, force_insert=False, validate=True, + def save(self, safe=True, force_insert=False, validate=True, clean=True, write_options=None, cascade=None, cascade_kwargs=None, _refs=None): """Save the :class:`~mongoengine.Document` to the database. If the @@ -179,6 +179,8 @@ class Document(BaseDocument): :param force_insert: only try to create a new document, don't allow updates of existing documents :param validate: validates the document; set to ``False`` to skip. + :param clean: call the document clean method, requires `validate` to be + True. :param write_options: Extra keyword arguments are passed down to :meth:`~pymongo.collection.Collection.save` OR :meth:`~pymongo.collection.Collection.insert` @@ -208,7 +210,7 @@ class Document(BaseDocument): signals.pre_save.send(self.__class__, document=self) if validate: - self.validate() + self.validate(clean=clean) if not write_options: write_options = {} diff --git a/mongoengine/fields.py b/mongoengine/fields.py index 94e11556..8aa7f641 100644 --- a/mongoengine/fields.py +++ b/mongoengine/fields.py @@ -461,7 +461,7 @@ class EmbeddedDocumentField(BaseField): return value return self.document_type.to_mongo(value) - def validate(self, value): + def validate(self, value, clean=True): """Make sure that the document instance is an instance of the EmbeddedDocument subclass provided when the document was defined. """ @@ -469,7 +469,7 @@ class EmbeddedDocumentField(BaseField): if not isinstance(value, self.document_type): self.error('Invalid embedded document instance provided to an ' 'EmbeddedDocumentField') - self.document_type.validate(value) + self.document_type.validate(value, clean) def lookup_member(self, member_name): return self.document_type._fields.get(member_name) @@ -499,12 +499,12 @@ class GenericEmbeddedDocumentField(BaseField): return value - def validate(self, value): + def validate(self, value, clean=True): if not isinstance(value, EmbeddedDocument): self.error('Invalid embedded document instance provided to an ' 'GenericEmbeddedDocumentField') - value.validate() + value.validate(clean=clean) def to_mongo(self, document): if document is None: diff --git a/tests/document/instance.py b/tests/document/instance.py index 48ddc10d..2e07eb26 100644 --- a/tests/document/instance.py +++ b/tests/document/instance.py @@ -490,6 +490,76 @@ class InstanceTest(unittest.TestCase): self.assertTrue('id' in keys) self.assertTrue('e' in keys) + def test_document_clean(self): + class TestDocument(Document): + status = StringField() + pub_date = DateTimeField() + + def clean(self): + if self.status == 'draft' and self.pub_date is not None: + msg = 'Draft entries may not have a publication date.' + raise ValidationError(msg) + # Set the pub_date for published items if not set. + if self.status == 'published' and self.pub_date is None: + self.pub_date = datetime.now() + + TestDocument.drop_collection() + + t = TestDocument(status="draft", pub_date=datetime.now()) + + try: + t.save() + except ValidationError, e: + expect_msg = "Draft entries may not have a publication date." + self.assertTrue(expect_msg in e.message) + self.assertEqual(e.to_dict(), {'__all__': expect_msg}) + + t = TestDocument(status="published") + t.save(clean=False) + + self.assertEquals(t.pub_date, None) + + t = TestDocument(status="published") + t.save(clean=True) + + self.assertEquals(type(t.pub_date), datetime) + + def test_document_embedded_clean(self): + class TestEmbeddedDocument(EmbeddedDocument): + x = IntField(required=True) + y = IntField(required=True) + z = IntField(required=True) + + meta = {'allow_inheritance': False} + + def clean(self): + if self.z: + if self.z != self.x + self.y: + raise ValidationError('Value of z != x + y') + else: + self.z = self.x + self.y + + class TestDocument(Document): + doc = EmbeddedDocumentField(TestEmbeddedDocument) + status = StringField() + + TestDocument.drop_collection() + + t = TestDocument(doc=TestEmbeddedDocument(x=10, y=25, z=15)) + try: + t.save() + except ValidationError, e: + expect_msg = "Value of z != x + y" + self.assertTrue(expect_msg in e.message) + self.assertEqual(e.to_dict(), {'doc': {'__all__': expect_msg}}) + + t = TestDocument(doc=TestEmbeddedDocument(x=10, y=25)).save() + self.assertEquals(t.doc.z, 35) + + # Asserts not raises + t = TestDocument(doc=TestEmbeddedDocument(x=15, y=35, z=5)) + t.save(clean=False) + def test_save(self): """Ensure that a document may be saved in the database. """ @@ -1935,7 +2005,5 @@ class ValidatorErrorTest(unittest.TestCase): self.assertRaises(OperationError, change_shard_key) - - if __name__ == '__main__': unittest.main() From 99fe1da34564b29cb2505fe938b09ca4253a884c Mon Sep 17 00:00:00 2001 From: Ross Lawley Date: Wed, 7 Nov 2012 13:20:34 +0000 Subject: [PATCH 015/189] Add value_decorator into SequenceField Allows post processing of the calculated counter value. --- docs/changelog.rst | 1 + docs/upgrade.rst | 7 +++++++ mongoengine/fields.py | 33 ++++++++++++++++++--------------- tests/test_fields.py | 24 ++++++++++++++++++++++++ 4 files changed, 50 insertions(+), 15 deletions(-) diff --git a/docs/changelog.rst b/docs/changelog.rst index ca18d3e9..550cc8dc 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -4,6 +4,7 @@ Changelog Changes in 0.8 ============== +- Updated SequenceFields to allow post processing of the calculated counter value (MongoEngine/mongoengine#141) - Added clean method to documents for pre validation data cleaning (MongoEngine/mongoengine#60) - Added support setting for read prefrence at a query level (MongoEngine/mongoengine#157) - Added _instance to EmbeddedDocuments pointing to the parent (MongoEngine/mongoengine#139) diff --git a/docs/upgrade.rst b/docs/upgrade.rst index bf0a8421..daf09126 100644 --- a/docs/upgrade.rst +++ b/docs/upgrade.rst @@ -57,6 +57,13 @@ you will need to declare :attr:`allow_inheritance` in the meta data like so: meta = {'allow_inheritance': True} +SequenceFields +-------------- + +:class:`~mongoengine.fields.SequenceField`s now inherit from `BaseField` to +allow flexible storage of the calculated value. As such MIN and MAX settings +are no longer handled. + 0.6 to 0.7 ========== diff --git a/mongoengine/fields.py b/mongoengine/fields.py index 8aa7f641..e2ce33cd 100644 --- a/mongoengine/fields.py +++ b/mongoengine/fields.py @@ -1329,7 +1329,7 @@ class GeoPointField(BaseField): self.error('Both values in point must be float or int') -class SequenceField(IntField): +class SequenceField(BaseField): """Provides a sequental counter see: http://www.mongodb.org/display/DOCS/Object+IDs#ObjectIDs-SequenceNumbers @@ -1341,15 +1341,26 @@ class SequenceField(IntField): cluster of machines, it is easier to create an object ID than have global, uniformly increasing sequence numbers. + Use any callable as `value_decorator` to transform calculated counter into + any value suitable for your needs, e.g. string or hexadecimal + representation of the default integer counter value. + .. versionadded:: 0.5 + + .. versionchanged:: 0.8 added `value_decorator` """ + _auto_gen = True + COLLECTION_NAME = 'mongoengine.counters' + VALUE_DECORATOR = int def __init__(self, collection_name=None, db_alias=None, - sequence_name=None, *args, **kwargs): - self.collection_name = collection_name or 'mongoengine.counters' + sequence_name=None, value_decorator=None, *args, **kwargs): + self.collection_name = collection_name or self.COLLECTION_NAME self.db_alias = db_alias or DEFAULT_CONNECTION_NAME self.sequence_name = sequence_name + self.value_decorator = (callable(value_decorator) and + value_decorator or self.VALUE_DECORATOR) return super(SequenceField, self).__init__(*args, **kwargs) def generate(self): @@ -1364,24 +1375,16 @@ class SequenceField(IntField): update={"$inc": {"next": 1}}, new=True, upsert=True) - return counter['next'] + return self.value_decorator(counter['next']) def __get__(self, instance, owner): - - if instance is None: - return self - - if not instance._data: - return - - value = instance._data.get(self.name) - - if not value and instance._initialised: + value = super(SequenceField, self).__get__(instance, owner) + if value is None and instance._initialised: value = self.generate() instance._data[self.name] = value instance._mark_as_changed(self.name) - return int(value) if value else None + return value def __set__(self, instance, value): diff --git a/tests/test_fields.py b/tests/test_fields.py index 1c13a58c..f1a36ed7 100644 --- a/tests/test_fields.py +++ b/tests/test_fields.py @@ -1,5 +1,8 @@ # -*- coding: utf-8 -*- from __future__ import with_statement +import sys +sys.path[0:0] = [""] + import datetime import os import unittest @@ -2184,6 +2187,27 @@ class FieldTest(unittest.TestCase): c = self.db['mongoengine.counters'].find_one({'_id': 'animal.id'}) self.assertEqual(c['next'], 10) + def test_sequence_field_value_decorator(self): + class Person(Document): + id = SequenceField(primary_key=True, value_decorator=str) + name = StringField() + + self.db['mongoengine.counters'].drop() + Person.drop_collection() + + for x in xrange(10): + p = Person(name="Person %s" % x) + p.save() + + c = self.db['mongoengine.counters'].find_one({'_id': 'person.id'}) + self.assertEqual(c['next'], 10) + + ids = [i.id for i in Person.objects] + self.assertEqual(ids, map(str, range(1, 11))) + + c = self.db['mongoengine.counters'].find_one({'_id': 'person.id'}) + self.assertEqual(c['next'], 10) + def test_generic_embedded_document(self): class Car(EmbeddedDocument): name = StringField() From 9ca96e4e17f4d954abd4163c5a3e13b0ff094f96 Mon Sep 17 00:00:00 2001 From: Ross Lawley Date: Wed, 7 Nov 2012 13:51:02 +0000 Subject: [PATCH 016/189] Added none() to queryset (MongoEngine/mongoengine#127) --- docs/changelog.rst | 1 + mongoengine/queryset/queryset.py | 4 ++++ tests/test_queryset.py | 8 ++++++++ 3 files changed, 13 insertions(+) diff --git a/docs/changelog.rst b/docs/changelog.rst index 550cc8dc..9bd822b8 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -4,6 +4,7 @@ Changelog Changes in 0.8 ============== +- Added none() to queryset (MongoEngine/mongoengine#127) - Updated SequenceFields to allow post processing of the calculated counter value (MongoEngine/mongoengine#141) - Added clean method to documents for pre validation data cleaning (MongoEngine/mongoengine#60) - Added support setting for read prefrence at a query level (MongoEngine/mongoengine#157) diff --git a/mongoengine/queryset/queryset.py b/mongoengine/queryset/queryset.py index cf4b4f82..65c71e13 100644 --- a/mongoengine/queryset/queryset.py +++ b/mongoengine/queryset/queryset.py @@ -489,6 +489,10 @@ class QuerySet(object): self._iter = False self._cursor.rewind() + def none(self): + """Helper that just returns a list""" + return [] + def count(self): """Count the selected elements in the query. """ diff --git a/tests/test_queryset.py b/tests/test_queryset.py index dcb25241..a3e64d21 100644 --- a/tests/test_queryset.py +++ b/tests/test_queryset.py @@ -252,6 +252,14 @@ class QuerySetTest(unittest.TestCase): Blog.drop_collection() + def test_none(self): + class A(Document): + pass + + A.drop_collection() + A().save() + self.assertEqual(A.objects.none(), []) + def test_chaining(self): class A(Document): pass From 8706fbe461c1bb3f5ea8d9ee23434a6aeaf86fc5 Mon Sep 17 00:00:00 2001 From: Ross Lawley Date: Wed, 7 Nov 2012 15:04:45 +0000 Subject: [PATCH 017/189] Updated index creation now tied to Document class ((MongoEngine/mongoengine#102) --- docs/changelog.rst | 1 + docs/upgrade.rst | 9 +- mongoengine/document.py | 83 ++++++++++++- mongoengine/queryset/queryset.py | 114 +++--------------- tests/document/indexes.py | 10 +- .../test_convert_to_new_inheritance_model.py | 2 +- tests/migration/turn_off_inheritance.py | 2 +- tests/test_queryset.py | 3 +- 8 files changed, 115 insertions(+), 109 deletions(-) diff --git a/docs/changelog.rst b/docs/changelog.rst index 9bd822b8..ca450f11 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -4,6 +4,7 @@ Changelog Changes in 0.8 ============== +- Updated index creation now tied to Document class ((MongoEngine/mongoengine#102) - Added none() to queryset (MongoEngine/mongoengine#127) - Updated SequenceFields to allow post processing of the calculated counter value (MongoEngine/mongoengine#141) - Added clean method to documents for pre validation data cleaning (MongoEngine/mongoengine#60) diff --git a/docs/upgrade.rst b/docs/upgrade.rst index daf09126..44c69beb 100644 --- a/docs/upgrade.rst +++ b/docs/upgrade.rst @@ -40,7 +40,7 @@ inherited classes like so: collection.drop_index(index) # 5. Recreate indexes - Animal.objects._ensure_indexes() + Animal.ensure_indexes() Document Definition @@ -56,6 +56,13 @@ you will need to declare :attr:`allow_inheritance` in the meta data like so: meta = {'allow_inheritance': True} +Indexes +------- + +Index methods are no longer tied to querysets but rather to the document class. +Although `QuerySet._ensure_indexes` and `QuerySet.ensure_index` still exist. +They should be replaced with :func:`~mongoengine.Document.ensure_indexes` / +:func:`~mongoengine.Document.ensure_index`. SequenceFields -------------- diff --git a/mongoengine/document.py b/mongoengine/document.py index fcf82563..cda3a9ca 100644 --- a/mongoengine/document.py +++ b/mongoengine/document.py @@ -7,7 +7,7 @@ from bson.dbref import DBRef from mongoengine import signals, queryset from base import (DocumentMetaclass, TopLevelDocumentMetaclass, BaseDocument, - BaseDict, BaseList) + BaseDict, BaseList, ALLOW_INHERITANCE) from queryset import OperationError, NotUniqueError from connection import get_db, DEFAULT_CONNECTION_NAME @@ -163,6 +163,8 @@ class Document(BaseDocument): ) else: cls._collection = db[collection_name] + if cls._meta.get('auto_create_index', True): + cls.ensure_indexes() return cls._collection def save(self, safe=True, force_insert=False, validate=True, clean=True, @@ -418,9 +420,86 @@ class Document(BaseDocument): """Drops the entire collection associated with this :class:`~mongoengine.Document` type from the database. """ + cls._collection = None db = cls._get_db() db.drop_collection(cls._get_collection_name()) - queryset.QuerySet._reset_already_indexed(cls) + + @classmethod + def ensure_index(cls, key_or_list, drop_dups=False, background=False, + **kwargs): + """Ensure that the given indexes are in place. + + :param key_or_list: a single index key or a list of index keys (to + construct a multi-field index); keys may be prefixed with a **+** + or a **-** to determine the index ordering + """ + index_spec = cls._build_index_spec(key_or_list) + index_spec = index_spec.copy() + fields = index_spec.pop('fields') + index_spec['drop_dups'] = drop_dups + index_spec['background'] = background + index_spec.update(kwargs) + + return cls._get_collection().ensure_index(fields, **index_spec) + + @classmethod + def ensure_indexes(cls): + """Checks the document meta data and ensures all the indexes exist. + + .. note:: You can disable automatic index creation by setting + `auto_create_index` to False in the documents meta data + """ + background = cls._meta.get('index_background', False) + drop_dups = cls._meta.get('index_drop_dups', False) + index_opts = cls._meta.get('index_opts') or {} + index_cls = cls._meta.get('index_cls', True) + + collection = cls._get_collection() + + # determine if an index which we are creating includes + # _cls as its first field; if so, we can avoid creating + # an extra index on _cls, as mongodb will use the existing + # index to service queries against _cls + cls_indexed = False + + def includes_cls(fields): + first_field = None + if len(fields): + if isinstance(fields[0], basestring): + first_field = fields[0] + elif isinstance(fields[0], (list, tuple)) and len(fields[0]): + first_field = fields[0][0] + return first_field == '_cls' + + # Ensure indexes created by uniqueness constraints + for index in cls._meta['unique_indexes']: + cls_indexed = cls_indexed or includes_cls(index) + collection.ensure_index(index, unique=True, background=background, + drop_dups=drop_dups, **index_opts) + + # Ensure document-defined indexes are created + if cls._meta['index_specs']: + index_spec = cls._meta['index_specs'] + for spec in index_spec: + spec = spec.copy() + fields = spec.pop('fields') + cls_indexed = cls_indexed or includes_cls(fields) + opts = index_opts.copy() + opts.update(spec) + collection.ensure_index(fields, background=background, **opts) + + # If _cls is being used (for polymorphism), it needs an index, + # only if another index doesn't begin with _cls + if (index_cls and not cls_indexed and + cls._meta.get('allow_inheritance', ALLOW_INHERITANCE) == True): + collection.ensure_index('_cls', background=background, + **index_opts) + + # Add geo indicies + for field in cls._geo_indices(): + index_spec = [(field.db_field, pymongo.GEO2D)] + collection.ensure_index(index_spec, background=background, + **index_opts) class DynamicDocument(Document): diff --git a/mongoengine/queryset/queryset.py b/mongoengine/queryset/queryset.py index 65c71e13..1122123b 100644 --- a/mongoengine/queryset/queryset.py +++ b/mongoengine/queryset/queryset.py @@ -1,11 +1,12 @@ -import pprint -import re import copy import itertools import operator +import pprint +import re +import warnings -import pymongo from bson.code import Code +import pymongo from pymongo.common import validate_read_preference from mongoengine import signals @@ -37,8 +38,6 @@ class QuerySet(object): """A set of results returned from a query. Wraps a MongoDB cursor, providing :class:`~mongoengine.Document` objects as the results. """ - - __already_indexed = set() __dereference = False def __init__(self, document, collection): @@ -95,24 +94,6 @@ class QuerySet(object): self._mongo_query.update(self._initial_query) return self._mongo_query - def ensure_index(self, key_or_list, drop_dups=False, background=False, - **kwargs): - """Ensure that the given indexes are in place. - - :param key_or_list: a single index key or a list of index keys (to - construct a multi-field index); keys may be prefixed with a **+** - or a **-** to determine the index ordering - """ - index_spec = self._document._build_index_spec(key_or_list) - index_spec = index_spec.copy() - fields = index_spec.pop('fields') - index_spec['drop_dups'] = drop_dups - index_spec['background'] = background - index_spec.update(kwargs) - - self._collection.ensure_index(fields, **index_spec) - return self - def __call__(self, q_obj=None, class_check=True, slave_okay=False, read_preference=None, **query): """Filter the selected documents by calling the @@ -150,87 +131,26 @@ class QuerySet(object): """Returns all documents.""" return self.__call__() + def ensure_index(self, **kwargs): + """Deprecated use :func:`~Document.ensure_index`""" + msg = ("Doc.objects()._ensure_index() is deprecated. " + "Use Doc.ensure_index() instead.") + warnings.warn(msg, DeprecationWarning) + self._document.__class__.ensure_index(**kwargs) + return self + def _ensure_indexes(self): - """Checks the document meta data and ensures all the indexes exist. - - .. note:: You can disable automatic index creation by setting - `auto_create_index` to False in the documents meta data - """ - background = self._document._meta.get('index_background', False) - drop_dups = self._document._meta.get('index_drop_dups', False) - index_opts = self._document._meta.get('index_opts') or {} - index_cls = self._document._meta.get('index_cls', True) - - # determine if an index which we are creating includes - # _cls as its first field; if so, we can avoid creating - # an extra index on _cls, as mongodb will use the existing - # index to service queries against _cls - cls_indexed = False - - def includes_cls(fields): - first_field = None - if len(fields): - if isinstance(fields[0], basestring): - first_field = fields[0] - elif isinstance(fields[0], (list, tuple)) and len(fields[0]): - first_field = fields[0][0] - return first_field == '_cls' - - # Ensure indexes created by uniqueness constraints - for index in self._document._meta['unique_indexes']: - cls_indexed = cls_indexed or includes_cls(index) - self._collection.ensure_index(index, unique=True, - background=background, drop_dups=drop_dups, **index_opts) - - # Ensure document-defined indexes are created - if self._document._meta['index_specs']: - index_spec = self._document._meta['index_specs'] - for spec in index_spec: - spec = spec.copy() - fields = spec.pop('fields') - cls_indexed = cls_indexed or includes_cls(fields) - opts = index_opts.copy() - opts.update(spec) - self._collection.ensure_index(fields, - background=background, **opts) - - # If _cls is being used (for polymorphism), it needs an index, - # only if another index doesn't begin with _cls - if index_cls and '_cls' in self._query and not cls_indexed: - self._collection.ensure_index('_cls', - background=background, **index_opts) - - # Add geo indicies - for field in self._document._geo_indices(): - index_spec = [(field.db_field, pymongo.GEO2D)] - self._collection.ensure_index(index_spec, - background=background, **index_opts) - - @classmethod - def _reset_already_indexed(cls, document=None): - """Helper to reset already indexed, can be useful for testing purposes - """ - if document: - cls.__already_indexed.discard(document) - cls.__already_indexed.clear() + """Deprecated use :func:`~Document.ensure_indexes`""" + msg = ("Doc.objects()._ensure_indexes() is deprecated. " + "Use Doc.ensure_indexes() instead.") + warnings.warn(msg, DeprecationWarning) + self._document.__class__.ensure_indexes() @property def _collection(self): """Property that returns the collection object. This allows us to perform operations only if the collection is accessed. """ - if self._document not in QuerySet.__already_indexed: - # Ensure collection exists - db = self._document._get_db() - if self._collection_obj.name not in db.collection_names(): - self._document._collection = None - self._collection_obj = self._document._get_collection() - - QuerySet.__already_indexed.add(self._document) - - if self._document._meta.get('auto_create_index', True): - self._ensure_indexes() - return self._collection_obj @property diff --git a/tests/document/indexes.py b/tests/document/indexes.py index a6b74cd0..8f83afc7 100644 --- a/tests/document/indexes.py +++ b/tests/document/indexes.py @@ -80,7 +80,7 @@ class InstanceTest(unittest.TestCase): ('addDate', -1)]}] self.assertEqual(expected_specs, BlogPost._meta['index_specs']) - BlogPost.objects._ensure_indexes() + BlogPost.ensure_indexes() info = BlogPost.objects._collection.index_information() # _id, '-date', 'tags', ('cat', 'date') # NB: there is no index on _cls by itself, since @@ -100,7 +100,7 @@ class InstanceTest(unittest.TestCase): BlogPost.drop_collection() - ExtendedBlogPost.objects._ensure_indexes() + ExtendedBlogPost.ensure_indexes() info = ExtendedBlogPost.objects._collection.index_information() info = [value['key'] for key, value in info.iteritems()] for expected in expected_specs: @@ -141,7 +141,7 @@ class InstanceTest(unittest.TestCase): [{'fields': [('keywords', 1)]}]) # Force index creation - MyDoc.objects._ensure_indexes() + MyDoc.ensure_indexes() self.assertEqual(MyDoc._meta['index_specs'], [{'fields': [('keywords', 1)]}]) @@ -189,7 +189,7 @@ class InstanceTest(unittest.TestCase): self.assertEqual([{'fields': [('location.point', '2d')]}], Place._meta['index_specs']) - Place.objects()._ensure_indexes() + Place.ensure_indexes() info = Place._get_collection().index_information() info = [value['key'] for key, value in info.iteritems()] self.assertTrue([('location.point', '2d')] in info) @@ -335,7 +335,7 @@ class InstanceTest(unittest.TestCase): recursive_obj = EmbeddedDocumentField(RecursiveObject) meta = {'allow_inheritance': True} - RecursiveDocument.objects._ensure_indexes() + RecursiveDocument.ensure_indexes() info = RecursiveDocument._get_collection().index_information() self.assertEqual(info.keys(), ['_id_', '_cls_1']) diff --git a/tests/migration/test_convert_to_new_inheritance_model.py b/tests/migration/test_convert_to_new_inheritance_model.py index 0ef37f74..d4337bf3 100644 --- a/tests/migration/test_convert_to_new_inheritance_model.py +++ b/tests/migration/test_convert_to_new_inheritance_model.py @@ -48,4 +48,4 @@ class ConvertToNewInheritanceModel(unittest.TestCase): collection.drop_index(index) # 5. Recreate indexes - Animal.objects._ensure_indexes() + Animal.ensure_indexes() diff --git a/tests/migration/turn_off_inheritance.py b/tests/migration/turn_off_inheritance.py index 5d0f7d73..ee461a84 100644 --- a/tests/migration/turn_off_inheritance.py +++ b/tests/migration/turn_off_inheritance.py @@ -59,4 +59,4 @@ class TurnOffInheritanceTest(unittest.TestCase): collection.drop_index(index) # 5. Recreate indexes - Animal.objects._ensure_indexes() + Animal.ensure_indexes() diff --git a/tests/test_queryset.py b/tests/test_queryset.py index a3e64d21..378b4899 100644 --- a/tests/test_queryset.py +++ b/tests/test_queryset.py @@ -3078,7 +3078,6 @@ class QuerySetTest(unittest.TestCase): self.assertEqual([1, 2, 3], numbers) Number.drop_collection() - def test_ensure_index(self): """Ensure that manual creation of indexes works. """ @@ -3086,7 +3085,7 @@ class QuerySetTest(unittest.TestCase): message = StringField() meta = {'allow_inheritance': True} - Comment.objects.ensure_index('message') + Comment.ensure_index('message') info = Comment.objects._collection.index_information() info = [(value['key'], From e7c0da38c2b512fabe1e3dc885a4aaf9919bcdef Mon Sep 17 00:00:00 2001 From: Ross Lawley Date: Wed, 7 Nov 2012 15:09:11 +0000 Subject: [PATCH 018/189] Better implementation for none - MongoEngine/mongoengine#127 --- mongoengine/queryset/queryset.py | 6 ++++-- tests/test_queryset.py | 4 +++- 2 files changed, 7 insertions(+), 3 deletions(-) diff --git a/mongoengine/queryset/queryset.py b/mongoengine/queryset/queryset.py index 1122123b..058bdd86 100644 --- a/mongoengine/queryset/queryset.py +++ b/mongoengine/queryset/queryset.py @@ -39,6 +39,7 @@ class QuerySet(object): providing :class:`~mongoengine.Document` objects as the results. """ __dereference = False + __none = False def __init__(self, document, collection): self._document = document @@ -391,7 +392,7 @@ class QuerySet(object): """ self._iter = True try: - if self._limit == 0: + if self._limit == 0 or self.__none: raise StopIteration if self._scalar: return self._get_scalar(self._document._from_son( @@ -411,7 +412,8 @@ class QuerySet(object): def none(self): """Helper that just returns a list""" - return [] + self.__none = True + return self def count(self): """Count the selected elements in the query. diff --git a/tests/test_queryset.py b/tests/test_queryset.py index 378b4899..09b6b3ff 100644 --- a/tests/test_queryset.py +++ b/tests/test_queryset.py @@ -258,7 +258,9 @@ class QuerySetTest(unittest.TestCase): A.drop_collection() A().save() - self.assertEqual(A.objects.none(), []) + + self.assertEqual(list(A.objects.none()), []) + self.assertEqual(list(A.objects.none().all()), []) def test_chaining(self): class A(Document): From 4b45c0cd14182b3abf4b1af8cc9f2296e7bdbcda Mon Sep 17 00:00:00 2001 From: Ross Lawley Date: Wed, 7 Nov 2012 15:15:04 +0000 Subject: [PATCH 019/189] Removed deprecation warning #55 --- mongoengine/queryset/manager.py | 4 ---- 1 file changed, 4 deletions(-) diff --git a/mongoengine/queryset/manager.py b/mongoengine/queryset/manager.py index 7376e3c6..08d4d3ab 100644 --- a/mongoengine/queryset/manager.py +++ b/mongoengine/queryset/manager.py @@ -54,8 +54,4 @@ def queryset_manager(func): function should return a :class:`~mongoengine.queryset.QuerySet`, probably the same one that was passed in, but modified in some way. """ - if func.func_code.co_argcount == 1: - import warnings - msg = 'Methods decorated with queryset_manager should take 2 arguments' - warnings.warn(msg, DeprecationWarning) return QuerySetManager(func) From b8d53a6f0d69c81e0566f5cadbe9237e1f530131 Mon Sep 17 00:00:00 2001 From: Ross Lawley Date: Thu, 8 Nov 2012 12:04:14 +0000 Subject: [PATCH 020/189] Added json serialisation support - Added to_json and from_json to Document (MongoEngine/mongoengine#1) - Added to_json and from_json to QuerySet (MongoEngine/mongoengine#131) --- docs/changelog.rst | 4 +- mongoengine/base/document.py | 10 ++++ mongoengine/queryset/queryset.py | 10 ++++ tests/document/__init__.py | 5 +- tests/document/instance.py | 2 +- tests/document/json_serialisation.py | 81 ++++++++++++++++++++++++++++ tests/test_fields.py | 39 ++++++++------ tests/test_queryset.py | 70 +++++++++++++++++++++++- 8 files changed, 201 insertions(+), 20 deletions(-) create mode 100644 tests/document/json_serialisation.py diff --git a/docs/changelog.rst b/docs/changelog.rst index ca450f11..26108b5d 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -4,7 +4,9 @@ Changelog Changes in 0.8 ============== -- Updated index creation now tied to Document class ((MongoEngine/mongoengine#102) +- Added to_json and from_json to Document (MongoEngine/mongoengine#1) +- Added to_json and from_json to QuerySet (MongoEngine/mongoengine#131) +- Updated index creation now tied to Document class (MongoEngine/mongoengine#102) - Added none() to queryset (MongoEngine/mongoengine#127) - Updated SequenceFields to allow post processing of the calculated counter value (MongoEngine/mongoengine#141) - Added clean method to documents for pre validation data cleaning (MongoEngine/mongoengine#60) diff --git a/mongoengine/base/document.py b/mongoengine/base/document.py index 46f53205..939c9fbc 100644 --- a/mongoengine/base/document.py +++ b/mongoengine/base/document.py @@ -2,6 +2,7 @@ import operator from functools import partial import pymongo +from bson import json_util from bson.dbref import DBRef from mongoengine import signals @@ -253,6 +254,15 @@ class BaseDocument(object): if errors: raise ValidationError('ValidationError', errors=errors) + def to_json(self): + """Converts a document to JSON""" + return json_util.dumps(self.to_mongo()) + + @classmethod + def from_json(cls, json_data): + """Converts json data to an unsaved document instance""" + return cls._from_son(json_util.loads(json_data)) + def __expand_dynamic_values(self, name, value): """expand any dynamic values to their correct types / values""" if not isinstance(value, (dict, list, tuple)): diff --git a/mongoengine/queryset/queryset.py b/mongoengine/queryset/queryset.py index 058bdd86..3c44f012 100644 --- a/mongoengine/queryset/queryset.py +++ b/mongoengine/queryset/queryset.py @@ -6,6 +6,7 @@ import re import warnings from bson.code import Code +from bson import json_util import pymongo from pymongo.common import validate_read_preference @@ -1216,6 +1217,15 @@ class QuerySet(object): max_depth += 1 return self._dereference(self, max_depth=max_depth) + def to_json(self): + """Converts a queryset to JSON""" + return json_util.dumps(self._collection_obj.find(self._query)) + + def from_json(self, json_data): + """Converts json data to unsaved objects""" + son_data = json_util.loads(json_data) + return [self._document._from_son(data) for data in son_data] + @property def _dereference(self): if not self.__dereference: diff --git a/tests/document/__init__.py b/tests/document/__init__.py index 1ef25201..7774ee19 100644 --- a/tests/document/__init__.py +++ b/tests/document/__init__.py @@ -1,4 +1,6 @@ -# TODO EXPLICT IMPORTS +import sys +sys.path[0:0] = [""] +import unittest from class_methods import * from delta import * @@ -6,6 +8,7 @@ from dynamic import * from indexes import * from inheritance import * from instance import * +from json_serialisation import * if __name__ == '__main__': unittest.main() diff --git a/tests/document/instance.py b/tests/document/instance.py index 2e07eb26..2118575e 100644 --- a/tests/document/instance.py +++ b/tests/document/instance.py @@ -346,7 +346,7 @@ class InstanceTest(unittest.TestCase): meta = {'shard_key': ('superphylum',)} Animal.drop_collection() - doc = Animal(superphylum = 'Deuterostomia') + doc = Animal(superphylum='Deuterostomia') doc.save() doc.reload() Animal.drop_collection() diff --git a/tests/document/json_serialisation.py b/tests/document/json_serialisation.py new file mode 100644 index 00000000..dbc09d83 --- /dev/null +++ b/tests/document/json_serialisation.py @@ -0,0 +1,81 @@ +import sys +sys.path[0:0] = [""] + +import unittest +import uuid + +from nose.plugins.skip import SkipTest +from datetime import datetime +from bson import ObjectId + +import pymongo + +from mongoengine import * + +__all__ = ("TestJson",) + + +class TestJson(unittest.TestCase): + + def setUp(self): + connect(db='mongoenginetest') + + def test_json_simple(self): + + class Embedded(EmbeddedDocument): + string = StringField() + + class Doc(Document): + string = StringField() + embedded_field = EmbeddedDocumentField(Embedded) + + doc = Doc(string="Hi", embedded_field=Embedded(string="Hi")) + + self.assertEqual(doc, Doc.from_json(doc.to_json())) + + def test_json_complex(self): + + if pymongo.version_tuple[0] <= 2 and pymongo.version_tuple[1] <= 3: + raise SkipTest("Need pymongo 2.4 as has a fix for DBRefs") + + class EmbeddedDoc(EmbeddedDocument): + pass + + class Simple(Document): + pass + + class Doc(Document): + string_field = StringField(default='1') + int_field = IntField(default=1) + float_field = FloatField(default=1.1) + boolean_field = BooleanField(default=True) + datetime_field = DateTimeField(default=datetime.now) + embedded_document_field = EmbeddedDocumentField(EmbeddedDoc, + default=lambda: EmbeddedDoc()) + list_field = ListField(default=lambda: [1, 2, 3]) + dict_field = DictField(default=lambda: {"hello": "world"}) + objectid_field = ObjectIdField(default=ObjectId) + reference_field = ReferenceField(Simple, default=lambda: + Simple().save()) + map_field = MapField(IntField(), default=lambda: {"simple": 1}) + decimal_field = DecimalField(default=1.0) + complex_datetime_field = ComplexDateTimeField(default=datetime.now) + url_field = URLField(default="http://mongoengine.org") + dynamic_field = DynamicField(default=1) + generic_reference_field = GenericReferenceField( + default=lambda: Simple().save()) + sorted_list_field = SortedListField(IntField(), + default=lambda: [1, 2, 3]) + email_field = EmailField(default="ross@example.com") + geo_point_field = GeoPointField(default=lambda: [1, 2]) + sequence_field = SequenceField() + uuid_field = UUIDField(default=uuid.uuid4) + generic_embedded_document_field = GenericEmbeddedDocumentField( + default=lambda: EmbeddedDoc()) + + doc = Doc() + self.assertEqual(doc, Doc.from_json(doc.to_json())) + + +if __name__ == '__main__': + unittest.main() diff --git a/tests/test_fields.py b/tests/test_fields.py index f1a36ed7..69cce871 100644 --- a/tests/test_fields.py +++ b/tests/test_fields.py @@ -606,7 +606,8 @@ class FieldTest(unittest.TestCase): name = StringField() class CategoryList(Document): - categories = SortedListField(EmbeddedDocumentField(Category), ordering='count', reverse=True) + categories = SortedListField(EmbeddedDocumentField(Category), + ordering='count', reverse=True) name = StringField() catlist = CategoryList(name="Top categories") @@ -1616,8 +1617,9 @@ class FieldTest(unittest.TestCase): """Ensure that value is in a container of allowed values. """ class Shirt(Document): - size = StringField(max_length=3, choices=(('S', 'Small'), ('M', 'Medium'), ('L', 'Large'), - ('XL', 'Extra Large'), ('XXL', 'Extra Extra Large'))) + size = StringField(max_length=3, choices=( + ('S', 'Small'), ('M', 'Medium'), ('L', 'Large'), + ('XL', 'Extra Large'), ('XXL', 'Extra Extra Large'))) Shirt.drop_collection() @@ -1633,12 +1635,15 @@ class FieldTest(unittest.TestCase): Shirt.drop_collection() def test_choices_get_field_display(self): - """Test dynamic helper for returning the display value of a choices field. + """Test dynamic helper for returning the display value of a choices + field. """ class Shirt(Document): - size = StringField(max_length=3, choices=(('S', 'Small'), ('M', 'Medium'), ('L', 'Large'), - ('XL', 'Extra Large'), ('XXL', 'Extra Extra Large'))) - style = StringField(max_length=3, choices=(('S', 'Small'), ('B', 'Baggy'), ('W', 'wide')), default='S') + size = StringField(max_length=3, choices=( + ('S', 'Small'), ('M', 'Medium'), ('L', 'Large'), + ('XL', 'Extra Large'), ('XXL', 'Extra Extra Large'))) + style = StringField(max_length=3, choices=( + ('S', 'Small'), ('B', 'Baggy'), ('W', 'wide')), default='S') Shirt.drop_collection() @@ -1665,7 +1670,8 @@ class FieldTest(unittest.TestCase): """Ensure that value is in a container of allowed values. """ class Shirt(Document): - size = StringField(max_length=3, choices=('S', 'M', 'L', 'XL', 'XXL')) + size = StringField(max_length=3, + choices=('S', 'M', 'L', 'XL', 'XXL')) Shirt.drop_collection() @@ -1681,11 +1687,15 @@ class FieldTest(unittest.TestCase): Shirt.drop_collection() def test_simple_choices_get_field_display(self): - """Test dynamic helper for returning the display value of a choices field. + """Test dynamic helper for returning the display value of a choices + field. """ class Shirt(Document): - size = StringField(max_length=3, choices=('S', 'M', 'L', 'XL', 'XXL')) - style = StringField(max_length=3, choices=('Small', 'Baggy', 'wide'), default='Small') + size = StringField(max_length=3, + choices=('S', 'M', 'L', 'XL', 'XXL')) + style = StringField(max_length=3, + choices=('Small', 'Baggy', 'wide'), + default='Small') Shirt.drop_collection() @@ -1736,7 +1746,7 @@ class FieldTest(unittest.TestCase): self.assertTrue(putfile == result) self.assertEqual(result.the_file.read(), text) self.assertEqual(result.the_file.content_type, content_type) - result.the_file.delete() # Remove file from GridFS + result.the_file.delete() # Remove file from GridFS PutFile.objects.delete() # Ensure file-like objects are stored @@ -1801,7 +1811,6 @@ class FieldTest(unittest.TestCase): the_file = FileField() DemoFile.objects.create() - def test_file_field_no_default(self): class GridDocument(Document): @@ -1817,7 +1826,6 @@ class FieldTest(unittest.TestCase): doc_a = GridDocument() doc_a.save() - doc_b = GridDocument.objects.with_id(doc_a.id) doc_b.the_file.replace(f, filename='doc_b') doc_b.save() @@ -1859,7 +1867,7 @@ class FieldTest(unittest.TestCase): # Second instance test_file_dupe = TestFile() - data = test_file_dupe.the_file.read() # Should be None + data = test_file_dupe.the_file.read() # Should be None self.assertTrue(test_file.name != test_file_dupe.name) self.assertTrue(test_file.the_file.read() != data) @@ -2328,7 +2336,6 @@ class FieldTest(unittest.TestCase): self.assertEqual(error_dict['comments'][1]['content'], u'Field is required') - post.comments[1].content = 'here we go' post.validate() diff --git a/tests/test_queryset.py b/tests/test_queryset.py index 09b6b3ff..9dfe9a27 100644 --- a/tests/test_queryset.py +++ b/tests/test_queryset.py @@ -1,7 +1,10 @@ from __future__ import with_statement import sys sys.path[0:0] = [""] + import unittest +import uuid +from nose.plugins.skip import SkipTest from datetime import datetime, timedelta @@ -74,7 +77,6 @@ class QuerySetTest(unittest.TestCase): def test_generic_reference(): list(BlogPost.objects(author2__name="test")) - def test_find(self): """Ensure that a query returns a valid set of results. """ @@ -3672,6 +3674,72 @@ class QueryFieldListTest(unittest.TestCase): self.assertRaises(ConfigurationError, Bar.objects, read_preference='Primary') + def test_json_simple(self): + + class Embedded(EmbeddedDocument): + string = StringField() + + class Doc(Document): + string = StringField() + embedded_field = EmbeddedDocumentField(Embedded) + + Doc.drop_collection() + Doc(string="Hi", embedded_field=Embedded(string="Hi")).save() + Doc(string="Bye", embedded_field=Embedded(string="Bye")).save() + + Doc().save() + json_data = Doc.objects.to_json() + doc_objects = list(Doc.objects) + + self.assertEqual(doc_objects, Doc.objects.from_json(json_data)) + + def test_json_complex(self): + if pymongo.version_tuple[0] <= 2 and pymongo.version_tuple[1] <= 3: + raise SkipTest("Need pymongo 2.4 as has a fix for DBRefs") + + class EmbeddedDoc(EmbeddedDocument): + pass + + class Simple(Document): + pass + + class Doc(Document): + string_field = StringField(default='1') + int_field = IntField(default=1) + float_field = FloatField(default=1.1) + boolean_field = BooleanField(default=True) + datetime_field = DateTimeField(default=datetime.now) + embedded_document_field = EmbeddedDocumentField(EmbeddedDoc, + default=lambda: EmbeddedDoc()) + list_field = ListField(default=lambda: [1, 2, 3]) + dict_field = DictField(default=lambda: {"hello": "world"}) + objectid_field = ObjectIdField(default=ObjectId) + reference_field = ReferenceField(Simple, default=lambda: + Simple().save()) + map_field = MapField(IntField(), default=lambda: {"simple": 1}) + decimal_field = DecimalField(default=1.0) + complex_datetime_field = ComplexDateTimeField(default=datetime.now) + url_field = URLField(default="http://mongoengine.org") + dynamic_field = DynamicField(default=1) + generic_reference_field = GenericReferenceField( + default=lambda: Simple().save()) + sorted_list_field = SortedListField(IntField(), + default=lambda: [1, 2, 3]) + email_field = EmailField(default="ross@example.com") + geo_point_field = GeoPointField(default=lambda: [1, 2]) + sequence_field = SequenceField() + uuid_field = UUIDField(default=uuid.uuid4) + generic_embedded_document_field = GenericEmbeddedDocumentField( + default=lambda: EmbeddedDoc()) + + Simple.drop_collection() + Doc.drop_collection() + + Doc().save() + json_data = Doc.objects.to_json() + doc_objects = list(Doc.objects) + + self.assertEqual(doc_objects, Doc.objects.from_json(json_data)) if __name__ == '__main__': From 363e50abbe1db318472de82ad583c98cef3e61c3 Mon Sep 17 00:00:00 2001 From: Ross Lawley Date: Thu, 8 Nov 2012 14:46:56 +0000 Subject: [PATCH 021/189] Updated documents with embedded documents can be created in a single operation (MongoEngine/mongoengine#6) --- docs/changelog.rst | 1 + mongoengine/base/document.py | 18 ++++++++++++++++-- mongoengine/common.py | 5 +++-- tests/document/instance.py | 36 ++++++++++++++++++++++++++++++++++++ 4 files changed, 56 insertions(+), 4 deletions(-) diff --git a/docs/changelog.rst b/docs/changelog.rst index 26108b5d..778a047f 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -4,6 +4,7 @@ Changelog Changes in 0.8 ============== +- Added support for creating documents with embedded documents in a single operation (MongoEngine/mongoengine#6) - Added to_json and from_json to Document (MongoEngine/mongoengine#1) - Added to_json and from_json to QuerySet (MongoEngine/mongoengine#131) - Updated index creation now tied to Document class (MongoEngine/mongoengine#102) diff --git a/mongoengine/base/document.py b/mongoengine/base/document.py index 939c9fbc..2dd4b03b 100644 --- a/mongoengine/base/document.py +++ b/mongoengine/base/document.py @@ -28,7 +28,14 @@ class BaseDocument(object): _dynamic_lock = True _initialised = False - def __init__(self, **values): + def __init__(self, __auto_convert=True, **values): + """ + Initialise a document or embedded document + + :param __auto_convert: Try and will cast python objects to Object types + :param values: A dictionary of values for the document + """ + signals.pre_init.send(self.__class__, document=self, values=values) self._data = {} @@ -50,9 +57,16 @@ class BaseDocument(object): elif self._dynamic: dynamic_data[key] = value else: + FileField = _import_class('FileField') for key, value in values.iteritems(): key = self._reverse_db_field_map.get(key, key) + if (value is not None and __auto_convert and + key in self._fields): + field = self._fields.get(key) + if not isinstance(field, FileField): + value = field.to_python(value) setattr(self, key, value) + # Set any get_fieldname_display methods self.__set_field_display() @@ -487,7 +501,7 @@ class BaseDocument(object): % (cls._class_name, errors)) raise InvalidDocumentError(msg) - obj = cls(**data) + obj = cls(__auto_convert=False, **data) obj._changed_fields = changed_fields obj._created = False return obj diff --git a/mongoengine/common.py b/mongoengine/common.py index c76801ce..a8422c09 100644 --- a/mongoengine/common.py +++ b/mongoengine/common.py @@ -9,8 +9,9 @@ def _import_class(cls_name): doc_classes = ('Document', 'DynamicEmbeddedDocument', 'EmbeddedDocument', 'MapReduceDocument') field_classes = ('DictField', 'DynamicField', 'EmbeddedDocumentField', - 'GenericReferenceField', 'GenericEmbeddedDocumentField', - 'GeoPointField', 'ReferenceField', 'StringField') + 'FileField', 'GenericReferenceField', + 'GenericEmbeddedDocumentField', 'GeoPointField', + 'ReferenceField', 'StringField') queryset_classes = ('OperationError',) deref_classes = ('DeReference',) diff --git a/tests/document/instance.py b/tests/document/instance.py index 2118575e..8fb4fd72 100644 --- a/tests/document/instance.py +++ b/tests/document/instance.py @@ -2005,5 +2005,41 @@ class ValidatorErrorTest(unittest.TestCase): self.assertRaises(OperationError, change_shard_key) + def test_kwargs_simple(self): + + class Embedded(EmbeddedDocument): + name = StringField() + + class Doc(Document): + doc_name = StringField() + doc = EmbeddedDocumentField(Embedded) + + classic_doc = Doc(doc_name="my doc", doc=Embedded(name="embedded doc")) + dict_doc = Doc(**{"doc_name": "my doc", + "doc": {"name": "embedded doc"}}) + + self.assertEqual(classic_doc, dict_doc) + self.assertEqual(classic_doc._data, dict_doc._data) + + def test_kwargs_complex(self): + + class Embedded(EmbeddedDocument): + name = StringField() + + class Doc(Document): + doc_name = StringField() + docs = ListField(EmbeddedDocumentField(Embedded)) + + classic_doc = Doc(doc_name="my doc", docs=[ + Embedded(name="embedded doc1"), + Embedded(name="embedded doc2")]) + dict_doc = Doc(**{"doc_name": "my doc", + "docs": [{"name": "embedded doc1"}, + {"name": "embedded doc2"}]}) + + self.assertEqual(classic_doc, dict_doc) + self.assertEqual(classic_doc._data, dict_doc._data) + + if __name__ == '__main__': unittest.main() From 1a93b9b2263a71f179a35760b84b5d740fdd4f57 Mon Sep 17 00:00:00 2001 From: helduel Date: Thu, 8 Nov 2012 16:30:29 +0100 Subject: [PATCH 022/189] More precise "created" keyword argument signals If a document has a user given id value, the post_save signal always got the "created" keyword argument with False value (unless force_insert is True). This patch uses the result of getlasterror to check whether the save was an update or not. --- mongoengine/document.py | 19 +++++++++++++++---- tests/test_signals.py | 29 +++++++++++++++++++++++++++++ 2 files changed, 44 insertions(+), 4 deletions(-) diff --git a/mongoengine/document.py b/mongoengine/document.py index 7b3afafb..694d1ed4 100644 --- a/mongoengine/document.py +++ b/mongoengine/document.py @@ -233,13 +233,24 @@ class Document(BaseDocument): actual_key = self._db_field_map.get(k, k) select_dict[actual_key] = doc[actual_key] + def is_new_object(last_error): + if last_error is not None: + updated = last_error.get("updatedExisting") + if updated is not None: + return not updated + return created + upsert = self._created if updates: - collection.update(select_dict, {"$set": updates}, - upsert=upsert, safe=safe, **write_options) + last_error = collection.update(select_dict, + {"$set": updates}, upsert=upsert, safe=safe, + **write_options) + created = is_new_object(last_error) if removals: - collection.update(select_dict, {"$unset": removals}, - upsert=upsert, safe=safe, **write_options) + last_error = collection.update(select_dict, + {"$unset": removals}, upsert=upsert, safe=safe, + **write_options) + created = created or is_new_object(last_error) warn_cascade = not cascade and 'cascade' not in self._meta cascade = (self._meta.get('cascade', True) diff --git a/tests/test_signals.py b/tests/test_signals.py index d1199248..2ca820da 100644 --- a/tests/test_signals.py +++ b/tests/test_signals.py @@ -108,6 +108,20 @@ class SignalTests(unittest.TestCase): signal_output.append('post_delete Another signal, %s' % document) self.Another = Another + + class ExplicitId(Document): + id = IntField(primary_key=True) + + @classmethod + def post_save(cls, sender, document, **kwargs): + if 'created' in kwargs: + if kwargs['created']: + signal_output.append('Is created') + else: + signal_output.append('Is updated') + + self.ExplicitId = ExplicitId + self.ExplicitId.objects.delete() # Save up the number of connected signals so that we can check at the end # that all the signals we register get properly unregistered self.pre_signals = ( @@ -137,6 +151,8 @@ class SignalTests(unittest.TestCase): signals.pre_delete.connect(Another.pre_delete, sender=Another) signals.post_delete.connect(Another.post_delete, sender=Another) + signals.post_save.connect(ExplicitId.post_save, sender=ExplicitId) + def tearDown(self): signals.pre_init.disconnect(self.Author.pre_init) signals.post_init.disconnect(self.Author.post_init) @@ -154,6 +170,8 @@ class SignalTests(unittest.TestCase): signals.post_save.disconnect(self.Another.post_save) signals.pre_save.disconnect(self.Another.pre_save) + signals.post_save.disconnect(self.ExplicitId.post_save) + # Check that all our signals got disconnected properly. post_signals = ( len(signals.pre_init.receivers), @@ -166,6 +184,8 @@ class SignalTests(unittest.TestCase): len(signals.post_bulk_insert.receivers), ) + self.ExplicitId.objects.delete() + self.assertEqual(self.pre_signals, post_signals) def test_model_signals(self): @@ -228,3 +248,12 @@ class SignalTests(unittest.TestCase): ]) self.Author.objects.delete() + + def test_signals_with_explicit_doc_ids(self): + """ Model saves must have a created flag the first time.""" + ei = self.ExplicitId(id=123) + # post save must received the created flag, even if there's already + # an object id present + self.assertEqual(self.get_signal_output(ei.save), ['Is created']) + # second time, it must be an update + self.assertEqual(self.get_signal_output(ei.save), ['Is updated']) From f265915aa227f941fa07fae2df85155be9e0f3d1 Mon Sep 17 00:00:00 2001 From: Ross Lawley Date: Thu, 8 Nov 2012 16:35:20 +0000 Subject: [PATCH 023/189] Updated inheritable objects created by upsert now contain _cls (MongoEngine/mongoengine#118) --- docs/changelog.rst | 1 + mongoengine/queryset/queryset.py | 12 ++++++++++-- tests/test_queryset.py | 19 +++++++++++++++++++ 3 files changed, 30 insertions(+), 2 deletions(-) diff --git a/docs/changelog.rst b/docs/changelog.rst index 778a047f..e8d3d574 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -4,6 +4,7 @@ Changelog Changes in 0.8 ============== +- Updated inheritable objects created by upsert now contain _cls (MongoEngine/mongoengine#118) - Added support for creating documents with embedded documents in a single operation (MongoEngine/mongoengine#6) - Added to_json and from_json to Document (MongoEngine/mongoengine#1) - Added to_json and from_json to QuerySet (MongoEngine/mongoengine#131) diff --git a/mongoengine/queryset/queryset.py b/mongoengine/queryset/queryset.py index 3c44f012..bfd15a8c 100644 --- a/mongoengine/queryset/queryset.py +++ b/mongoengine/queryset/queryset.py @@ -824,8 +824,16 @@ class QuerySet(object): if not write_options: write_options = {} - update = transform.update(self._document, **update) query = self._query + update = transform.update(self._document, **update) + + # If doing an atomic upsert on an inheritable class + # then ensure we add _cls to the update operation + if upsert and '_cls' in query: + if '$set' in update: + update["$set"]["_cls"] = self._document._class_name + else: + update["$set"] = {"_cls": self._document._class_name} try: ret = self._collection.update(query, update, multi=multi, @@ -852,7 +860,7 @@ class QuerySet(object): .. versionadded:: 0.2 """ - return self.update(safe_update=True, upsert=False, multi=False, + return self.update(safe_update=True, upsert=upsert, multi=False, write_options=None, **update) def __iter__(self): diff --git a/tests/test_queryset.py b/tests/test_queryset.py index 9dfe9a27..a86920e9 100644 --- a/tests/test_queryset.py +++ b/tests/test_queryset.py @@ -3741,6 +3741,25 @@ class QueryFieldListTest(unittest.TestCase): self.assertEqual(doc_objects, Doc.objects.from_json(json_data)) + def test_upsert_includes_cls(self): + """Upserts should include _cls information for inheritable classes + """ + + class Test(Document): + test = StringField() + + Test.drop_collection() + Test.objects(test='foo').update_one(upsert=True, set__test='foo') + self.assertFalse('_cls' in Test._collection.find_one()) + + class Test(Document): + meta = {'allow_inheritance': True} + test = StringField() + + Test.drop_collection() + + Test.objects(test='foo').update_one(upsert=True, set__test='foo') + self.assertTrue('_cls' in Test._collection.find_one()) if __name__ == '__main__': unittest.main() From dfdc0d92c3f95e9b6788c14c8fb9facaa4bfbfc9 Mon Sep 17 00:00:00 2001 From: Ross Lawley Date: Thu, 8 Nov 2012 16:40:58 +0000 Subject: [PATCH 024/189] Updated docs --- docs/guide/defining-documents.rst | 2 +- docs/guide/document-instances.rst | 2 +- docs/upgrade.rst | 6 +++--- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/docs/guide/defining-documents.rst b/docs/guide/defining-documents.rst index ea8e05b2..9abea9ba 100644 --- a/docs/guide/defining-documents.rst +++ b/docs/guide/defining-documents.rst @@ -597,7 +597,7 @@ Working with existing data As MongoEngine no longer defaults to needing :attr:`_cls` you can quickly and easily get working with existing data. Just define the document to match the expected schema in your database. If you have wildly varying schemas then -a :class:`~mongoengine.DynamicDocument` might be more appropriate. +a :class:`~mongoengine.DynamicDocument` might be more appropriate. :: # Will work with data in an existing collection named 'cmsPage' class Page(Document): diff --git a/docs/guide/document-instances.rst b/docs/guide/document-instances.rst index b3bf687b..e8e7d63c 100644 --- a/docs/guide/document-instances.rst +++ b/docs/guide/document-instances.rst @@ -64,7 +64,7 @@ document values for example:: .. note:: Cleaning is only called if validation is turned on and when calling -:meth:`~mongoengine.Document.save`. + :meth:`~mongoengine.Document.save`. Cascading Saves --------------- diff --git a/docs/upgrade.rst b/docs/upgrade.rst index 44c69beb..bf48527c 100644 --- a/docs/upgrade.rst +++ b/docs/upgrade.rst @@ -14,7 +14,7 @@ Data Model The inheritance model has changed, we no longer need to store an array of :attr:`types` with the model we can just use the classname in :attr:`_cls`. This means that you will have to update your indexes for each of your -inherited classes like so: +inherited classes like so: :: # 1. Declaration of the class class Animal(Document): @@ -49,7 +49,7 @@ Document Definition The default for inheritance has changed - its now off by default and :attr:`_cls` will not be stored automatically with the class. So if you extend your :class:`~mongoengine.Document` or :class:`~mongoengine.EmbeddedDocuments` -you will need to declare :attr:`allow_inheritance` in the meta data like so: +you will need to declare :attr:`allow_inheritance` in the meta data like so: :: class Animal(Document): name = StringField() @@ -67,7 +67,7 @@ They should be replaced with :func:`~mongoengine.Document.ensure_indexes` / SequenceFields -------------- -:class:`~mongoengine.fields.SequenceField`s now inherit from `BaseField` to +:class:`~mongoengine.fields.SequenceField` now inherits from `BaseField` to allow flexible storage of the calculated value. As such MIN and MAX settings are no longer handled. From 28ef54986d0847dac129f5bb4b66e644e4fe947b Mon Sep 17 00:00:00 2001 From: Ross Lawley Date: Wed, 21 Nov 2012 16:53:06 +0000 Subject: [PATCH 025/189] Deprecated `get_or_create` (MongoEngine/mongoengine#35) --- docs/changelog.rst | 1 + mongoengine/queryset/queryset.py | 10 ++++++++-- 2 files changed, 9 insertions(+), 2 deletions(-) diff --git a/docs/changelog.rst b/docs/changelog.rst index e8d3d574..c3c6340f 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -4,6 +4,7 @@ Changelog Changes in 0.8 ============== +- Deprecated `get_or_create` (MongoEngine/mongoengine#35) - Updated inheritable objects created by upsert now contain _cls (MongoEngine/mongoengine#118) - Added support for creating documents with embedded documents in a single operation (MongoEngine/mongoengine#6) - Added to_json and from_json to Document (MongoEngine/mongoengine#1) diff --git a/mongoengine/queryset/queryset.py b/mongoengine/queryset/queryset.py index bfd15a8c..dde7d55e 100644 --- a/mongoengine/queryset/queryset.py +++ b/mongoengine/queryset/queryset.py @@ -232,10 +232,11 @@ class QuerySet(object): dictionary of default values for the new document may be provided as a keyword argument called :attr:`defaults`. - .. warning:: This requires two separate operations and therefore a + .. note:: This requires two separate operations and therefore a race condition exists. Because there are no transactions in mongoDB other approaches should be investigated, to ensure you - don't accidently duplicate data when using this method. + don't accidently duplicate data when using this method. This is + now scheduled to be removed before 1.0 :param write_options: optional extra keyword arguments used if we have to create a new document. @@ -244,9 +245,14 @@ class QuerySet(object): :param auto_save: if the object is to be saved automatically if not found. + .. deprecated:: 0.8 .. versionchanged:: 0.6 - added `auto_save` .. versionadded:: 0.3 """ + msg = ("get_or_create is scheduled to be deprecated. The approach is " + "flawed without transactions. Upserts should be preferred.") + raise DeprecationWarning(msg) + defaults = query.get('defaults', {}) if 'defaults' in query: del query['defaults'] From aa5a9ff1f428027a5eb6adec54fa6f596ae4e32d Mon Sep 17 00:00:00 2001 From: Ross Lawley Date: Wed, 21 Nov 2012 17:03:32 +0000 Subject: [PATCH 026/189] Documentation update for document errors (MongoEngine/mongoengine#124) --- docs/changelog.rst | 1 + docs/guide/querying.rst | 8 +++++--- 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/docs/changelog.rst b/docs/changelog.rst index c3c6340f..a13a5f1d 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -4,6 +4,7 @@ Changelog Changes in 0.8 ============== +- Documentation update for document errors (MongoEngine/mongoengine#124) - Deprecated `get_or_create` (MongoEngine/mongoengine#35) - Updated inheritable objects created by upsert now contain _cls (MongoEngine/mongoengine#118) - Added support for creating documents with embedded documents in a single operation (MongoEngine/mongoengine#6) diff --git a/docs/guide/querying.rst b/docs/guide/querying.rst index 14498017..d5829432 100644 --- a/docs/guide/querying.rst +++ b/docs/guide/querying.rst @@ -179,9 +179,11 @@ Retrieving unique results ------------------------- To retrieve a result that should be unique in the collection, use :meth:`~mongoengine.queryset.QuerySet.get`. This will raise -:class:`~mongoengine.queryset.DoesNotExist` if no document matches the query, -and :class:`~mongoengine.queryset.MultipleObjectsReturned` if more than one -document matched the query. +:class:`~mongoengine.queryset.DoesNotExist` if +no document matches the query, and +:class:`~mongoengine.queryset.MultipleObjectsReturned` +if more than one document matched the query. These exceptions are merged into +your document defintions eg: `MyDoc.DoesNotExist` A variation of this method exists, :meth:`~mongoengine.queryset.Queryset.get_or_create`, that will create a new From 003454573ce86614344e11e6252d9accaa392101 Mon Sep 17 00:00:00 2001 From: Ross Lawley Date: Wed, 21 Nov 2012 17:14:53 +0000 Subject: [PATCH 027/189] Making django user sparse (MongoEngine/mongoengine#128) --- mongoengine/django/auth.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mongoengine/django/auth.py b/mongoengine/django/auth.py index 65afacfd..1685f6f0 100644 --- a/mongoengine/django/auth.py +++ b/mongoengine/django/auth.py @@ -70,7 +70,7 @@ class User(Document): meta = { 'allow_inheritance': True, 'indexes': [ - {'fields': ['username'], 'unique': True} + {'fields': ['username'], 'unique': True, 'sparse': True} ] } From 027b3d36de9a42d31757353150dd39a94a6da584 Mon Sep 17 00:00:00 2001 From: Ross Lawley Date: Tue, 27 Nov 2012 14:01:58 +0000 Subject: [PATCH 028/189] Fixed deprecation warning --- mongoengine/queryset/queryset.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mongoengine/queryset/queryset.py b/mongoengine/queryset/queryset.py index dde7d55e..3acee36e 100644 --- a/mongoengine/queryset/queryset.py +++ b/mongoengine/queryset/queryset.py @@ -251,7 +251,7 @@ class QuerySet(object): """ msg = ("get_or_create is scheduled to be deprecated. The approach is " "flawed without transactions. Upserts should be preferred.") - raise DeprecationWarning(msg) + warnings.warn(msg, DeprecationWarning) defaults = query.get('defaults', {}) if 'defaults' in query: From b5e868655e5ff056150b358510f8aea4a8125881 Mon Sep 17 00:00:00 2001 From: Ross Lawley Date: Tue, 27 Nov 2012 14:02:49 +0000 Subject: [PATCH 029/189] Updated travis.yml --- .travis.yml | 2 +- tests/test_dereference.py | 6 ++++++ 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index c10a1f36..1aa97746 100644 --- a/.travis.yml +++ b/.travis.yml @@ -26,4 +26,4 @@ notifications: branches: only: - master - - 0.7 \ No newline at end of file + - 0.8 \ No newline at end of file diff --git a/tests/test_dereference.py b/tests/test_dereference.py index c9631ebb..8f617924 100644 --- a/tests/test_dereference.py +++ b/tests/test_dereference.py @@ -42,6 +42,12 @@ class FieldTest(unittest.TestCase): group_obj = Group.objects.first() self.assertEqual(q, 1) + len(group_obj._data['members']) + self.assertEqual(q, 1) + + len(group_obj.members) + self.assertEqual(q, 2) + [m for m in group_obj.members] self.assertEqual(q, 2) From 59e7617e82a89a0afe61f24180f8873efbb316bf Mon Sep 17 00:00:00 2001 From: Ross Lawley Date: Tue, 27 Nov 2012 14:02:49 +0000 Subject: [PATCH 030/189] Trying to fix seesaw test on travis --- tests/all_warnings/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/all_warnings/__init__.py b/tests/all_warnings/__init__.py index 4609c5a5..7d4db0d6 100644 --- a/tests/all_warnings/__init__.py +++ b/tests/all_warnings/__init__.py @@ -45,7 +45,7 @@ class AllWarnings(unittest.TestCase): p2.parent = p1 p2.save(cascade=False) - self.assertEqual(len(self.warning_list), 1) + self.assertTrue(len(self.warning_list) > 0) warning = self.warning_list[0] self.assertEqual(FutureWarning, warning["category"]) self.assertTrue("ReferenceFields will default to using ObjectId" From b849c719a8c336d388a5a341e2ffb9afd9bf86eb Mon Sep 17 00:00:00 2001 From: Ross Lawley Date: Tue, 27 Nov 2012 14:02:50 +0000 Subject: [PATCH 031/189] Adding some debugging --- tests/all_warnings/__init__.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tests/all_warnings/__init__.py b/tests/all_warnings/__init__.py index 7d4db0d6..220b0bbf 100644 --- a/tests/all_warnings/__init__.py +++ b/tests/all_warnings/__init__.py @@ -69,6 +69,8 @@ class AllWarnings(unittest.TestCase): p2.save() self.assertEqual(len(self.warning_list), 1) + if len(self.warning_list) > 1: + print self.warning_list warning = self.warning_list[0] self.assertEqual(FutureWarning, warning["category"]) self.assertTrue("Cascading saves will default to off in 0.8" From 68e4a27aaff24ca60b851da71f5c4aff45e87341 Mon Sep 17 00:00:00 2001 From: Ross Lawley Date: Tue, 27 Nov 2012 14:02:50 +0000 Subject: [PATCH 032/189] Fixed handling for old style types --- docs/changelog.rst | 4 ++++ python-mongoengine.spec | 2 +- 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/docs/changelog.rst b/docs/changelog.rst index a13a5f1d..756b1cd6 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -19,6 +19,10 @@ Changes in 0.8 - Inheritance is off by default (MongoEngine/mongoengine#122) - Remove _types and just use _cls for inheritance (MongoEngine/mongoengine#148) +Changes in 0.7.7 +================ +- Fix handling for old style _types + Changes in 0.7.6 ================ - Unicode fix for repr (MongoEngine/mongoengine#133) diff --git a/python-mongoengine.spec b/python-mongoengine.spec index d796f993..9a376ec7 100644 --- a/python-mongoengine.spec +++ b/python-mongoengine.spec @@ -5,7 +5,7 @@ %define srcname mongoengine Name: python-%{srcname} -Version: 0.7.6 +Version: 0.7.7 Release: 1%{?dist} Summary: A Python Document-Object Mapper for working with MongoDB From f9dd051ec90c9fc17d2cc1221d43e6549427b915 Mon Sep 17 00:00:00 2001 From: Ross Lawley Date: Tue, 27 Nov 2012 14:02:50 +0000 Subject: [PATCH 033/189] Merged get_document fix --- mongoengine/base/common.py | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/mongoengine/base/common.py b/mongoengine/base/common.py index 82728d1e..dc43d405 100644 --- a/mongoengine/base/common.py +++ b/mongoengine/base/common.py @@ -9,11 +9,10 @@ _document_registry = {} def get_document(name): doc = _document_registry.get(name, None) - if not doc: - # Possible old style names - end = ".%s" % name - possible_match = [k for k in _document_registry.keys() - if k.endswith(end)] + if not doc and '.' in name: + # Possible old style name + end = name.split('.')[-1] + possible_match = [k for k in _document_registry.keys() if k == end] if len(possible_match) == 1: doc = _document_registry.get(possible_match.pop(), None) if not doc: From 3598fe0fb45d969ef936fedc33db4a63371266a8 Mon Sep 17 00:00:00 2001 From: Ross Lawley Date: Tue, 27 Nov 2012 14:02:50 +0000 Subject: [PATCH 034/189] Adding _collection to _cls --- mongoengine/base/metaclasses.py | 3 +++ tests/document/instance.py | 25 +++++++++++++++++++++++++ 2 files changed, 28 insertions(+) diff --git a/mongoengine/base/metaclasses.py b/mongoengine/base/metaclasses.py index e68ec13d..c6c4db1f 100644 --- a/mongoengine/base/metaclasses.py +++ b/mongoengine/base/metaclasses.py @@ -169,6 +169,9 @@ class DocumentMetaclass(type): "field name" % field.name) raise InvalidDocumentError(msg) + if issubclass(new_class, Document): + new_class._collection = None + # Add class to the _document_registry _document_registry[new_class._class_name] = new_class diff --git a/tests/document/instance.py b/tests/document/instance.py index 8fb4fd72..de677e21 100644 --- a/tests/document/instance.py +++ b/tests/document/instance.py @@ -1762,6 +1762,31 @@ class InstanceTest(unittest.TestCase): self.assertEqual(Book._get_collection(), get_db("testdb-2")[Book._get_collection_name()]) self.assertEqual(AuthorBooks._get_collection(), get_db("testdb-3")[AuthorBooks._get_collection_name()]) + def test_db_alias_overrides(self): + """db_alias can be overriden + """ + # Register a connection with db_alias testdb-2 + register_connection('testdb-2', 'mongoenginetest2') + + class A(Document): + """Uses default db_alias + """ + name = StringField() + meta = {"allow_inheritance": True} + + class B(A): + """Uses testdb-2 db_alias + """ + meta = {"db_alias": "testdb-2"} + + A.objects.all() + + self.assertEquals('testdb-2', B._meta.get('db_alias')) + self.assertEquals('mongoenginetest', + A._get_collection().database.name) + self.assertEquals('mongoenginetest2', + B._get_collection().database.name) + def test_db_alias_propagates(self): """db_alias propagates? """ From 219b28c97b846b90e0d755558be526bb51773a92 Mon Sep 17 00:00:00 2001 From: Ross Lawley Date: Tue, 27 Nov 2012 14:04:57 +0000 Subject: [PATCH 035/189] Updated docs regarding 3598fe0fb45d969ef936fedc33db4a63371266a8 Fixed db_alias and inherited Documents (MongoEngine/mongoengine#143) --- AUTHORS | 1 + docs/changelog.rst | 1 + 2 files changed, 2 insertions(+) diff --git a/AUTHORS b/AUTHORS index 6ba2f88d..5801981a 100644 --- a/AUTHORS +++ b/AUTHORS @@ -124,3 +124,4 @@ that much better: * Stefan Wójcik * dimonb * Garry Polley + * James Slagle diff --git a/docs/changelog.rst b/docs/changelog.rst index 756b1cd6..a56f33ac 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -4,6 +4,7 @@ Changelog Changes in 0.8 ============== +- Fixed db_alias and inherited Documents (MongoEngine/mongoengine#143) - Documentation update for document errors (MongoEngine/mongoengine#124) - Deprecated `get_or_create` (MongoEngine/mongoengine#35) - Updated inheritable objects created by upsert now contain _cls (MongoEngine/mongoengine#118) From f6f7c12f0ecadb7ae50f29287622b9a42507dacb Mon Sep 17 00:00:00 2001 From: Ross Lawley Date: Tue, 27 Nov 2012 14:37:13 +0000 Subject: [PATCH 036/189] Added test case checking type with dbref=False Ensures when dbref=False the data is stored as the same type as the primary key of the item stored. MongoEngine/mongoengine#160 --- tests/test_dereference.py | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) diff --git a/tests/test_dereference.py b/tests/test_dereference.py index 8f617924..41f8aebf 100644 --- a/tests/test_dereference.py +++ b/tests/test_dereference.py @@ -123,6 +123,27 @@ class FieldTest(unittest.TestCase): User.drop_collection() Group.drop_collection() + def test_list_item_dereference_dref_false_stores_as_type(self): + """Ensure that DBRef items are stored as their type + """ + class User(Document): + my_id = IntField(primary_key=True) + name = StringField() + + class Group(Document): + members = ListField(ReferenceField(User, dbref=False)) + + User.drop_collection() + Group.drop_collection() + + user = User(my_id=1, name='user 1').save() + + Group(members=User.objects).save() + group = Group.objects.first() + + self.assertEqual(Group._get_collection().find_one()['members'], [1]) + self.assertEqual(group.members, [user]) + def test_handle_old_style_references(self): """Ensure that DBRef items in ListFields are dereferenced. """ From 148f8b8a3a2f53b2e086b48b2294b537224c4d33 Mon Sep 17 00:00:00 2001 From: Stefan Wojcik Date: Mon, 17 Dec 2012 21:13:45 -0800 Subject: [PATCH 037/189] Only allow QNode instances to be passed as query objects --- mongoengine/queryset.py | 3 +++ tests/test_queryset.py | 8 ++++++++ 2 files changed, 11 insertions(+) diff --git a/mongoengine/queryset.py b/mongoengine/queryset.py index 5a1aa718..e987da91 100644 --- a/mongoengine/queryset.py +++ b/mongoengine/queryset.py @@ -423,6 +423,9 @@ class QuerySet(object): """ query = Q(**query) if q_obj: + # make sure proper query object is passed + if not isinstance(q_obj, QNode): + raise InvalidQueryError('Not a query object: %s. Did you intend to use key=value?' % q_obj) query &= q_obj self._query_obj &= query self._mongo_query = None diff --git a/tests/test_queryset.py b/tests/test_queryset.py index 55531a1e..c49566cf 100644 --- a/tests/test_queryset.py +++ b/tests/test_queryset.py @@ -1289,6 +1289,14 @@ class QuerySetTest(unittest.TestCase): self.assertEqual(len(self.Person.objects(Q(age__in=[20]))), 2) self.assertEqual(len(self.Person.objects(Q(age__in=[20, 30]))), 3) + # Test invalid query objs + def wrong_query_objs(): + self.Person.objects('user1') + def wrong_query_objs_filter(): + self.Person.objects('user1') + self.assertRaises(InvalidQueryError, wrong_query_objs) + self.assertRaises(InvalidQueryError, wrong_query_objs_filter) + def test_q_regex(self): """Ensure that Q objects can be queried using regexes. """ From 3e8f02c64babd024c39a913a946f3b6e3dc6fd75 Mon Sep 17 00:00:00 2001 From: Ross Lawley Date: Wed, 19 Dec 2012 11:39:19 +0000 Subject: [PATCH 038/189] Merge sequence field changes --- mongoengine/fields.py | 3 +-- tests/test_fields.py | 1 - 2 files changed, 1 insertion(+), 3 deletions(-) diff --git a/mongoengine/fields.py b/mongoengine/fields.py index 65996a4b..fe94d351 100644 --- a/mongoengine/fields.py +++ b/mongoengine/fields.py @@ -1368,8 +1368,7 @@ class SequenceField(BaseField): """ Generate and Increment the counter """ - sequence_name = (self.sequence_name or - self.owner_document._get_collection_name()) + sequence_name = self.get_sequence_name() sequence_id = "%s.%s" % (sequence_name, self.name) collection = get_db(alias=self.db_alias)[self.collection_name] counter = collection.find_and_modify(query={"_id": sequence_id}, diff --git a/tests/test_fields.py b/tests/test_fields.py index 6e3dc8b8..97a2d5fd 100644 --- a/tests/test_fields.py +++ b/tests/test_fields.py @@ -2241,7 +2241,6 @@ class FieldTest(unittest.TestCase): Post(title="MongoEngine", comments=[Comment(content="NoSQL Rocks"), Comment(content="MongoEngine Rocks")]).save() - import ipdb; ipdb.set_trace(); c = self.db['mongoengine.counters'].find_one({'_id': 'comment.id'}) self.assertEqual(c['next'], 2) post = Post.objects.first() From 1a131ff1207cc116e5b90bcf3f70268dd6f0061f Mon Sep 17 00:00:00 2001 From: Ross Lawley Date: Wed, 19 Dec 2012 12:16:12 +0000 Subject: [PATCH 039/189] Only allow QNode instances to be passed as query objects (MongoEngine/mongoengine#199) --- AUTHORS | 1 + docs/changelog.rst | 1 + 2 files changed, 2 insertions(+) diff --git a/AUTHORS b/AUTHORS index 23afe6f6..2519c89d 100644 --- a/AUTHORS +++ b/AUTHORS @@ -129,3 +129,4 @@ that much better: * Peter Teichman * Jakub Kot * Jorge Bastida + * Stefan Wójcik \ No newline at end of file diff --git a/docs/changelog.rst b/docs/changelog.rst index 109940ea..845f3a9e 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -19,6 +19,7 @@ Changes in 0.8 - Added _instance to EmbeddedDocuments pointing to the parent (MongoEngine/mongoengine#139) - Inheritance is off by default (MongoEngine/mongoengine#122) - Remove _types and just use _cls for inheritance (MongoEngine/mongoengine#148) +- Only allow QNode instances to be passed as query objects (MongoEngine/mongoengine#199) Changes in 0.7.9 ================ From c528ac09d6ab25fdb2bf7972d51da42995a464e2 Mon Sep 17 00:00:00 2001 From: Ross Lawley Date: Wed, 19 Dec 2012 12:29:46 +0000 Subject: [PATCH 040/189] Fix merge for QNode checks --- mongoengine/queryset/queryset.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mongoengine/queryset/queryset.py b/mongoengine/queryset/queryset.py index 24836eb5..fda8a755 100644 --- a/mongoengine/queryset/queryset.py +++ b/mongoengine/queryset/queryset.py @@ -17,7 +17,7 @@ from mongoengine.errors import (OperationError, NotUniqueError, from . import transform from .field_list import QueryFieldList -from .visitor import Q +from .visitor import Q, QNode __all__ = ('QuerySet', 'DO_NOTHING', 'NULLIFY', 'CASCADE', 'DENY', 'PULL') From 9cc02d4dbe27b6a8b12f958b4fae58573264dde0 Mon Sep 17 00:00:00 2001 From: Ross Lawley Date: Wed, 19 Dec 2012 12:32:06 +0000 Subject: [PATCH 041/189] Dynamic fields are now validated on save (MongoEngine/mongoengine#153) (MongoEngine/mongoengine#154) --- AUTHORS | 3 ++- docs/changelog.rst | 1 + mongoengine/base/document.py | 3 +++ mongoengine/fields.py | 4 ++++ tests/document/dynamic.py | 23 +++++++++++++++++++++++ 5 files changed, 33 insertions(+), 1 deletion(-) diff --git a/AUTHORS b/AUTHORS index 2519c89d..794f297d 100644 --- a/AUTHORS +++ b/AUTHORS @@ -129,4 +129,5 @@ that much better: * Peter Teichman * Jakub Kot * Jorge Bastida - * Stefan Wójcik \ No newline at end of file + * Stefan Wójcik + * Pete Campton \ No newline at end of file diff --git a/docs/changelog.rst b/docs/changelog.rst index 845f3a9e..fa0fe100 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -20,6 +20,7 @@ Changes in 0.8 - Inheritance is off by default (MongoEngine/mongoengine#122) - Remove _types and just use _cls for inheritance (MongoEngine/mongoengine#148) - Only allow QNode instances to be passed as query objects (MongoEngine/mongoengine#199) +- Dynamic fields are now validated on save (MongoEngine/mongoengine#153) (MongoEngine/mongoengine#154) Changes in 0.7.9 ================ diff --git a/mongoengine/base/document.py b/mongoengine/base/document.py index 2dd4b03b..a3f10f52 100644 --- a/mongoengine/base/document.py +++ b/mongoengine/base/document.py @@ -245,6 +245,9 @@ class BaseDocument(object): # Get a list of tuples of field names and their current values fields = [(field, self._data.get(name)) for name, field in self._fields.items()] + if self._dynamic: + fields += [(field, self._data.get(name)) + for name, field in self._dynamic_fields.items()] EmbeddedDocumentField = _import_class("EmbeddedDocumentField") GenericEmbeddedDocumentField = _import_class("GenericEmbeddedDocumentField") diff --git a/mongoengine/fields.py b/mongoengine/fields.py index fe94d351..73c0db4d 100644 --- a/mongoengine/fields.py +++ b/mongoengine/fields.py @@ -564,6 +564,10 @@ class DynamicField(BaseField): return StringField().prepare_query_value(op, value) return self.to_mongo(value) + def validate(self, value, clean=True): + if hasattr(value, "validate"): + value.validate(clean=clean) + class ListField(ComplexBaseField): """A list field that wraps a standard field, allowing multiple instances diff --git a/tests/document/dynamic.py b/tests/document/dynamic.py index d879b54c..ca0db0a3 100644 --- a/tests/document/dynamic.py +++ b/tests/document/dynamic.py @@ -122,6 +122,29 @@ class DynamicTest(unittest.TestCase): self.assertEqual(1, self.Person.objects(misc__hello='world').count()) + def test_complex_embedded_document_validation(self): + """Ensure embedded dynamic documents may be validated""" + class Embedded(DynamicEmbeddedDocument): + content = URLField() + + class Doc(DynamicDocument): + pass + + Doc.drop_collection() + doc = Doc() + + embedded_doc_1 = Embedded(content='http://mongoengine.org') + embedded_doc_1.validate() + + embedded_doc_2 = Embedded(content='this is not a url') + with self.assertRaises(ValidationError): + embedded_doc_2.validate() + + doc.embedded_field_1 = embedded_doc_1 + doc.embedded_field_2 = embedded_doc_2 + with self.assertRaises(ValidationError): + doc.validate() + def test_inheritance(self): """Ensure that dynamic document plays nice with inheritance""" class Employee(self.Person): From 7f732459a130d2c0a870c1803c9694002f325e17 Mon Sep 17 00:00:00 2001 From: Ross Lawley Date: Wed, 19 Dec 2012 12:34:02 +0000 Subject: [PATCH 042/189] Updated tickets links as now default to MongoEngine/mongoengine --- docs/changelog.rst | 104 ++++++++++++++++++++++----------------------- 1 file changed, 52 insertions(+), 52 deletions(-) diff --git a/docs/changelog.rst b/docs/changelog.rst index fa0fe100..8fc279e7 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -4,23 +4,23 @@ Changelog Changes in 0.8 ============== -- Fixed db_alias and inherited Documents (MongoEngine/mongoengine#143) -- Documentation update for document errors (MongoEngine/mongoengine#124) -- Deprecated `get_or_create` (MongoEngine/mongoengine#35) -- Updated inheritable objects created by upsert now contain _cls (MongoEngine/mongoengine#118) -- Added support for creating documents with embedded documents in a single operation (MongoEngine/mongoengine#6) -- Added to_json and from_json to Document (MongoEngine/mongoengine#1) -- Added to_json and from_json to QuerySet (MongoEngine/mongoengine#131) -- Updated index creation now tied to Document class (MongoEngine/mongoengine#102) -- Added none() to queryset (MongoEngine/mongoengine#127) -- Updated SequenceFields to allow post processing of the calculated counter value (MongoEngine/mongoengine#141) -- Added clean method to documents for pre validation data cleaning (MongoEngine/mongoengine#60) -- Added support setting for read prefrence at a query level (MongoEngine/mongoengine#157) -- Added _instance to EmbeddedDocuments pointing to the parent (MongoEngine/mongoengine#139) -- Inheritance is off by default (MongoEngine/mongoengine#122) -- Remove _types and just use _cls for inheritance (MongoEngine/mongoengine#148) -- Only allow QNode instances to be passed as query objects (MongoEngine/mongoengine#199) -- Dynamic fields are now validated on save (MongoEngine/mongoengine#153) (MongoEngine/mongoengine#154) +- Fixed db_alias and inherited Documents (#143) +- Documentation update for document errors (#124) +- Deprecated `get_or_create` (#35) +- Updated inheritable objects created by upsert now contain _cls (#118) +- Added support for creating documents with embedded documents in a single operation (#6) +- Added to_json and from_json to Document (#1) +- Added to_json and from_json to QuerySet (#131) +- Updated index creation now tied to Document class (#102) +- Added none() to queryset (#127) +- Updated SequenceFields to allow post processing of the calculated counter value (#141) +- Added clean method to documents for pre validation data cleaning (#60) +- Added support setting for read prefrence at a query level (#157) +- Added _instance to EmbeddedDocuments pointing to the parent (#139) +- Inheritance is off by default (#122) +- Remove _types and just use _cls for inheritance (#148) +- Only allow QNode instances to be passed as query objects (#199) +- Dynamic fields are now validated on save (#153) (#154) Changes in 0.7.9 ================ @@ -29,12 +29,12 @@ Changes in 0.7.9 Changes in 0.7.8 ================ -- Fix sequence fields in embedded documents (MongoEngine/mongoengine#166) -- Fix query chaining with .order_by() (MongoEngine/mongoengine#176) -- Added optional encoding and collection config for Django sessions (MongoEngine/mongoengine#180, MongoEngine/mongoengine#181, MongoEngine/mongoengine#183) -- Fixed EmailField so can add extra validation (MongoEngine/mongoengine#173, MongoEngine/mongoengine#174, MongoEngine/mongoengine#187) -- Fixed bulk inserts can now handle custom pk's (MongoEngine/mongoengine#192) -- Added as_pymongo method to return raw or cast results from pymongo (MongoEngine/mongoengine#193) +- Fix sequence fields in embedded documents (#166) +- Fix query chaining with .order_by() (#176) +- Added optional encoding and collection config for Django sessions (#180, #181, #183) +- Fixed EmailField so can add extra validation (#173, #174, #187) +- Fixed bulk inserts can now handle custom pk's (#192) +- Added as_pymongo method to return raw or cast results from pymongo (#193) Changes in 0.7.7 ================ @@ -42,70 +42,70 @@ Changes in 0.7.7 Changes in 0.7.6 ================ -- Unicode fix for repr (MongoEngine/mongoengine#133) -- Allow updates with match operators (MongoEngine/mongoengine#144) -- Updated URLField - now can have a override the regex (MongoEngine/mongoengine#136) +- Unicode fix for repr (#133) +- Allow updates with match operators (#144) +- Updated URLField - now can have a override the regex (#136) - Allow Django AuthenticationBackends to work with Django user (hmarr/mongoengine#573) -- Fixed reload issue with ReferenceField where dbref=False (MongoEngine/mongoengine#138) +- Fixed reload issue with ReferenceField where dbref=False (#138) Changes in 0.7.5 ================ -- ReferenceFields with dbref=False use ObjectId instead of strings (MongoEngine/mongoengine#134) - See ticket for upgrade notes (https://github.com/MongoEngine/mongoengine/issues/134) +- ReferenceFields with dbref=False use ObjectId instead of strings (#134) + See ticket for upgrade notes (#134) Changes in 0.7.4 ================ -- Fixed index inheritance issues - firmed up testcases (MongoEngine/mongoengine#123) (MongoEngine/mongoengine#125) +- Fixed index inheritance issues - firmed up testcases (#123) (#125) Changes in 0.7.3 ================ -- Reverted EmbeddedDocuments meta handling - now can turn off inheritance (MongoEngine/mongoengine#119) +- Reverted EmbeddedDocuments meta handling - now can turn off inheritance (#119) Changes in 0.7.2 ================ -- Update index spec generation so its not destructive (MongoEngine/mongoengine#113) +- Update index spec generation so its not destructive (#113) Changes in 0.7.1 ================= -- Fixed index spec inheritance (MongoEngine/mongoengine#111) +- Fixed index spec inheritance (#111) Changes in 0.7.0 ================= -- Updated queryset.delete so you can use with skip / limit (MongoEngine/mongoengine#107) -- Updated index creation allows kwargs to be passed through refs (MongoEngine/mongoengine#104) -- Fixed Q object merge edge case (MongoEngine/mongoengine#109) +- Updated queryset.delete so you can use with skip / limit (#107) +- Updated index creation allows kwargs to be passed through refs (#104) +- Fixed Q object merge edge case (#109) - Fixed reloading on sharded documents (hmarr/mongoengine#569) -- Added NotUniqueError for duplicate keys (MongoEngine/mongoengine#62) -- Added custom collection / sequence naming for SequenceFields (MongoEngine/mongoengine#92) -- Fixed UnboundLocalError in composite index with pk field (MongoEngine/mongoengine#88) +- Added NotUniqueError for duplicate keys (#62) +- Added custom collection / sequence naming for SequenceFields (#92) +- Fixed UnboundLocalError in composite index with pk field (#88) - Updated ReferenceField's to optionally store ObjectId strings - this will become the default in 0.8 (MongoEngine/mongoengine#89) + this will become the default in 0.8 (#89) - Added FutureWarning - save will default to `cascade=False` in 0.8 -- Added example of indexing embedded document fields (MongoEngine/mongoengine#75) -- Fixed ImageField resizing when forcing size (MongoEngine/mongoengine#80) -- Add flexibility for fields handling bad data (MongoEngine/mongoengine#78) +- Added example of indexing embedded document fields (#75) +- Fixed ImageField resizing when forcing size (#80) +- Add flexibility for fields handling bad data (#78) - Embedded Documents no longer handle meta definitions -- Use weakref proxies in base lists / dicts (MongoEngine/mongoengine#74) +- Use weakref proxies in base lists / dicts (#74) - Improved queryset filtering (hmarr/mongoengine#554) - Fixed Dynamic Documents and Embedded Documents (hmarr/mongoengine#561) -- Fixed abstract classes and shard keys (MongoEngine/mongoengine#64) +- Fixed abstract classes and shard keys (#64) - Fixed Python 2.5 support - Added Python 3 support (thanks to Laine Heron) Changes in 0.6.20 ================= -- Added support for distinct and db_alias (MongoEngine/mongoengine#59) +- Added support for distinct and db_alias (#59) - Improved support for chained querysets when constraining the same fields (hmarr/mongoengine#554) -- Fixed BinaryField lookup re (MongoEngine/mongoengine#48) +- Fixed BinaryField lookup re (#48) Changes in 0.6.19 ================= -- Added Binary support to UUID (MongoEngine/mongoengine#47) -- Fixed MapField lookup for fields without declared lookups (MongoEngine/mongoengine#46) -- Fixed BinaryField python value issue (MongoEngine/mongoengine#48) -- Fixed SequenceField non numeric value lookup (MongoEngine/mongoengine#41) -- Fixed queryset manager issue (MongoEngine/mongoengine#52) +- Added Binary support to UUID (#47) +- Fixed MapField lookup for fields without declared lookups (#46) +- Fixed BinaryField python value issue (#48) +- Fixed SequenceField non numeric value lookup (#41) +- Fixed queryset manager issue (#52) - Fixed FileField comparision (hmarr/mongoengine#547) Changes in 0.6.18 From 420c3e0073c9a055b4db062863352b93a4b2911f Mon Sep 17 00:00:00 2001 From: Ross Lawley Date: Wed, 19 Dec 2012 12:51:42 +0000 Subject: [PATCH 043/189] Fixing for python2.5 closes #188 --- mongoengine/base/__init__.py | 10 +++++----- mongoengine/base/document.py | 6 +++--- mongoengine/base/fields.py | 4 ++-- mongoengine/base/metaclasses.py | 4 ++-- mongoengine/errors.py | 2 +- mongoengine/queryset/__init__.py | 10 +++++----- mongoengine/queryset/manager.py | 2 +- mongoengine/queryset/queryset.py | 8 +++++--- tests/__init__.py | 4 ++-- tests/document/dynamic.py | 6 ++---- 10 files changed, 28 insertions(+), 28 deletions(-) diff --git a/mongoengine/base/__init__.py b/mongoengine/base/__init__.py index 1d4a6ebe..ce119b3a 100644 --- a/mongoengine/base/__init__.py +++ b/mongoengine/base/__init__.py @@ -1,5 +1,5 @@ -from .common import * -from .datastructures import * -from .document import * -from .fields import * -from .metaclasses import * +from mongoengine.base.common import * +from mongoengine.base.datastructures import * +from mongoengine.base.document import * +from mongoengine.base.fields import * +from mongoengine.base.metaclasses import * diff --git a/mongoengine/base/document.py b/mongoengine/base/document.py index a3f10f52..affc20e0 100644 --- a/mongoengine/base/document.py +++ b/mongoengine/base/document.py @@ -12,9 +12,9 @@ from mongoengine.errors import (ValidationError, InvalidDocumentError, from mongoengine.python_support import (PY3, UNICODE_KWARGS, txt_type, to_str_keys_recursive) -from .common import get_document, ALLOW_INHERITANCE -from .datastructures import BaseDict, BaseList -from .fields import ComplexBaseField +from mongoengine.base.common import get_document, ALLOW_INHERITANCE +from mongoengine.base.datastructures import BaseDict, BaseList +from mongoengine.base.fields import ComplexBaseField __all__ = ('BaseDocument', 'NON_FIELD_ERRORS') diff --git a/mongoengine/base/fields.py b/mongoengine/base/fields.py index 11719b55..a892fbd2 100644 --- a/mongoengine/base/fields.py +++ b/mongoengine/base/fields.py @@ -7,8 +7,8 @@ from bson import DBRef, ObjectId from mongoengine.common import _import_class from mongoengine.errors import ValidationError -from .common import ALLOW_INHERITANCE -from .datastructures import BaseDict, BaseList +from mongoengine.base.common import ALLOW_INHERITANCE +from mongoengine.base.datastructures import BaseDict, BaseList __all__ = ("BaseField", "ComplexBaseField", "ObjectIdField") diff --git a/mongoengine/base/metaclasses.py b/mongoengine/base/metaclasses.py index c6c4db1f..af39e144 100644 --- a/mongoengine/base/metaclasses.py +++ b/mongoengine/base/metaclasses.py @@ -9,8 +9,8 @@ from mongoengine.queryset import (DO_NOTHING, DoesNotExist, MultipleObjectsReturned, QuerySet, QuerySetManager) -from .common import _document_registry, ALLOW_INHERITANCE -from .fields import BaseField, ComplexBaseField, ObjectIdField +from mongoengine.base.common import _document_registry, ALLOW_INHERITANCE +from mongoengine.base.fields import BaseField, ComplexBaseField, ObjectIdField __all__ = ('DocumentMetaclass', 'TopLevelDocumentMetaclass') diff --git a/mongoengine/errors.py b/mongoengine/errors.py index eb72503d..9cfcd1d2 100644 --- a/mongoengine/errors.py +++ b/mongoengine/errors.py @@ -1,6 +1,6 @@ from collections import defaultdict -from .python_support import txt_type +from mongoengine.python_support import txt_type __all__ = ('NotRegistered', 'InvalidDocumentError', 'ValidationError') diff --git a/mongoengine/queryset/__init__.py b/mongoengine/queryset/__init__.py index f6feeab7..026a7acd 100644 --- a/mongoengine/queryset/__init__.py +++ b/mongoengine/queryset/__init__.py @@ -1,11 +1,11 @@ from mongoengine.errors import (DoesNotExist, MultipleObjectsReturned, InvalidQueryError, OperationError, NotUniqueError) -from .field_list import * -from .manager import * -from .queryset import * -from .transform import * -from .visitor import * +from mongoengine.queryset.field_list import * +from mongoengine.queryset.manager import * +from mongoengine.queryset.queryset import * +from mongoengine.queryset.transform import * +from mongoengine.queryset.visitor import * __all__ = (field_list.__all__ + manager.__all__ + queryset.__all__ + transform.__all__ + visitor.__all__) diff --git a/mongoengine/queryset/manager.py b/mongoengine/queryset/manager.py index 08d4d3ab..d9f9992f 100644 --- a/mongoengine/queryset/manager.py +++ b/mongoengine/queryset/manager.py @@ -1,5 +1,5 @@ from functools import partial -from .queryset import QuerySet +from mongoengine.queryset.queryset import QuerySet __all__ = ('queryset_manager', 'QuerySetManager') diff --git a/mongoengine/queryset/queryset.py b/mongoengine/queryset/queryset.py index fda8a755..3ea9f232 100644 --- a/mongoengine/queryset/queryset.py +++ b/mongoengine/queryset/queryset.py @@ -1,3 +1,5 @@ +from __future__ import absolute_import + import copy import itertools import operator @@ -15,9 +17,9 @@ from mongoengine.common import _import_class from mongoengine.errors import (OperationError, NotUniqueError, InvalidQueryError) -from . import transform -from .field_list import QueryFieldList -from .visitor import Q, QNode +from mongoengine.queryset import transform +from mongoengine.queryset.field_list import QueryFieldList +from mongoengine.queryset.visitor import Q, QNode __all__ = ('QuerySet', 'DO_NOTHING', 'NULLIFY', 'CASCADE', 'DENY', 'PULL') diff --git a/tests/__init__.py b/tests/__init__.py index f2a43b05..ccc90f79 100644 --- a/tests/__init__.py +++ b/tests/__init__.py @@ -1,2 +1,2 @@ -from .all_warnings import AllWarnings -from .document import * \ No newline at end of file +from all_warnings import AllWarnings +from document import * \ No newline at end of file diff --git a/tests/document/dynamic.py b/tests/document/dynamic.py index ca0db0a3..4848b8fa 100644 --- a/tests/document/dynamic.py +++ b/tests/document/dynamic.py @@ -137,13 +137,11 @@ class DynamicTest(unittest.TestCase): embedded_doc_1.validate() embedded_doc_2 = Embedded(content='this is not a url') - with self.assertRaises(ValidationError): - embedded_doc_2.validate() + self.assertRaises(ValidationError, embedded_doc_2.validate) doc.embedded_field_1 = embedded_doc_1 doc.embedded_field_2 = embedded_doc_2 - with self.assertRaises(ValidationError): - doc.validate() + self.assertRaises(ValidationError, doc.validate) def test_inheritance(self): """Ensure that dynamic document plays nice with inheritance""" From 50b755db0c535fd96b374be88d79410ebc356838 Mon Sep 17 00:00:00 2001 From: Ross Lawley Date: Wed, 19 Dec 2012 13:35:37 +0000 Subject: [PATCH 044/189] Split out queryset tests --- tests/__init__.py | 3 +- tests/queryset/__init__.py | 5 + tests/queryset/field_list.py | 67 ++ .../queryset.py} | 571 +----------------- tests/queryset/transform.py | 148 +++++ tests/queryset/visitor.py | 310 ++++++++++ 6 files changed, 565 insertions(+), 539 deletions(-) create mode 100644 tests/queryset/__init__.py create mode 100644 tests/queryset/field_list.py rename tests/{test_queryset.py => queryset/queryset.py} (85%) create mode 100644 tests/queryset/transform.py create mode 100644 tests/queryset/visitor.py diff --git a/tests/__init__.py b/tests/__init__.py index ccc90f79..152a8ce0 100644 --- a/tests/__init__.py +++ b/tests/__init__.py @@ -1,2 +1,3 @@ from all_warnings import AllWarnings -from document import * \ No newline at end of file +from document import * +from queryset import * \ No newline at end of file diff --git a/tests/queryset/__init__.py b/tests/queryset/__init__.py new file mode 100644 index 00000000..93cb8c23 --- /dev/null +++ b/tests/queryset/__init__.py @@ -0,0 +1,5 @@ + +from transform import * +from field_list import * +from queryset import * +from visitor import * \ No newline at end of file diff --git a/tests/queryset/field_list.py b/tests/queryset/field_list.py new file mode 100644 index 00000000..f3b457b4 --- /dev/null +++ b/tests/queryset/field_list.py @@ -0,0 +1,67 @@ +import sys +sys.path[0:0] = [""] + +import unittest + +from mongoengine import * +from mongoengine.queryset import QueryFieldList + +__all__ = ("QueryFieldListTest",) + +class QueryFieldListTest(unittest.TestCase): + + def test_empty(self): + q = QueryFieldList() + self.assertFalse(q) + + q = QueryFieldList(always_include=['_cls']) + self.assertFalse(q) + + def test_include_include(self): + q = QueryFieldList() + q += QueryFieldList(fields=['a', 'b'], value=QueryFieldList.ONLY) + self.assertEqual(q.as_dict(), {'a': True, 'b': True}) + q += QueryFieldList(fields=['b', 'c'], value=QueryFieldList.ONLY) + self.assertEqual(q.as_dict(), {'b': True}) + + def test_include_exclude(self): + q = QueryFieldList() + q += QueryFieldList(fields=['a', 'b'], value=QueryFieldList.ONLY) + self.assertEqual(q.as_dict(), {'a': True, 'b': True}) + q += QueryFieldList(fields=['b', 'c'], value=QueryFieldList.EXCLUDE) + self.assertEqual(q.as_dict(), {'a': True}) + + def test_exclude_exclude(self): + q = QueryFieldList() + q += QueryFieldList(fields=['a', 'b'], value=QueryFieldList.EXCLUDE) + self.assertEqual(q.as_dict(), {'a': False, 'b': False}) + q += QueryFieldList(fields=['b', 'c'], value=QueryFieldList.EXCLUDE) + self.assertEqual(q.as_dict(), {'a': False, 'b': False, 'c': False}) + + def test_exclude_include(self): + q = QueryFieldList() + q += QueryFieldList(fields=['a', 'b'], value=QueryFieldList.EXCLUDE) + self.assertEqual(q.as_dict(), {'a': False, 'b': False}) + q += QueryFieldList(fields=['b', 'c'], value=QueryFieldList.ONLY) + self.assertEqual(q.as_dict(), {'c': True}) + + def test_always_include(self): + q = QueryFieldList(always_include=['x', 'y']) + q += QueryFieldList(fields=['a', 'b', 'x'], value=QueryFieldList.EXCLUDE) + q += QueryFieldList(fields=['b', 'c'], value=QueryFieldList.ONLY) + self.assertEqual(q.as_dict(), {'x': True, 'y': True, 'c': True}) + + def test_reset(self): + q = QueryFieldList(always_include=['x', 'y']) + q += QueryFieldList(fields=['a', 'b', 'x'], value=QueryFieldList.EXCLUDE) + q += QueryFieldList(fields=['b', 'c'], value=QueryFieldList.ONLY) + self.assertEqual(q.as_dict(), {'x': True, 'y': True, 'c': True}) + q.reset() + self.assertFalse(q) + q += QueryFieldList(fields=['b', 'c'], value=QueryFieldList.ONLY) + self.assertEqual(q.as_dict(), {'x': True, 'y': True, 'b': True, 'c': True}) + + def test_using_a_slice(self): + q = QueryFieldList() + q += QueryFieldList(fields=['a'], value={"$slice": 5}) + self.assertEqual(q.as_dict(), {'a': {"$slice": 5}}) diff --git a/tests/test_queryset.py b/tests/queryset/queryset.py similarity index 85% rename from tests/test_queryset.py rename to tests/queryset/queryset.py index 07725d4c..e3e02158 100644 --- a/tests/test_queryset.py +++ b/tests/queryset/queryset.py @@ -18,12 +18,13 @@ from mongoengine import * from mongoengine.connection import get_connection from mongoengine.python_support import PY3 from mongoengine.tests import query_counter -from mongoengine.queryset import (Q, QuerySet, QuerySetManager, +from mongoengine.queryset import (QuerySet, QuerySetManager, MultipleObjectsReturned, DoesNotExist, QueryFieldList, queryset_manager) -from mongoengine.queryset import transform from mongoengine.errors import InvalidQueryError +__all__ = ("QuerySetTest",) + class QuerySetTest(unittest.TestCase): @@ -47,22 +48,6 @@ class QuerySetTest(unittest.TestCase): self.assertTrue(isinstance(self.Person.objects._collection, pymongo.collection.Collection)) - def test_transform_query(self): - """Ensure that the _transform_query function operates correctly. - """ - self.assertEqual(transform.query(name='test', age=30), - {'name': 'test', 'age': 30}) - self.assertEqual(transform.query(age__lt=30), - {'age': {'$lt': 30}}) - self.assertEqual(transform.query(age__gt=20, age__lt=50), - {'age': {'$gt': 20, '$lt': 50}}) - self.assertEqual(transform.query(age=20, age__gt=50), - {'age': 20}) - self.assertEqual(transform.query(friend__age__gte=30), - {'friend.age': {'$gte': 30}}) - self.assertEqual(transform.query(name__exists=True), - {'name': {'$exists': True}}) - def test_cannot_perform_joins_references(self): class BlogPost(Document): @@ -264,30 +249,6 @@ class QuerySetTest(unittest.TestCase): self.assertEqual(list(A.objects.none()), []) self.assertEqual(list(A.objects.none().all()), []) - def test_chaining(self): - class A(Document): - pass - - class B(Document): - a = ReferenceField(A) - - A.drop_collection() - B.drop_collection() - - a1 = A().save() - a2 = A().save() - - B(a=a1).save() - - # Works - q1 = B.objects.filter(a__in=[a1, a2], a=a1)._query - - # Doesn't work - q2 = B.objects.filter(a__in=[a1, a2]) - q2 = q2.filter(a=a1)._query - - self.assertEqual(q1, q2) - def test_update_write_options(self): """Test that passing write_options works""" @@ -830,48 +791,30 @@ class QuerySetTest(unittest.TestCase): self.assertEqual(obj, person) obj = self.Person.objects(name__contains='Van').first() self.assertEqual(obj, None) - obj = self.Person.objects(Q(name__contains='van')).first() - self.assertEqual(obj, person) - obj = self.Person.objects(Q(name__contains='Van')).first() - self.assertEqual(obj, None) # Test icontains obj = self.Person.objects(name__icontains='Van').first() self.assertEqual(obj, person) - obj = self.Person.objects(Q(name__icontains='Van')).first() - self.assertEqual(obj, person) # Test startswith obj = self.Person.objects(name__startswith='Guido').first() self.assertEqual(obj, person) obj = self.Person.objects(name__startswith='guido').first() self.assertEqual(obj, None) - obj = self.Person.objects(Q(name__startswith='Guido')).first() - self.assertEqual(obj, person) - obj = self.Person.objects(Q(name__startswith='guido')).first() - self.assertEqual(obj, None) # Test istartswith obj = self.Person.objects(name__istartswith='guido').first() self.assertEqual(obj, person) - obj = self.Person.objects(Q(name__istartswith='guido')).first() - self.assertEqual(obj, person) # Test endswith obj = self.Person.objects(name__endswith='Rossum').first() self.assertEqual(obj, person) obj = self.Person.objects(name__endswith='rossuM').first() self.assertEqual(obj, None) - obj = self.Person.objects(Q(name__endswith='Rossum')).first() - self.assertEqual(obj, person) - obj = self.Person.objects(Q(name__endswith='rossuM')).first() - self.assertEqual(obj, None) # Test iendswith obj = self.Person.objects(name__iendswith='rossuM').first() self.assertEqual(obj, person) - obj = self.Person.objects(Q(name__iendswith='rossuM')).first() - self.assertEqual(obj, person) # Test exact obj = self.Person.objects(name__exact='Guido van Rossum').first() @@ -880,28 +823,18 @@ class QuerySetTest(unittest.TestCase): self.assertEqual(obj, None) obj = self.Person.objects(name__exact='Guido van Rossu').first() self.assertEqual(obj, None) - obj = self.Person.objects(Q(name__exact='Guido van Rossum')).first() - self.assertEqual(obj, person) - obj = self.Person.objects(Q(name__exact='Guido van rossum')).first() - self.assertEqual(obj, None) - obj = self.Person.objects(Q(name__exact='Guido van Rossu')).first() - self.assertEqual(obj, None) # Test iexact obj = self.Person.objects(name__iexact='gUIDO VAN rOSSUM').first() self.assertEqual(obj, person) obj = self.Person.objects(name__iexact='gUIDO VAN rOSSU').first() self.assertEqual(obj, None) - obj = self.Person.objects(Q(name__iexact='gUIDO VAN rOSSUM')).first() - self.assertEqual(obj, person) - obj = self.Person.objects(Q(name__iexact='gUIDO VAN rOSSU')).first() - self.assertEqual(obj, None) # Test unsafe expressions person = self.Person(name='Guido van Rossum [.\'Geek\']') person.save() - obj = self.Person.objects(Q(name__icontains='[.\'Geek')).first() + obj = self.Person.objects(name__icontains='[.\'Geek').first() self.assertEqual(obj, person) def test_not(self): @@ -944,14 +877,14 @@ class QuerySetTest(unittest.TestCase): blog_3.save() blog_post_1 = BlogPost(blog=blog_1, title="Blog Post #1", - is_published = True, - published_date=datetime(2010, 1, 5, 0, 0 ,0)) + is_published=True, + published_date=datetime(2010, 1, 5, 0, 0, 0)) blog_post_2 = BlogPost(blog=blog_2, title="Blog Post #2", - is_published = True, - published_date=datetime(2010, 1, 6, 0, 0 ,0)) + is_published=True, + published_date=datetime(2010, 1, 6, 0, 0, 0)) blog_post_3 = BlogPost(blog=blog_3, title="Blog Post #3", - is_published = True, - published_date=datetime(2010, 1, 7, 0, 0 ,0)) + is_published=True, + published_date=datetime(2010, 1, 7, 0, 0, 0)) blog_post_1.save() blog_post_2.save() @@ -971,21 +904,6 @@ class QuerySetTest(unittest.TestCase): BlogPost.drop_collection() Blog.drop_collection() - def test_raw_and_merging(self): - class Doc(Document): - meta = {'allow_inheritance': False} - - raw_query = Doc.objects(__raw__={'deleted': False, - 'scraped': 'yes', - '$nor': [{'views.extracted': 'no'}, - {'attachments.views.extracted':'no'}] - })._query - - expected = {'deleted': False, 'scraped': 'yes', - '$nor': [{'views.extracted': 'no'}, - {'attachments.views.extracted': 'no'}]} - self.assertEqual(expected, raw_query) - def test_ordering(self): """Ensure default ordering is applied and can be overridden. """ @@ -1000,11 +918,11 @@ class QuerySetTest(unittest.TestCase): BlogPost.drop_collection() blog_post_1 = BlogPost(title="Blog Post #1", - published_date=datetime(2010, 1, 5, 0, 0 ,0)) + published_date=datetime(2010, 1, 5, 0, 0, 0)) blog_post_2 = BlogPost(title="Blog Post #2", - published_date=datetime(2010, 1, 6, 0, 0 ,0)) + published_date=datetime(2010, 1, 6, 0, 0, 0)) blog_post_3 = BlogPost(title="Blog Post #3", - published_date=datetime(2010, 1, 7, 0, 0 ,0)) + published_date=datetime(2010, 1, 7, 0, 0, 0)) blog_post_1.save() blog_post_2.save() @@ -1310,151 +1228,6 @@ class QuerySetTest(unittest.TestCase): BlogPost.drop_collection() - def test_q(self): - """Ensure that Q objects may be used to query for documents. - """ - class BlogPost(Document): - title = StringField() - publish_date = DateTimeField() - published = BooleanField() - - BlogPost.drop_collection() - - post1 = BlogPost(title='Test 1', publish_date=datetime(2010, 1, 8), published=False) - post1.save() - - post2 = BlogPost(title='Test 2', publish_date=datetime(2010, 1, 15), published=True) - post2.save() - - post3 = BlogPost(title='Test 3', published=True) - post3.save() - - post4 = BlogPost(title='Test 4', publish_date=datetime(2010, 1, 8)) - post4.save() - - post5 = BlogPost(title='Test 1', publish_date=datetime(2010, 1, 15)) - post5.save() - - post6 = BlogPost(title='Test 1', published=False) - post6.save() - - # Check ObjectId lookup works - obj = BlogPost.objects(id=post1.id).first() - self.assertEqual(obj, post1) - - # Check Q object combination with one does not exist - q = BlogPost.objects(Q(title='Test 5') | Q(published=True)) - posts = [post.id for post in q] - - published_posts = (post2, post3) - self.assertTrue(all(obj.id in posts for obj in published_posts)) - - q = BlogPost.objects(Q(title='Test 1') | Q(published=True)) - posts = [post.id for post in q] - published_posts = (post1, post2, post3, post5, post6) - self.assertTrue(all(obj.id in posts for obj in published_posts)) - - # Check Q object combination - date = datetime(2010, 1, 10) - q = BlogPost.objects(Q(publish_date__lte=date) | Q(published=True)) - posts = [post.id for post in q] - - published_posts = (post1, post2, post3, post4) - self.assertTrue(all(obj.id in posts for obj in published_posts)) - - self.assertFalse(any(obj.id in posts for obj in [post5, post6])) - - BlogPost.drop_collection() - - # Check the 'in' operator - self.Person(name='user1', age=20).save() - self.Person(name='user2', age=20).save() - self.Person(name='user3', age=30).save() - self.Person(name='user4', age=40).save() - - self.assertEqual(len(self.Person.objects(Q(age__in=[20]))), 2) - self.assertEqual(len(self.Person.objects(Q(age__in=[20, 30]))), 3) - - # Test invalid query objs - def wrong_query_objs(): - self.Person.objects('user1') - def wrong_query_objs_filter(): - self.Person.objects('user1') - self.assertRaises(InvalidQueryError, wrong_query_objs) - self.assertRaises(InvalidQueryError, wrong_query_objs_filter) - - def test_q_regex(self): - """Ensure that Q objects can be queried using regexes. - """ - person = self.Person(name='Guido van Rossum') - person.save() - - import re - obj = self.Person.objects(Q(name=re.compile('^Gui'))).first() - self.assertEqual(obj, person) - obj = self.Person.objects(Q(name=re.compile('^gui'))).first() - self.assertEqual(obj, None) - - obj = self.Person.objects(Q(name=re.compile('^gui', re.I))).first() - self.assertEqual(obj, person) - - obj = self.Person.objects(Q(name__not=re.compile('^bob'))).first() - self.assertEqual(obj, person) - - obj = self.Person.objects(Q(name__not=re.compile('^Gui'))).first() - self.assertEqual(obj, None) - - def test_q_lists(self): - """Ensure that Q objects query ListFields correctly. - """ - class BlogPost(Document): - tags = ListField(StringField()) - - BlogPost.drop_collection() - - BlogPost(tags=['python', 'mongo']).save() - BlogPost(tags=['python']).save() - - self.assertEqual(len(BlogPost.objects(Q(tags='mongo'))), 1) - self.assertEqual(len(BlogPost.objects(Q(tags='python'))), 2) - - BlogPost.drop_collection() - - def test_raw_query_and_Q_objects(self): - """ - Test raw plays nicely - """ - class Foo(Document): - name = StringField() - a = StringField() - b = StringField() - c = StringField() - - meta = { - 'allow_inheritance': False - } - - query = Foo.objects(__raw__={'$nor': [{'name': 'bar'}]})._query - self.assertEqual(query, {'$nor': [{'name': 'bar'}]}) - - q1 = {'$or': [{'a': 1}, {'b': 1}]} - query = Foo.objects(Q(__raw__=q1) & Q(c=1))._query - self.assertEqual(query, {'$or': [{'a': 1}, {'b': 1}], 'c': 1}) - - def test_q_merge_queries_edge_case(self): - - class User(Document): - email = EmailField(required=False) - name = StringField() - - User.drop_collection() - pk = ObjectId() - User(email='example@example.com', pk=pk).save() - - self.assertEqual(1, User.objects.filter( - Q(email='example@example.com') | - Q(name='John Doe') - ).limit(2).filter(pk=pk).count()) def test_exec_js_query(self): """Ensure that queries are properly formed for use in exec_js. @@ -1491,13 +1264,6 @@ class QuerySetTest(unittest.TestCase): c = BlogPost.objects(published=False).exec_js(js_func, 'hits') self.assertEqual(c, 1) - # Ensure that Q object queries work - c = BlogPost.objects(Q(published=True)).exec_js(js_func, 'hits') - self.assertEqual(c, 2) - - c = BlogPost.objects(Q(published=False)).exec_js(js_func, 'hits') - self.assertEqual(c, 1) - BlogPost.drop_collection() def test_exec_js_field_sub(self): @@ -2558,56 +2324,6 @@ class QuerySetTest(unittest.TestCase): BlogPost.drop_collection() - def test_query_field_name(self): - """Ensure that the correct field name is used when querying. - """ - class Comment(EmbeddedDocument): - content = StringField(db_field='commentContent') - - class BlogPost(Document): - title = StringField(db_field='postTitle') - comments = ListField(EmbeddedDocumentField(Comment), - db_field='postComments') - - - BlogPost.drop_collection() - - data = {'title': 'Post 1', 'comments': [Comment(content='test')]} - post = BlogPost(**data) - post.save() - - self.assertTrue('postTitle' in - BlogPost.objects(title=data['title'])._query) - self.assertFalse('title' in - BlogPost.objects(title=data['title'])._query) - self.assertEqual(len(BlogPost.objects(title=data['title'])), 1) - - self.assertTrue('_id' in BlogPost.objects(pk=post.id)._query) - self.assertEqual(len(BlogPost.objects(pk=post.id)), 1) - - self.assertTrue('postComments.commentContent' in - BlogPost.objects(comments__content='test')._query) - self.assertEqual(len(BlogPost.objects(comments__content='test')), 1) - - BlogPost.drop_collection() - - def test_query_pk_field_name(self): - """Ensure that the correct "primary key" field name is used when querying - """ - class BlogPost(Document): - title = StringField(primary_key=True, db_field='postTitle') - - BlogPost.drop_collection() - - data = { 'title':'Post 1' } - post = BlogPost(**data) - post.save() - - self.assertTrue('_id' in BlogPost.objects(pk=data['title'])._query) - self.assertTrue('_id' in BlogPost.objects(title=data['title'])._query) - self.assertEqual(len(BlogPost.objects(pk=data['title'])), 1) - - BlogPost.drop_collection() def test_query_value_conversion(self): """Ensure that query values are properly converted when necessary. @@ -3446,227 +3162,6 @@ class QuerySetTest(unittest.TestCase): else: self.assertEqual("[u'A1', u'A2']", "%s" % sorted(self.Person.objects.scalar('name').in_bulk(list(pks)).values())) - -class QTest(unittest.TestCase): - - def setUp(self): - connect(db='mongoenginetest') - - def test_empty_q(self): - """Ensure that empty Q objects won't hurt. - """ - q1 = Q() - q2 = Q(age__gte=18) - q3 = Q() - q4 = Q(name='test') - q5 = Q() - - class Person(Document): - name = StringField() - age = IntField() - - query = {'$or': [{'age': {'$gte': 18}}, {'name': 'test'}]} - self.assertEqual((q1 | q2 | q3 | q4 | q5).to_query(Person), query) - - query = {'age': {'$gte': 18}, 'name': 'test'} - self.assertEqual((q1 & q2 & q3 & q4 & q5).to_query(Person), query) - - def test_q_with_dbref(self): - """Ensure Q objects handle DBRefs correctly""" - connect(db='mongoenginetest') - - class User(Document): - pass - - class Post(Document): - created_user = ReferenceField(User) - - user = User.objects.create() - Post.objects.create(created_user=user) - - self.assertEqual(Post.objects.filter(created_user=user).count(), 1) - self.assertEqual(Post.objects.filter(Q(created_user=user)).count(), 1) - - def test_and_combination(self): - """Ensure that Q-objects correctly AND together. - """ - class TestDoc(Document): - x = IntField() - y = StringField() - - # Check than an error is raised when conflicting queries are anded - def invalid_combination(): - query = Q(x__lt=7) & Q(x__lt=3) - query.to_query(TestDoc) - self.assertRaises(InvalidQueryError, invalid_combination) - - # Check normal cases work without an error - query = Q(x__lt=7) & Q(x__gt=3) - - q1 = Q(x__lt=7) - q2 = Q(x__gt=3) - query = (q1 & q2).to_query(TestDoc) - self.assertEqual(query, {'x': {'$lt': 7, '$gt': 3}}) - - # More complex nested example - query = Q(x__lt=100) & Q(y__ne='NotMyString') - query &= Q(y__in=['a', 'b', 'c']) & Q(x__gt=-100) - mongo_query = { - 'x': {'$lt': 100, '$gt': -100}, - 'y': {'$ne': 'NotMyString', '$in': ['a', 'b', 'c']}, - } - self.assertEqual(query.to_query(TestDoc), mongo_query) - - def test_or_combination(self): - """Ensure that Q-objects correctly OR together. - """ - class TestDoc(Document): - x = IntField() - - q1 = Q(x__lt=3) - q2 = Q(x__gt=7) - query = (q1 | q2).to_query(TestDoc) - self.assertEqual(query, { - '$or': [ - {'x': {'$lt': 3}}, - {'x': {'$gt': 7}}, - ] - }) - - def test_and_or_combination(self): - """Ensure that Q-objects handle ANDing ORed components. - """ - class TestDoc(Document): - x = IntField() - y = BooleanField() - - query = (Q(x__gt=0) | Q(x__exists=False)) - query &= Q(x__lt=100) - self.assertEqual(query.to_query(TestDoc), { - '$or': [ - {'x': {'$lt': 100, '$gt': 0}}, - {'x': {'$lt': 100, '$exists': False}}, - ] - }) - - q1 = (Q(x__gt=0) | Q(x__exists=False)) - q2 = (Q(x__lt=100) | Q(y=True)) - query = (q1 & q2).to_query(TestDoc) - - self.assertEqual(['$or'], query.keys()) - conditions = [ - {'x': {'$lt': 100, '$gt': 0}}, - {'x': {'$lt': 100, '$exists': False}}, - {'x': {'$gt': 0}, 'y': True}, - {'x': {'$exists': False}, 'y': True}, - ] - self.assertEqual(len(conditions), len(query['$or'])) - for condition in conditions: - self.assertTrue(condition in query['$or']) - - def test_or_and_or_combination(self): - """Ensure that Q-objects handle ORing ANDed ORed components. :) - """ - class TestDoc(Document): - x = IntField() - y = BooleanField() - - q1 = (Q(x__gt=0) & (Q(y=True) | Q(y__exists=False))) - q2 = (Q(x__lt=100) & (Q(y=False) | Q(y__exists=False))) - query = (q1 | q2).to_query(TestDoc) - - self.assertEqual(['$or'], query.keys()) - conditions = [ - {'x': {'$gt': 0}, 'y': True}, - {'x': {'$gt': 0}, 'y': {'$exists': False}}, - {'x': {'$lt': 100}, 'y':False}, - {'x': {'$lt': 100}, 'y': {'$exists': False}}, - ] - self.assertEqual(len(conditions), len(query['$or'])) - for condition in conditions: - self.assertTrue(condition in query['$or']) - - - def test_q_clone(self): - - class TestDoc(Document): - x = IntField() - - TestDoc.drop_collection() - for i in xrange(1, 101): - t = TestDoc(x=i) - t.save() - - # Check normal cases work without an error - test = TestDoc.objects(Q(x__lt=7) & Q(x__gt=3)) - - self.assertEqual(test.count(), 3) - - test2 = test.clone() - self.assertEqual(test2.count(), 3) - self.assertFalse(test2 == test) - - test2.filter(x=6) - self.assertEqual(test2.count(), 1) - self.assertEqual(test.count(), 3) - -class QueryFieldListTest(unittest.TestCase): - def test_empty(self): - q = QueryFieldList() - self.assertFalse(q) - - q = QueryFieldList(always_include=['_cls']) - self.assertFalse(q) - - def test_include_include(self): - q = QueryFieldList() - q += QueryFieldList(fields=['a', 'b'], value=QueryFieldList.ONLY) - self.assertEqual(q.as_dict(), {'a': True, 'b': True}) - q += QueryFieldList(fields=['b', 'c'], value=QueryFieldList.ONLY) - self.assertEqual(q.as_dict(), {'b': True}) - - def test_include_exclude(self): - q = QueryFieldList() - q += QueryFieldList(fields=['a', 'b'], value=QueryFieldList.ONLY) - self.assertEqual(q.as_dict(), {'a': True, 'b': True}) - q += QueryFieldList(fields=['b', 'c'], value=QueryFieldList.EXCLUDE) - self.assertEqual(q.as_dict(), {'a': True}) - - def test_exclude_exclude(self): - q = QueryFieldList() - q += QueryFieldList(fields=['a', 'b'], value=QueryFieldList.EXCLUDE) - self.assertEqual(q.as_dict(), {'a': False, 'b': False}) - q += QueryFieldList(fields=['b', 'c'], value=QueryFieldList.EXCLUDE) - self.assertEqual(q.as_dict(), {'a': False, 'b': False, 'c': False}) - - def test_exclude_include(self): - q = QueryFieldList() - q += QueryFieldList(fields=['a', 'b'], value=QueryFieldList.EXCLUDE) - self.assertEqual(q.as_dict(), {'a': False, 'b': False}) - q += QueryFieldList(fields=['b', 'c'], value=QueryFieldList.ONLY) - self.assertEqual(q.as_dict(), {'c': True}) - - def test_always_include(self): - q = QueryFieldList(always_include=['x', 'y']) - q += QueryFieldList(fields=['a', 'b', 'x'], value=QueryFieldList.EXCLUDE) - q += QueryFieldList(fields=['b', 'c'], value=QueryFieldList.ONLY) - self.assertEqual(q.as_dict(), {'x': True, 'y': True, 'c': True}) - - def test_reset(self): - q = QueryFieldList(always_include=['x', 'y']) - q += QueryFieldList(fields=['a', 'b', 'x'], value=QueryFieldList.EXCLUDE) - q += QueryFieldList(fields=['b', 'c'], value=QueryFieldList.ONLY) - self.assertEqual(q.as_dict(), {'x': True, 'y': True, 'c': True}) - q.reset() - self.assertFalse(q) - q += QueryFieldList(fields=['b', 'c'], value=QueryFieldList.ONLY) - self.assertEqual(q.as_dict(), {'x': True, 'y': True, 'b': True, 'c': True}) - - def test_using_a_slice(self): - q = QueryFieldList() - q += QueryFieldList(fields=['a'], value={"$slice": 5}) - self.assertEqual(q.as_dict(), {'a': {"$slice": 5}}) - def test_elem_match(self): class Foo(EmbeddedDocument): shape = StringField() @@ -3691,6 +3186,26 @@ class QueryFieldListTest(unittest.TestCase): ak = list(Bar.objects(foo__match={'shape': "square", "color": "purple"})) self.assertEqual([b1], ak) + def test_upsert_includes_cls(self): + """Upserts should include _cls information for inheritable classes + """ + + class Test(Document): + test = StringField() + + Test.drop_collection() + Test.objects(test='foo').update_one(upsert=True, set__test='foo') + self.assertFalse('_cls' in Test._collection.find_one()) + + class Test(Document): + meta = {'allow_inheritance': True} + test = StringField() + + Test.drop_collection() + + Test.objects(test='foo').update_one(upsert=True, set__test='foo') + self.assertTrue('_cls' in Test._collection.find_one()) + def test_read_preference(self): class Bar(Document): pass @@ -3769,26 +3284,6 @@ class QueryFieldListTest(unittest.TestCase): self.assertEqual(doc_objects, Doc.objects.from_json(json_data)) - def test_upsert_includes_cls(self): - """Upserts should include _cls information for inheritable classes - """ - - class Test(Document): - test = StringField() - - Test.drop_collection() - Test.objects(test='foo').update_one(upsert=True, set__test='foo') - self.assertFalse('_cls' in Test._collection.find_one()) - - class Test(Document): - meta = {'allow_inheritance': True} - test = StringField() - - Test.drop_collection() - - Test.objects(test='foo').update_one(upsert=True, set__test='foo') - self.assertTrue('_cls' in Test._collection.find_one()) - def test_as_pymongo(self): from decimal import Decimal diff --git a/tests/queryset/transform.py b/tests/queryset/transform.py new file mode 100644 index 00000000..666b3451 --- /dev/null +++ b/tests/queryset/transform.py @@ -0,0 +1,148 @@ +from __future__ import with_statement +import sys +sys.path[0:0] = [""] + +import unittest + +from mongoengine import * +from mongoengine.queryset import Q +from mongoengine.queryset import transform + +__all__ = ("TransformTest",) + + +class TransformTest(unittest.TestCase): + + def setUp(self): + connect(db='mongoenginetest') + + def test_transform_query(self): + """Ensure that the _transform_query function operates correctly. + """ + self.assertEqual(transform.query(name='test', age=30), + {'name': 'test', 'age': 30}) + self.assertEqual(transform.query(age__lt=30), + {'age': {'$lt': 30}}) + self.assertEqual(transform.query(age__gt=20, age__lt=50), + {'age': {'$gt': 20, '$lt': 50}}) + self.assertEqual(transform.query(age=20, age__gt=50), + {'age': 20}) + self.assertEqual(transform.query(friend__age__gte=30), + {'friend.age': {'$gte': 30}}) + self.assertEqual(transform.query(name__exists=True), + {'name': {'$exists': True}}) + + def test_query_field_name(self): + """Ensure that the correct field name is used when querying. + """ + class Comment(EmbeddedDocument): + content = StringField(db_field='commentContent') + + class BlogPost(Document): + title = StringField(db_field='postTitle') + comments = ListField(EmbeddedDocumentField(Comment), + db_field='postComments') + + BlogPost.drop_collection() + + data = {'title': 'Post 1', 'comments': [Comment(content='test')]} + post = BlogPost(**data) + post.save() + + self.assertTrue('postTitle' in + BlogPost.objects(title=data['title'])._query) + self.assertFalse('title' in + BlogPost.objects(title=data['title'])._query) + self.assertEqual(len(BlogPost.objects(title=data['title'])), 1) + + self.assertTrue('_id' in BlogPost.objects(pk=post.id)._query) + self.assertEqual(len(BlogPost.objects(pk=post.id)), 1) + + self.assertTrue('postComments.commentContent' in + BlogPost.objects(comments__content='test')._query) + self.assertEqual(len(BlogPost.objects(comments__content='test')), 1) + + BlogPost.drop_collection() + + def test_query_pk_field_name(self): + """Ensure that the correct "primary key" field name is used when + querying + """ + class BlogPost(Document): + title = StringField(primary_key=True, db_field='postTitle') + + BlogPost.drop_collection() + + data = {'title': 'Post 1'} + post = BlogPost(**data) + post.save() + + self.assertTrue('_id' in BlogPost.objects(pk=data['title'])._query) + self.assertTrue('_id' in BlogPost.objects(title=data['title'])._query) + self.assertEqual(len(BlogPost.objects(pk=data['title'])), 1) + + BlogPost.drop_collection() + + def test_chaining(self): + class A(Document): + pass + + class B(Document): + a = ReferenceField(A) + + A.drop_collection() + B.drop_collection() + + a1 = A().save() + a2 = A().save() + + B(a=a1).save() + + # Works + q1 = B.objects.filter(a__in=[a1, a2], a=a1)._query + + # Doesn't work + q2 = B.objects.filter(a__in=[a1, a2]) + q2 = q2.filter(a=a1)._query + + self.assertEqual(q1, q2) + + def test_raw_query_and_Q_objects(self): + """ + Test raw plays nicely + """ + class Foo(Document): + name = StringField() + a = StringField() + b = StringField() + c = StringField() + + meta = { + 'allow_inheritance': False + } + + query = Foo.objects(__raw__={'$nor': [{'name': 'bar'}]})._query + self.assertEqual(query, {'$nor': [{'name': 'bar'}]}) + + q1 = {'$or': [{'a': 1}, {'b': 1}]} + query = Foo.objects(Q(__raw__=q1) & Q(c=1))._query + self.assertEqual(query, {'$or': [{'a': 1}, {'b': 1}], 'c': 1}) + + def test_raw_and_merging(self): + class Doc(Document): + meta = {'allow_inheritance': False} + + raw_query = Doc.objects(__raw__={'deleted': False, + 'scraped': 'yes', + '$nor': [{'views.extracted': 'no'}, + {'attachments.views.extracted':'no'}] + })._query + + expected = {'deleted': False, 'scraped': 'yes', + '$nor': [{'views.extracted': 'no'}, + {'attachments.views.extracted': 'no'}]} + self.assertEqual(expected, raw_query) + + +if __name__ == '__main__': + unittest.main() diff --git a/tests/queryset/visitor.py b/tests/queryset/visitor.py new file mode 100644 index 00000000..71c35615 --- /dev/null +++ b/tests/queryset/visitor.py @@ -0,0 +1,310 @@ +from __future__ import with_statement +import sys +sys.path[0:0] = [""] + +import unittest + +from bson import ObjectId +from datetime import datetime + +from mongoengine import * +from mongoengine.queryset import Q +from mongoengine.errors import InvalidQueryError + +__all__ = ("QTest",) + + +class QTest(unittest.TestCase): + + def setUp(self): + connect(db='mongoenginetest') + + class Person(Document): + name = StringField() + age = IntField() + meta = {'allow_inheritance': True} + + Person.drop_collection() + self.Person = Person + + def test_empty_q(self): + """Ensure that empty Q objects won't hurt. + """ + q1 = Q() + q2 = Q(age__gte=18) + q3 = Q() + q4 = Q(name='test') + q5 = Q() + + class Person(Document): + name = StringField() + age = IntField() + + query = {'$or': [{'age': {'$gte': 18}}, {'name': 'test'}]} + self.assertEqual((q1 | q2 | q3 | q4 | q5).to_query(Person), query) + + query = {'age': {'$gte': 18}, 'name': 'test'} + self.assertEqual((q1 & q2 & q3 & q4 & q5).to_query(Person), query) + + def test_q_with_dbref(self): + """Ensure Q objects handle DBRefs correctly""" + connect(db='mongoenginetest') + + class User(Document): + pass + + class Post(Document): + created_user = ReferenceField(User) + + user = User.objects.create() + Post.objects.create(created_user=user) + + self.assertEqual(Post.objects.filter(created_user=user).count(), 1) + self.assertEqual(Post.objects.filter(Q(created_user=user)).count(), 1) + + def test_and_combination(self): + """Ensure that Q-objects correctly AND together. + """ + class TestDoc(Document): + x = IntField() + y = StringField() + + # Check than an error is raised when conflicting queries are anded + def invalid_combination(): + query = Q(x__lt=7) & Q(x__lt=3) + query.to_query(TestDoc) + self.assertRaises(InvalidQueryError, invalid_combination) + + # Check normal cases work without an error + query = Q(x__lt=7) & Q(x__gt=3) + + q1 = Q(x__lt=7) + q2 = Q(x__gt=3) + query = (q1 & q2).to_query(TestDoc) + self.assertEqual(query, {'x': {'$lt': 7, '$gt': 3}}) + + # More complex nested example + query = Q(x__lt=100) & Q(y__ne='NotMyString') + query &= Q(y__in=['a', 'b', 'c']) & Q(x__gt=-100) + mongo_query = { + 'x': {'$lt': 100, '$gt': -100}, + 'y': {'$ne': 'NotMyString', '$in': ['a', 'b', 'c']}, + } + self.assertEqual(query.to_query(TestDoc), mongo_query) + + def test_or_combination(self): + """Ensure that Q-objects correctly OR together. + """ + class TestDoc(Document): + x = IntField() + + q1 = Q(x__lt=3) + q2 = Q(x__gt=7) + query = (q1 | q2).to_query(TestDoc) + self.assertEqual(query, { + '$or': [ + {'x': {'$lt': 3}}, + {'x': {'$gt': 7}}, + ] + }) + + def test_and_or_combination(self): + """Ensure that Q-objects handle ANDing ORed components. + """ + class TestDoc(Document): + x = IntField() + y = BooleanField() + + query = (Q(x__gt=0) | Q(x__exists=False)) + query &= Q(x__lt=100) + self.assertEqual(query.to_query(TestDoc), { + '$or': [ + {'x': {'$lt': 100, '$gt': 0}}, + {'x': {'$lt': 100, '$exists': False}}, + ] + }) + + q1 = (Q(x__gt=0) | Q(x__exists=False)) + q2 = (Q(x__lt=100) | Q(y=True)) + query = (q1 & q2).to_query(TestDoc) + + self.assertEqual(['$or'], query.keys()) + conditions = [ + {'x': {'$lt': 100, '$gt': 0}}, + {'x': {'$lt': 100, '$exists': False}}, + {'x': {'$gt': 0}, 'y': True}, + {'x': {'$exists': False}, 'y': True}, + ] + self.assertEqual(len(conditions), len(query['$or'])) + for condition in conditions: + self.assertTrue(condition in query['$or']) + + def test_or_and_or_combination(self): + """Ensure that Q-objects handle ORing ANDed ORed components. :) + """ + class TestDoc(Document): + x = IntField() + y = BooleanField() + + q1 = (Q(x__gt=0) & (Q(y=True) | Q(y__exists=False))) + q2 = (Q(x__lt=100) & (Q(y=False) | Q(y__exists=False))) + query = (q1 | q2).to_query(TestDoc) + + self.assertEqual(['$or'], query.keys()) + conditions = [ + {'x': {'$gt': 0}, 'y': True}, + {'x': {'$gt': 0}, 'y': {'$exists': False}}, + {'x': {'$lt': 100}, 'y':False}, + {'x': {'$lt': 100}, 'y': {'$exists': False}}, + ] + self.assertEqual(len(conditions), len(query['$or'])) + for condition in conditions: + self.assertTrue(condition in query['$or']) + + def test_q_clone(self): + + class TestDoc(Document): + x = IntField() + + TestDoc.drop_collection() + for i in xrange(1, 101): + t = TestDoc(x=i) + t.save() + + # Check normal cases work without an error + test = TestDoc.objects(Q(x__lt=7) & Q(x__gt=3)) + + self.assertEqual(test.count(), 3) + + test2 = test.clone() + self.assertEqual(test2.count(), 3) + self.assertFalse(test2 == test) + + test2.filter(x=6) + self.assertEqual(test2.count(), 1) + self.assertEqual(test.count(), 3) + + def test_q(self): + """Ensure that Q objects may be used to query for documents. + """ + class BlogPost(Document): + title = StringField() + publish_date = DateTimeField() + published = BooleanField() + + BlogPost.drop_collection() + + post1 = BlogPost(title='Test 1', publish_date=datetime(2010, 1, 8), published=False) + post1.save() + + post2 = BlogPost(title='Test 2', publish_date=datetime(2010, 1, 15), published=True) + post2.save() + + post3 = BlogPost(title='Test 3', published=True) + post3.save() + + post4 = BlogPost(title='Test 4', publish_date=datetime(2010, 1, 8)) + post4.save() + + post5 = BlogPost(title='Test 1', publish_date=datetime(2010, 1, 15)) + post5.save() + + post6 = BlogPost(title='Test 1', published=False) + post6.save() + + # Check ObjectId lookup works + obj = BlogPost.objects(id=post1.id).first() + self.assertEqual(obj, post1) + + # Check Q object combination with one does not exist + q = BlogPost.objects(Q(title='Test 5') | Q(published=True)) + posts = [post.id for post in q] + + published_posts = (post2, post3) + self.assertTrue(all(obj.id in posts for obj in published_posts)) + + q = BlogPost.objects(Q(title='Test 1') | Q(published=True)) + posts = [post.id for post in q] + published_posts = (post1, post2, post3, post5, post6) + self.assertTrue(all(obj.id in posts for obj in published_posts)) + + # Check Q object combination + date = datetime(2010, 1, 10) + q = BlogPost.objects(Q(publish_date__lte=date) | Q(published=True)) + posts = [post.id for post in q] + + published_posts = (post1, post2, post3, post4) + self.assertTrue(all(obj.id in posts for obj in published_posts)) + + self.assertFalse(any(obj.id in posts for obj in [post5, post6])) + + BlogPost.drop_collection() + + # Check the 'in' operator + self.Person(name='user1', age=20).save() + self.Person(name='user2', age=20).save() + self.Person(name='user3', age=30).save() + self.Person(name='user4', age=40).save() + + self.assertEqual(len(self.Person.objects(Q(age__in=[20]))), 2) + self.assertEqual(len(self.Person.objects(Q(age__in=[20, 30]))), 3) + + # Test invalid query objs + def wrong_query_objs(): + self.Person.objects('user1') + def wrong_query_objs_filter(): + self.Person.objects('user1') + self.assertRaises(InvalidQueryError, wrong_query_objs) + self.assertRaises(InvalidQueryError, wrong_query_objs_filter) + + def test_q_regex(self): + """Ensure that Q objects can be queried using regexes. + """ + person = self.Person(name='Guido van Rossum') + person.save() + + import re + obj = self.Person.objects(Q(name=re.compile('^Gui'))).first() + self.assertEqual(obj, person) + obj = self.Person.objects(Q(name=re.compile('^gui'))).first() + self.assertEqual(obj, None) + + obj = self.Person.objects(Q(name=re.compile('^gui', re.I))).first() + self.assertEqual(obj, person) + + obj = self.Person.objects(Q(name__not=re.compile('^bob'))).first() + self.assertEqual(obj, person) + + obj = self.Person.objects(Q(name__not=re.compile('^Gui'))).first() + self.assertEqual(obj, None) + + def test_q_lists(self): + """Ensure that Q objects query ListFields correctly. + """ + class BlogPost(Document): + tags = ListField(StringField()) + + BlogPost.drop_collection() + + BlogPost(tags=['python', 'mongo']).save() + BlogPost(tags=['python']).save() + + self.assertEqual(len(BlogPost.objects(Q(tags='mongo'))), 1) + self.assertEqual(len(BlogPost.objects(Q(tags='python'))), 2) + + BlogPost.drop_collection() + + def test_q_merge_queries_edge_case(self): + + class User(Document): + email = EmailField(required=False) + name = StringField() + + User.drop_collection() + pk = ObjectId() + User(email='example@example.com', pk=pk).save() + + self.assertEqual(1, User.objects.filter( + Q(email='example@example.com') | + Q(name='John Doe') + ).limit(2).filter(pk=pk).count()) \ No newline at end of file From 42f506adc6875f31c5081b1067b13c16f71bdb29 Mon Sep 17 00:00:00 2001 From: Ross Lawley Date: Wed, 19 Dec 2012 13:58:04 +0000 Subject: [PATCH 045/189] Updates to test suite --- tests/queryset/field_list.py | 271 ++++++++++++++++++++++++++++++++++- tests/queryset/queryset.py | 251 -------------------------------- tests/queryset/visitor.py | 5 +- 3 files changed, 274 insertions(+), 253 deletions(-) diff --git a/tests/queryset/field_list.py b/tests/queryset/field_list.py index f3b457b4..6a9c6a9f 100644 --- a/tests/queryset/field_list.py +++ b/tests/queryset/field_list.py @@ -6,7 +6,8 @@ import unittest from mongoengine import * from mongoengine.queryset import QueryFieldList -__all__ = ("QueryFieldListTest",) +__all__ = ("QueryFieldListTest", "OnlyExcludeAllTest") + class QueryFieldListTest(unittest.TestCase): @@ -65,3 +66,271 @@ class QueryFieldListTest(unittest.TestCase): q = QueryFieldList() q += QueryFieldList(fields=['a'], value={"$slice": 5}) self.assertEqual(q.as_dict(), {'a': {"$slice": 5}}) + + +class OnlyExcludeAllTest(unittest.TestCase): + + def setUp(self): + connect(db='mongoenginetest') + + class Person(Document): + name = StringField() + age = IntField() + meta = {'allow_inheritance': True} + + Person.drop_collection() + self.Person = Person + + def test_only(self): + """Ensure that QuerySet.only only returns the requested fields. + """ + person = self.Person(name='test', age=25) + person.save() + + obj = self.Person.objects.only('name').get() + self.assertEqual(obj.name, person.name) + self.assertEqual(obj.age, None) + + obj = self.Person.objects.only('age').get() + self.assertEqual(obj.name, None) + self.assertEqual(obj.age, person.age) + + obj = self.Person.objects.only('name', 'age').get() + self.assertEqual(obj.name, person.name) + self.assertEqual(obj.age, person.age) + + # Check polymorphism still works + class Employee(self.Person): + salary = IntField(db_field='wage') + + employee = Employee(name='test employee', age=40, salary=30000) + employee.save() + + obj = self.Person.objects(id=employee.id).only('age').get() + self.assertTrue(isinstance(obj, Employee)) + + # Check field names are looked up properly + obj = Employee.objects(id=employee.id).only('salary').get() + self.assertEqual(obj.salary, employee.salary) + self.assertEqual(obj.name, None) + + def test_only_with_subfields(self): + class User(EmbeddedDocument): + name = StringField() + email = StringField() + + class Comment(EmbeddedDocument): + title = StringField() + text = StringField() + + class BlogPost(Document): + content = StringField() + author = EmbeddedDocumentField(User) + comments = ListField(EmbeddedDocumentField(Comment)) + + BlogPost.drop_collection() + + post = BlogPost(content='Had a good coffee today...') + post.author = User(name='Test User') + post.comments = [Comment(title='I aggree', text='Great post!'), Comment(title='Coffee', text='I hate coffee')] + post.save() + + obj = BlogPost.objects.only('author.name',).get() + self.assertEqual(obj.content, None) + self.assertEqual(obj.author.email, None) + self.assertEqual(obj.author.name, 'Test User') + self.assertEqual(obj.comments, []) + + obj = BlogPost.objects.only('content', 'comments.title',).get() + self.assertEqual(obj.content, 'Had a good coffee today...') + self.assertEqual(obj.author, None) + self.assertEqual(obj.comments[0].title, 'I aggree') + self.assertEqual(obj.comments[1].title, 'Coffee') + self.assertEqual(obj.comments[0].text, None) + self.assertEqual(obj.comments[1].text, None) + + obj = BlogPost.objects.only('comments',).get() + self.assertEqual(obj.content, None) + self.assertEqual(obj.author, None) + self.assertEqual(obj.comments[0].title, 'I aggree') + self.assertEqual(obj.comments[1].title, 'Coffee') + self.assertEqual(obj.comments[0].text, 'Great post!') + self.assertEqual(obj.comments[1].text, 'I hate coffee') + + BlogPost.drop_collection() + + def test_exclude(self): + class User(EmbeddedDocument): + name = StringField() + email = StringField() + + class Comment(EmbeddedDocument): + title = StringField() + text = StringField() + + class BlogPost(Document): + content = StringField() + author = EmbeddedDocumentField(User) + comments = ListField(EmbeddedDocumentField(Comment)) + + BlogPost.drop_collection() + + post = BlogPost(content='Had a good coffee today...') + post.author = User(name='Test User') + post.comments = [Comment(title='I aggree', text='Great post!'), Comment(title='Coffee', text='I hate coffee')] + post.save() + + obj = BlogPost.objects.exclude('author', 'comments.text').get() + self.assertEqual(obj.author, None) + self.assertEqual(obj.content, 'Had a good coffee today...') + self.assertEqual(obj.comments[0].title, 'I aggree') + self.assertEqual(obj.comments[0].text, None) + + BlogPost.drop_collection() + + def test_exclude_only_combining(self): + class Attachment(EmbeddedDocument): + name = StringField() + content = StringField() + + class Email(Document): + sender = StringField() + to = StringField() + subject = StringField() + body = StringField() + content_type = StringField() + attachments = ListField(EmbeddedDocumentField(Attachment)) + + Email.drop_collection() + email = Email(sender='me', to='you', subject='From Russia with Love', body='Hello!', content_type='text/plain') + email.attachments = [ + Attachment(name='file1.doc', content='ABC'), + Attachment(name='file2.doc', content='XYZ'), + ] + email.save() + + obj = Email.objects.exclude('content_type').exclude('body').get() + self.assertEqual(obj.sender, 'me') + self.assertEqual(obj.to, 'you') + self.assertEqual(obj.subject, 'From Russia with Love') + self.assertEqual(obj.body, None) + self.assertEqual(obj.content_type, None) + + obj = Email.objects.only('sender', 'to').exclude('body', 'sender').get() + self.assertEqual(obj.sender, None) + self.assertEqual(obj.to, 'you') + self.assertEqual(obj.subject, None) + self.assertEqual(obj.body, None) + self.assertEqual(obj.content_type, None) + + obj = Email.objects.exclude('attachments.content').exclude('body').only('to', 'attachments.name').get() + self.assertEqual(obj.attachments[0].name, 'file1.doc') + self.assertEqual(obj.attachments[0].content, None) + self.assertEqual(obj.sender, None) + self.assertEqual(obj.to, 'you') + self.assertEqual(obj.subject, None) + self.assertEqual(obj.body, None) + self.assertEqual(obj.content_type, None) + + Email.drop_collection() + + def test_all_fields(self): + + class Email(Document): + sender = StringField() + to = StringField() + subject = StringField() + body = StringField() + content_type = StringField() + + Email.drop_collection() + + email = Email(sender='me', to='you', subject='From Russia with Love', body='Hello!', content_type='text/plain') + email.save() + + obj = Email.objects.exclude('content_type', 'body').only('to', 'body').all_fields().get() + self.assertEqual(obj.sender, 'me') + self.assertEqual(obj.to, 'you') + self.assertEqual(obj.subject, 'From Russia with Love') + self.assertEqual(obj.body, 'Hello!') + self.assertEqual(obj.content_type, 'text/plain') + + Email.drop_collection() + + def test_slicing_fields(self): + """Ensure that query slicing an array works. + """ + class Numbers(Document): + n = ListField(IntField()) + + Numbers.drop_collection() + + numbers = Numbers(n=[0,1,2,3,4,5,-5,-4,-3,-2,-1]) + numbers.save() + + # first three + numbers = Numbers.objects.fields(slice__n=3).get() + self.assertEqual(numbers.n, [0, 1, 2]) + + # last three + numbers = Numbers.objects.fields(slice__n=-3).get() + self.assertEqual(numbers.n, [-3, -2, -1]) + + # skip 2, limit 3 + numbers = Numbers.objects.fields(slice__n=[2, 3]).get() + self.assertEqual(numbers.n, [2, 3, 4]) + + # skip to fifth from last, limit 4 + numbers = Numbers.objects.fields(slice__n=[-5, 4]).get() + self.assertEqual(numbers.n, [-5, -4, -3, -2]) + + # skip to fifth from last, limit 10 + numbers = Numbers.objects.fields(slice__n=[-5, 10]).get() + self.assertEqual(numbers.n, [-5, -4, -3, -2, -1]) + + # skip to fifth from last, limit 10 dict method + numbers = Numbers.objects.fields(n={"$slice": [-5, 10]}).get() + self.assertEqual(numbers.n, [-5, -4, -3, -2, -1]) + + def test_slicing_nested_fields(self): + """Ensure that query slicing an embedded array works. + """ + + class EmbeddedNumber(EmbeddedDocument): + n = ListField(IntField()) + + class Numbers(Document): + embedded = EmbeddedDocumentField(EmbeddedNumber) + + Numbers.drop_collection() + + numbers = Numbers() + numbers.embedded = EmbeddedNumber(n=[0,1,2,3,4,5,-5,-4,-3,-2,-1]) + numbers.save() + + # first three + numbers = Numbers.objects.fields(slice__embedded__n=3).get() + self.assertEqual(numbers.embedded.n, [0, 1, 2]) + + # last three + numbers = Numbers.objects.fields(slice__embedded__n=-3).get() + self.assertEqual(numbers.embedded.n, [-3, -2, -1]) + + # skip 2, limit 3 + numbers = Numbers.objects.fields(slice__embedded__n=[2, 3]).get() + self.assertEqual(numbers.embedded.n, [2, 3, 4]) + + # skip to fifth from last, limit 4 + numbers = Numbers.objects.fields(slice__embedded__n=[-5, 4]).get() + self.assertEqual(numbers.embedded.n, [-5, -4, -3, -2]) + + # skip to fifth from last, limit 10 + numbers = Numbers.objects.fields(slice__embedded__n=[-5, 10]).get() + self.assertEqual(numbers.embedded.n, [-5, -4, -3, -2, -1]) + + # skip to fifth from last, limit 10 dict method + numbers = Numbers.objects.fields(embedded__n={"$slice": [-5, 10]}).get() + self.assertEqual(numbers.embedded.n, [-5, -4, -3, -2, -1]) + +if __name__ == '__main__': + unittest.main() diff --git a/tests/queryset/queryset.py b/tests/queryset/queryset.py index e3e02158..bad3d360 100644 --- a/tests/queryset/queryset.py +++ b/tests/queryset/queryset.py @@ -939,257 +939,6 @@ class QuerySetTest(unittest.TestCase): BlogPost.drop_collection() - def test_only(self): - """Ensure that QuerySet.only only returns the requested fields. - """ - person = self.Person(name='test', age=25) - person.save() - - obj = self.Person.objects.only('name').get() - self.assertEqual(obj.name, person.name) - self.assertEqual(obj.age, None) - - obj = self.Person.objects.only('age').get() - self.assertEqual(obj.name, None) - self.assertEqual(obj.age, person.age) - - obj = self.Person.objects.only('name', 'age').get() - self.assertEqual(obj.name, person.name) - self.assertEqual(obj.age, person.age) - - # Check polymorphism still works - class Employee(self.Person): - salary = IntField(db_field='wage') - - employee = Employee(name='test employee', age=40, salary=30000) - employee.save() - - obj = self.Person.objects(id=employee.id).only('age').get() - self.assertTrue(isinstance(obj, Employee)) - - # Check field names are looked up properly - obj = Employee.objects(id=employee.id).only('salary').get() - self.assertEqual(obj.salary, employee.salary) - self.assertEqual(obj.name, None) - - def test_only_with_subfields(self): - class User(EmbeddedDocument): - name = StringField() - email = StringField() - - class Comment(EmbeddedDocument): - title = StringField() - text = StringField() - - class BlogPost(Document): - content = StringField() - author = EmbeddedDocumentField(User) - comments = ListField(EmbeddedDocumentField(Comment)) - - BlogPost.drop_collection() - - post = BlogPost(content='Had a good coffee today...') - post.author = User(name='Test User') - post.comments = [Comment(title='I aggree', text='Great post!'), Comment(title='Coffee', text='I hate coffee')] - post.save() - - obj = BlogPost.objects.only('author.name',).get() - self.assertEqual(obj.content, None) - self.assertEqual(obj.author.email, None) - self.assertEqual(obj.author.name, 'Test User') - self.assertEqual(obj.comments, []) - - obj = BlogPost.objects.only('content', 'comments.title',).get() - self.assertEqual(obj.content, 'Had a good coffee today...') - self.assertEqual(obj.author, None) - self.assertEqual(obj.comments[0].title, 'I aggree') - self.assertEqual(obj.comments[1].title, 'Coffee') - self.assertEqual(obj.comments[0].text, None) - self.assertEqual(obj.comments[1].text, None) - - obj = BlogPost.objects.only('comments',).get() - self.assertEqual(obj.content, None) - self.assertEqual(obj.author, None) - self.assertEqual(obj.comments[0].title, 'I aggree') - self.assertEqual(obj.comments[1].title, 'Coffee') - self.assertEqual(obj.comments[0].text, 'Great post!') - self.assertEqual(obj.comments[1].text, 'I hate coffee') - - BlogPost.drop_collection() - - def test_exclude(self): - class User(EmbeddedDocument): - name = StringField() - email = StringField() - - class Comment(EmbeddedDocument): - title = StringField() - text = StringField() - - class BlogPost(Document): - content = StringField() - author = EmbeddedDocumentField(User) - comments = ListField(EmbeddedDocumentField(Comment)) - - BlogPost.drop_collection() - - post = BlogPost(content='Had a good coffee today...') - post.author = User(name='Test User') - post.comments = [Comment(title='I aggree', text='Great post!'), Comment(title='Coffee', text='I hate coffee')] - post.save() - - obj = BlogPost.objects.exclude('author', 'comments.text').get() - self.assertEqual(obj.author, None) - self.assertEqual(obj.content, 'Had a good coffee today...') - self.assertEqual(obj.comments[0].title, 'I aggree') - self.assertEqual(obj.comments[0].text, None) - - BlogPost.drop_collection() - - def test_exclude_only_combining(self): - class Attachment(EmbeddedDocument): - name = StringField() - content = StringField() - - class Email(Document): - sender = StringField() - to = StringField() - subject = StringField() - body = StringField() - content_type = StringField() - attachments = ListField(EmbeddedDocumentField(Attachment)) - - Email.drop_collection() - email = Email(sender='me', to='you', subject='From Russia with Love', body='Hello!', content_type='text/plain') - email.attachments = [ - Attachment(name='file1.doc', content='ABC'), - Attachment(name='file2.doc', content='XYZ'), - ] - email.save() - - obj = Email.objects.exclude('content_type').exclude('body').get() - self.assertEqual(obj.sender, 'me') - self.assertEqual(obj.to, 'you') - self.assertEqual(obj.subject, 'From Russia with Love') - self.assertEqual(obj.body, None) - self.assertEqual(obj.content_type, None) - - obj = Email.objects.only('sender', 'to').exclude('body', 'sender').get() - self.assertEqual(obj.sender, None) - self.assertEqual(obj.to, 'you') - self.assertEqual(obj.subject, None) - self.assertEqual(obj.body, None) - self.assertEqual(obj.content_type, None) - - obj = Email.objects.exclude('attachments.content').exclude('body').only('to', 'attachments.name').get() - self.assertEqual(obj.attachments[0].name, 'file1.doc') - self.assertEqual(obj.attachments[0].content, None) - self.assertEqual(obj.sender, None) - self.assertEqual(obj.to, 'you') - self.assertEqual(obj.subject, None) - self.assertEqual(obj.body, None) - self.assertEqual(obj.content_type, None) - - Email.drop_collection() - - def test_all_fields(self): - - class Email(Document): - sender = StringField() - to = StringField() - subject = StringField() - body = StringField() - content_type = StringField() - - Email.drop_collection() - - email = Email(sender='me', to='you', subject='From Russia with Love', body='Hello!', content_type='text/plain') - email.save() - - obj = Email.objects.exclude('content_type', 'body').only('to', 'body').all_fields().get() - self.assertEqual(obj.sender, 'me') - self.assertEqual(obj.to, 'you') - self.assertEqual(obj.subject, 'From Russia with Love') - self.assertEqual(obj.body, 'Hello!') - self.assertEqual(obj.content_type, 'text/plain') - - Email.drop_collection() - - def test_slicing_fields(self): - """Ensure that query slicing an array works. - """ - class Numbers(Document): - n = ListField(IntField()) - - Numbers.drop_collection() - - numbers = Numbers(n=[0,1,2,3,4,5,-5,-4,-3,-2,-1]) - numbers.save() - - # first three - numbers = Numbers.objects.fields(slice__n=3).get() - self.assertEqual(numbers.n, [0, 1, 2]) - - # last three - numbers = Numbers.objects.fields(slice__n=-3).get() - self.assertEqual(numbers.n, [-3, -2, -1]) - - # skip 2, limit 3 - numbers = Numbers.objects.fields(slice__n=[2, 3]).get() - self.assertEqual(numbers.n, [2, 3, 4]) - - # skip to fifth from last, limit 4 - numbers = Numbers.objects.fields(slice__n=[-5, 4]).get() - self.assertEqual(numbers.n, [-5, -4, -3, -2]) - - # skip to fifth from last, limit 10 - numbers = Numbers.objects.fields(slice__n=[-5, 10]).get() - self.assertEqual(numbers.n, [-5, -4, -3, -2, -1]) - - # skip to fifth from last, limit 10 dict method - numbers = Numbers.objects.fields(n={"$slice": [-5, 10]}).get() - self.assertEqual(numbers.n, [-5, -4, -3, -2, -1]) - - def test_slicing_nested_fields(self): - """Ensure that query slicing an embedded array works. - """ - - class EmbeddedNumber(EmbeddedDocument): - n = ListField(IntField()) - - class Numbers(Document): - embedded = EmbeddedDocumentField(EmbeddedNumber) - - Numbers.drop_collection() - - numbers = Numbers() - numbers.embedded = EmbeddedNumber(n=[0,1,2,3,4,5,-5,-4,-3,-2,-1]) - numbers.save() - - # first three - numbers = Numbers.objects.fields(slice__embedded__n=3).get() - self.assertEqual(numbers.embedded.n, [0, 1, 2]) - - # last three - numbers = Numbers.objects.fields(slice__embedded__n=-3).get() - self.assertEqual(numbers.embedded.n, [-3, -2, -1]) - - # skip 2, limit 3 - numbers = Numbers.objects.fields(slice__embedded__n=[2, 3]).get() - self.assertEqual(numbers.embedded.n, [2, 3, 4]) - - # skip to fifth from last, limit 4 - numbers = Numbers.objects.fields(slice__embedded__n=[-5, 4]).get() - self.assertEqual(numbers.embedded.n, [-5, -4, -3, -2]) - - # skip to fifth from last, limit 10 - numbers = Numbers.objects.fields(slice__embedded__n=[-5, 10]).get() - self.assertEqual(numbers.embedded.n, [-5, -4, -3, -2, -1]) - - # skip to fifth from last, limit 10 dict method - numbers = Numbers.objects.fields(embedded__n={"$slice": [-5, 10]}).get() - self.assertEqual(numbers.embedded.n, [-5, -4, -3, -2, -1]) - def test_find_embedded(self): """Ensure that an embedded document is properly returned from a query. """ diff --git a/tests/queryset/visitor.py b/tests/queryset/visitor.py index 71c35615..82a99136 100644 --- a/tests/queryset/visitor.py +++ b/tests/queryset/visitor.py @@ -307,4 +307,7 @@ class QTest(unittest.TestCase): self.assertEqual(1, User.objects.filter( Q(email='example@example.com') | Q(name='John Doe') - ).limit(2).filter(pk=pk).count()) \ No newline at end of file + ).limit(2).filter(pk=pk).count()) + +if __name__ == '__main__': + unittest.main() From 3074dad293f7c0e7753fd194247675c9b955a011 Mon Sep 17 00:00:00 2001 From: Ross Lawley Date: Wed, 19 Dec 2012 13:58:22 +0000 Subject: [PATCH 046/189] Test mixing only, include and exclude #191 --- tests/queryset/field_list.py | 34 ++++++++++++++++++++++++++++++++++ 1 file changed, 34 insertions(+) diff --git a/tests/queryset/field_list.py b/tests/queryset/field_list.py index 6a9c6a9f..9e711336 100644 --- a/tests/queryset/field_list.py +++ b/tests/queryset/field_list.py @@ -81,6 +81,40 @@ class OnlyExcludeAllTest(unittest.TestCase): Person.drop_collection() self.Person = Person + def test_mixing_only_exclude(self): + + class MyDoc(Document): + a = StringField() + b = StringField() + c = StringField() + d = StringField() + e = StringField() + f = StringField() + + include = ['a', 'b', 'c', 'd', 'e'] + exclude = ['d', 'e'] + only = ['b', 'c'] + + qs = MyDoc.objects.fields(**dict(((i, 1) for i in include))) + self.assertEqual(qs._loaded_fields.as_dict(), + {'a': 1, 'b': 1, 'c': 1, 'd': 1, 'e': 1}) + qs = qs.only(*only) + self.assertEqual(qs._loaded_fields.as_dict(), {'b': 1, 'c': 1}) + qs = qs.exclude(*exclude) + self.assertEqual(qs._loaded_fields.as_dict(), {'b': 1, 'c': 1}) + + qs = MyDoc.objects.fields(**dict(((i, 1) for i in include))) + qs = qs.exclude(*exclude) + self.assertEqual(qs._loaded_fields.as_dict(), {'a': 1, 'b': 1, 'c': 1}) + qs = qs.only(*only) + self.assertEqual(qs._loaded_fields.as_dict(), {'b': 1, 'c': 1}) + + qs = MyDoc.objects.exclude(*exclude) + qs = qs.fields(**dict(((i, 1) for i in include))) + self.assertEqual(qs._loaded_fields.as_dict(), {'a': 1, 'b': 1, 'c': 1}) + qs = qs.only(*only) + self.assertEqual(qs._loaded_fields.as_dict(), {'b': 1, 'c': 1}) + def test_only(self): """Ensure that QuerySet.only only returns the requested fields. """ From 1c10f3020b079247ce1d0bca269fd03635c2ca70 Mon Sep 17 00:00:00 2001 From: Ross Lawley Date: Wed, 19 Dec 2012 14:56:15 +0000 Subject: [PATCH 047/189] Added support for multiple slices Also made slicing chainable. (#170) (#190) (#191) --- docs/changelog.rst | 1 + mongoengine/queryset/field_list.py | 21 ++++++++++++++++++++- tests/queryset/field_list.py | 29 +++++++++++++++++++++++++++++ 3 files changed, 50 insertions(+), 1 deletion(-) diff --git a/docs/changelog.rst b/docs/changelog.rst index 8fc279e7..9e1cec80 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -21,6 +21,7 @@ Changes in 0.8 - Remove _types and just use _cls for inheritance (#148) - Only allow QNode instances to be passed as query objects (#199) - Dynamic fields are now validated on save (#153) (#154) +- Added support for multiple slices and made slicing chainable. (#170) (#190) (#191) Changes in 0.7.9 ================ diff --git a/mongoengine/queryset/field_list.py b/mongoengine/queryset/field_list.py index 1c825fa9..7b2b0cb5 100644 --- a/mongoengine/queryset/field_list.py +++ b/mongoengine/queryset/field_list.py @@ -12,20 +12,31 @@ class QueryFieldList(object): self.fields = set(fields) self.always_include = set(always_include) self._id = None + self.slice = {} def __add__(self, f): - if not self.fields: + if isinstance(f.value, dict): + for field in f.fields: + self.slice[field] = f.value + if not self.fields: + self.fields = f.fields + elif not self.fields: self.fields = f.fields self.value = f.value + self.slice = {} elif self.value is self.ONLY and f.value is self.ONLY: + self._clean_slice() self.fields = self.fields.intersection(f.fields) elif self.value is self.EXCLUDE and f.value is self.EXCLUDE: self.fields = self.fields.union(f.fields) + self._clean_slice() elif self.value is self.ONLY and f.value is self.EXCLUDE: self.fields -= f.fields + self._clean_slice() elif self.value is self.EXCLUDE and f.value is self.ONLY: self.value = self.ONLY self.fields = f.fields - self.fields + self._clean_slice() if '_id' in f.fields: self._id = f.value @@ -42,10 +53,18 @@ class QueryFieldList(object): def as_dict(self): field_list = dict((field, self.value) for field in self.fields) + if self.slice: + field_list.update(self.slice) if self._id is not None: field_list['_id'] = self._id return field_list def reset(self): self.fields = set([]) + self.slice = {} self.value = self.ONLY + + def _clean_slice(self): + if self.slice: + for field in set(self.slice.keys()) - self.fields: + del self.slice[field] diff --git a/tests/queryset/field_list.py b/tests/queryset/field_list.py index 9e711336..4a8a72b3 100644 --- a/tests/queryset/field_list.py +++ b/tests/queryset/field_list.py @@ -115,6 +115,35 @@ class OnlyExcludeAllTest(unittest.TestCase): qs = qs.only(*only) self.assertEqual(qs._loaded_fields.as_dict(), {'b': 1, 'c': 1}) + def test_slicing(self): + + class MyDoc(Document): + a = ListField() + b = ListField() + c = ListField() + d = ListField() + e = ListField() + f = ListField() + + include = ['a', 'b', 'c', 'd', 'e'] + exclude = ['d', 'e'] + only = ['b', 'c'] + + qs = MyDoc.objects.fields(**dict(((i, 1) for i in include))) + qs = qs.exclude(*exclude) + qs = qs.only(*only) + qs = qs.fields(slice__b=5) + self.assertEqual(qs._loaded_fields.as_dict(), + {'b': {'$slice': 5}, 'c': 1}) + + qs = qs.fields(slice__c=[5, 1]) + self.assertEqual(qs._loaded_fields.as_dict(), + {'b': {'$slice': 5}, 'c': {'$slice': [5, 1]}}) + + qs = qs.exclude('c') + self.assertEqual(qs._loaded_fields.as_dict(), + {'b': {'$slice': 5}}) + def test_only(self): """Ensure that QuerySet.only only returns the requested fields. """ From f335591045e01ed12b12dca4964c3a8c2200a35f Mon Sep 17 00:00:00 2001 From: Ross Lawley Date: Wed, 19 Dec 2012 16:55:14 +0000 Subject: [PATCH 048/189] Fix index build_spec #177 --- docs/changelog.rst | 4 +- mongoengine/base/document.py | 2 +- mongoengine/fields.py | 2 +- tests/document/indexes.py | 108 +++++++++++++++++++++++------------ 4 files changed, 76 insertions(+), 40 deletions(-) diff --git a/docs/changelog.rst b/docs/changelog.rst index 9e1cec80..279abc98 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -2,8 +2,8 @@ Changelog ========= -Changes in 0.8 -============== +Changes in 0.8.X +================ - Fixed db_alias and inherited Documents (#143) - Documentation update for document errors (#124) - Deprecated `get_or_create` (#35) diff --git a/mongoengine/base/document.py b/mongoengine/base/document.py index affc20e0..93bde8ec 100644 --- a/mongoengine/base/document.py +++ b/mongoengine/base/document.py @@ -525,7 +525,7 @@ class BaseDocument(object): # Check to see if we need to include _cls allow_inheritance = cls._meta.get('allow_inheritance', - ALLOW_INHERITANCE) != False + ALLOW_INHERITANCE) include_cls = allow_inheritance and not spec.get('sparse', False) for key in spec['fields']: diff --git a/mongoengine/fields.py b/mongoengine/fields.py index 73c0db4d..3f9810ff 100644 --- a/mongoengine/fields.py +++ b/mongoengine/fields.py @@ -749,7 +749,7 @@ class ReferenceField(BaseField): if dbref is None: msg = ("ReferenceFields will default to using ObjectId " - " strings in 0.8, set DBRef=True if this isn't desired") + "in 0.8, set DBRef=True if this isn't desired") warnings.warn(msg, FutureWarning) self.dbref = dbref if dbref is not None else True # To change in 0.8 diff --git a/tests/document/indexes.py b/tests/document/indexes.py index 8f83afc7..9ebd9cb0 100644 --- a/tests/document/indexes.py +++ b/tests/document/indexes.py @@ -1,31 +1,23 @@ # -*- coding: utf-8 -*- from __future__ import with_statement -import bson -import os -import pickle -import pymongo -import sys import unittest -import uuid -import warnings +import sys + +sys.path[0:0] = [""] + +import os +import pymongo from nose.plugins.skip import SkipTest from datetime import datetime -from tests.fixtures import Base, Mixin, PickleEmbedded, PickleTest - from mongoengine import * -from mongoengine.errors import (NotRegistered, InvalidDocumentError, - InvalidQueryError) -from mongoengine.queryset import NULLIFY, Q from mongoengine.connection import get_db, get_connection -TEST_IMAGE_PATH = os.path.join(os.path.dirname(__file__), 'mongoengine.png') - -__all__ = ("InstanceTest", ) +__all__ = ("IndexesTest", ) -class InstanceTest(unittest.TestCase): +class IndexesTest(unittest.TestCase): def setUp(self): connect(db='mongoenginetest') @@ -47,20 +39,59 @@ class InstanceTest(unittest.TestCase): continue self.db.drop_collection(collection) - def test_indexes_document(self, ): + def ztest_indexes_document(self, ): """Ensure that indexes are used when meta[indexes] is specified for Documents """ - index_test(Document) + self.index_test(Document) def test_indexes_dynamic_document(self, ): """Ensure that indexes are used when meta[indexes] is specified for Dynamic Documents """ - index_test(DynamicDocument) + self.index_test(DynamicDocument) def index_test(self, InheritFrom): + class BlogPost(InheritFrom): + date = DateTimeField(db_field='addDate', default=datetime.now) + category = StringField() + tags = ListField(StringField()) + meta = { + 'indexes': [ + '-date', + 'tags', + ('category', '-date') + ] + } + + expected_specs = [{'fields': [('addDate', -1)]}, + {'fields': [('tags', 1)]}, + {'fields': [('category', 1), ('addDate', -1)]}] + self.assertEqual(expected_specs, BlogPost._meta['index_specs']) + + BlogPost.ensure_indexes() + info = BlogPost.objects._collection.index_information() + # _id, '-date', 'tags', ('cat', 'date') + self.assertEqual(len(info), 4) + info = [value['key'] for key, value in info.iteritems()] + for expected in expected_specs: + self.assertTrue(expected['fields'] in info) + + def test_indexes_document_inheritance(self): + """Ensure that indexes are used when meta[indexes] is specified for + Documents + """ + self.index_test_inheritance(Document) + + def test_indexes_dynamic_document_inheritance(self): + """Ensure that indexes are used when meta[indexes] is specified for + Dynamic Documents + """ + self.index_test_inheritance(DynamicDocument) + + def index_test_inheritance(self, InheritFrom): + class BlogPost(InheritFrom): date = DateTimeField(db_field='addDate', default=datetime.now) category = StringField() @@ -217,7 +248,7 @@ class InstanceTest(unittest.TestCase): info = BlogPost.objects._collection.index_information() # _id, '-date' - self.assertEqual(len(info), 3) + self.assertEqual(len(info), 2) # Indexes are lazy so use list() to perform query list(BlogPost.objects) @@ -265,7 +296,6 @@ class InstanceTest(unittest.TestCase): } user_guid = StringField(required=True) - User.drop_collection() u = User(user_guid='123') @@ -295,7 +325,7 @@ class InstanceTest(unittest.TestCase): BlogPost.drop_collection() info = BlogPost.objects._collection.index_information() - self.assertEqual(info.keys(), ['_cls_1_date.yr_-1', '_id_']) + self.assertEqual(info.keys(), ['date.yr_-1', '_id_']) BlogPost.drop_collection() def test_list_embedded_document_index(self): @@ -318,7 +348,7 @@ class InstanceTest(unittest.TestCase): info = BlogPost.objects._collection.index_information() # we don't use _cls in with list fields by default - self.assertEqual(info.keys(), ['_id_', '_cls_1_tags.tag_1']) + self.assertEqual(info.keys(), ['_id_', 'tags.tag_1']) post1 = BlogPost(title="Embedded Indexes tests in place", tags=[Tag(name="about"), Tag(name="time")] @@ -347,7 +377,7 @@ class InstanceTest(unittest.TestCase): class Parent(Document): name = StringField() - location = ReferenceField(Location) + location = ReferenceField(Location, dbref=False) Location.drop_collection() Parent.drop_collection() @@ -396,8 +426,7 @@ class InstanceTest(unittest.TestCase): meta = { 'indexes': [ ['categories', 'id'] - ], - 'allow_inheritance': False + ] } title = StringField(required=True) @@ -498,15 +527,18 @@ class InstanceTest(unittest.TestCase): BlogPost.drop_collection() - post1 = BlogPost(title='test1', sub=SubDocument(year=2009, slug="test")) + post1 = BlogPost(title='test1', + sub=SubDocument(year=2009, slug="test")) post1.save() # sub.slug is different so won't raise exception - post2 = BlogPost(title='test2', sub=SubDocument(year=2010, slug='another-slug')) + post2 = BlogPost(title='test2', + sub=SubDocument(year=2010, slug='another-slug')) post2.save() # Now there will be two docs with the same sub.slug - post3 = BlogPost(title='test3', sub=SubDocument(year=2010, slug='test')) + post3 = BlogPost(title='test3', + sub=SubDocument(year=2010, slug='test')) self.assertRaises(NotUniqueError, post3.save) BlogPost.drop_collection() @@ -525,19 +557,23 @@ class InstanceTest(unittest.TestCase): BlogPost.drop_collection() - post1 = BlogPost(title='test1', sub=SubDocument(year=2009, slug="test")) + post1 = BlogPost(title='test1', + sub=SubDocument(year=2009, slug="test")) post1.save() # sub.slug is different so won't raise exception - post2 = BlogPost(title='test2', sub=SubDocument(year=2010, slug='another-slug')) + post2 = BlogPost(title='test2', + sub=SubDocument(year=2010, slug='another-slug')) post2.save() # Now there will be two docs with the same sub.slug - post3 = BlogPost(title='test3', sub=SubDocument(year=2010, slug='test')) + post3 = BlogPost(title='test3', + sub=SubDocument(year=2010, slug='test')) self.assertRaises(NotUniqueError, post3.save) # Now there will be two docs with the same title and year - post3 = BlogPost(title='test1', sub=SubDocument(year=2009, slug='test-1')) + post3 = BlogPost(title='test1', + sub=SubDocument(year=2009, slug='test-1')) self.assertRaises(NotUniqueError, post3.save) BlogPost.drop_collection() @@ -566,7 +602,7 @@ class InstanceTest(unittest.TestCase): list(Log.objects) info = Log.objects._collection.index_information() self.assertEqual(3600, - info['_cls_1_created_1']['expireAfterSeconds']) + info['created_1']['expireAfterSeconds']) def test_unique_and_indexes(self): """Ensure that 'unique' constraints aren't overridden by @@ -586,7 +622,7 @@ class InstanceTest(unittest.TestCase): cust_dupe = Customer(cust_id=1) try: cust_dupe.save() - raise AssertionError, "We saved a dupe!" + raise AssertionError("We saved a dupe!") except NotUniqueError: pass Customer.drop_collection() @@ -630,7 +666,7 @@ class InstanceTest(unittest.TestCase): info = BlogPost.objects._collection.index_information() info = [value['key'] for key, value in info.iteritems()] - index_item = [('_cls', 1), ('_id', 1), ('comments.comment_id', 1)] + index_item = [('_id', 1), ('comments.comment_id', 1)] self.assertTrue(index_item in info) if __name__ == '__main__': From 485b811bd00c4c96486b8537347bbc33f05b87d1 Mon Sep 17 00:00:00 2001 From: Ross Lawley Date: Wed, 19 Dec 2012 17:05:27 +0000 Subject: [PATCH 049/189] Test case for embedded docs and 2d indexes #183 --- tests/document/indexes.py | 24 ++++++++++++++++++++++++ 1 file changed, 24 insertions(+) diff --git a/tests/document/indexes.py b/tests/document/indexes.py index 9ebd9cb0..285d8c6c 100644 --- a/tests/document/indexes.py +++ b/tests/document/indexes.py @@ -225,6 +225,30 @@ class IndexesTest(unittest.TestCase): info = [value['key'] for key, value in info.iteritems()] self.assertTrue([('location.point', '2d')] in info) + def test_explicit_geo2d_index_embedded(self): + """Ensure that geo2d indexes work when created via meta[indexes] + """ + class EmbeddedLocation(EmbeddedDocument): + location = DictField() + + class Place(Document): + current = DictField( + field=EmbeddedDocumentField('EmbeddedLocation')) + meta = { + 'allow_inheritance': True, + 'indexes': [ + '*current.location.point', + ] + } + + self.assertEqual([{'fields': [('current.location.point', '2d')]}], + Place._meta['index_specs']) + + Place.ensure_indexes() + info = Place._get_collection().index_information() + info = [value['key'] for key, value in info.iteritems()] + self.assertTrue([('current.location.point', '2d')] in info) + def test_dictionary_indexes(self): """Ensure that indexes are used when meta[indexes] contains dictionaries instead of lists. From c5b047d0cde0e40c844183a5e5f0c500e63ff8f7 Mon Sep 17 00:00:00 2001 From: Ross Lawley Date: Fri, 21 Dec 2012 11:55:05 +0000 Subject: [PATCH 050/189] Fixed GridFSProxy __getattr__ behaviour (#196) --- AUTHORS | 3 +- docs/changelog.rst | 1 + docs/guide/gridfs.rst | 16 +- mongoengine/fields.py | 2 +- tests/document/instance.py | 3 +- tests/fields/__init__.py | 2 + tests/{test_fields.py => fields/fields.py} | 305 +---------------- tests/fields/file.py | 370 +++++++++++++++++++++ tests/{document => fields}/mongoengine.png | Bin 9 files changed, 384 insertions(+), 318 deletions(-) create mode 100644 tests/fields/__init__.py rename tests/{test_fields.py => fields/fields.py} (87%) create mode 100644 tests/fields/file.py rename tests/{document => fields}/mongoengine.png (100%) diff --git a/AUTHORS b/AUTHORS index 794f297d..b49ddabd 100644 --- a/AUTHORS +++ b/AUTHORS @@ -130,4 +130,5 @@ that much better: * Jakub Kot * Jorge Bastida * Stefan Wójcik - * Pete Campton \ No newline at end of file + * Pete Campton + * Martyn Smith \ No newline at end of file diff --git a/docs/changelog.rst b/docs/changelog.rst index 279abc98..352d0c77 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -22,6 +22,7 @@ Changes in 0.8.X - Only allow QNode instances to be passed as query objects (#199) - Dynamic fields are now validated on save (#153) (#154) - Added support for multiple slices and made slicing chainable. (#170) (#190) (#191) +- Fixed GridFSProxy __getattr__ behaviour (#196) Changes in 0.7.9 ================ diff --git a/docs/guide/gridfs.rst b/docs/guide/gridfs.rst index 9c80a99e..11259477 100644 --- a/docs/guide/gridfs.rst +++ b/docs/guide/gridfs.rst @@ -18,20 +18,10 @@ a document is created to store details about animals, including a photo:: family = StringField() photo = FileField() - marmot = Animal('Marmota', 'Sciuridae') - - marmot_photo = open('marmot.jpg', 'r') # Retrieve a photo from disk - marmot.photo = marmot_photo # Store photo in the document - marmot.photo.content_type = 'image/jpeg' # Store metadata - - marmot.save() - -Another way of writing to a :class:`~mongoengine.FileField` is to use the -:func:`put` method. This allows for metadata to be stored in the same call as -the file:: - - marmot.photo.put(marmot_photo, content_type='image/jpeg') + marmot = Animal(genus='Marmota', family='Sciuridae') + marmot_photo = open('marmot.jpg', 'r') + marmot.photo.put(marmot_photo, content_type = 'image/jpeg') marmot.save() Retrieval diff --git a/mongoengine/fields.py b/mongoengine/fields.py index 3f9810ff..1e1a5ce8 100644 --- a/mongoengine/fields.py +++ b/mongoengine/fields.py @@ -969,7 +969,7 @@ class GridFSProxy(object): if name in attrs: return self.__getattribute__(name) obj = self.get() - if name in dir(obj): + if hasattr(obj, name): return getattr(obj, name) raise AttributeError diff --git a/tests/document/instance.py b/tests/document/instance.py index 5e29dc31..0054480f 100644 --- a/tests/document/instance.py +++ b/tests/document/instance.py @@ -19,7 +19,8 @@ from mongoengine.queryset import NULLIFY, Q from mongoengine.connection import get_db from mongoengine.base import get_document -TEST_IMAGE_PATH = os.path.join(os.path.dirname(__file__), 'mongoengine.png') +TEST_IMAGE_PATH = os.path.join(os.path.dirname(__file__), + '../fields/mongoengine.png') __all__ = ("InstanceTest",) diff --git a/tests/fields/__init__.py b/tests/fields/__init__.py new file mode 100644 index 00000000..86dfa840 --- /dev/null +++ b/tests/fields/__init__.py @@ -0,0 +1,2 @@ +from fields import * +from file import * \ No newline at end of file diff --git a/tests/test_fields.py b/tests/fields/fields.py similarity index 87% rename from tests/test_fields.py rename to tests/fields/fields.py index 97a2d5fd..a96ff0b9 100644 --- a/tests/test_fields.py +++ b/tests/fields/fields.py @@ -4,24 +4,21 @@ import sys sys.path[0:0] = [""] import datetime -import os import unittest import uuid -import tempfile from decimal import Decimal from bson import Binary, DBRef, ObjectId -import gridfs -from nose.plugins.skip import SkipTest from mongoengine import * from mongoengine.connection import get_db from mongoengine.base import _document_registry from mongoengine.errors import NotRegistered -from mongoengine.python_support import PY3, b, StringIO, bin_type +from mongoengine.python_support import PY3, b, bin_type + +__all__ = ("FieldTest", ) -TEST_IMAGE_PATH = os.path.join(os.path.dirname(__file__), 'document/mongoengine.png') class FieldTest(unittest.TestCase): @@ -1728,302 +1725,6 @@ class FieldTest(unittest.TestCase): Shirt.drop_collection() - def test_file_fields(self): - """Ensure that file fields can be written to and their data retrieved - """ - class PutFile(Document): - the_file = FileField() - - class StreamFile(Document): - the_file = FileField() - - class SetFile(Document): - the_file = FileField() - - text = b('Hello, World!') - more_text = b('Foo Bar') - content_type = 'text/plain' - - PutFile.drop_collection() - StreamFile.drop_collection() - SetFile.drop_collection() - - putfile = PutFile() - putfile.the_file.put(text, content_type=content_type) - putfile.save() - putfile.validate() - result = PutFile.objects.first() - self.assertTrue(putfile == result) - self.assertEqual(result.the_file.read(), text) - self.assertEqual(result.the_file.content_type, content_type) - result.the_file.delete() # Remove file from GridFS - PutFile.objects.delete() - - # Ensure file-like objects are stored - putfile = PutFile() - putstring = StringIO() - putstring.write(text) - putstring.seek(0) - putfile.the_file.put(putstring, content_type=content_type) - putfile.save() - putfile.validate() - result = PutFile.objects.first() - self.assertTrue(putfile == result) - self.assertEqual(result.the_file.read(), text) - self.assertEqual(result.the_file.content_type, content_type) - result.the_file.delete() - - streamfile = StreamFile() - streamfile.the_file.new_file(content_type=content_type) - streamfile.the_file.write(text) - streamfile.the_file.write(more_text) - streamfile.the_file.close() - streamfile.save() - streamfile.validate() - result = StreamFile.objects.first() - self.assertTrue(streamfile == result) - self.assertEqual(result.the_file.read(), text + more_text) - self.assertEqual(result.the_file.content_type, content_type) - result.the_file.seek(0) - self.assertEqual(result.the_file.tell(), 0) - self.assertEqual(result.the_file.read(len(text)), text) - self.assertEqual(result.the_file.tell(), len(text)) - self.assertEqual(result.the_file.read(len(more_text)), more_text) - self.assertEqual(result.the_file.tell(), len(text + more_text)) - result.the_file.delete() - - # Ensure deleted file returns None - self.assertTrue(result.the_file.read() == None) - - setfile = SetFile() - setfile.the_file = text - setfile.save() - setfile.validate() - result = SetFile.objects.first() - self.assertTrue(setfile == result) - self.assertEqual(result.the_file.read(), text) - - # Try replacing file with new one - result.the_file.replace(more_text) - result.save() - result.validate() - result = SetFile.objects.first() - self.assertTrue(setfile == result) - self.assertEqual(result.the_file.read(), more_text) - result.the_file.delete() - - PutFile.drop_collection() - StreamFile.drop_collection() - SetFile.drop_collection() - - # Make sure FileField is optional and not required - class DemoFile(Document): - the_file = FileField() - DemoFile.objects.create() - - def test_file_field_no_default(self): - - class GridDocument(Document): - the_file = FileField() - - GridDocument.drop_collection() - - with tempfile.TemporaryFile() as f: - f.write(b("Hello World!")) - f.flush() - - # Test without default - doc_a = GridDocument() - doc_a.save() - - doc_b = GridDocument.objects.with_id(doc_a.id) - doc_b.the_file.replace(f, filename='doc_b') - doc_b.save() - self.assertNotEqual(doc_b.the_file.grid_id, None) - - # Test it matches - doc_c = GridDocument.objects.with_id(doc_b.id) - self.assertEqual(doc_b.the_file.grid_id, doc_c.the_file.grid_id) - - # Test with default - doc_d = GridDocument(the_file=b('')) - doc_d.save() - - doc_e = GridDocument.objects.with_id(doc_d.id) - self.assertEqual(doc_d.the_file.grid_id, doc_e.the_file.grid_id) - - doc_e.the_file.replace(f, filename='doc_e') - doc_e.save() - - doc_f = GridDocument.objects.with_id(doc_e.id) - self.assertEqual(doc_e.the_file.grid_id, doc_f.the_file.grid_id) - - db = GridDocument._get_db() - grid_fs = gridfs.GridFS(db) - self.assertEqual(['doc_b', 'doc_e'], grid_fs.list()) - - def test_file_uniqueness(self): - """Ensure that each instance of a FileField is unique - """ - class TestFile(Document): - name = StringField() - the_file = FileField() - - # First instance - test_file = TestFile() - test_file.name = "Hello, World!" - test_file.the_file.put(b('Hello, World!')) - test_file.save() - - # Second instance - test_file_dupe = TestFile() - data = test_file_dupe.the_file.read() # Should be None - - self.assertTrue(test_file.name != test_file_dupe.name) - self.assertTrue(test_file.the_file.read() != data) - - TestFile.drop_collection() - - def test_file_boolean(self): - """Ensure that a boolean test of a FileField indicates its presence - """ - class TestFile(Document): - the_file = FileField() - - test_file = TestFile() - self.assertFalse(bool(test_file.the_file)) - test_file.the_file = b('Hello, World!') - test_file.the_file.content_type = 'text/plain' - test_file.save() - self.assertTrue(bool(test_file.the_file)) - - TestFile.drop_collection() - - def test_file_cmp(self): - """Test comparing against other types""" - class TestFile(Document): - the_file = FileField() - - test_file = TestFile() - self.assertFalse(test_file.the_file in [{"test": 1}]) - - def test_image_field(self): - if PY3: - raise SkipTest('PIL does not have Python 3 support') - - class TestImage(Document): - image = ImageField() - - TestImage.drop_collection() - - t = TestImage() - t.image.put(open(TEST_IMAGE_PATH, 'r')) - t.save() - - t = TestImage.objects.first() - - self.assertEqual(t.image.format, 'PNG') - - w, h = t.image.size - self.assertEqual(w, 371) - self.assertEqual(h, 76) - - t.image.delete() - - def test_image_field_resize(self): - if PY3: - raise SkipTest('PIL does not have Python 3 support') - - class TestImage(Document): - image = ImageField(size=(185, 37)) - - TestImage.drop_collection() - - t = TestImage() - t.image.put(open(TEST_IMAGE_PATH, 'r')) - t.save() - - t = TestImage.objects.first() - - self.assertEqual(t.image.format, 'PNG') - w, h = t.image.size - - self.assertEqual(w, 185) - self.assertEqual(h, 37) - - t.image.delete() - - def test_image_field_resize_force(self): - if PY3: - raise SkipTest('PIL does not have Python 3 support') - - class TestImage(Document): - image = ImageField(size=(185, 37, True)) - - TestImage.drop_collection() - - t = TestImage() - t.image.put(open(TEST_IMAGE_PATH, 'r')) - t.save() - - t = TestImage.objects.first() - - self.assertEqual(t.image.format, 'PNG') - w, h = t.image.size - - self.assertEqual(w, 185) - self.assertEqual(h, 37) - - t.image.delete() - - def test_image_field_thumbnail(self): - if PY3: - raise SkipTest('PIL does not have Python 3 support') - - class TestImage(Document): - image = ImageField(thumbnail_size=(92, 18)) - - TestImage.drop_collection() - - t = TestImage() - t.image.put(open(TEST_IMAGE_PATH, 'r')) - t.save() - - t = TestImage.objects.first() - - self.assertEqual(t.image.thumbnail.format, 'PNG') - self.assertEqual(t.image.thumbnail.width, 92) - self.assertEqual(t.image.thumbnail.height, 18) - - t.image.delete() - - def test_file_multidb(self): - register_connection('test_files', 'test_files') - class TestFile(Document): - name = StringField() - the_file = FileField(db_alias="test_files", - collection_name="macumba") - - TestFile.drop_collection() - - # delete old filesystem - get_db("test_files").macumba.files.drop() - get_db("test_files").macumba.chunks.drop() - - # First instance - test_file = TestFile() - test_file.name = "Hello, World!" - test_file.the_file.put(b('Hello, World!'), - name="hello.txt") - test_file.save() - - data = get_db("test_files").macumba.files.find_one() - self.assertEqual(data.get('name'), 'hello.txt') - - test_file = TestFile.objects.first() - self.assertEqual(test_file.the_file.read(), - b('Hello, World!')) - def test_geo_indexes(self): """Ensure that indexes are created automatically for GeoPointFields. """ diff --git a/tests/fields/file.py b/tests/fields/file.py new file mode 100644 index 00000000..17d9ec37 --- /dev/null +++ b/tests/fields/file.py @@ -0,0 +1,370 @@ +# -*- coding: utf-8 -*- +from __future__ import with_statement +import sys +sys.path[0:0] = [""] + +import datetime +import os +import unittest +import uuid +import tempfile + +from decimal import Decimal + +from bson import Binary, DBRef, ObjectId +import gridfs + +from nose.plugins.skip import SkipTest +from mongoengine import * +from mongoengine.connection import get_db +from mongoengine.base import _document_registry +from mongoengine.errors import NotRegistered +from mongoengine.python_support import PY3, b, StringIO, bin_type + +TEST_IMAGE_PATH = os.path.join(os.path.dirname(__file__), 'mongoengine.png') + + +class FileTest(unittest.TestCase): + + def setUp(self): + connect(db='mongoenginetest') + self.db = get_db() + + def tearDown(self): + self.db.drop_collection('fs.files') + self.db.drop_collection('fs.chunks') + + def test_file_field_optional(self): + # Make sure FileField is optional and not required + class DemoFile(Document): + the_file = FileField() + DemoFile.objects.create() + + def test_file_fields(self): + """Ensure that file fields can be written to and their data retrieved + """ + + class PutFile(Document): + the_file = FileField() + + PutFile.drop_collection() + + text = b('Hello, World!') + more_text = b('Foo Bar') + content_type = 'text/plain' + + putfile = PutFile() + putfile.the_file.put(text, content_type=content_type) + putfile.save() + putfile.validate() + result = PutFile.objects.first() + self.assertTrue(putfile == result) + self.assertEqual(result.the_file.read(), text) + self.assertEqual(result.the_file.content_type, content_type) + result.the_file.delete() # Remove file from GridFS + PutFile.objects.delete() + + # Ensure file-like objects are stored + PutFile.drop_collection() + + putfile = PutFile() + putstring = StringIO() + putstring.write(text) + putstring.seek(0) + putfile.the_file.put(putstring, content_type=content_type) + putfile.save() + putfile.validate() + result = PutFile.objects.first() + self.assertTrue(putfile == result) + self.assertEqual(result.the_file.read(), text) + self.assertEqual(result.the_file.content_type, content_type) + result.the_file.delete() + + def test_file_fields_stream(self): + """Ensure that file fields can be written to and their data retrieved + """ + class StreamFile(Document): + the_file = FileField() + + StreamFile.drop_collection() + + text = b('Hello, World!') + more_text = b('Foo Bar') + content_type = 'text/plain' + + streamfile = StreamFile() + streamfile.the_file.new_file(content_type=content_type) + streamfile.the_file.write(text) + streamfile.the_file.write(more_text) + streamfile.the_file.close() + streamfile.save() + streamfile.validate() + result = StreamFile.objects.first() + self.assertTrue(streamfile == result) + self.assertEqual(result.the_file.read(), text + more_text) + self.assertEqual(result.the_file.content_type, content_type) + result.the_file.seek(0) + self.assertEqual(result.the_file.tell(), 0) + self.assertEqual(result.the_file.read(len(text)), text) + self.assertEqual(result.the_file.tell(), len(text)) + self.assertEqual(result.the_file.read(len(more_text)), more_text) + self.assertEqual(result.the_file.tell(), len(text + more_text)) + result.the_file.delete() + + # Ensure deleted file returns None + self.assertTrue(result.the_file.read() == None) + + def test_file_fields_set(self): + + class SetFile(Document): + the_file = FileField() + + text = b('Hello, World!') + more_text = b('Foo Bar') + + SetFile.drop_collection() + + setfile = SetFile() + setfile.the_file = text + setfile.save() + + result = SetFile.objects.first() + self.assertTrue(setfile == result) + self.assertEqual(result.the_file.read(), text) + + # Try replacing file with new one + result.the_file.replace(more_text) + result.save() + result.validate() + result = SetFile.objects.first() + self.assertTrue(setfile == result) + self.assertEqual(result.the_file.read(), more_text) + result.the_file.delete() + + def test_file_field_no_default(self): + + class GridDocument(Document): + the_file = FileField() + + GridDocument.drop_collection() + + with tempfile.TemporaryFile() as f: + f.write(b("Hello World!")) + f.flush() + + # Test without default + doc_a = GridDocument() + doc_a.save() + + doc_b = GridDocument.objects.with_id(doc_a.id) + doc_b.the_file.replace(f, filename='doc_b') + doc_b.save() + self.assertNotEqual(doc_b.the_file.grid_id, None) + + # Test it matches + doc_c = GridDocument.objects.with_id(doc_b.id) + self.assertEqual(doc_b.the_file.grid_id, doc_c.the_file.grid_id) + + # Test with default + doc_d = GridDocument(the_file=b('')) + doc_d.save() + + doc_e = GridDocument.objects.with_id(doc_d.id) + self.assertEqual(doc_d.the_file.grid_id, doc_e.the_file.grid_id) + + doc_e.the_file.replace(f, filename='doc_e') + doc_e.save() + + doc_f = GridDocument.objects.with_id(doc_e.id) + self.assertEqual(doc_e.the_file.grid_id, doc_f.the_file.grid_id) + + db = GridDocument._get_db() + grid_fs = gridfs.GridFS(db) + self.assertEqual(['doc_b', 'doc_e'], grid_fs.list()) + + def test_file_uniqueness(self): + """Ensure that each instance of a FileField is unique + """ + class TestFile(Document): + name = StringField() + the_file = FileField() + + # First instance + test_file = TestFile() + test_file.name = "Hello, World!" + test_file.the_file.put(b('Hello, World!')) + test_file.save() + + # Second instance + test_file_dupe = TestFile() + data = test_file_dupe.the_file.read() # Should be None + + self.assertTrue(test_file.name != test_file_dupe.name) + self.assertTrue(test_file.the_file.read() != data) + + TestFile.drop_collection() + + def test_file_saving(self): + """Ensure you can add meta data to file""" + + class Animal(Document): + genus = StringField() + family = StringField() + photo = FileField() + + Animal.drop_collection() + marmot = Animal(genus='Marmota', family='Sciuridae') + + marmot_photo = open(TEST_IMAGE_PATH, 'r') # Retrieve a photo from disk + marmot.photo.put(marmot_photo, content_type='image/jpeg', foo='bar') + marmot.photo.close() + marmot.save() + + marmot = Animal.objects.get() + self.assertEqual(marmot.photo.content_type, 'image/jpeg') + self.assertEqual(marmot.photo.foo, 'bar') + + def test_file_boolean(self): + """Ensure that a boolean test of a FileField indicates its presence + """ + class TestFile(Document): + the_file = FileField() + TestFile.drop_collection() + + test_file = TestFile() + self.assertFalse(bool(test_file.the_file)) + test_file.the_file.put(b('Hello, World!'), content_type='text/plain') + test_file.save() + self.assertTrue(bool(test_file.the_file)) + + test_file = TestFile.objects.first() + self.assertEqual(test_file.the_file.content_type, "text/plain") + + def test_file_cmp(self): + """Test comparing against other types""" + class TestFile(Document): + the_file = FileField() + + test_file = TestFile() + self.assertFalse(test_file.the_file in [{"test": 1}]) + + def test_image_field(self): + if PY3: + raise SkipTest('PIL does not have Python 3 support') + + class TestImage(Document): + image = ImageField() + + TestImage.drop_collection() + + t = TestImage() + t.image.put(open(TEST_IMAGE_PATH, 'r')) + t.save() + + t = TestImage.objects.first() + + self.assertEqual(t.image.format, 'PNG') + + w, h = t.image.size + self.assertEqual(w, 371) + self.assertEqual(h, 76) + + t.image.delete() + + def test_image_field_resize(self): + if PY3: + raise SkipTest('PIL does not have Python 3 support') + + class TestImage(Document): + image = ImageField(size=(185, 37)) + + TestImage.drop_collection() + + t = TestImage() + t.image.put(open(TEST_IMAGE_PATH, 'r')) + t.save() + + t = TestImage.objects.first() + + self.assertEqual(t.image.format, 'PNG') + w, h = t.image.size + + self.assertEqual(w, 185) + self.assertEqual(h, 37) + + t.image.delete() + + def test_image_field_resize_force(self): + if PY3: + raise SkipTest('PIL does not have Python 3 support') + + class TestImage(Document): + image = ImageField(size=(185, 37, True)) + + TestImage.drop_collection() + + t = TestImage() + t.image.put(open(TEST_IMAGE_PATH, 'r')) + t.save() + + t = TestImage.objects.first() + + self.assertEqual(t.image.format, 'PNG') + w, h = t.image.size + + self.assertEqual(w, 185) + self.assertEqual(h, 37) + + t.image.delete() + + def test_image_field_thumbnail(self): + if PY3: + raise SkipTest('PIL does not have Python 3 support') + + class TestImage(Document): + image = ImageField(thumbnail_size=(92, 18)) + + TestImage.drop_collection() + + t = TestImage() + t.image.put(open(TEST_IMAGE_PATH, 'r')) + t.save() + + t = TestImage.objects.first() + + self.assertEqual(t.image.thumbnail.format, 'PNG') + self.assertEqual(t.image.thumbnail.width, 92) + self.assertEqual(t.image.thumbnail.height, 18) + + t.image.delete() + + def test_file_multidb(self): + register_connection('test_files', 'test_files') + + class TestFile(Document): + name = StringField() + the_file = FileField(db_alias="test_files", + collection_name="macumba") + + TestFile.drop_collection() + + # delete old filesystem + get_db("test_files").macumba.files.drop() + get_db("test_files").macumba.chunks.drop() + + # First instance + test_file = TestFile() + test_file.name = "Hello, World!" + test_file.the_file.put(b('Hello, World!'), + name="hello.txt") + test_file.save() + + data = get_db("test_files").macumba.files.find_one() + self.assertEqual(data.get('name'), 'hello.txt') + + test_file = TestFile.objects.first() + self.assertEqual(test_file.the_file.read(), + b('Hello, World!')) + +if __name__ == '__main__': + unittest.main() diff --git a/tests/document/mongoengine.png b/tests/fields/mongoengine.png similarity index 100% rename from tests/document/mongoengine.png rename to tests/fields/mongoengine.png From 286beca6c5798c0d5c25e7981a8a0c448743eb58 Mon Sep 17 00:00:00 2001 From: Ross Lawley Date: Fri, 21 Dec 2012 12:07:15 +0000 Subject: [PATCH 051/189] Added Marcelo Anton to authors #152 --- AUTHORS | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/AUTHORS b/AUTHORS index b49ddabd..4b3a2973 100644 --- a/AUTHORS +++ b/AUTHORS @@ -131,4 +131,5 @@ that much better: * Jorge Bastida * Stefan Wójcik * Pete Campton - * Martyn Smith \ No newline at end of file + * Martyn Smith + * Marcelo Anton \ No newline at end of file From 0c2fb6807e425638cc92a92a0ef058932a9b1c05 Mon Sep 17 00:00:00 2001 From: Ross Lawley Date: Fri, 21 Dec 2012 12:14:32 +0000 Subject: [PATCH 052/189] Added Aleksey Porfirov to AUTHORS #151 --- AUTHORS | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/AUTHORS b/AUTHORS index 4b3a2973..989fd68c 100644 --- a/AUTHORS +++ b/AUTHORS @@ -132,4 +132,5 @@ that much better: * Stefan Wójcik * Pete Campton * Martyn Smith - * Marcelo Anton \ No newline at end of file + * Marcelo Anton + * Aleksey Porfirov \ No newline at end of file From bf74d7537cafc9f9baf15a87eb8c2d7e569dd5c9 Mon Sep 17 00:00:00 2001 From: Ross Lawley Date: Fri, 21 Dec 2012 16:20:01 +0000 Subject: [PATCH 053/189] Fix Django timezone support - update field for callable #151 --- mongoengine/fields.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/mongoengine/fields.py b/mongoengine/fields.py index 1e1a5ce8..f6c03119 100644 --- a/mongoengine/fields.py +++ b/mongoengine/fields.py @@ -300,6 +300,8 @@ class DateTimeField(BaseField): return value if isinstance(value, datetime.date): return datetime.datetime(value.year, value.month, value.day) + if callable(value): + return value() # Attempt to parse a datetime: # value = smart_str(value) From 3aff4610394e31c64ac2a3ed2f256fd1f3e90da6 Mon Sep 17 00:00:00 2001 From: Ross Lawley Date: Fri, 21 Dec 2012 16:29:27 +0000 Subject: [PATCH 054/189] Fix test discovery --- tests/document/indexes.py | 37 +++++++++++++++++++------------------ 1 file changed, 19 insertions(+), 18 deletions(-) diff --git a/tests/document/indexes.py b/tests/document/indexes.py index 285d8c6c..445cfe2b 100644 --- a/tests/document/indexes.py +++ b/tests/document/indexes.py @@ -39,19 +39,19 @@ class IndexesTest(unittest.TestCase): continue self.db.drop_collection(collection) - def ztest_indexes_document(self, ): + def test_indexes_document(self): """Ensure that indexes are used when meta[indexes] is specified for Documents """ - self.index_test(Document) + self._index_test(Document) - def test_indexes_dynamic_document(self, ): + def test_indexes_dynamic_document(self): """Ensure that indexes are used when meta[indexes] is specified for Dynamic Documents """ - self.index_test(DynamicDocument) + self._index_test(DynamicDocument) - def index_test(self, InheritFrom): + def _index_test(self, InheritFrom): class BlogPost(InheritFrom): date = DateTimeField(db_field='addDate', default=datetime.now) @@ -78,19 +78,7 @@ class IndexesTest(unittest.TestCase): for expected in expected_specs: self.assertTrue(expected['fields'] in info) - def test_indexes_document_inheritance(self): - """Ensure that indexes are used when meta[indexes] is specified for - Documents - """ - self.index_test_inheritance(Document) - - def test_indexes_dynamic_document_inheritance(self): - """Ensure that indexes are used when meta[indexes] is specified for - Dynamic Documents - """ - self.index_test_inheritance(DynamicDocument) - - def index_test_inheritance(self, InheritFrom): + def _index_test_inheritance(self, InheritFrom): class BlogPost(InheritFrom): date = DateTimeField(db_field='addDate', default=datetime.now) @@ -137,6 +125,18 @@ class IndexesTest(unittest.TestCase): for expected in expected_specs: self.assertTrue(expected['fields'] in info) + def test_indexes_document_inheritance(self): + """Ensure that indexes are used when meta[indexes] is specified for + Documents + """ + self._index_test_inheritance(Document) + + def test_indexes_dynamic_document_inheritance(self): + """Ensure that indexes are used when meta[indexes] is specified for + Dynamic Documents + """ + self._index_test_inheritance(DynamicDocument) + def test_inherited_index(self): """Ensure index specs are inhertited correctly""" @@ -301,6 +301,7 @@ class IndexesTest(unittest.TestCase): meta = { 'indexes': ['name'], } + Person.drop_collection() Person(name="test", user_guid='123').save() From 1cdf71b647f31a8c7cf05b0256a9795038c8808d Mon Sep 17 00:00:00 2001 From: Ross Lawley Date: Fri, 21 Dec 2012 16:35:09 +0000 Subject: [PATCH 055/189] Simplified Q objects Removed QueryTreeTransformerVisitor (#98) (#171) --- docs/changelog.rst | 1 + mongoengine/queryset/visitor.py | 92 ++------------------------------- tests/queryset/visitor.py | 72 +++++++++++++++++--------- 3 files changed, 53 insertions(+), 112 deletions(-) diff --git a/docs/changelog.rst b/docs/changelog.rst index f934f5e0..b9ab42c3 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -24,6 +24,7 @@ Changes in 0.8.X - Added support for multiple slices and made slicing chainable. (#170) (#190) (#191) - Fixed GridFSProxy __getattr__ behaviour (#196) - Fix Django timezone support (#151) +- Simplified Q objects, removed QueryTreeTransformerVisitor (#98) (#171) Changes in 0.7.9 ================ diff --git a/mongoengine/queryset/visitor.py b/mongoengine/queryset/visitor.py index 94d6a5e1..8932a54f 100644 --- a/mongoengine/queryset/visitor.py +++ b/mongoengine/queryset/visitor.py @@ -55,57 +55,6 @@ class SimplificationVisitor(QNodeVisitor): return combined_query -class QueryTreeTransformerVisitor(QNodeVisitor): - """Transforms the query tree in to a form that may be used with MongoDB. - """ - - def visit_combination(self, combination): - if combination.operation == combination.AND: - # MongoDB doesn't allow us to have too many $or operations in our - # queries, so the aim is to move the ORs up the tree to one - # 'master' $or. Firstly, we must find all the necessary parts (part - # of an AND combination or just standard Q object), and store them - # separately from the OR parts. - or_groups = [] - and_parts = [] - for node in combination.children: - if isinstance(node, QCombination): - if node.operation == node.OR: - # Any of the children in an $or component may cause - # the query to succeed - or_groups.append(node.children) - elif node.operation == node.AND: - and_parts.append(node) - elif isinstance(node, Q): - and_parts.append(node) - - # Now we combine the parts into a usable query. AND together all of - # the necessary parts. Then for each $or part, create a new query - # that ANDs the necessary part with the $or part. - clauses = [] - for or_group in product(*or_groups): - q_object = reduce(lambda a, b: a & b, and_parts, Q()) - q_object = reduce(lambda a, b: a & b, or_group, q_object) - clauses.append(q_object) - # Finally, $or the generated clauses in to one query. Each of the - # clauses is sufficient for the query to succeed. - return reduce(lambda a, b: a | b, clauses, Q()) - - if combination.operation == combination.OR: - children = [] - # Crush any nested ORs in to this combination as MongoDB doesn't - # support nested $or operations - for node in combination.children: - if (isinstance(node, QCombination) and - node.operation == combination.OR): - children += node.children - else: - children.append(node) - combination.children = children - - return combination - - class QueryCompilerVisitor(QNodeVisitor): """Compiles the nodes in a query tree to a PyMongo-compatible query dictionary. @@ -115,45 +64,14 @@ class QueryCompilerVisitor(QNodeVisitor): self.document = document def visit_combination(self, combination): + operator = "$and" if combination.operation == combination.OR: - return {'$or': combination.children} - elif combination.operation == combination.AND: - return self._mongo_query_conjunction(combination.children) - return combination + operator = "$or" + return {operator: combination.children} def visit_query(self, query): return transform.query(self.document, **query.query) - def _mongo_query_conjunction(self, queries): - """Merges Mongo query dicts - effectively &ing them together. - """ - combined_query = {} - for query in queries: - for field, ops in query.items(): - if field not in combined_query: - combined_query[field] = ops - else: - # The field is already present in the query the only way - # we can merge is if both the existing value and the new - # value are operation dicts, reject anything else - if (not isinstance(combined_query[field], dict) or - not isinstance(ops, dict)): - message = 'Conflicting values for ' + field - raise InvalidQueryError(message) - - current_ops = set(combined_query[field].keys()) - new_ops = set(ops.keys()) - # Make sure that the same operation isn't applied more than - # once to a single field - intersection = current_ops.intersection(new_ops) - if intersection: - msg = 'Duplicate query conditions: ' - raise InvalidQueryError(msg + ', '.join(intersection)) - - # Right! We've got two non-overlapping dicts of operations! - combined_query[field].update(copy.deepcopy(ops)) - return combined_query - class QNode(object): """Base class for nodes in query trees. @@ -164,7 +82,6 @@ class QNode(object): def to_query(self, document): query = self.accept(SimplificationVisitor()) - query = query.accept(QueryTreeTransformerVisitor()) query = query.accept(QueryCompilerVisitor(document)) return query @@ -205,7 +122,8 @@ class QCombination(QNode): # If the child is a combination of the same type, we can merge its # children directly into this combinations children if isinstance(node, QCombination) and node.operation == operation: - self.children += node.children + # self.children += node.children + self.children.append(node) else: self.children.append(node) diff --git a/tests/queryset/visitor.py b/tests/queryset/visitor.py index 82a99136..4af39e87 100644 --- a/tests/queryset/visitor.py +++ b/tests/queryset/visitor.py @@ -115,29 +115,31 @@ class QTest(unittest.TestCase): x = IntField() y = BooleanField() + TestDoc.drop_collection() + query = (Q(x__gt=0) | Q(x__exists=False)) query &= Q(x__lt=100) - self.assertEqual(query.to_query(TestDoc), { - '$or': [ - {'x': {'$lt': 100, '$gt': 0}}, - {'x': {'$lt': 100, '$exists': False}}, - ] + self.assertEqual(query.to_query(TestDoc), {'$and': [ + {'$or': [{'x': {'$gt': 0}}, + {'x': {'$exists': False}}]}, + {'x': {'$lt': 100}}] }) q1 = (Q(x__gt=0) | Q(x__exists=False)) q2 = (Q(x__lt=100) | Q(y=True)) query = (q1 & q2).to_query(TestDoc) - self.assertEqual(['$or'], query.keys()) - conditions = [ - {'x': {'$lt': 100, '$gt': 0}}, - {'x': {'$lt': 100, '$exists': False}}, - {'x': {'$gt': 0}, 'y': True}, - {'x': {'$exists': False}, 'y': True}, - ] - self.assertEqual(len(conditions), len(query['$or'])) - for condition in conditions: - self.assertTrue(condition in query['$or']) + TestDoc(x=101).save() + TestDoc(x=10).save() + TestDoc(y=True).save() + + self.assertEqual(query, + {'$and': [ + {'$or': [{'x': {'$gt': 0}}, {'x': {'$exists': False}}]}, + {'$or': [{'x': {'$lt': 100}}, {'y': True}]} + ]}) + + self.assertEqual(2, TestDoc.objects(q1 & q2).count()) def test_or_and_or_combination(self): """Ensure that Q-objects handle ORing ANDed ORed components. :) @@ -146,20 +148,40 @@ class QTest(unittest.TestCase): x = IntField() y = BooleanField() + TestDoc.drop_collection() + TestDoc(x=-1, y=True).save() + TestDoc(x=101, y=True).save() + TestDoc(x=99, y=False).save() + TestDoc(x=101, y=False).save() + q1 = (Q(x__gt=0) & (Q(y=True) | Q(y__exists=False))) q2 = (Q(x__lt=100) & (Q(y=False) | Q(y__exists=False))) query = (q1 | q2).to_query(TestDoc) - self.assertEqual(['$or'], query.keys()) - conditions = [ - {'x': {'$gt': 0}, 'y': True}, - {'x': {'$gt': 0}, 'y': {'$exists': False}}, - {'x': {'$lt': 100}, 'y':False}, - {'x': {'$lt': 100}, 'y': {'$exists': False}}, - ] - self.assertEqual(len(conditions), len(query['$or'])) - for condition in conditions: - self.assertTrue(condition in query['$or']) + self.assertEqual(query, + {'$or': [ + {'$and': [{'x': {'$gt': 0}}, + {'$or': [{'y': True}, {'y': {'$exists': False}}]}]}, + {'$and': [{'x': {'$lt': 100}}, + {'$or': [{'y': False}, {'y': {'$exists': False}}]}]} + ]} + ) + + self.assertEqual(2, TestDoc.objects(q1 | q2).count()) + + def test_multiple_occurence_in_field(self): + class Test(Document): + name = StringField(max_length=40) + title = StringField(max_length=40) + + q1 = Q(name__contains='te') | Q(title__contains='te') + q2 = Q(name__contains='12') | Q(title__contains='12') + + q3 = q1 & q2 + + query = q3.to_query(Test) + self.assertEqual(query["$and"][0], q1.to_query(Test)) + self.assertEqual(query["$and"][1], q2.to_query(Test)) def test_q_clone(self): From b9e0f525262111433158db16307424086efb71e0 Mon Sep 17 00:00:00 2001 From: Ross Lawley Date: Fri, 21 Dec 2012 17:09:10 +0000 Subject: [PATCH 056/189] FileFields now copyable (#198) --- docs/changelog.rst | 1 + mongoengine/fields.py | 14 +++++++++++--- tests/fields/file.py | 39 ++++++++++++++++++++++++++------------- 3 files changed, 38 insertions(+), 16 deletions(-) diff --git a/docs/changelog.rst b/docs/changelog.rst index b9ab42c3..c6493038 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -25,6 +25,7 @@ Changes in 0.8.X - Fixed GridFSProxy __getattr__ behaviour (#196) - Fix Django timezone support (#151) - Simplified Q objects, removed QueryTreeTransformerVisitor (#98) (#171) +- FileFields now copyable (#198) Changes in 0.7.9 ================ diff --git a/mongoengine/fields.py b/mongoengine/fields.py index f6c03119..5f11ae3b 100644 --- a/mongoengine/fields.py +++ b/mongoengine/fields.py @@ -986,14 +986,22 @@ class GridFSProxy(object): self_dict['_fs'] = None return self_dict + def __copy__(self): + copied = GridFSProxy() + copied.__dict__.update(self.__getstate__()) + return copied + + def __deepcopy__(self, memo): + return self.__copy__() + def __repr__(self): return '<%s: %s>' % (self.__class__.__name__, self.grid_id) def __eq__(self, other): if isinstance(other, GridFSProxy): - return ((self.grid_id == other.grid_id) and - (self.collection_name == other.collection_name) and - (self.db_alias == other.db_alias)) + return ((self.grid_id == other.grid_id) and + (self.collection_name == other.collection_name) and + (self.db_alias == other.db_alias)) else: return False diff --git a/tests/fields/file.py b/tests/fields/file.py index 17d9ec37..a39dadbf 100644 --- a/tests/fields/file.py +++ b/tests/fields/file.py @@ -3,23 +3,17 @@ from __future__ import with_statement import sys sys.path[0:0] = [""] -import datetime +import copy import os import unittest -import uuid import tempfile -from decimal import Decimal - -from bson import Binary, DBRef, ObjectId import gridfs from nose.plugins.skip import SkipTest from mongoengine import * from mongoengine.connection import get_db -from mongoengine.base import _document_registry -from mongoengine.errors import NotRegistered -from mongoengine.python_support import PY3, b, StringIO, bin_type +from mongoengine.python_support import PY3, b, StringIO TEST_IMAGE_PATH = os.path.join(os.path.dirname(__file__), 'mongoengine.png') @@ -50,13 +44,12 @@ class FileTest(unittest.TestCase): PutFile.drop_collection() text = b('Hello, World!') - more_text = b('Foo Bar') content_type = 'text/plain' putfile = PutFile() putfile.the_file.put(text, content_type=content_type) putfile.save() - putfile.validate() + result = PutFile.objects.first() self.assertTrue(putfile == result) self.assertEqual(result.the_file.read(), text) @@ -73,7 +66,7 @@ class FileTest(unittest.TestCase): putstring.seek(0) putfile.the_file.put(putstring, content_type=content_type) putfile.save() - putfile.validate() + result = PutFile.objects.first() self.assertTrue(putfile == result) self.assertEqual(result.the_file.read(), text) @@ -98,7 +91,7 @@ class FileTest(unittest.TestCase): streamfile.the_file.write(more_text) streamfile.the_file.close() streamfile.save() - streamfile.validate() + result = StreamFile.objects.first() self.assertTrue(streamfile == result) self.assertEqual(result.the_file.read(), text + more_text) @@ -135,7 +128,7 @@ class FileTest(unittest.TestCase): # Try replacing file with new one result.the_file.replace(more_text) result.save() - result.validate() + result = SetFile.objects.first() self.assertTrue(setfile == result) self.assertEqual(result.the_file.read(), more_text) @@ -366,5 +359,25 @@ class FileTest(unittest.TestCase): self.assertEqual(test_file.the_file.read(), b('Hello, World!')) + def test_copyable(self): + class PutFile(Document): + the_file = FileField() + + PutFile.drop_collection() + + text = b('Hello, World!') + content_type = 'text/plain' + + putfile = PutFile() + putfile.the_file.put(text, content_type=content_type) + putfile.save() + + class TestFile(Document): + name = StringField() + + self.assertEqual(putfile, copy.copy(putfile)) + self.assertEqual(putfile, copy.deepcopy(putfile)) + + if __name__ == '__main__': unittest.main() From 09a5f5c8f33f036fa7c12caf614f799bdc4046c2 Mon Sep 17 00:00:00 2001 From: Ross Lawley Date: Thu, 3 Jan 2013 12:56:42 +0000 Subject: [PATCH 057/189] Added note for django and sites config issues --- docs/django.rst | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/docs/django.rst b/docs/django.rst index 144baab5..a4f05602 100644 --- a/docs/django.rst +++ b/docs/django.rst @@ -10,6 +10,10 @@ In your **settings.py** file, ignore the standard database settings (unless you also plan to use the ORM in your project), and instead call :func:`~mongoengine.connect` somewhere in the settings module. +.. note :: If getting an ``ImproperlyConfigured: settings.DATABASES is + improperly configured`` error you may need to remove + ``django.contrib.sites`` from ``INSTALLED_APPS`` in settings.py. + Authentication ============== MongoEngine includes a Django authentication backend, which uses MongoDB. The From 9bbd8dbe624c385c68404834884f20b304b5b64f Mon Sep 17 00:00:00 2001 From: Ross Lawley Date: Fri, 4 Jan 2013 09:41:08 +0000 Subject: [PATCH 058/189] Querysets now return clones and are no longer edit in place Fixes #56 --- docs/changelog.rst | 1 + docs/upgrade.rst | 22 + mongoengine/connection.py | 6 +- mongoengine/queryset/queryset.py | 1182 ++++++++++++++++-------------- tests/queryset/queryset.py | 11 +- tests/queryset/visitor.py | 4 +- 6 files changed, 656 insertions(+), 570 deletions(-) diff --git a/docs/changelog.rst b/docs/changelog.rst index c6493038..4fd3e143 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -26,6 +26,7 @@ Changes in 0.8.X - Fix Django timezone support (#151) - Simplified Q objects, removed QueryTreeTransformerVisitor (#98) (#171) - FileFields now copyable (#198) +- Querysets now return clones and are no longer edit in place (#56) Changes in 0.7.9 ================ diff --git a/docs/upgrade.rst b/docs/upgrade.rst index bf48527c..9c6c9a9d 100644 --- a/docs/upgrade.rst +++ b/docs/upgrade.rst @@ -56,6 +56,28 @@ you will need to declare :attr:`allow_inheritance` in the meta data like so: :: meta = {'allow_inheritance': True} +Querysets +~~~~~~~~~ + +Querysets now return clones and should no longer be considered editable in +place. This brings us in line with how Django's querysets work and removes a +long running gotcha. If you edit your querysets inplace you will have to +update your code like so: :: + + # Old code: + mammals = Animal.objects(type="mammal") + mammals.filter(order="Carnivora") # Returns a cloned queryset that isn't assigned to anything - so this will break in 0.8 + [m for m in mammals] # This will return all mammals in 0.8 as the 2nd filter returned a new queryset + + # Update example a) assign queryset after a change: + mammals = Animal.objects(type="mammal") + carnivores = mammals.filter(order="Carnivora") # Reassign the new queryset so fitler can be applied + [m for m in carnivores] # This will return all carnivores + + # Update example b) chain the queryset: + mammals = Animal.objects(type="mammal").filter(order="Carnivora") # The final queryset is assgined to mammals + [m for m in mammals] # This will return all carnivores + Indexes ------- diff --git a/mongoengine/connection.py b/mongoengine/connection.py index 1ccbbe31..87308ba3 100644 --- a/mongoengine/connection.py +++ b/mongoengine/connection.py @@ -28,8 +28,10 @@ def register_connection(alias, name, host='localhost', port=27017, :param name: the name of the specific database to use :param host: the host name of the :program:`mongod` instance to connect to :param port: the port that the :program:`mongod` instance is running on - :param is_slave: whether the connection can act as a slave ** Depreciated pymongo 2.0.1+ - :param read_preference: The read preference for the collection ** Added pymongo 2.1 + :param is_slave: whether the connection can act as a slave + ** Depreciated pymongo 2.0.1+ + :param read_preference: The read preference for the collection + ** Added pymongo 2.1 :param slaves: a list of aliases of slave connections; each of these must be a registered connection that has :attr:`is_slave` set to ``True`` :param username: username to authenticate with diff --git a/mongoengine/queryset/queryset.py b/mongoengine/queryset/queryset.py index 3ea9f232..239975f3 100644 --- a/mongoengine/queryset/queryset.py +++ b/mongoengine/queryset/queryset.py @@ -42,7 +42,6 @@ class QuerySet(object): providing :class:`~mongoengine.Document` objects as the results. """ __dereference = False - __none = False def __init__(self, document, collection): self._document = document @@ -60,6 +59,7 @@ class QuerySet(object): self._read_preference = None self._iter = False self._scalar = [] + self._none = False self._as_pymongo = False self._as_pymongo_coerce = False @@ -71,35 +71,9 @@ class QuerySet(object): self._cursor_obj = None self._limit = None self._skip = None + self._slice = None self._hint = -1 # Using -1 as None is a valid value for hint - def clone(self): - """Creates a copy of the current - :class:`~mongoengine.queryset.QuerySet` - - .. versionadded:: 0.5 - """ - c = self.__class__(self._document, self._collection_obj) - - copy_props = ('_initial_query', '_query_obj', '_where_clause', - '_loaded_fields', '_ordering', '_snapshot', - '_timeout', '_limit', '_skip', '_slave_okay', '_hint', - '_read_preference') - - for prop in copy_props: - val = getattr(self, prop) - setattr(c, prop, copy.deepcopy(val)) - - return c - - @property - def _query(self): - if self._mongo_query is None: - self._mongo_query = self._query_obj.to_query(self._document) - if self._class_check: - self._mongo_query.update(self._initial_query) - return self._mongo_query - def __call__(self, q_obj=None, class_check=True, slave_okay=False, read_preference=None, **query): """Filter the selected documents by calling the @@ -121,87 +95,94 @@ class QuerySet(object): if q_obj: # make sure proper query object is passed if not isinstance(q_obj, QNode): - raise InvalidQueryError('Not a query object: %s. Did you intend to use key=value?' % q_obj) + msg = ("Not a query object: %s. " + "Did you intend to use key=value?" % q_obj) + raise InvalidQueryError(msg) query &= q_obj - self._query_obj &= query - self._mongo_query = None - self._cursor_obj = None + + queryset = self.clone() + queryset._query_obj &= query + queryset._mongo_query = None + queryset._cursor_obj = None if read_preference is not None: - self.read_preference(read_preference) - self._class_check = class_check + queryset.read_preference(read_preference) + queryset._class_check = class_check + return queryset + + + def __iter__(self): + """Support iterator protocol""" + self.rewind() return self - def filter(self, *q_objs, **query): - """An alias of :meth:`~mongoengine.queryset.QuerySet.__call__` + def __len__(self): + return self.count() + + def __getitem__(self, key): + """Support skip and limit using getitem and slicing syntax. """ - return self.__call__(*q_objs, **query) + queryset = self.clone() + + # Slice provided + if isinstance(key, slice): + try: + queryset._cursor_obj = queryset._cursor[key] + queryset._slice = key + queryset._skip, queryset._limit = key.start, key.stop + except IndexError, err: + # PyMongo raises an error if key.start == key.stop, catch it, + # bin it, kill it. + start = key.start or 0 + if start >= 0 and key.stop >= 0 and key.step is None: + if start == key.stop: + queryset.limit(0) + queryset._skip = key.start + queryset._limit = key.stop - start + return queryset + raise err + # Allow further QuerySet modifications to be performed + return queryset + # Integer index provided + elif isinstance(key, int): + if queryset._scalar: + return queryset._get_scalar( + queryset._document._from_son(queryset._cursor[key])) + if queryset._as_pymongo: + return queryset._get_as_pymongo(queryset._cursor.next()) + return queryset._document._from_son(queryset._cursor[key]) + raise AttributeError + + def __repr__(self): + """Provides the string representation of the QuerySet + + .. versionchanged:: 0.6.13 Now doesnt modify the cursor + """ + + if self._iter: + return '.. queryset mid-iteration ..' + + data = [] + for i in xrange(REPR_OUTPUT_SIZE + 1): + try: + data.append(self.next()) + except StopIteration: + break + if len(data) > REPR_OUTPUT_SIZE: + data[-1] = "...(remaining elements truncated)..." + + self.rewind() + return repr(data) + + # Core functions def all(self): """Returns all documents.""" return self.__call__() - def ensure_index(self, **kwargs): - """Deprecated use :func:`~Document.ensure_index`""" - msg = ("Doc.objects()._ensure_index() is deprecated. " - "Use Doc.ensure_index() instead.") - warnings.warn(msg, DeprecationWarning) - self._document.__class__.ensure_index(**kwargs) - return self - - def _ensure_indexes(self): - """Deprecated use :func:`~Document.ensure_indexes`""" - msg = ("Doc.objects()._ensure_indexes() is deprecated. " - "Use Doc.ensure_indexes() instead.") - warnings.warn(msg, DeprecationWarning) - self._document.__class__.ensure_indexes() - - @property - def _collection(self): - """Property that returns the collection object. This allows us to - perform operations only if the collection is accessed. + def filter(self, *q_objs, **query): + """An alias of :meth:`~mongoengine.queryset.QuerySet.__call__` """ - return self._collection_obj - - @property - def _cursor_args(self): - cursor_args = { - 'snapshot': self._snapshot, - 'timeout': self._timeout, - 'slave_okay': self._slave_okay, - } - if self._read_preference is not None: - cursor_args['read_preference'] = self._read_preference - if self._loaded_fields: - cursor_args['fields'] = self._loaded_fields.as_dict() - return cursor_args - - @property - def _cursor(self): - if self._cursor_obj is None: - - self._cursor_obj = self._collection.find(self._query, - **self._cursor_args) - # Apply where clauses to cursor - if self._where_clause: - self._cursor_obj.where(self._where_clause) - - if self._ordering: - # Apply query ordering - self._cursor_obj.sort(self._ordering) - elif self._document._meta['ordering']: - # Otherwise, apply the ordering from the document model - self.order_by(*self._document._meta['ordering']) - self._cursor_obj.sort(self._ordering) - - if self._limit is not None: - self._cursor_obj.limit(self._limit - (self._skip or 0)) - - if self._skip is not None: - self._cursor_obj.skip(self._skip) - - if self._hint != -1: - self._cursor_obj.hint(self._hint) - return self._cursor_obj + return self.__call__(*q_objs, **query) def get(self, *q_objs, **query): """Retrieve the the matching object raising @@ -212,22 +193,29 @@ class QuerySet(object): .. versionadded:: 0.3 """ - self.limit(2) - self.__call__(*q_objs, **query) + queryset = self.__call__(*q_objs, **query) + queryset = queryset.limit(2) try: - result = self.next() + result = queryset.next() except StopIteration: msg = ("%s matching query does not exist." - % self._document._class_name) - raise self._document.DoesNotExist(msg) + % queryset._document._class_name) + raise queryset._document.DoesNotExist(msg) try: - self.next() + queryset.next() except StopIteration: return result - self.rewind() - message = u'%d items returned, instead of 1' % self.count() - raise self._document.MultipleObjectsReturned(message) + queryset.rewind() + message = u'%d items returned, instead of 1' % queryset.count() + raise queryset._document.MultipleObjectsReturned(message) + + def create(self, **kwargs): + """Create new object. Returns the saved object instance. + + .. versionadded:: 0.4 + """ + return self._document(**kwargs).save() def get_or_create(self, write_options=None, auto_save=True, *q_objs, **query): @@ -277,20 +265,12 @@ class QuerySet(object): doc.save(write_options=write_options) return doc, True - def create(self, **kwargs): - """Create new object. Returns the saved object instance. - - .. versionadded:: 0.4 - """ - doc = self._document(**kwargs) - doc.save() - return doc - def first(self): """Retrieve the first object matching the query. """ + queryset = self.clone() try: - result = self[0] + result = queryset[0] except IndexError: result = None return result @@ -367,6 +347,117 @@ class QuerySet(object): self._document, documents=results, loaded=True) return return_one and results[0] or results + def count(self): + """Count the selected elements in the query. + """ + if self._limit == 0: + return 0 + return self._cursor.count(with_limit_and_skip=True) + + def delete(self, safe=False): + """Delete the documents matched by the query. + + :param safe: check if the operation succeeded before returning + """ + queryset = self.clone() + doc = queryset._document + + # Handle deletes where skips or limits have been applied + if queryset._skip or queryset._limit: + for doc in queryset: + doc.delete() + return + + delete_rules = doc._meta.get('delete_rules') or {} + # Check for DENY rules before actually deleting/nullifying any other + # references + for rule_entry in delete_rules: + document_cls, field_name = rule_entry + rule = doc._meta['delete_rules'][rule_entry] + if rule == DENY and document_cls.objects( + **{field_name + '__in': self}).count() > 0: + msg = ("Could not delete document (%s.%s refers to it)" + % (document_cls.__name__, field_name)) + raise OperationError(msg) + + for rule_entry in delete_rules: + document_cls, field_name = rule_entry + rule = doc._meta['delete_rules'][rule_entry] + if rule == CASCADE: + ref_q = document_cls.objects(**{field_name + '__in': self}) + ref_q_count = ref_q.count() + if (doc != document_cls and ref_q_count > 0 + or (doc == document_cls and ref_q_count > 0)): + ref_q.delete(safe=safe) + elif rule == NULLIFY: + document_cls.objects(**{field_name + '__in': self}).update( + safe_update=safe, + **{'unset__%s' % field_name: 1}) + elif rule == PULL: + document_cls.objects(**{field_name + '__in': self}).update( + safe_update=safe, + **{'pull_all__%s' % field_name: self}) + + queryset._collection.remove(queryset._query, safe=safe) + + def update(self, safe_update=True, upsert=False, multi=True, + write_options=None, **update): + """Perform an atomic update on the fields matched by the query. When + ``safe_update`` is used, the number of affected documents is returned. + + :param safe_update: check if the operation succeeded before returning + :param upsert: Any existing document with that "_id" is overwritten. + :param write_options: extra keyword arguments for + :meth:`~pymongo.collection.Collection.update` + + .. versionadded:: 0.2 + """ + if not update: + raise OperationError("No update parameters, would remove data") + + if not write_options: + write_options = {} + + queryset = self.clone() + query = queryset._query + update = transform.update(queryset._document, **update) + + # If doing an atomic upsert on an inheritable class + # then ensure we add _cls to the update operation + if upsert and '_cls' in query: + if '$set' in update: + update["$set"]["_cls"] = queryset._document._class_name + else: + update["$set"] = {"_cls": queryset._document._class_name} + + try: + ret = queryset._collection.update(query, update, multi=multi, + upsert=upsert, safe=safe_update, + **write_options) + if ret is not None and 'n' in ret: + return ret['n'] + except pymongo.errors.OperationFailure, err: + if unicode(err) == u'multi not coded yet': + message = u'update() method requires MongoDB 1.1.3+' + raise OperationError(message) + raise OperationError(u'Update failed (%s)' % unicode(err)) + + def update_one(self, safe_update=True, upsert=False, write_options=None, + **update): + """Perform an atomic update on first field matched by the query. When + ``safe_update`` is used, the number of affected documents is returned. + + :param safe_update: check if the operation succeeded before returning + :param upsert: Any existing document with that "_id" is overwritten. + :param write_options: extra keyword arguments for + :meth:`~pymongo.collection.Collection.update` + :param update: Django-style update keyword arguments + + .. versionadded:: 0.2 + """ + return self.update(safe_update=True, upsert=upsert, multi=False, + write_options=None, **update) + def with_id(self, object_id): """Retrieve the object matching the id provided. Uses `object_id` only and raises InvalidQueryError if a filter has been applied. @@ -375,10 +466,11 @@ class QuerySet(object): .. versionchanged:: 0.6 Raises InvalidQueryError if filter has been set """ - if not self._query_obj.empty: + queryset = self.clone() + if not queryset._query_obj.empty: msg = "Cannot use a filter whilst using `with_id`" raise InvalidQueryError(msg) - return self.filter(pk=object_id).first() + return queryset.filter(pk=object_id).first() def in_bulk(self, object_ids): """Retrieve a set of documents by their ids. @@ -406,139 +498,48 @@ class QuerySet(object): return doc_map - def next(self): - """Wrap the result in a :class:`~mongoengine.Document` object. - """ - self._iter = True - try: - if self._limit == 0 or self.__none: - raise StopIteration - if self._scalar: - return self._get_scalar(self._document._from_son( - self._cursor.next())) - if self._as_pymongo: - return self._get_as_pymongo(self._cursor.next()) - - return self._document._from_son(self._cursor.next()) - except StopIteration, e: - self.rewind() - raise e - - def rewind(self): - """Rewind the cursor to its unevaluated state. - - .. versionadded:: 0.3 - """ - self._iter = False - self._cursor.rewind() - def none(self): """Helper that just returns a list""" - self.__none = True - return self + queryset = self.clone() + queryset._none = True + return queryset - def count(self): - """Count the selected elements in the query. + def clone(self): + """Creates a copy of the current + :class:`~mongoengine.queryset.QuerySet` + + .. versionadded:: 0.5 """ - if self._limit == 0: - return 0 - return self._cursor.count(with_limit_and_skip=True) + c = self.__class__(self._document, self._collection_obj) - def __len__(self): - return self.count() + copy_props = ('_mongo_query', '_initial_query', '_none', '_query_obj', + '_where_clause', '_loaded_fields', '_ordering', '_snapshot', + '_timeout', '_class_check', '_slave_okay', '_read_preference', + '_iter', '_scalar', '_as_pymongo', '_as_pymongo_coerce', + '_limit', '_skip', '_slice', '_hint') - def map_reduce(self, map_f, reduce_f, output, finalize_f=None, limit=None, - scope=None): - """Perform a map/reduce query using the current query spec - and ordering. While ``map_reduce`` respects ``QuerySet`` chaining, - it must be the last call made, as it does not return a maleable - ``QuerySet``. + for prop in copy_props: + val = getattr(self, prop) + setattr(c, prop, copy.copy(val)) - See the :meth:`~mongoengine.tests.QuerySetTest.test_map_reduce` - and :meth:`~mongoengine.tests.QuerySetTest.test_map_advanced` - tests in ``tests.queryset.QuerySetTest`` for usage examples. + if self._cursor_obj: + c._cursor_obj = self._cursor_obj.clone() - :param map_f: map function, as :class:`~bson.code.Code` or string - :param reduce_f: reduce function, as - :class:`~bson.code.Code` or string - :param output: output collection name, if set to 'inline' will try to - use :class:`~pymongo.collection.Collection.inline_map_reduce` - This can also be a dictionary containing output options - see: http://docs.mongodb.org/manual/reference/commands/#mapReduce - :param finalize_f: finalize function, an optional function that - performs any post-reduction processing. - :param scope: values to insert into map/reduce global scope. Optional. - :param limit: number of objects from current query to provide - to map/reduce method + if self._slice: + c._cursor_obj[self._slice] - Returns an iterator yielding - :class:`~mongoengine.document.MapReduceDocument`. + return c - .. note:: + def select_related(self, max_depth=1): + """Handles dereferencing of :class:`~bson.dbref.DBRef` objects to + a maximum depth in order to cut down the number queries to mongodb. - Map/Reduce changed in server version **>= 1.7.4**. The PyMongo - :meth:`~pymongo.collection.Collection.map_reduce` helper requires - PyMongo version **>= 1.11**. - - .. versionchanged:: 0.5 - - removed ``keep_temp`` keyword argument, which was only relevant - for MongoDB server versions older than 1.7.4 - - .. versionadded:: 0.3 + .. versionadded:: 0.5 """ - MapReduceDocument = _import_class('MapReduceDocument') - - if not hasattr(self._collection, "map_reduce"): - raise NotImplementedError("Requires MongoDB >= 1.7.1") - - map_f_scope = {} - if isinstance(map_f, Code): - map_f_scope = map_f.scope - map_f = unicode(map_f) - map_f = Code(self._sub_js_fields(map_f), map_f_scope) - - reduce_f_scope = {} - if isinstance(reduce_f, Code): - reduce_f_scope = reduce_f.scope - reduce_f = unicode(reduce_f) - reduce_f_code = self._sub_js_fields(reduce_f) - reduce_f = Code(reduce_f_code, reduce_f_scope) - - mr_args = {'query': self._query} - - if finalize_f: - finalize_f_scope = {} - if isinstance(finalize_f, Code): - finalize_f_scope = finalize_f.scope - finalize_f = unicode(finalize_f) - finalize_f_code = self._sub_js_fields(finalize_f) - finalize_f = Code(finalize_f_code, finalize_f_scope) - mr_args['finalize'] = finalize_f - - if scope: - mr_args['scope'] = scope - - if limit: - mr_args['limit'] = limit - - if output == 'inline' and not self._ordering: - map_reduce_function = 'inline_map_reduce' - else: - map_reduce_function = 'map_reduce' - mr_args['out'] = output - - results = getattr(self._collection, map_reduce_function)( - map_f, reduce_f, **mr_args) - - if map_reduce_function == 'map_reduce': - results = results.find() - - if self._ordering: - results = results.sort(self._ordering) - - for doc in results: - yield MapReduceDocument(self._document, self._collection, - doc['_id'], doc['value']) + # Make select related work the same for querysets + max_depth += 1 + queryset = self.clone() + return queryset._dereference(queryset, max_depth=max_depth) def limit(self, n): """Limit the number of returned documents to `n`. This may also be @@ -546,14 +547,15 @@ class QuerySet(object): :param n: the maximum number of objects to return """ + queryset = self.clone() if n == 0: - self._cursor.limit(1) + queryset._cursor.limit(1) else: - self._cursor.limit(n) - self._limit = n + queryset._cursor.limit(n) + queryset._limit = n # Return self to allow chaining - return self + return queryset def skip(self, n): """Skip `n` documents before returning the results. This may also be @@ -561,9 +563,10 @@ class QuerySet(object): :param n: the number of objects to skip before returning results """ - self._cursor.skip(n) - self._skip = n - return self + queryset = self.clone() + queryset._cursor.skip(n) + queryset._skip = n + return queryset def hint(self, index=None): """Added 'hint' support, telling Mongo the proper index to use for the @@ -578,39 +581,10 @@ class QuerySet(object): .. versionadded:: 0.5 """ - self._cursor.hint(index) - self._hint = index - return self - - def __getitem__(self, key): - """Support skip and limit using getitem and slicing syntax. - """ - # Slice provided - if isinstance(key, slice): - try: - self._cursor_obj = self._cursor[key] - self._skip, self._limit = key.start, key.stop - except IndexError, err: - # PyMongo raises an error if key.start == key.stop, catch it, - # bin it, kill it. - start = key.start or 0 - if start >= 0 and key.stop >= 0 and key.step is None: - if start == key.stop: - self.limit(0) - self._skip, self._limit = key.start, key.stop - start - return self - raise err - # Allow further QuerySet modifications to be performed - return self - # Integer index provided - elif isinstance(key, int): - if self._scalar: - return self._get_scalar(self._document._from_son( - self._cursor[key])) - if self._as_pymongo: - return self._get_as_pymongo(self._cursor.next()) - return self._document._from_son(self._cursor[key]) - raise AttributeError + queryset = self.clone() + queryset._cursor.hint(index) + queryset._hint = index + return queryset def distinct(self, field): """Return a list of distinct values for a given field. @@ -621,8 +595,9 @@ class QuerySet(object): .. versionchanged:: 0.5 - Fixed handling references .. versionchanged:: 0.6 - Improved db_field refrence handling """ - return self._dereference(self._cursor.distinct(field), 1, - name=field, instance=self._document) + queryset = self.clone() + return queryset._dereference(queryset._cursor.distinct(field), 1, + name=field, instance=queryset._document) def only(self, *fields): """Load only a subset of this document's fields. :: @@ -679,11 +654,12 @@ class QuerySet(object): cleaned_fields.append((key, value)) fields = sorted(cleaned_fields, key=operator.itemgetter(1)) + queryset = self.clone() for value, group in itertools.groupby(fields, lambda x: x[1]): fields = [field for field, value in group] - fields = self._fields_to_dbfields(fields) - self._loaded_fields += QueryFieldList(fields, value=value) - return self + fields = queryset._fields_to_dbfields(fields) + queryset._loaded_fields += QueryFieldList(fields, value=value) + return queryset def all_fields(self): """Include all fields. Reset all previously calls of .only() or @@ -693,18 +669,10 @@ class QuerySet(object): .. versionadded:: 0.5 """ - self._loaded_fields = QueryFieldList( - always_include=self._loaded_fields.always_include) - return self - - def _fields_to_dbfields(self, fields): - """Translate fields paths to its db equivalents""" - ret = [] - for field in fields: - field = ".".join(f.db_field for f in - self._document._lookup_field(field.split('.'))) - ret.append(field) - return ret + queryset = self.clone() + queryset._loaded_fields = QueryFieldList( + always_include=queryset._loaded_fields.always_include) + return queryset def order_by(self, *keys): """Order the :class:`~mongoengine.queryset.QuerySet` by the keys. The @@ -714,25 +682,9 @@ class QuerySet(object): :param keys: fields to order the query results by; keys may be prefixed with **+** or **-** to determine the ordering direction """ - key_list = [] - for key in keys: - if not key: - continue - direction = pymongo.ASCENDING - if key[0] == '-': - direction = pymongo.DESCENDING - if key[0] in ('-', '+'): - key = key[1:] - key = key.replace('__', '.') - try: - key = self._document._translate_field_name(key) - except: - pass - key_list.append((key, direction)) - - self._ordering = key_list - - return self + queryset = self.clone() + queryset._ordering = self._get_order_by(keys) + return queryset def explain(self, format=False): """Return an explain plan record for the @@ -740,7 +692,6 @@ class QuerySet(object): :param format: format the plan before returning it """ - plan = self._cursor.explain() if format: plan = pprint.pformat(plan) @@ -753,8 +704,9 @@ class QuerySet(object): ..versionchanged:: 0.5 - made chainable """ - self._snapshot = enabled - return self + queryset = self.clone() + queryset._snapshot = enabled + return queryset def timeout(self, enabled): """Enable or disable the default mongod timeout when querying. @@ -763,16 +715,18 @@ class QuerySet(object): ..versionchanged:: 0.5 - made chainable """ - self._timeout = enabled - return self + queryset = self.clone() + queryset._timeout = enabled + return queryset def slave_okay(self, enabled): """Enable or disable the slave_okay when querying. :param enabled: whether or not the slave_okay is enabled """ - self._slave_okay = enabled - return self + queryset = self.clone() + queryset._slave_okay = enabled + return queryset def read_preference(self, read_preference): """Change the read_preference when querying. @@ -781,170 +735,9 @@ class QuerySet(object): preference. """ validate_read_preference('read_preference', read_preference) - self._read_preference = read_preference - return self - - def delete(self, safe=False): - """Delete the documents matched by the query. - - :param safe: check if the operation succeeded before returning - """ - doc = self._document - - # Handle deletes where skips or limits have been applied - if self._skip or self._limit: - for doc in self: - doc.delete() - return - - delete_rules = doc._meta.get('delete_rules') or {} - # Check for DENY rules before actually deleting/nullifying any other - # references - for rule_entry in delete_rules: - document_cls, field_name = rule_entry - rule = doc._meta['delete_rules'][rule_entry] - if rule == DENY and document_cls.objects( - **{field_name + '__in': self}).count() > 0: - msg = ("Could not delete document (%s.%s refers to it)" - % (document_cls.__name__, field_name)) - raise OperationError(msg) - - for rule_entry in delete_rules: - document_cls, field_name = rule_entry - rule = doc._meta['delete_rules'][rule_entry] - if rule == CASCADE: - ref_q = document_cls.objects(**{field_name + '__in': self}) - ref_q_count = ref_q.count() - if (doc != document_cls and ref_q_count > 0 - or (doc == document_cls and ref_q_count > 0)): - ref_q.delete(safe=safe) - elif rule == NULLIFY: - document_cls.objects(**{field_name + '__in': self}).update( - safe_update=safe, - **{'unset__%s' % field_name: 1}) - elif rule == PULL: - document_cls.objects(**{field_name + '__in': self}).update( - safe_update=safe, - **{'pull_all__%s' % field_name: self}) - - self._collection.remove(self._query, safe=safe) - - def update(self, safe_update=True, upsert=False, multi=True, - write_options=None, **update): - """Perform an atomic update on the fields matched by the query. When - ``safe_update`` is used, the number of affected documents is returned. - - :param safe_update: check if the operation succeeded before returning - :param upsert: Any existing document with that "_id" is overwritten. - :param write_options: extra keyword arguments for - :meth:`~pymongo.collection.Collection.update` - - .. versionadded:: 0.2 - """ - if not update: - raise OperationError("No update parameters, would remove data") - - if not write_options: - write_options = {} - - query = self._query - update = transform.update(self._document, **update) - - # If doing an atomic upsert on an inheritable class - # then ensure we add _cls to the update operation - if upsert and '_cls' in query: - if '$set' in update: - update["$set"]["_cls"] = self._document._class_name - else: - update["$set"] = {"_cls": self._document._class_name} - - try: - ret = self._collection.update(query, update, multi=multi, - upsert=upsert, safe=safe_update, - **write_options) - if ret is not None and 'n' in ret: - return ret['n'] - except pymongo.errors.OperationFailure, err: - if unicode(err) == u'multi not coded yet': - message = u'update() method requires MongoDB 1.1.3+' - raise OperationError(message) - raise OperationError(u'Update failed (%s)' % unicode(err)) - - def update_one(self, safe_update=True, upsert=False, write_options=None, - **update): - """Perform an atomic update on first field matched by the query. When - ``safe_update`` is used, the number of affected documents is returned. - - :param safe_update: check if the operation succeeded before returning - :param upsert: Any existing document with that "_id" is overwritten. - :param write_options: extra keyword arguments for - :meth:`~pymongo.collection.Collection.update` - :param update: Django-style update keyword arguments - - .. versionadded:: 0.2 - """ - return self.update(safe_update=True, upsert=upsert, multi=False, - write_options=None, **update) - - def __iter__(self): - self.rewind() - return self - - def _get_scalar(self, doc): - - def lookup(obj, name): - chunks = name.split('__') - for chunk in chunks: - obj = getattr(obj, chunk) - return obj - - data = [lookup(doc, n) for n in self._scalar] - if len(data) == 1: - return data[0] - - return tuple(data) - - def _get_as_pymongo(self, row): - # Extract which fields paths we should follow if .fields(...) was - # used. If not, handle all fields. - if not getattr(self, '__as_pymongo_fields', None): - self.__as_pymongo_fields = [] - for field in self._loaded_fields.fields - set(['_cls', '_id', '_types']): - self.__as_pymongo_fields.append(field) - while '.' in field: - field, _ = field.rsplit('.', 1) - self.__as_pymongo_fields.append(field) - - all_fields = not self.__as_pymongo_fields - - def clean(data, path=None): - path = path or '' - - if isinstance(data, dict): - new_data = {} - for key, value in data.iteritems(): - new_path = '%s.%s' % (path, key) if path else key - if all_fields or new_path in self.__as_pymongo_fields: - new_data[key] = clean(value, path=new_path) - data = new_data - elif isinstance(data, list): - data = [clean(d, path=path) for d in data] - else: - if self._as_pymongo_coerce: - # If we need to coerce types, we need to determine the - # type of this field and use the corresponding .to_python(...) - from mongoengine.fields import EmbeddedDocumentField - obj = self._document - for chunk in path.split('.'): - obj = getattr(obj, chunk, None) - if obj is None: - break - elif isinstance(obj, EmbeddedDocumentField): - obj = obj.document_type - if obj and data is not None: - data = obj.to_python(data) - return data - return clean(row) + queryset = self.clone() + queryset._read_preference = read_preference + return queryset def scalar(self, *fields): """Instead of returning Document instances, return either a specific @@ -955,14 +748,15 @@ class QuerySet(object): :param fields: One or more fields to return instead of a Document. """ - self._scalar = list(fields) + queryset = self.clone() + queryset._scalar = list(fields) if fields: - self.only(*fields) + queryset = queryset.only(*fields) else: - self.all_fields() + queryset = queryset.all_fields() - return self + return queryset def values_list(self, *fields): """An alias for scalar""" @@ -972,36 +766,122 @@ class QuerySet(object): """Instead of returning Document instances, return raw values from pymongo. - :param coerce_type: Field types (if applicable) would be use to coerce types. + :param coerce_type: Field types (if applicable) would be use to + coerce types. """ - self._as_pymongo = True - self._as_pymongo_coerce = coerce_types - return self + queryset = self.clone() + queryset._as_pymongo = True + queryset._as_pymongo_coerce = coerce_types + return queryset - def _sub_js_fields(self, code): - """When fields are specified with [~fieldname] syntax, where - *fieldname* is the Python name of a field, *fieldname* will be - substituted for the MongoDB name of the field (specified using the - :attr:`name` keyword argument in a field's constructor). + # JSON Helpers + + def to_json(self): + """Converts a queryset to JSON""" + queryset = self.clone() + return json_util.dumps(queryset._collection_obj.find(queryset._query)) + + def from_json(self, json_data): + """Converts json data to unsaved objects""" + son_data = json_util.loads(json_data) + return [self._document._from_son(data) for data in son_data] + + # JS functionality + + def map_reduce(self, map_f, reduce_f, output, finalize_f=None, limit=None, + scope=None): + """Perform a map/reduce query using the current query spec + and ordering. While ``map_reduce`` respects ``QuerySet`` chaining, + it must be the last call made, as it does not return a maleable + ``QuerySet``. + + See the :meth:`~mongoengine.tests.QuerySetTest.test_map_reduce` + and :meth:`~mongoengine.tests.QuerySetTest.test_map_advanced` + tests in ``tests.queryset.QuerySetTest`` for usage examples. + + :param map_f: map function, as :class:`~bson.code.Code` or string + :param reduce_f: reduce function, as + :class:`~bson.code.Code` or string + :param output: output collection name, if set to 'inline' will try to + use :class:`~pymongo.collection.Collection.inline_map_reduce` + This can also be a dictionary containing output options + see: http://docs.mongodb.org/manual/reference/commands/#mapReduce + :param finalize_f: finalize function, an optional function that + performs any post-reduction processing. + :param scope: values to insert into map/reduce global scope. Optional. + :param limit: number of objects from current query to provide + to map/reduce method + + Returns an iterator yielding + :class:`~mongoengine.document.MapReduceDocument`. + + .. note:: + + Map/Reduce changed in server version **>= 1.7.4**. The PyMongo + :meth:`~pymongo.collection.Collection.map_reduce` helper requires + PyMongo version **>= 1.11**. + + .. versionchanged:: 0.5 + - removed ``keep_temp`` keyword argument, which was only relevant + for MongoDB server versions older than 1.7.4 + + .. versionadded:: 0.3 """ - def field_sub(match): - # Extract just the field name, and look up the field objects - field_name = match.group(1).split('.') - fields = self._document._lookup_field(field_name) - # Substitute the correct name for the field into the javascript - return u'["%s"]' % fields[-1].db_field + queryset = self.clone() - def field_path_sub(match): - # Extract just the field name, and look up the field objects - field_name = match.group(1).split('.') - fields = self._document._lookup_field(field_name) - # Substitute the correct name for the field into the javascript - return ".".join([f.db_field for f in fields]) + MapReduceDocument = _import_class('MapReduceDocument') - code = re.sub(u'\[\s*~([A-z_][A-z_0-9.]+?)\s*\]', field_sub, code) - code = re.sub(u'\{\{\s*~([A-z_][A-z_0-9.]+?)\s*\}\}', field_path_sub, - code) - return code + if not hasattr(self._collection, "map_reduce"): + raise NotImplementedError("Requires MongoDB >= 1.7.1") + + map_f_scope = {} + if isinstance(map_f, Code): + map_f_scope = map_f.scope + map_f = unicode(map_f) + map_f = Code(queryset._sub_js_fields(map_f), map_f_scope) + + reduce_f_scope = {} + if isinstance(reduce_f, Code): + reduce_f_scope = reduce_f.scope + reduce_f = unicode(reduce_f) + reduce_f_code = queryset._sub_js_fields(reduce_f) + reduce_f = Code(reduce_f_code, reduce_f_scope) + + mr_args = {'query': queryset._query} + + if finalize_f: + finalize_f_scope = {} + if isinstance(finalize_f, Code): + finalize_f_scope = finalize_f.scope + finalize_f = unicode(finalize_f) + finalize_f_code = queryset._sub_js_fields(finalize_f) + finalize_f = Code(finalize_f_code, finalize_f_scope) + mr_args['finalize'] = finalize_f + + if scope: + mr_args['scope'] = scope + + if limit: + mr_args['limit'] = limit + + if output == 'inline' and not queryset._ordering: + map_reduce_function = 'inline_map_reduce' + else: + map_reduce_function = 'map_reduce' + mr_args['out'] = output + + results = getattr(queryset._collection, map_reduce_function)( + map_f, reduce_f, **mr_args) + + if map_reduce_function == 'map_reduce': + results = results.find() + + if queryset._ordering: + results = results.sort(queryset._ordering) + + for doc in results: + yield MapReduceDocument(queryset._document, queryset._collection, + doc['_id'], doc['value']) def exec_js(self, code, *fields, **options): """Execute a Javascript function on the server. A list of fields may be @@ -1025,24 +905,26 @@ class QuerySet(object): :param options: options that you want available to the function (accessed in Javascript through the ``options`` object) """ - code = self._sub_js_fields(code) + queryset = self.clone() - fields = [self._document._translate_field_name(f) for f in fields] - collection = self._document._get_collection_name() + code = queryset._sub_js_fields(code) + + fields = [queryset._document._translate_field_name(f) for f in fields] + collection = queryset._document._get_collection_name() scope = { 'collection': collection, 'options': options or {}, } - query = self._query - if self._where_clause: - query['$where'] = self._where_clause + query = queryset._query + if queryset._where_clause: + query['$where'] = queryset._where_clause scope['query'] = query code = Code(code, scope=scope) - db = self._document._get_db() + db = queryset._document._get_db() return db.eval(code, *fields) def where(self, where_clause): @@ -1056,9 +938,10 @@ class QuerySet(object): .. versionadded:: 0.5 """ - where_clause = self._sub_js_fields(where_clause) - self._where_clause = where_clause - return self + queryset = self.clone() + where_clause = queryset._sub_js_fields(where_clause) + queryset._where_clause = where_clause + return queryset def sum(self, field): """Sum over the values of the specified field. @@ -1157,6 +1040,101 @@ class QuerySet(object): normalize=normalize) return self._item_frequencies_exec_js(field, normalize=normalize) + # Iterator helpers + + def next(self): + """Wrap the result in a :class:`~mongoengine.Document` object. + """ + self._iter = True + try: + if self._limit == 0 or self._none: + raise StopIteration + if self._scalar: + return self._get_scalar(self._document._from_son( + self._cursor.next())) + if self._as_pymongo: + return self._get_as_pymongo(self._cursor.next()) + + return self._document._from_son(self._cursor.next()) + except StopIteration, e: + self.rewind() + raise e + + def rewind(self): + """Rewind the cursor to its unevaluated state. + + .. versionadded:: 0.3 + """ + self._iter = False + self._cursor.rewind() + + # Properties + + @property + def _collection(self): + """Property that returns the collection object. This allows us to + perform operations only if the collection is accessed. + """ + return self._collection_obj + + @property + def _cursor_args(self): + cursor_args = { + 'snapshot': self._snapshot, + 'timeout': self._timeout, + 'slave_okay': self._slave_okay, + } + if self._read_preference is not None: + cursor_args['read_preference'] = self._read_preference + if self._loaded_fields: + cursor_args['fields'] = self._loaded_fields.as_dict() + return cursor_args + + @property + def _cursor(self): + if self._cursor_obj is None: + + self._cursor_obj = self._collection.find(self._query, + **self._cursor_args) + # Apply where clauses to cursor + if self._where_clause: + where_clause = self._sub_js_fields(self._where_clause) + self._cursor_obj.where(where_clause) + + if self._ordering: + # Apply query ordering + self._cursor_obj.sort(self._ordering) + elif self._document._meta['ordering']: + # Otherwise, apply the ordering from the document model + order = self._get_order_by(self._document._meta['ordering']) + self._cursor_obj.sort(order) + + if self._limit is not None: + self._cursor_obj.limit(self._limit - (self._skip or 0)) + + if self._skip is not None: + self._cursor_obj.skip(self._skip) + + if self._hint != -1: + self._cursor_obj.hint(self._hint) + return self._cursor_obj + + @property + def _query(self): + if self._mongo_query is None: + self._mongo_query = self._query_obj.to_query(self._document) + if self._class_check: + self._mongo_query.update(self._initial_query) + return self._mongo_query + + @property + def _dereference(self): + if not self.__dereference: + self.__dereference = _import_class('DeReference')() + return self.__dereference + + # Helper Functions + def _item_frequencies_map_reduce(self, field, normalize=False): map_func = """ function() { @@ -1269,48 +1247,130 @@ class QuerySet(object): return frequencies - def __repr__(self): - """Provides the string representation of the QuerySet + def _fields_to_dbfields(self, fields): + """Translate fields paths to its db equivalents""" + ret = [] + for field in fields: + field = ".".join(f.db_field for f in + self._document._lookup_field(field.split('.'))) + ret.append(field) + return ret - .. versionchanged:: 0.6.13 Now doesnt modify the cursor + def _get_order_by(self, keys): + """Creates a list of order by fields """ - - if self._iter: - return '.. queryset mid-iteration ..' - - data = [] - for i in xrange(REPR_OUTPUT_SIZE + 1): + key_list = [] + for key in keys: + if not key: + continue + direction = pymongo.ASCENDING + if key[0] == '-': + direction = pymongo.DESCENDING + if key[0] in ('-', '+'): + key = key[1:] + key = key.replace('__', '.') try: - data.append(self.next()) - except StopIteration: - break - if len(data) > REPR_OUTPUT_SIZE: - data[-1] = "...(remaining elements truncated)..." + key = self._document._translate_field_name(key) + except: + pass + key_list.append((key, direction)) + return key_list - self.rewind() - return repr(data) + def _get_scalar(self, doc): - def select_related(self, max_depth=1): - """Handles dereferencing of :class:`~bson.dbref.DBRef` objects to - a maximum depth in order to cut down the number queries to mongodb. + def lookup(obj, name): + chunks = name.split('__') + for chunk in chunks: + obj = getattr(obj, chunk) + return obj - .. versionadded:: 0.5 + data = [lookup(doc, n) for n in self._scalar] + if len(data) == 1: + return data[0] + + return tuple(data) + + def _get_as_pymongo(self, row): + # Extract which fields paths we should follow if .fields(...) was + # used. If not, handle all fields. + if not getattr(self, '__as_pymongo_fields', None): + self.__as_pymongo_fields = [] + for field in self._loaded_fields.fields - set(['_cls', '_id']): + self.__as_pymongo_fields.append(field) + while '.' in field: + field, _ = field.rsplit('.', 1) + self.__as_pymongo_fields.append(field) + + all_fields = not self.__as_pymongo_fields + + def clean(data, path=None): + path = path or '' + + if isinstance(data, dict): + new_data = {} + for key, value in data.iteritems(): + new_path = '%s.%s' % (path, key) if path else key + if all_fields or new_path in self.__as_pymongo_fields: + new_data[key] = clean(value, path=new_path) + data = new_data + elif isinstance(data, list): + data = [clean(d, path=path) for d in data] + else: + if self._as_pymongo_coerce: + # If we need to coerce types, we need to determine the + # type of this field and use the corresponding + # .to_python(...) + from mongoengine.fields import EmbeddedDocumentField + obj = self._document + for chunk in path.split('.'): + obj = getattr(obj, chunk, None) + if obj is None: + break + elif isinstance(obj, EmbeddedDocumentField): + obj = obj.document_type + if obj and data is not None: + data = obj.to_python(data) + return data + return clean(row) + + def _sub_js_fields(self, code): + """When fields are specified with [~fieldname] syntax, where + *fieldname* is the Python name of a field, *fieldname* will be + substituted for the MongoDB name of the field (specified using the + :attr:`name` keyword argument in a field's constructor). """ - # Make select related work the same for querysets - max_depth += 1 - return self._dereference(self, max_depth=max_depth) + def field_sub(match): + # Extract just the field name, and look up the field objects + field_name = match.group(1).split('.') + fields = self._document._lookup_field(field_name) + # Substitute the correct name for the field into the javascript + return u'["%s"]' % fields[-1].db_field - def to_json(self): - """Converts a queryset to JSON""" - return json_util.dumps(self._collection_obj.find(self._query)) + def field_path_sub(match): + # Extract just the field name, and look up the field objects + field_name = match.group(1).split('.') + fields = self._document._lookup_field(field_name) + # Substitute the correct name for the field into the javascript + return ".".join([f.db_field for f in fields]) - def from_json(self, json_data): - """Converts json data to unsaved objects""" - son_data = json_util.loads(json_data) - return [self._document._from_son(data) for data in son_data] + code = re.sub(u'\[\s*~([A-z_][A-z_0-9.]+?)\s*\]', field_sub, code) + code = re.sub(u'\{\{\s*~([A-z_][A-z_0-9.]+?)\s*\}\}', field_path_sub, + code) + return code - @property - def _dereference(self): - if not self.__dereference: - self.__dereference = _import_class('DeReference')() - return self.__dereference + # Deprecated + + def ensure_index(self, **kwargs): + """Deprecated use :func:`~Document.ensure_index`""" + msg = ("Doc.objects()._ensure_index() is deprecated. " + "Use Doc.ensure_index() instead.") + warnings.warn(msg, DeprecationWarning) + self._document.__class__.ensure_index(**kwargs) + return self + + def _ensure_indexes(self): + """Deprecated use :func:`~Document.ensure_indexes`""" + msg = ("Doc.objects()._ensure_indexes() is deprecated. " + "Use Doc.ensure_indexes() instead.") + warnings.warn(msg, DeprecationWarning) + self._document.__class__.ensure_indexes() diff --git a/tests/queryset/queryset.py b/tests/queryset/queryset.py index bad3d360..bf64a565 100644 --- a/tests/queryset/queryset.py +++ b/tests/queryset/queryset.py @@ -713,19 +713,19 @@ class QuerySetTest(unittest.TestCase): self.assertEqual(p._cursor_args, {'snapshot': False, 'slave_okay': False, 'timeout': True}) - p.snapshot(False).slave_okay(False).timeout(False) + p = p.snapshot(False).slave_okay(False).timeout(False) self.assertEqual(p._cursor_args, {'snapshot': False, 'slave_okay': False, 'timeout': False}) - p.snapshot(True).slave_okay(False).timeout(False) + p = p.snapshot(True).slave_okay(False).timeout(False) self.assertEqual(p._cursor_args, {'snapshot': True, 'slave_okay': False, 'timeout': False}) - p.snapshot(True).slave_okay(True).timeout(False) + p = p.snapshot(True).slave_okay(True).timeout(False) self.assertEqual(p._cursor_args, {'snapshot': True, 'slave_okay': True, 'timeout': False}) - p.snapshot(True).slave_okay(True).timeout(True) + p = p.snapshot(True).slave_okay(True).timeout(True) self.assertEqual(p._cursor_args, {'snapshot': True, 'slave_okay': True, 'timeout': True}) @@ -773,7 +773,8 @@ class QuerySetTest(unittest.TestCase): self.assertEqual(len(docs), 1000) # Limit and skip - self.assertEqual('[, , ]', "%s" % docs[1:4]) + docs = docs[1:4] + self.assertEqual('[, , ]', "%s" % docs) self.assertEqual(docs.count(), 3) self.assertEqual(len(docs), 3) diff --git a/tests/queryset/visitor.py b/tests/queryset/visitor.py index 4af39e87..98815dbc 100644 --- a/tests/queryset/visitor.py +++ b/tests/queryset/visitor.py @@ -202,8 +202,8 @@ class QTest(unittest.TestCase): self.assertEqual(test2.count(), 3) self.assertFalse(test2 == test) - test2.filter(x=6) - self.assertEqual(test2.count(), 1) + test3 = test2.filter(x=6) + self.assertEqual(test3.count(), 1) self.assertEqual(test.count(), 3) def test_q(self): From e6ac8cab53a3269d2dee91d529f646d59f1b98c8 Mon Sep 17 00:00:00 2001 From: Ross Lawley Date: Fri, 4 Jan 2013 14:28:42 +0000 Subject: [PATCH 059/189] Fixing python 2.5 support --- mongoengine/queryset/queryset.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/mongoengine/queryset/queryset.py b/mongoengine/queryset/queryset.py index 239975f3..e6373700 100644 --- a/mongoengine/queryset/queryset.py +++ b/mongoengine/queryset/queryset.py @@ -516,12 +516,15 @@ class QuerySet(object): '_where_clause', '_loaded_fields', '_ordering', '_snapshot', '_timeout', '_class_check', '_slave_okay', '_read_preference', '_iter', '_scalar', '_as_pymongo', '_as_pymongo_coerce', - '_limit', '_skip', '_slice', '_hint') + '_limit', '_skip', '_hint') for prop in copy_props: val = getattr(self, prop) setattr(c, prop, copy.copy(val)) + if self._slice: + c._slice = self._slice + if self._cursor_obj: c._cursor_obj = self._cursor_obj.clone() From d9ed33d1b191f075a5370a7f9f199205496ea953 Mon Sep 17 00:00:00 2001 From: Ross Lawley Date: Fri, 4 Jan 2013 14:33:08 +0000 Subject: [PATCH 060/189] Added python 3.3 support to travis --- .travis.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.travis.yml b/.travis.yml index 806d8b41..c5f3961c 100644 --- a/.travis.yml +++ b/.travis.yml @@ -7,6 +7,7 @@ python: - 2.7 - 3.1 - 3.2 + - 3.3 env: - PYMONGO=dev - PYMONGO=2.4.1 From 85173d188ba8038cf6713e94d25c43e53e49db1d Mon Sep 17 00:00:00 2001 From: Ross Lawley Date: Fri, 4 Jan 2013 15:57:08 +0000 Subject: [PATCH 061/189] Add simplejson to python 2.5 build --- .travis.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.travis.yml b/.travis.yml index c5f3961c..bf34bab2 100644 --- a/.travis.yml +++ b/.travis.yml @@ -17,6 +17,7 @@ install: - if [[ $TRAVIS_PYTHON_VERSION == '2.'* ]]; then sudo ln -s /usr/lib/i386-linux-gnu/libz.so /usr/lib/; fi - if [[ $TRAVIS_PYTHON_VERSION == '2.'* ]]; then pip install PIL --use-mirrors ; true; fi - if [[ $TRAVIS_PYTHON_VERSION == '2.'* ]]; then pip install PIL --use-mirrors ; true; fi + - if [[ $TRAVIS_PYTHON_VERSION == '2.5' ]]; then pip install simplejson --use-mirrors ; true; fi - if [[ $PYMONGO == 'dev' ]]; then pip install https://github.com/mongodb/mongo-python-driver/tarball/master; true; fi - if [[ $PYMONGO != 'dev' ]]; then pip install pymongo==$PYMONGO --use-mirrors; true; fi - python setup.py install From 0f9e4ef352fa14b52ddc37b3bf9f17f03f8f0ab5 Mon Sep 17 00:00:00 2001 From: Ross Lawley Date: Fri, 4 Jan 2013 16:27:58 +0000 Subject: [PATCH 062/189] Add mongoengine.png asset in the build --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 35ca5793..4b42e71c 100644 --- a/setup.py +++ b/setup.py @@ -56,7 +56,7 @@ if sys.version_info[0] == 3: extra_opts['packages'] = find_packages(exclude=('tests',)) if "test" in sys.argv or "nosetests" in sys.argv: extra_opts['packages'].append("tests") - extra_opts['package_data'] = {"tests": ["mongoengine.png"]} + extra_opts['package_data'] = {"tests": ["fields/mongoengine.png"]} else: extra_opts['tests_require'] = ['nose', 'coverage', 'blinker', 'django>=1.3', 'PIL'] extra_opts['packages'] = find_packages(exclude=('tests',)) From 5c45eee817a680851f990e0ca0e1fc26c5cb6e9c Mon Sep 17 00:00:00 2001 From: Ross Lawley Date: Fri, 4 Jan 2013 16:28:26 +0000 Subject: [PATCH 063/189] Whitespace --- tests/document/instance.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/tests/document/instance.py b/tests/document/instance.py index 0054480f..c8a1b110 100644 --- a/tests/document/instance.py +++ b/tests/document/instance.py @@ -654,8 +654,7 @@ class InstanceTest(unittest.TestCase): Foo.drop_collection() - a = Foo(name='hello') - a.save() + a = Foo(name='hello').save() a.bar = a with open(TEST_IMAGE_PATH, 'rb') as test_image: @@ -665,7 +664,7 @@ class InstanceTest(unittest.TestCase): # Confirm can save and it resets the changed fields without hitting # max recursion error b = Foo.objects.with_id(a.id) - b.name='world' + b.name = 'world' b.save() self.assertEqual(b.picture, b.bar.picture, b.bar.bar.picture) From 7bb9c7d47f18cdee5c68eceb7cbe6792a962dbae Mon Sep 17 00:00:00 2001 From: Nick Joyce Date: Mon, 7 Jan 2013 14:49:39 +0000 Subject: [PATCH 064/189] Ensure that the update actions are grouped rather than serial. MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This is a performance update. When multiple properties of the same entity have been deleted and modified, 2 calls to update the entity are made, one {"$set": … } and another {"$unset": … }. This is 2 network interface calls which is a performance killer (even at lan speeds). Fixes: #210 --- mongoengine/document.py | 11 ++++++++--- tests/test_document.py | 30 ++++++++++++++++++++++++++++++ 2 files changed, 38 insertions(+), 3 deletions(-) diff --git a/mongoengine/document.py b/mongoengine/document.py index 7b3afafb..2bd1d221 100644 --- a/mongoengine/document.py +++ b/mongoengine/document.py @@ -234,11 +234,16 @@ class Document(BaseDocument): select_dict[actual_key] = doc[actual_key] upsert = self._created + work = {} + if updates: - collection.update(select_dict, {"$set": updates}, - upsert=upsert, safe=safe, **write_options) + work["$set"] = updates + if removals: - collection.update(select_dict, {"$unset": removals}, + work["$unset"] = removals + + if work: + collection.update(select_dict, work, upsert=upsert, safe=safe, **write_options) warn_cascade = not cascade and 'cascade' not in self._meta diff --git a/tests/test_document.py b/tests/test_document.py index cd0ab8fb..fa6eb282 100644 --- a/tests/test_document.py +++ b/tests/test_document.py @@ -3515,6 +3515,36 @@ class ValidatorErrorTest(unittest.TestCase): self.assertRaises(OperationError, change_shard_key) + def test_set_unset_one_operation(self): + """Ensure that $set and $unset actions are performed in the same + operation. + """ + class FooBar(Document): + meta = { + 'collection': 'foobar', + } + + foo = StringField(default=None) + bar = StringField(default=None) + + FooBar.drop_collection() + + # write an entity with a single prop + foo = FooBar(foo='foo') + foo.save() + + self.assertEqual(foo.foo, 'foo') + + # set foo to the default causing mongoengine to $unset foo. + foo.foo = None + foo.bar = 'bar' + + foo.save() + foo.reload() + + self.assertIsNone(foo.foo) + self.assertEqual(foo.bar, 'bar') + if __name__ == '__main__': unittest.main() From 50905ab459e729d2136a47c0e7be0c9d10daef14 Mon Sep 17 00:00:00 2001 From: Ross Lawley Date: Wed, 9 Jan 2013 08:41:03 +0000 Subject: [PATCH 065/189] Test update --- tests/test_dereference.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/tests/test_dereference.py b/tests/test_dereference.py index 41f8aebf..8557ec5c 100644 --- a/tests/test_dereference.py +++ b/tests/test_dereference.py @@ -198,6 +198,10 @@ class FieldTest(unittest.TestCase): raw_data = Group._get_collection().find_one() self.assertTrue(isinstance(raw_data['author'], DBRef)) self.assertTrue(isinstance(raw_data['members'][0], DBRef)) + group = Group.objects.first() + + self.assertEqual(group.author, user) + self.assertEqual(group.members, [user]) # Migrate the model definition class Group(Document): From 72dd9daa23ea4e9adf44df6a3c3ab2499e971ad6 Mon Sep 17 00:00:00 2001 From: Ross Lawley Date: Wed, 9 Jan 2013 16:16:48 +0000 Subject: [PATCH 066/189] Fixing py3.3 tests --- mongoengine/queryset/transform.py | 2 +- tests/queryset/transform.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/mongoengine/queryset/transform.py b/mongoengine/queryset/transform.py index 8ee84eed..9fe87802 100644 --- a/mongoengine/queryset/transform.py +++ b/mongoengine/queryset/transform.py @@ -26,7 +26,7 @@ def query(_doc_cls=None, _field_operation=False, **query): """ mongo_query = {} merge_query = defaultdict(list) - for key, value in query.items(): + for key, value in sorted(query.items()): if key == "__raw__": mongo_query.update(value) continue diff --git a/tests/queryset/transform.py b/tests/queryset/transform.py index 666b3451..d38cbfd5 100644 --- a/tests/queryset/transform.py +++ b/tests/queryset/transform.py @@ -26,7 +26,7 @@ class TransformTest(unittest.TestCase): self.assertEqual(transform.query(age__gt=20, age__lt=50), {'age': {'$gt': 20, '$lt': 50}}) self.assertEqual(transform.query(age=20, age__gt=50), - {'age': 20}) + {'$and': [{'age': {'$gt': 50}}, {'age': 20}]}) self.assertEqual(transform.query(friend__age__gte=30), {'friend.age': {'$gte': 30}}) self.assertEqual(transform.query(name__exists=True), From f5d02e1b1015856d013e73d01c7035359f0e9edb Mon Sep 17 00:00:00 2001 From: Martin Alderete Date: Tue, 15 Jan 2013 02:40:15 -0300 Subject: [PATCH 067/189] Fixed issue with choices validation when they are simple list/tuple, after model.validate() did not get any error message. Added test to ensure that model.validate() set the correct error messages. --- AUTHORS | 1 + mongoengine/base.py | 2 +- tests/test_fields.py | 35 +++++++++++++++++++++++++++++++++++ 3 files changed, 37 insertions(+), 1 deletion(-) diff --git a/AUTHORS b/AUTHORS index 82a1dfa4..ac48d7b7 100644 --- a/AUTHORS +++ b/AUTHORS @@ -128,3 +128,4 @@ that much better: * Peter Teichman * Jakub Kot * Jorge Bastida + * Martin Alderete https://github.com/malderete diff --git a/mongoengine/base.py b/mongoengine/base.py index 013afe78..f73af4cc 100644 --- a/mongoengine/base.py +++ b/mongoengine/base.py @@ -252,7 +252,7 @@ class BaseField(object): elif value_to_check not in self.choices: msg = ('Value must be %s of %s' % (err_msg, unicode(self.choices))) - self.error() + self.error(msg) # check validation argument if self.validation is not None: diff --git a/tests/test_fields.py b/tests/test_fields.py index 28af1b23..fa7e3965 100644 --- a/tests/test_fields.py +++ b/tests/test_fields.py @@ -1706,6 +1706,41 @@ class FieldTest(unittest.TestCase): Shirt.drop_collection() + + def test_simple_choices_validation_invalid_value(self): + """Ensure that error messages are correct. + """ + SIZES = ('S', 'M', 'L', 'XL', 'XXL') + COLORS = (('R', 'Red'), ('B', 'Blue')) + SIZE_MESSAGE = u"Value must be one of ('S', 'M', 'L', 'XL', 'XXL')" + COLOR_MESSAGE = u"Value must be one of ['R', 'B']" + + class Shirt(Document): + size = StringField(max_length=3, choices=SIZES) + color = StringField(max_length=1, choices=COLORS) + + Shirt.drop_collection() + + shirt = Shirt() + shirt.validate() + + shirt.size = "S" + shirt.color = "R" + shirt.validate() + + shirt.size = "XS" + shirt.color = "G" + + try: + shirt.validate() + except ValidationError, error: + # get the validation rules + error_dict = error.to_dict() + self.assertEqual(error_dict['size'], SIZE_MESSAGE) + self.assertEqual(error_dict['color'], COLOR_MESSAGE) + + Shirt.drop_collection() + def test_file_fields(self): """Ensure that file fields can be written to and their data retrieved """ From 17eeeb75366ab4625ee5c1a11089aaf9c21fee18 Mon Sep 17 00:00:00 2001 From: lcya86 Date: Thu, 17 Jan 2013 15:31:27 +0800 Subject: [PATCH 068/189] Update mongoengine/django/sessions.py MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit added the "get_decoded" method to the MongoSession class --- mongoengine/django/sessions.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/mongoengine/django/sessions.py b/mongoengine/django/sessions.py index 810b6265..8a35e896 100644 --- a/mongoengine/django/sessions.py +++ b/mongoengine/django/sessions.py @@ -34,6 +34,9 @@ class MongoSession(Document): meta = {'collection': MONGOENGINE_SESSION_COLLECTION, 'db_alias': MONGOENGINE_SESSION_DB_ALIAS, 'allow_inheritance': False} + + def get_decoded(self): + return SessionStore().decode(self.session_data) class SessionStore(SessionBase): From 2c7b12c022130727c25c3c0686a53a3762639f8e Mon Sep 17 00:00:00 2001 From: Ross Lawley Date: Tue, 22 Jan 2013 13:31:53 +0000 Subject: [PATCH 069/189] Added support for $maxDistance (#179) --- docs/changelog.rst | 1 + docs/guide/querying.rst | 2 ++ mongoengine/queryset/transform.py | 5 ++++- tests/queryset/queryset.py | 10 ++++++++++ 4 files changed, 17 insertions(+), 1 deletion(-) diff --git a/docs/changelog.rst b/docs/changelog.rst index 4fd3e143..3e8f7821 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -27,6 +27,7 @@ Changes in 0.8.X - Simplified Q objects, removed QueryTreeTransformerVisitor (#98) (#171) - FileFields now copyable (#198) - Querysets now return clones and are no longer edit in place (#56) +- Added support for $maxDistance (#179) Changes in 0.7.9 ================ diff --git a/docs/guide/querying.rst b/docs/guide/querying.rst index d5829432..40e36e32 100644 --- a/docs/guide/querying.rst +++ b/docs/guide/querying.rst @@ -92,6 +92,8 @@ may used with :class:`~mongoengine.GeoPointField`\ s: * ``within_polygon`` -- filter documents to those within a given polygon (e.g. [(41.91,-87.69), (41.92,-87.68), (41.91,-87.65), (41.89,-87.65)]). .. note:: Requires Mongo Server 2.0 +* ``max_distance`` -- can be added to your location queries to set a maximum +distance. Querying lists diff --git a/mongoengine/queryset/transform.py b/mongoengine/queryset/transform.py index 9fe87802..5707cecb 100644 --- a/mongoengine/queryset/transform.py +++ b/mongoengine/queryset/transform.py @@ -9,7 +9,8 @@ __all__ = ('query', 'update') COMPARISON_OPERATORS = ('ne', 'gt', 'gte', 'lt', 'lte', 'in', 'nin', 'mod', 'all', 'size', 'exists', 'not') GEO_OPERATORS = ('within_distance', 'within_spherical_distance', - 'within_box', 'within_polygon', 'near', 'near_sphere') + 'within_box', 'within_polygon', 'near', 'near_sphere', + 'max_distance') STRING_OPERATORS = ('contains', 'icontains', 'startswith', 'istartswith', 'endswith', 'iendswith', 'exact', 'iexact') @@ -97,6 +98,8 @@ def query(_doc_cls=None, _field_operation=False, **query): value = {'$nearSphere': value} elif op == 'within_box': value = {'$within': {'$box': value}} + elif op == "max_distance": + value = {'$maxDistance': value} else: raise NotImplementedError("Geo method '%s' has not " "been implemented" % op) diff --git a/tests/queryset/queryset.py b/tests/queryset/queryset.py index bf64a565..b5b0b280 100644 --- a/tests/queryset/queryset.py +++ b/tests/queryset/queryset.py @@ -2238,6 +2238,12 @@ class QuerySetTest(unittest.TestCase): self.assertEqual(events.count(), 3) self.assertEqual(list(events), [event3, event1, event2]) + # find events within 10 degrees of san francisco + point = [37.7566023, -122.415579] + events = Event.objects(location__near=point, location__max_distance=10) + self.assertEqual(events.count(), 1) + self.assertEqual(events[0], event2) + # find events within 10 degrees of san francisco point_and_distance = [[37.7566023, -122.415579], 10] events = Event.objects(location__within_distance=point_and_distance) @@ -2317,6 +2323,10 @@ class QuerySetTest(unittest.TestCase): ); self.assertEqual(points.count(), 2) + points = Point.objects(location__near_sphere=[-122, 37.5], + location__max_distance=60 / earth_radius); + self.assertEqual(points.count(), 2) + # Finds both points, but orders the north point first because it's # closer to the reference point to the north. points = Point.objects(location__near_sphere=[-122, 38.5]) From 3ba58ebaae51a708ccf02bd3bd7f6e75e6ce3258 Mon Sep 17 00:00:00 2001 From: Ross Lawley Date: Tue, 22 Jan 2013 13:39:53 +0000 Subject: [PATCH 070/189] Added Nicolas Trippar to AUTHORS #179 --- AUTHORS | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/AUTHORS b/AUTHORS index 989fd68c..7c2f8c83 100644 --- a/AUTHORS +++ b/AUTHORS @@ -133,4 +133,5 @@ that much better: * Pete Campton * Martyn Smith * Marcelo Anton - * Aleksey Porfirov \ No newline at end of file + * Aleksey Porfirov + * Nicolas Trippar \ No newline at end of file From 344dc64df85cbcbcfccaad4cbce7351465170c43 Mon Sep 17 00:00:00 2001 From: Ross Lawley Date: Tue, 22 Jan 2013 14:05:06 +0000 Subject: [PATCH 071/189] Updated authors and changelog #163 --- AUTHORS | 3 ++- docs/changelog.rst | 1 + mongoengine/document.py | 7 +++---- 3 files changed, 6 insertions(+), 5 deletions(-) diff --git a/AUTHORS b/AUTHORS index 7c2f8c83..aa7f833d 100644 --- a/AUTHORS +++ b/AUTHORS @@ -134,4 +134,5 @@ that much better: * Martyn Smith * Marcelo Anton * Aleksey Porfirov - * Nicolas Trippar \ No newline at end of file + * Nicolas Trippar + * Manuel Hermann \ No newline at end of file diff --git a/docs/changelog.rst b/docs/changelog.rst index 3e8f7821..1905a9d3 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -28,6 +28,7 @@ Changes in 0.8.X - FileFields now copyable (#198) - Querysets now return clones and are no longer edit in place (#56) - Added support for $maxDistance (#179) +- Uses getlasterror to test created on updated saves (#163) Changes in 0.7.9 ================ diff --git a/mongoengine/document.py b/mongoengine/document.py index 03a838bf..69d4d406 100644 --- a/mongoengine/document.py +++ b/mongoengine/document.py @@ -219,11 +219,11 @@ class Document(BaseDocument): doc = self.to_mongo() - find_delta = ('_id' not in doc or self._created or force_insert) + created = ('_id' not in doc or self._created or force_insert) try: collection = self.__class__.objects._collection - if find_delta: + if created: if force_insert: object_id = collection.insert(doc, safe=safe, **write_options) @@ -289,8 +289,7 @@ class Document(BaseDocument): self._changed_fields = [] self._created = False - signals.post_save.send(self.__class__, document=self, - created=find_delta) + signals.post_save.send(self.__class__, document=self, created=created) return self def cascade_save(self, warn_cascade=None, *args, **kwargs): From 692f00864d982a8c54f08bcda3712b43d9708751 Mon Sep 17 00:00:00 2001 From: Ross Lawley Date: Tue, 22 Jan 2013 15:16:58 +0000 Subject: [PATCH 072/189] Fixed inheritance and unique index creation (#140) --- docs/changelog.rst | 1 + mongoengine/base/document.py | 79 +++++++++++++++++++++++---------- mongoengine/base/metaclasses.py | 5 +-- mongoengine/document.py | 17 ++----- tests/document/indexes.py | 9 ++-- 5 files changed, 65 insertions(+), 46 deletions(-) diff --git a/docs/changelog.rst b/docs/changelog.rst index 1905a9d3..cb0ac6c6 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -29,6 +29,7 @@ Changes in 0.8.X - Querysets now return clones and are no longer edit in place (#56) - Added support for $maxDistance (#179) - Uses getlasterror to test created on updated saves (#163) +- Fixed inheritance and unique index creation (#140) Changes in 0.7.9 ================ diff --git a/mongoengine/base/document.py b/mongoengine/base/document.py index 93bde8ec..9f400618 100644 --- a/mongoengine/base/document.py +++ b/mongoengine/base/document.py @@ -509,6 +509,34 @@ class BaseDocument(object): obj._created = False return obj + @classmethod + def _build_index_specs(cls, meta_indexes): + """Generate and merge the full index specs + """ + + geo_indices = cls._geo_indices() + unique_indices = cls._unique_with_indexes() + index_specs = [cls._build_index_spec(spec) + for spec in meta_indexes] + + def merge_index_specs(index_specs, indices): + if not indices: + return index_specs + + spec_fields = [v['fields'] + for k, v in enumerate(index_specs)] + # Merge unqiue_indexes with existing specs + for k, v in enumerate(indices): + if v['fields'] in spec_fields: + index_specs[spec_fields.index(v['fields'])].update(v) + else: + index_specs.append(v) + return index_specs + + index_specs = merge_index_specs(index_specs, geo_indices) + index_specs = merge_index_specs(index_specs, unique_indices) + return index_specs + @classmethod def _build_index_spec(cls, spec): """Build a PyMongo index spec from a MongoEngine index spec. @@ -576,6 +604,7 @@ class BaseDocument(object): """ unique_indexes = [] for field_name, field in cls._fields.items(): + sparse = False # Generate a list of indexes needed by uniqueness constraints if field.unique: field.required = True @@ -596,11 +625,14 @@ class BaseDocument(object): unique_with.append('.'.join(name_parts)) # Unique field should be required parts[-1].required = True + sparse = (not sparse and + parts[-1].name not in cls.__dict__) unique_fields += unique_with # Add the new index to the list - index = [("%s%s" % (namespace, f), pymongo.ASCENDING) + fields = [("%s%s" % (namespace, f), pymongo.ASCENDING) for f in unique_fields] + index = {'fields': fields, 'unique': True, 'sparse': sparse} unique_indexes.append(index) # Grab any embedded document field unique indexes @@ -612,6 +644,29 @@ class BaseDocument(object): return unique_indexes + @classmethod + def _geo_indices(cls, inspected=None): + inspected = inspected or [] + geo_indices = [] + inspected.append(cls) + + EmbeddedDocumentField = _import_class("EmbeddedDocumentField") + GeoPointField = _import_class("GeoPointField") + + for field in cls._fields.values(): + if not isinstance(field, (EmbeddedDocumentField, GeoPointField)): + continue + if hasattr(field, 'document_type'): + field_cls = field.document_type + if field_cls in inspected: + continue + if hasattr(field_cls, '_geo_indices'): + geo_indices += field_cls._geo_indices(inspected) + elif field._geo_index: + geo_indices.append({'fields': + [(field.db_field, pymongo.GEO2D)]}) + return geo_indices + @classmethod def _lookup_field(cls, parts): """Lookup a field based on its attribute and return a list containing @@ -671,28 +726,6 @@ class BaseDocument(object): parts = [f.db_field for f in cls._lookup_field(parts)] return '.'.join(parts) - @classmethod - def _geo_indices(cls, inspected=None): - inspected = inspected or [] - geo_indices = [] - inspected.append(cls) - - EmbeddedDocumentField = _import_class("EmbeddedDocumentField") - GeoPointField = _import_class("GeoPointField") - - for field in cls._fields.values(): - if not isinstance(field, (EmbeddedDocumentField, GeoPointField)): - continue - if hasattr(field, 'document_type'): - field_cls = field.document_type - if field_cls in inspected: - continue - if hasattr(field_cls, '_geo_indices'): - geo_indices += field_cls._geo_indices(inspected) - elif field._geo_index: - geo_indices.append(field) - return geo_indices - def __set_field_display(self): """Dynamically set the display value for a field with choices""" for attr_name, field in self._fields.items(): diff --git a/mongoengine/base/metaclasses.py b/mongoengine/base/metaclasses.py index af39e144..2b63bfa8 100644 --- a/mongoengine/base/metaclasses.py +++ b/mongoengine/base/metaclasses.py @@ -329,10 +329,7 @@ class TopLevelDocumentMetaclass(DocumentMetaclass): meta = new_class._meta # Set index specifications - meta['index_specs'] = [new_class._build_index_spec(spec) - for spec in meta['indexes']] - unique_indexes = new_class._unique_with_indexes() - new_class._meta['unique_indexes'] = unique_indexes + meta['index_specs'] = new_class._build_index_specs(meta['indexes']) # If collection is a callable - call it and set the value collection = meta.get('collection') diff --git a/mongoengine/document.py b/mongoengine/document.py index 69d4d406..fff7efad 100644 --- a/mongoengine/document.py +++ b/mongoengine/document.py @@ -105,7 +105,7 @@ class Document(BaseDocument): By default, _cls will be added to the start of every index (that doesn't contain a list) if allow_inheritance is True. This can be - disabled by either setting types to False on the specific index or + disabled by either setting cls to False on the specific index or by setting index_cls to False on the meta dictionary for the document. """ @@ -481,12 +481,6 @@ class Document(BaseDocument): first_field = fields[0][0] return first_field == '_cls' - # Ensure indexes created by uniqueness constraints - for index in cls._meta['unique_indexes']: - cls_indexed = cls_indexed or includes_cls(index) - collection.ensure_index(index, unique=True, background=background, - drop_dups=drop_dups, **index_opts) - # Ensure document-defined indexes are created if cls._meta['index_specs']: index_spec = cls._meta['index_specs'] @@ -496,7 +490,8 @@ class Document(BaseDocument): cls_indexed = cls_indexed or includes_cls(fields) opts = index_opts.copy() opts.update(spec) - collection.ensure_index(fields, background=background, **opts) + collection.ensure_index(fields, background=background, + drop_dups=drop_dups, **opts) # If _cls is being used (for polymorphism), it needs an index, # only if another index doesn't begin with _cls @@ -505,12 +500,6 @@ class Document(BaseDocument): collection.ensure_index('_cls', background=background, **index_opts) - # Add geo indicies - for field in cls._geo_indices(): - index_spec = [(field.db_field, pymongo.GEO2D)] - collection.ensure_index(index_spec, background=background, - **index_opts) - class DynamicDocument(Document): """A Dynamic Document class allowing flexible, expandable and uncontrolled diff --git a/tests/document/indexes.py b/tests/document/indexes.py index cf25f61e..fb278aa7 100644 --- a/tests/document/indexes.py +++ b/tests/document/indexes.py @@ -259,13 +259,12 @@ class IndexesTest(unittest.TestCase): tags = ListField(StringField()) meta = { 'indexes': [ - {'fields': ['-date'], 'unique': True, - 'sparse': True, 'types': False}, + {'fields': ['-date'], 'unique': True, 'sparse': True}, ], } self.assertEqual([{'fields': [('addDate', -1)], 'unique': True, - 'sparse': True, 'types': False}], + 'sparse': True}], BlogPost._meta['index_specs']) BlogPost.drop_collection() @@ -674,7 +673,7 @@ class IndexesTest(unittest.TestCase): User.drop_collection() - def test_types_index_with_pk(self): + def test_index_with_pk(self): """Ensure you can use `pk` as part of a query""" class Comment(EmbeddedDocument): @@ -687,7 +686,7 @@ class IndexesTest(unittest.TestCase): {'fields': ['pk', 'comments.comment_id'], 'unique': True}]} except UnboundLocalError: - self.fail('Unbound local error at types index + pk definition') + self.fail('Unbound local error at index + pk definition') info = BlogPost.objects._collection.index_information() info = [value['key'] for key, value in info.iteritems()] From 3364e040c83412444dd19c00f714d33c27a0e526 Mon Sep 17 00:00:00 2001 From: Ross Lawley Date: Tue, 22 Jan 2013 16:05:44 +0000 Subject: [PATCH 073/189] Ensure $maxDistance is always the last part of the query (#179) --- mongoengine/queryset/transform.py | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/mongoengine/queryset/transform.py b/mongoengine/queryset/transform.py index 5707cecb..71f12e37 100644 --- a/mongoengine/queryset/transform.py +++ b/mongoengine/queryset/transform.py @@ -1,5 +1,7 @@ from collections import defaultdict +from bson import SON + from mongoengine.common import _import_class from mongoengine.errors import InvalidQueryError, LookUpError @@ -123,6 +125,16 @@ def query(_doc_cls=None, _field_operation=False, **query): elif key in mongo_query: if key in mongo_query and isinstance(mongo_query[key], dict): mongo_query[key].update(value) + # $maxDistance needs to come last - convert to SON + if '$maxDistance' in mongo_query[key]: + value_dict = mongo_query[key] + value_son = SON() + for k, v in value_dict.iteritems(): + if k == '$maxDistance': + continue + value_son[k] = v + value_son['$maxDistance'] = value_dict['$maxDistance'] + mongo_query[key] = value_son else: # Store for manually merging later merge_query[key].append(value) From 445f9453c4d305c394bb833dcd16ddce4b3ab2b8 Mon Sep 17 00:00:00 2001 From: Ross Lawley Date: Tue, 22 Jan 2013 16:38:07 +0000 Subject: [PATCH 074/189] Fixed reverse delete rule with inheritance (#197) --- mongoengine/document.py | 17 +++++++++++++---- tests/document/instance.py | 36 ++++++++++++++++++++++++++++++++++++ 2 files changed, 49 insertions(+), 4 deletions(-) diff --git a/mongoengine/document.py b/mongoengine/document.py index fff7efad..f40f1c9f 100644 --- a/mongoengine/document.py +++ b/mongoengine/document.py @@ -7,7 +7,7 @@ from bson.dbref import DBRef from mongoengine import signals, queryset from base import (DocumentMetaclass, TopLevelDocumentMetaclass, BaseDocument, - BaseDict, BaseList, ALLOW_INHERITANCE) + BaseDict, BaseList, ALLOW_INHERITANCE, get_document) from queryset import OperationError, NotUniqueError from connection import get_db, DEFAULT_CONNECTION_NAME @@ -421,9 +421,18 @@ class Document(BaseDocument): """This method registers the delete rules to apply when removing this object. """ - delete_rules = cls._meta.get('delete_rules') or {} - delete_rules[(document_cls, field_name)] = rule - cls._meta['delete_rules'] = delete_rules + classes = [get_document(class_name) + for class_name in cls._subclasses + if class_name != cls.__name__] + [cls] + documents = [get_document(class_name) + for class_name in document_cls._subclasses + if class_name != document_cls.__name__] + [document_cls] + + for cls in classes: + for document_cls in documents: + delete_rules = cls._meta.get('delete_rules') or {} + delete_rules[(document_cls, field_name)] = rule + cls._meta['delete_rules'] = delete_rules @classmethod def drop_collection(cls): diff --git a/tests/document/instance.py b/tests/document/instance.py index c8a1b110..07c4f0e0 100644 --- a/tests/document/instance.py +++ b/tests/document/instance.py @@ -1378,6 +1378,42 @@ class InstanceTest(unittest.TestCase): author.delete() self.assertEqual(len(BlogPost.objects), 0) + def test_reverse_delete_rule_with_document_inheritance(self): + """Ensure that a referenced document is also deleted upon deletion + of a child document. + """ + + class Writer(self.Person): + pass + + class BlogPost(Document): + content = StringField() + author = ReferenceField(self.Person, reverse_delete_rule=CASCADE) + reviewer = ReferenceField(self.Person, reverse_delete_rule=NULLIFY) + + self.Person.drop_collection() + BlogPost.drop_collection() + + author = Writer(name='Test User') + author.save() + + reviewer = Writer(name='Re Viewer') + reviewer.save() + + post = BlogPost(content='Watched some TV') + post.author = author + post.reviewer = reviewer + post.save() + + reviewer.delete() + self.assertEqual(len(BlogPost.objects), 1) + self.assertEqual(BlogPost.objects.get().reviewer, None) + + # Delete the Writer should lead to deletion of the BlogPost + author.delete() + self.assertEqual(len(BlogPost.objects), 0) + + def test_reverse_delete_rule_cascade_and_nullify_complex_field(self): """Ensure that a referenced document is also deleted upon deletion for complex fields. From c44b98a7e15d1c82f54e5c90ccd17e8d834cb4df Mon Sep 17 00:00:00 2001 From: Ross Lawley Date: Tue, 22 Jan 2013 17:54:35 +0000 Subject: [PATCH 075/189] Updated changelog --- docs/changelog.rst | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/changelog.rst b/docs/changelog.rst index cb0ac6c6..c0757fb1 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -30,6 +30,7 @@ Changes in 0.8.X - Added support for $maxDistance (#179) - Uses getlasterror to test created on updated saves (#163) - Fixed inheritance and unique index creation (#140) +- Fixed reverse delete rule with inheritance (#197) Changes in 0.7.9 ================ From 6d68ad735cfcb6f37b964612867f31ff188b622f Mon Sep 17 00:00:00 2001 From: Ross Lawley Date: Tue, 22 Jan 2013 17:56:15 +0000 Subject: [PATCH 076/189] Fixed validation for GenericReferences Where the references haven't been dereferenced --- docs/changelog.rst | 1 + mongoengine/fields.py | 8 ++- tests/test_dereference.py | 105 ++++++++++++++++++++++++++++++++++++++ 3 files changed, 112 insertions(+), 2 deletions(-) diff --git a/docs/changelog.rst b/docs/changelog.rst index c0757fb1..ba2c04c8 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -31,6 +31,7 @@ Changes in 0.8.X - Uses getlasterror to test created on updated saves (#163) - Fixed inheritance and unique index creation (#140) - Fixed reverse delete rule with inheritance (#197) +- Fixed validation for GenericReferences which havent been dereferenced Changes in 0.7.9 ================ diff --git a/mongoengine/fields.py b/mongoengine/fields.py index 5f11ae3b..f7817742 100644 --- a/mongoengine/fields.py +++ b/mongoengine/fields.py @@ -865,11 +865,15 @@ class GenericReferenceField(BaseField): return super(GenericReferenceField, self).__get__(instance, owner) def validate(self, value): - if not isinstance(value, (Document, DBRef)): + if not isinstance(value, (Document, DBRef, dict, SON)): self.error('GenericReferences can only contain documents') + if isinstance(value, (dict, SON)): + if '_ref' not in value or '_cls' not in value: + self.error('GenericReferences can only contain documents') + # We need the id from the saved object to create the DBRef - if isinstance(value, Document) and value.id is None: + elif isinstance(value, Document) and value.id is None: self.error('You can only reference documents once they have been' ' saved to the database') diff --git a/tests/test_dereference.py b/tests/test_dereference.py index 8557ec5c..f42482d1 100644 --- a/tests/test_dereference.py +++ b/tests/test_dereference.py @@ -1,4 +1,7 @@ from __future__ import with_statement +import sys +sys.path[0:0] = [""] + import unittest from bson import DBRef, ObjectId @@ -1018,3 +1021,105 @@ class FieldTest(unittest.TestCase): msg = Message.objects.get(id=1) self.assertEqual(0, msg.comments[0].id) self.assertEqual(1, msg.comments[1].id) + + def test_list_item_dereference_dref_false_save_doesnt_cause_extra_queries(self): + """Ensure that DBRef items in ListFields are dereferenced. + """ + class User(Document): + name = StringField() + + class Group(Document): + name = StringField() + members = ListField(ReferenceField(User, dbref=False)) + + User.drop_collection() + Group.drop_collection() + + for i in xrange(1, 51): + User(name='user %s' % i).save() + + Group(name="Test", members=User.objects).save() + + with query_counter() as q: + self.assertEqual(q, 0) + + group_obj = Group.objects.first() + self.assertEqual(q, 1) + + group_obj.name = "new test" + group_obj.save() + + self.assertEqual(q, 2) + + def test_list_item_dereference_dref_true_save_doesnt_cause_extra_queries(self): + """Ensure that DBRef items in ListFields are dereferenced. + """ + class User(Document): + name = StringField() + + class Group(Document): + name = StringField() + members = ListField(ReferenceField(User, dbref=True)) + + User.drop_collection() + Group.drop_collection() + + for i in xrange(1, 51): + User(name='user %s' % i).save() + + Group(name="Test", members=User.objects).save() + + with query_counter() as q: + self.assertEqual(q, 0) + + group_obj = Group.objects.first() + self.assertEqual(q, 1) + + group_obj.name = "new test" + group_obj.save() + + self.assertEqual(q, 2) + + def test_generic_reference_save_doesnt_cause_extra_queries(self): + + class UserA(Document): + name = StringField() + + class UserB(Document): + name = StringField() + + class UserC(Document): + name = StringField() + + class Group(Document): + name = StringField() + members = ListField(GenericReferenceField()) + + UserA.drop_collection() + UserB.drop_collection() + UserC.drop_collection() + Group.drop_collection() + + members = [] + for i in xrange(1, 51): + a = UserA(name='User A %s' % i).save() + b = UserB(name='User B %s' % i).save() + c = UserC(name='User C %s' % i).save() + + members += [a, b, c] + + Group(name="test", members=members).save() + + with query_counter() as q: + self.assertEqual(q, 0) + + group_obj = Group.objects.first() + self.assertEqual(q, 1) + + group_obj.name = "new test" + group_obj.save() + + self.assertEqual(q, 2) + +if __name__ == '__main__': + unittest.main() From e5e88d792e457e17040d803ab8c0592b174989bb Mon Sep 17 00:00:00 2001 From: Ross Lawley Date: Wed, 23 Jan 2013 12:54:14 +0000 Subject: [PATCH 077/189] Added SwitchDB context manager (#106) --- docs/apireference.rst | 1 + docs/changelog.rst | 1 + docs/guide/connecting.rst | 18 ++++++++++++++++ mongoengine/connection.py | 43 ++++++++++++++++++++++++++++++++++++++- tests/test_connection.py | 23 +++++++++++++++++++++ 5 files changed, 85 insertions(+), 1 deletion(-) diff --git a/docs/apireference.rst b/docs/apireference.rst index 0f8901a1..69b1db03 100644 --- a/docs/apireference.rst +++ b/docs/apireference.rst @@ -7,6 +7,7 @@ Connecting .. autofunction:: mongoengine.connect .. autofunction:: mongoengine.register_connection +.. autoclass:: mongoengine.SwitchDB Documents ========= diff --git a/docs/changelog.rst b/docs/changelog.rst index ba2c04c8..354d4718 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -32,6 +32,7 @@ Changes in 0.8.X - Fixed inheritance and unique index creation (#140) - Fixed reverse delete rule with inheritance (#197) - Fixed validation for GenericReferences which havent been dereferenced +- Added SwitchDB context manager (#106) Changes in 0.7.9 ================ diff --git a/docs/guide/connecting.rst b/docs/guide/connecting.rst index 657c46c2..b39ccda4 100644 --- a/docs/guide/connecting.rst +++ b/docs/guide/connecting.rst @@ -69,3 +69,21 @@ to point across databases and collections. Below is an example schema, using book = ReferenceField(Book) meta = {"db_alias": "users-books-db"} + + +Switch Database Context Manager +=============================== + +Sometimes you might want to switch the database to query against for a class. +The SwitchDB context manager allows you to change the database alias for a +class eg :: + + from mongoengine import SwitchDB + + class User(Document): + name = StringField() + + meta = {"db_alias": "user-db"} + + with SwitchDB(User, 'archive-user-db') as User: + User(name="Ross").save() # Saves the 'archive-user-db' diff --git a/mongoengine/connection.py b/mongoengine/connection.py index 87308ba3..b6c78e84 100644 --- a/mongoengine/connection.py +++ b/mongoengine/connection.py @@ -3,7 +3,7 @@ from pymongo import Connection, ReplicaSetConnection, uri_parser __all__ = ['ConnectionError', 'connect', 'register_connection', - 'DEFAULT_CONNECTION_NAME'] + 'DEFAULT_CONNECTION_NAME', 'SwitchDB'] DEFAULT_CONNECTION_NAME = 'default' @@ -163,6 +163,47 @@ def connect(db, alias=DEFAULT_CONNECTION_NAME, **kwargs): return get_connection(alias) + +class SwitchDB(object): + """ SwitchDB alias contextmanager. + + Example :: + # Register connections + register_connection('default', 'mongoenginetest') + register_connection('testdb-1', 'mongoenginetest2') + + class Group(Document): + name = StringField() + + Group(name="test").save() # Saves in the default db + + with SwitchDB(Group, 'testdb-1') as Group: + Group(name="hello testdb!").save() # Saves in testdb-1 + + """ + + def __init__(self, cls, db_alias): + """ Construct the query_counter. + + :param cls: the class to change the registered db + :param db_alias: the name of the specific database to use + """ + self.cls = cls + self.collection = cls._get_collection() + self.db_alias = db_alias + self.ori_db_alias = cls._meta.get("db_alias", DEFAULT_CONNECTION_NAME) + + def __enter__(self): + """ change the db_alias and clear the cached collection """ + self.cls._meta["db_alias"] = self.db_alias + self.cls._collection = None + return self.cls + + def __exit__(self, t, value, traceback): + """ Reset the db_alias and collection """ + self.cls._meta["db_alias"] = self.ori_db_alias + self.cls._collection = self.collection + # Support old naming convention _get_connection = get_connection _get_db = get_db diff --git a/tests/test_connection.py b/tests/test_connection.py index cd03df0b..4931dc9f 100644 --- a/tests/test_connection.py +++ b/tests/test_connection.py @@ -93,6 +93,29 @@ class ConnectionTest(unittest.TestCase): date_doc = DateDoc.objects.first() self.assertEqual(d, date_doc.the_date) + def test_switch_db_context_manager(self): + register_connection('testdb-1', 'mongoenginetest2') + + class Group(Document): + name = StringField() + + Group.drop_collection() + + Group(name="hello - default").save() + self.assertEqual(1, Group.objects.count()) + + with SwitchDB(Group, 'testdb-1') as Group: + + self.assertEqual(0, Group.objects.count()) + + Group(name="hello").save() + + self.assertEqual(1, Group.objects.count()) + + Group.drop_collection() + self.assertEqual(0, Group.objects.count()) + + self.assertEqual(1, Group.objects.count()) if __name__ == '__main__': unittest.main() From ea46edf50a712021d12249fd9e784bffbb505520 Mon Sep 17 00:00:00 2001 From: Ross Lawley Date: Wed, 23 Jan 2013 16:07:07 +0000 Subject: [PATCH 078/189] Added switch_db method to document instances (#106) --- docs/changelog.rst | 1 + mongoengine/connection.py | 1 + mongoengine/document.py | 46 +++++++++++++++++++++++++++++++---- tests/document/instance.py | 50 ++++++++++++++++++++++++++++++++++++++ tests/test_connection.py | 1 + 5 files changed, 94 insertions(+), 5 deletions(-) diff --git a/docs/changelog.rst b/docs/changelog.rst index 354d4718..65e11034 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -33,6 +33,7 @@ Changes in 0.8.X - Fixed reverse delete rule with inheritance (#197) - Fixed validation for GenericReferences which havent been dereferenced - Added SwitchDB context manager (#106) +- Added switch_db method to document instances (#106) Changes in 0.7.9 ================ diff --git a/mongoengine/connection.py b/mongoengine/connection.py index b6c78e84..80791e5f 100644 --- a/mongoengine/connection.py +++ b/mongoengine/connection.py @@ -168,6 +168,7 @@ class SwitchDB(object): """ SwitchDB alias contextmanager. Example :: + # Register connections register_connection('default', 'mongoenginetest') register_connection('testdb-1', 'mongoenginetest2') diff --git a/mongoengine/document.py b/mongoengine/document.py index f40f1c9f..3bc4caed 100644 --- a/mongoengine/document.py +++ b/mongoengine/document.py @@ -9,7 +9,7 @@ from mongoengine import signals, queryset from base import (DocumentMetaclass, TopLevelDocumentMetaclass, BaseDocument, BaseDict, BaseList, ALLOW_INHERITANCE, get_document) from queryset import OperationError, NotUniqueError -from connection import get_db, DEFAULT_CONNECTION_NAME +from connection import get_db, DEFAULT_CONNECTION_NAME, SwitchDB __all__ = ('Document', 'EmbeddedDocument', 'DynamicDocument', 'DynamicEmbeddedDocument', 'OperationError', @@ -222,7 +222,7 @@ class Document(BaseDocument): created = ('_id' not in doc or self._created or force_insert) try: - collection = self.__class__.objects._collection + collection = self._get_collection() if created: if force_insert: object_id = collection.insert(doc, safe=safe, @@ -321,6 +321,16 @@ class Document(BaseDocument): ref.save(**kwargs) ref._changed_fields = [] + @property + def _qs(self): + """ + Returns the queryset to use for updating / reloading / deletions + """ + qs = self.__class__.objects + if hasattr(self, '_objects'): + qs = self._objects + return qs + @property def _object_key(self): """Dict to identify object in collection @@ -342,7 +352,7 @@ class Document(BaseDocument): raise OperationError('attempt to update a document not yet saved') # Need to add shard key to query, or you get an error - return self.__class__.objects(**self._object_key).update_one(**kwargs) + return self._qs.filter(**self._object_key).update_one(**kwargs) def delete(self, safe=False): """Delete the :class:`~mongoengine.Document` from the database. This @@ -353,13 +363,39 @@ class Document(BaseDocument): signals.pre_delete.send(self.__class__, document=self) try: - self.__class__.objects(**self._object_key).delete(safe=safe) + self._qs.filter(**self._object_key).delete(safe=safe) except pymongo.errors.OperationFailure, err: message = u'Could not delete document (%s)' % err.message raise OperationError(message) signals.post_delete.send(self.__class__, document=self) + def switch_db(self, db_alias): + """ + Temporarily switch the database for a document instance. + + Only really useful for archiving off data and calling `save()`:: + + user = User.objects.get(id=user_id) + user.switch_db('archive-db') + user.save() + + If you need to read from another database see + :class:`~mongoengine.SwitchDB` + + :param db_alias: The database alias to use for saving the document + """ + with SwitchDB(self.__class__, db_alias) as cls: + collection = cls._get_collection() + db = cls._get_db + self._get_collection = lambda: collection + self._get_db = lambda: db + self._collection = collection + self._created = True + self._objects = self.__class__.objects + self._objects._collection_obj = collection + return self + def select_related(self, max_depth=1): """Handles dereferencing of :class:`~bson.dbref.DBRef` objects to a maximum depth in order to cut down the number queries to mongodb. @@ -377,7 +413,7 @@ class Document(BaseDocument): .. versionchanged:: 0.6 Now chainable """ id_field = self._meta['id_field'] - obj = self.__class__.objects( + obj = self._qs.filter( **{id_field: self[id_field]} ).limit(1).select_related(max_depth=max_depth) if obj: diff --git a/tests/document/instance.py b/tests/document/instance.py index 07c4f0e0..3b5a4bd9 100644 --- a/tests/document/instance.py +++ b/tests/document/instance.py @@ -2114,6 +2114,56 @@ class ValidatorErrorTest(unittest.TestCase): self.assertEqual(classic_doc, dict_doc) self.assertEqual(classic_doc._data, dict_doc._data) + def test_switch_db_instance(self): + register_connection('testdb-1', 'mongoenginetest2') + + class Group(Document): + name = StringField() + + Group.drop_collection() + with SwitchDB(Group, 'testdb-1') as Group: + Group.drop_collection() + + Group(name="hello - default").save() + self.assertEqual(1, Group.objects.count()) + + group = Group.objects.first() + group.switch_db('testdb-1') + group.name = "hello - testdb!" + group.save() + + with SwitchDB(Group, 'testdb-1') as Group: + group = Group.objects.first() + self.assertEqual("hello - testdb!", group.name) + + group = Group.objects.first() + self.assertEqual("hello - default", group.name) + + # Slightly contrived now - perform an update + # Only works as they have the same object_id + group.switch_db('testdb-1') + group.update(set__name="hello - update") + + with SwitchDB(Group, 'testdb-1') as Group: + group = Group.objects.first() + self.assertEqual("hello - update", group.name) + Group.drop_collection() + self.assertEqual(0, Group.objects.count()) + + group = Group.objects.first() + self.assertEqual("hello - default", group.name) + + # Totally contrived now - perform a delete + # Only works as they have the same object_id + group.switch_db('testdb-1') + group.delete() + + with SwitchDB(Group, 'testdb-1') as Group: + self.assertEqual(0, Group.objects.count()) + + group = Group.objects.first() + self.assertEqual("hello - default", group.name) + if __name__ == '__main__': unittest.main() diff --git a/tests/test_connection.py b/tests/test_connection.py index 4931dc9f..7ff18a38 100644 --- a/tests/test_connection.py +++ b/tests/test_connection.py @@ -94,6 +94,7 @@ class ConnectionTest(unittest.TestCase): self.assertEqual(d, date_doc.the_date) def test_switch_db_context_manager(self): + connect('mongoenginetest') register_connection('testdb-1', 'mongoenginetest2') class Group(Document): From 4f70c27b5695216839bd92d42eadd04518d95e58 Mon Sep 17 00:00:00 2001 From: Ross Lawley Date: Wed, 23 Jan 2013 16:19:07 +0000 Subject: [PATCH 079/189] Updated doc string --- mongoengine/connection.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/mongoengine/connection.py b/mongoengine/connection.py index 80791e5f..9f906a28 100644 --- a/mongoengine/connection.py +++ b/mongoengine/connection.py @@ -165,7 +165,7 @@ def connect(db, alias=DEFAULT_CONNECTION_NAME, **kwargs): class SwitchDB(object): - """ SwitchDB alias contextmanager. + """ SwitchDB alias context manager. Example :: @@ -184,7 +184,7 @@ class SwitchDB(object): """ def __init__(self, cls, db_alias): - """ Construct the query_counter. + """ Construct the SwitchDB context manager :param cls: the class to change the registered db :param db_alias: the name of the specific database to use From 3a6dc77d3630ca87033374ae8c094f5ee8c9cda2 Mon Sep 17 00:00:00 2001 From: Ross Lawley Date: Wed, 23 Jan 2013 19:05:44 +0000 Subject: [PATCH 080/189] Added no_dereference context manager (#82) Reorganised the context_managers as well --- docs/apireference.rst | 8 +- docs/changelog.rst | 3 +- docs/guide/connecting.rst | 8 +- docs/guide/querying.rst | 18 +++- mongoengine/base/fields.py | 12 ++- mongoengine/common.py | 2 +- mongoengine/connection.py | 43 +-------- mongoengine/context_managers.py | 159 +++++++++++++++++++++++++++++++ mongoengine/document.py | 18 ++-- mongoengine/fields.py | 2 +- mongoengine/queryset/manager.py | 4 +- mongoengine/queryset/queryset.py | 1 - mongoengine/tests.py | 59 ------------ tests/queryset/queryset.py | 2 +- tests/test_connection.py | 4 +- tests/test_dereference.py | 74 +++++++++++++- 16 files changed, 289 insertions(+), 128 deletions(-) create mode 100644 mongoengine/context_managers.py delete mode 100644 mongoengine/tests.py diff --git a/docs/apireference.rst b/docs/apireference.rst index 69b1db03..049cc303 100644 --- a/docs/apireference.rst +++ b/docs/apireference.rst @@ -7,7 +7,6 @@ Connecting .. autofunction:: mongoengine.connect .. autofunction:: mongoengine.register_connection -.. autoclass:: mongoengine.SwitchDB Documents ========= @@ -35,6 +34,13 @@ Documents .. autoclass:: mongoengine.ValidationError :members: +Context Managers +================ + +.. autoclass:: mongoengine.context_managers.switch_db +.. autoclass:: mongoengine.context_managers.no_dereference +.. autoclass:: mongoengine.context_managers.query_counter + Querying ======== diff --git a/docs/changelog.rst b/docs/changelog.rst index 65e11034..bead6935 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -32,8 +32,9 @@ Changes in 0.8.X - Fixed inheritance and unique index creation (#140) - Fixed reverse delete rule with inheritance (#197) - Fixed validation for GenericReferences which havent been dereferenced -- Added SwitchDB context manager (#106) +- Added switch_db context manager (#106) - Added switch_db method to document instances (#106) +- Added no_dereference context manager (#82) Changes in 0.7.9 ================ diff --git a/docs/guide/connecting.rst b/docs/guide/connecting.rst index b39ccda4..ebd61a97 100644 --- a/docs/guide/connecting.rst +++ b/docs/guide/connecting.rst @@ -75,15 +75,15 @@ Switch Database Context Manager =============================== Sometimes you might want to switch the database to query against for a class. -The SwitchDB context manager allows you to change the database alias for a -class eg :: +The :class:`~mongoengine.context_managers.switch_db` context manager allows +you to change the database alias for a class eg :: - from mongoengine import SwitchDB + from mongoengine.context_managers import switch_db class User(Document): name = StringField() meta = {"db_alias": "user-db"} - with SwitchDB(User, 'archive-user-db') as User: + with switch_db(User, 'archive-user-db') as User: User(name="Ross").save() # Saves the 'archive-user-db' diff --git a/docs/guide/querying.rst b/docs/guide/querying.rst index 40e36e32..7ccf1432 100644 --- a/docs/guide/querying.rst +++ b/docs/guide/querying.rst @@ -93,7 +93,7 @@ may used with :class:`~mongoengine.GeoPointField`\ s: [(41.91,-87.69), (41.92,-87.68), (41.91,-87.65), (41.89,-87.65)]). .. note:: Requires Mongo Server 2.0 * ``max_distance`` -- can be added to your location queries to set a maximum -distance. + distance. Querying lists @@ -369,6 +369,22 @@ references to the depth of 1 level. If you have more complicated documents and want to dereference more of the object at once then increasing the :attr:`max_depth` will dereference more levels of the document. +Turning off dereferencing +------------------------- + +Sometimes for performance reasons you don't want to automatically dereference +data . To turn off all dereferencing you can use the +:class:`~mongoengine.context_managers.no_dereference` context manager:: + + with no_dereference(Post) as Post: + post = Post.objects.first() + assert(isinstance(post.author, ObjectId)) + +.. note:: + + :class:`~mongoengine.context_managers.no_dereference` only works on the + Default QuerySet manager. + Advanced queries ================ Sometimes calling a :class:`~mongoengine.queryset.QuerySet` object with keyword diff --git a/mongoengine/base/fields.py b/mongoengine/base/fields.py index a892fbd2..82981e25 100644 --- a/mongoengine/base/fields.py +++ b/mongoengine/base/fields.py @@ -23,6 +23,7 @@ class BaseField(object): name = None _geo_index = False _auto_gen = False # Call `generate` to generate a value + _auto_dereference = True # These track each time a Field instance is created. Used to retain order. # The auto_creation_counter is used for fields that MongoEngine implicitly @@ -163,9 +164,11 @@ class ComplexBaseField(BaseField): ReferenceField = _import_class('ReferenceField') GenericReferenceField = _import_class('GenericReferenceField') - dereference = self.field is None or isinstance(self.field, - (GenericReferenceField, ReferenceField)) - if not self._dereference and instance._initialised and dereference: + dereference = (self._auto_dereference and + (self.field is None or isinstance(self.field, + (GenericReferenceField, ReferenceField)))) + + if not self.__dereference and instance._initialised and dereference: instance._data[self.name] = self._dereference( instance._data.get(self.name), max_depth=1, instance=instance, name=self.name @@ -182,7 +185,8 @@ class ComplexBaseField(BaseField): value = BaseDict(value, instance, self.name) instance._data[self.name] = value - if (instance._initialised and isinstance(value, (BaseList, BaseDict)) + if (self._auto_dereference and instance._initialised and + isinstance(value, (BaseList, BaseDict)) and not value._dereferenced): value = self._dereference( value, max_depth=1, instance=instance, name=self.name diff --git a/mongoengine/common.py b/mongoengine/common.py index a8422c09..718ac0b2 100644 --- a/mongoengine/common.py +++ b/mongoengine/common.py @@ -11,7 +11,7 @@ def _import_class(cls_name): field_classes = ('DictField', 'DynamicField', 'EmbeddedDocumentField', 'FileField', 'GenericReferenceField', 'GenericEmbeddedDocumentField', 'GeoPointField', - 'ReferenceField', 'StringField') + 'ReferenceField', 'StringField', 'ComplexBaseField') queryset_classes = ('OperationError',) deref_classes = ('DeReference',) diff --git a/mongoengine/connection.py b/mongoengine/connection.py index 9f906a28..a47be446 100644 --- a/mongoengine/connection.py +++ b/mongoengine/connection.py @@ -3,7 +3,7 @@ from pymongo import Connection, ReplicaSetConnection, uri_parser __all__ = ['ConnectionError', 'connect', 'register_connection', - 'DEFAULT_CONNECTION_NAME', 'SwitchDB'] + 'DEFAULT_CONNECTION_NAME'] DEFAULT_CONNECTION_NAME = 'default' @@ -164,47 +164,6 @@ def connect(db, alias=DEFAULT_CONNECTION_NAME, **kwargs): return get_connection(alias) -class SwitchDB(object): - """ SwitchDB alias context manager. - - Example :: - - # Register connections - register_connection('default', 'mongoenginetest') - register_connection('testdb-1', 'mongoenginetest2') - - class Group(Document): - name = StringField() - - Group(name="test").save() # Saves in the default db - - with SwitchDB(Group, 'testdb-1') as Group: - Group(name="hello testdb!").save() # Saves in testdb-1 - - """ - - def __init__(self, cls, db_alias): - """ Construct the SwitchDB context manager - - :param cls: the class to change the registered db - :param db_alias: the name of the specific database to use - """ - self.cls = cls - self.collection = cls._get_collection() - self.db_alias = db_alias - self.ori_db_alias = cls._meta.get("db_alias", DEFAULT_CONNECTION_NAME) - - def __enter__(self): - """ change the db_alias and clear the cached collection """ - self.cls._meta["db_alias"] = self.db_alias - self.cls._collection = None - return self.cls - - def __exit__(self, t, value, traceback): - """ Reset the db_alias and collection """ - self.cls._meta["db_alias"] = self.ori_db_alias - self.cls._collection = self.collection - # Support old naming convention _get_connection = get_connection _get_db = get_db diff --git a/mongoengine/context_managers.py b/mongoengine/context_managers.py new file mode 100644 index 00000000..7255d51c --- /dev/null +++ b/mongoengine/context_managers.py @@ -0,0 +1,159 @@ +from mongoengine.common import _import_class +from mongoengine.connection import DEFAULT_CONNECTION_NAME, get_db +from mongoengine.queryset import OperationError, QuerySet + +__all__ = ("switch_db", "no_dereference", "query_counter") + + +class switch_db(object): + """ switch_db alias context manager. + + Example :: + + # Register connections + register_connection('default', 'mongoenginetest') + register_connection('testdb-1', 'mongoenginetest2') + + class Group(Document): + name = StringField() + + Group(name="test").save() # Saves in the default db + + with switch_db(Group, 'testdb-1') as Group: + Group(name="hello testdb!").save() # Saves in testdb-1 + + """ + + def __init__(self, cls, db_alias): + """ Construct the switch_db context manager + + :param cls: the class to change the registered db + :param db_alias: the name of the specific database to use + """ + self.cls = cls + self.collection = cls._get_collection() + self.db_alias = db_alias + self.ori_db_alias = cls._meta.get("db_alias", DEFAULT_CONNECTION_NAME) + + def __enter__(self): + """ change the db_alias and clear the cached collection """ + self.cls._meta["db_alias"] = self.db_alias + self.cls._collection = None + return self.cls + + def __exit__(self, t, value, traceback): + """ Reset the db_alias and collection """ + self.cls._meta["db_alias"] = self.ori_db_alias + self.cls._collection = self.collection + + +class no_dereference(object): + """ no_dereference context manager. + + Turns off all dereferencing in Documents:: + + with no_dereference(Group) as Group: + Group.objects.find() + + """ + + def __init__(self, cls): + """ Construct the no_dereference context manager. + + :param cls: the class to turn dereferencing off on + """ + self.cls = cls + + ReferenceField = _import_class('ReferenceField') + GenericReferenceField = _import_class('GenericReferenceField') + ComplexBaseField = _import_class('ComplexBaseField') + + self.deref_fields = [k for k, v in self.cls._fields.iteritems() + if isinstance(v, (ReferenceField, + GenericReferenceField, + ComplexBaseField))] + + def __enter__(self): + """ change the objects default and _auto_dereference values""" + if 'queryset_class' in self.cls._meta: + raise OperationError("no_dereference context manager only works on" + " default queryset classes") + objects = self.cls.__dict__['objects'] + objects.default = QuerySetNoDeRef + self.cls.objects = objects + for field in self.deref_fields: + self.cls._fields[field]._auto_dereference = False + return self.cls + + def __exit__(self, t, value, traceback): + """ Reset the default and _auto_dereference values""" + objects = self.cls.__dict__['objects'] + objects.default = QuerySet + self.cls.objects = objects + for field in self.deref_fields: + self.cls._fields[field]._auto_dereference = True + return self.cls + + +class QuerySetNoDeRef(QuerySet): + """Special no_dereference QuerySet""" + def __dereference(items, max_depth=1, instance=None, name=None): + return items + + +class query_counter(object): + """ Query_counter contextmanager to get the number of queries. """ + + def __init__(self): + """ Construct the query_counter. """ + self.counter = 0 + self.db = get_db() + + def __enter__(self): + """ On every with block we need to drop the profile collection. """ + self.db.set_profiling_level(0) + self.db.system.profile.drop() + self.db.set_profiling_level(2) + return self + + def __exit__(self, t, value, traceback): + """ Reset the profiling level. """ + self.db.set_profiling_level(0) + + def __eq__(self, value): + """ == Compare querycounter. """ + return value == self._get_count() + + def __ne__(self, value): + """ != Compare querycounter. """ + return not self.__eq__(value) + + def __lt__(self, value): + """ < Compare querycounter. """ + return self._get_count() < value + + def __le__(self, value): + """ <= Compare querycounter. """ + return self._get_count() <= value + + def __gt__(self, value): + """ > Compare querycounter. """ + return self._get_count() > value + + def __ge__(self, value): + """ >= Compare querycounter. """ + return self._get_count() >= value + + def __int__(self): + """ int representation. """ + return self._get_count() + + def __repr__(self): + """ repr query_counter as the number of queries. """ + return u"%s" % self._get_count() + + def _get_count(self): + """ Get the number of queries. """ + count = self.db.system.profile.find().count() - self.counter + self.counter += 1 + return count diff --git a/mongoengine/document.py b/mongoengine/document.py index 3bc4caed..9d4a1e6e 100644 --- a/mongoengine/document.py +++ b/mongoengine/document.py @@ -1,15 +1,17 @@ +from __future__ import with_statement import warnings import pymongo import re from bson.dbref import DBRef -from mongoengine import signals, queryset - -from base import (DocumentMetaclass, TopLevelDocumentMetaclass, BaseDocument, - BaseDict, BaseList, ALLOW_INHERITANCE, get_document) -from queryset import OperationError, NotUniqueError -from connection import get_db, DEFAULT_CONNECTION_NAME, SwitchDB +from mongoengine import signals +from mongoengine.base import (DocumentMetaclass, TopLevelDocumentMetaclass, + BaseDocument, BaseDict, BaseList, + ALLOW_INHERITANCE, get_document) +from mongoengine.queryset import OperationError, NotUniqueError +from mongoengine.connection import get_db, DEFAULT_CONNECTION_NAME +from mongoengine.context_managers import switch_db __all__ = ('Document', 'EmbeddedDocument', 'DynamicDocument', 'DynamicEmbeddedDocument', 'OperationError', @@ -381,11 +383,11 @@ class Document(BaseDocument): user.save() If you need to read from another database see - :class:`~mongoengine.SwitchDB` + :class:`~mongoengine.context_managers.switch_db` :param db_alias: The database alias to use for saving the document """ - with SwitchDB(self.__class__, db_alias) as cls: + with switch_db(self.__class__, db_alias) as cls: collection = cls._get_collection() db = cls._get_db self._get_collection = lambda: collection diff --git a/mongoengine/fields.py b/mongoengine/fields.py index f7817742..1ccdb650 100644 --- a/mongoengine/fields.py +++ b/mongoengine/fields.py @@ -779,7 +779,7 @@ class ReferenceField(BaseField): value = instance._data.get(self.name) # Dereference DBRefs - if isinstance(value, DBRef): + if self._auto_dereference and isinstance(value, DBRef): value = self.document_type._get_db().dereference(value) if value is not None: instance._data[self.name] = self.document_type._from_son(value) diff --git a/mongoengine/queryset/manager.py b/mongoengine/queryset/manager.py index d9f9992f..47c2143d 100644 --- a/mongoengine/queryset/manager.py +++ b/mongoengine/queryset/manager.py @@ -18,11 +18,11 @@ class QuerySetManager(object): """ get_queryset = None + default = QuerySet def __init__(self, queryset_func=None): if queryset_func: self.get_queryset = queryset_func - self._collections = {} def __get__(self, instance, owner): """Descriptor for instantiating a new QuerySet object when @@ -33,7 +33,7 @@ class QuerySetManager(object): return self # owner is the document that contains the QuerySetManager - queryset_class = owner._meta.get('queryset_class') or QuerySet + queryset_class = owner._meta.get('queryset_class', self.default) queryset = queryset_class(owner, owner._get_collection()) if self.get_queryset: arg_count = self.get_queryset.func_code.co_argcount diff --git a/mongoengine/queryset/queryset.py b/mongoengine/queryset/queryset.py index e6373700..a9ff6e73 100644 --- a/mongoengine/queryset/queryset.py +++ b/mongoengine/queryset/queryset.py @@ -109,7 +109,6 @@ class QuerySet(object): queryset._class_check = class_check return queryset - def __iter__(self): """Support iterator protocol""" self.rewind() diff --git a/mongoengine/tests.py b/mongoengine/tests.py deleted file mode 100644 index 68663772..00000000 --- a/mongoengine/tests.py +++ /dev/null @@ -1,59 +0,0 @@ -from mongoengine.connection import get_db - - -class query_counter(object): - """ Query_counter contextmanager to get the number of queries. """ - - def __init__(self): - """ Construct the query_counter. """ - self.counter = 0 - self.db = get_db() - - def __enter__(self): - """ On every with block we need to drop the profile collection. """ - self.db.set_profiling_level(0) - self.db.system.profile.drop() - self.db.set_profiling_level(2) - return self - - def __exit__(self, t, value, traceback): - """ Reset the profiling level. """ - self.db.set_profiling_level(0) - - def __eq__(self, value): - """ == Compare querycounter. """ - return value == self._get_count() - - def __ne__(self, value): - """ != Compare querycounter. """ - return not self.__eq__(value) - - def __lt__(self, value): - """ < Compare querycounter. """ - return self._get_count() < value - - def __le__(self, value): - """ <= Compare querycounter. """ - return self._get_count() <= value - - def __gt__(self, value): - """ > Compare querycounter. """ - return self._get_count() > value - - def __ge__(self, value): - """ >= Compare querycounter. """ - return self._get_count() >= value - - def __int__(self): - """ int representation. """ - return self._get_count() - - def __repr__(self): - """ repr query_counter as the number of queries. """ - return u"%s" % self._get_count() - - def _get_count(self): - """ Get the number of queries. """ - count = self.db.system.profile.find().count() - self.counter - self.counter += 1 - return count diff --git a/tests/queryset/queryset.py b/tests/queryset/queryset.py index b5b0b280..35940447 100644 --- a/tests/queryset/queryset.py +++ b/tests/queryset/queryset.py @@ -17,7 +17,7 @@ from bson import ObjectId from mongoengine import * from mongoengine.connection import get_connection from mongoengine.python_support import PY3 -from mongoengine.tests import query_counter +from mongoengine.context_managers import query_counter from mongoengine.queryset import (QuerySet, QuerySetManager, MultipleObjectsReturned, DoesNotExist, QueryFieldList, queryset_manager) diff --git a/tests/test_connection.py b/tests/test_connection.py index 7ff18a38..2a216fef 100644 --- a/tests/test_connection.py +++ b/tests/test_connection.py @@ -1,3 +1,4 @@ +from __future__ import with_statement import datetime import pymongo import unittest @@ -8,6 +9,7 @@ from bson.tz_util import utc from mongoengine import * from mongoengine.connection import get_db, get_connection, ConnectionError +from mongoengine.context_managers import switch_db class ConnectionTest(unittest.TestCase): @@ -105,7 +107,7 @@ class ConnectionTest(unittest.TestCase): Group(name="hello - default").save() self.assertEqual(1, Group.objects.count()) - with SwitchDB(Group, 'testdb-1') as Group: + with switch_db(Group, 'testdb-1') as Group: self.assertEqual(0, Group.objects.count()) diff --git a/tests/test_dereference.py b/tests/test_dereference.py index f42482d1..8e4ffdd8 100644 --- a/tests/test_dereference.py +++ b/tests/test_dereference.py @@ -8,7 +8,7 @@ from bson import DBRef, ObjectId from mongoengine import * from mongoengine.connection import get_db -from mongoengine.tests import query_counter +from mongoengine.context_managers import query_counter, no_dereference class FieldTest(unittest.TestCase): @@ -1121,5 +1121,77 @@ class FieldTest(unittest.TestCase): self.assertEqual(q, 2) + def test_no_dereference_context_manager_object_id(self): + """Ensure that DBRef items in ListFields aren't dereferenced. + """ + class User(Document): + name = StringField() + + class Group(Document): + ref = ReferenceField(User, dbref=False) + generic = GenericReferenceField() + members = ListField(ReferenceField(User, dbref=False)) + + User.drop_collection() + Group.drop_collection() + + for i in xrange(1, 51): + User(name='user %s' % i).save() + + user = User.objects.first() + Group(ref=user, members=User.objects, generic=user).save() + + with no_dereference(Group) as NoDeRefGroup: + self.assertTrue(Group._fields['members']._auto_dereference) + self.assertFalse(NoDeRefGroup._fields['members']._auto_dereference) + + with no_dereference(Group) as Group: + group = Group.objects.first() + self.assertTrue(all([not isinstance(m, User) + for m in group.members])) + self.assertFalse(isinstance(group.ref, User)) + self.assertFalse(isinstance(group.generic, User)) + + self.assertTrue(all([isinstance(m, User) + for m in group.members])) + self.assertTrue(isinstance(group.ref, User)) + self.assertTrue(isinstance(group.generic, User)) + + def test_no_dereference_context_manager_dbref(self): + """Ensure that DBRef items in ListFields aren't dereferenced. + """ + class User(Document): + name = StringField() + + class Group(Document): + ref = ReferenceField(User, dbref=True) + generic = GenericReferenceField() + members = ListField(ReferenceField(User, dbref=True)) + + User.drop_collection() + Group.drop_collection() + + for i in xrange(1, 51): + User(name='user %s' % i).save() + + user = User.objects.first() + Group(ref=user, members=User.objects, generic=user).save() + + with no_dereference(Group) as NoDeRefGroup: + self.assertTrue(Group._fields['members']._auto_dereference) + self.assertFalse(NoDeRefGroup._fields['members']._auto_dereference) + + with no_dereference(Group) as Group: + group = Group.objects.first() + self.assertTrue(all([not isinstance(m, User) + for m in group.members])) + self.assertFalse(isinstance(group.ref, User)) + self.assertFalse(isinstance(group.generic, User)) + + self.assertTrue(all([isinstance(m, User) + for m in group.members])) + self.assertTrue(isinstance(group.ref, User)) + self.assertTrue(isinstance(group.generic, User)) + if __name__ == '__main__': unittest.main() From c8b65317effab9631b786e78f4592382a32c3658 Mon Sep 17 00:00:00 2001 From: Ross Lawley Date: Wed, 23 Jan 2013 20:15:05 +0000 Subject: [PATCH 081/189] Updated documentation instance tests --- tests/document/instance.py | 103 +++++++++++++++++++------------------ 1 file changed, 52 insertions(+), 51 deletions(-) diff --git a/tests/document/instance.py b/tests/document/instance.py index 3b5a4bd9..4c67046a 100644 --- a/tests/document/instance.py +++ b/tests/document/instance.py @@ -18,11 +18,12 @@ from mongoengine.errors import (NotRegistered, InvalidDocumentError, from mongoengine.queryset import NULLIFY, Q from mongoengine.connection import get_db from mongoengine.base import get_document +from mongoengine.context_managers import switch_db TEST_IMAGE_PATH = os.path.join(os.path.dirname(__file__), '../fields/mongoengine.png') -__all__ = ("InstanceTest",) +__all__ = ("InstanceTest", "ValidatorErrorTest") class InstanceTest(unittest.TestCase): @@ -1926,6 +1927,56 @@ class InstanceTest(unittest.TestCase): } )]), "1,2") + def test_switch_db_instance(self): + register_connection('testdb-1', 'mongoenginetest2') + + class Group(Document): + name = StringField() + + Group.drop_collection() + with switch_db(Group, 'testdb-1') as Group: + Group.drop_collection() + + Group(name="hello - default").save() + self.assertEqual(1, Group.objects.count()) + + group = Group.objects.first() + group.switch_db('testdb-1') + group.name = "hello - testdb!" + group.save() + + with switch_db(Group, 'testdb-1') as Group: + group = Group.objects.first() + self.assertEqual("hello - testdb!", group.name) + + group = Group.objects.first() + self.assertEqual("hello - default", group.name) + + # Slightly contrived now - perform an update + # Only works as they have the same object_id + group.switch_db('testdb-1') + group.update(set__name="hello - update") + + with switch_db(Group, 'testdb-1') as Group: + group = Group.objects.first() + self.assertEqual("hello - update", group.name) + Group.drop_collection() + self.assertEqual(0, Group.objects.count()) + + group = Group.objects.first() + self.assertEqual("hello - default", group.name) + + # Totally contrived now - perform a delete + # Only works as they have the same object_id + group.switch_db('testdb-1') + group.delete() + + with switch_db(Group, 'testdb-1') as Group: + self.assertEqual(0, Group.objects.count()) + + group = Group.objects.first() + self.assertEqual("hello - default", group.name) + class ValidatorErrorTest(unittest.TestCase): @@ -2114,56 +2165,6 @@ class ValidatorErrorTest(unittest.TestCase): self.assertEqual(classic_doc, dict_doc) self.assertEqual(classic_doc._data, dict_doc._data) - def test_switch_db_instance(self): - register_connection('testdb-1', 'mongoenginetest2') - - class Group(Document): - name = StringField() - - Group.drop_collection() - with SwitchDB(Group, 'testdb-1') as Group: - Group.drop_collection() - - Group(name="hello - default").save() - self.assertEqual(1, Group.objects.count()) - - group = Group.objects.first() - group.switch_db('testdb-1') - group.name = "hello - testdb!" - group.save() - - with SwitchDB(Group, 'testdb-1') as Group: - group = Group.objects.first() - self.assertEqual("hello - testdb!", group.name) - - group = Group.objects.first() - self.assertEqual("hello - default", group.name) - - # Slightly contrived now - perform an update - # Only works as they have the same object_id - group.switch_db('testdb-1') - group.update(set__name="hello - update") - - with SwitchDB(Group, 'testdb-1') as Group: - group = Group.objects.first() - self.assertEqual("hello - update", group.name) - Group.drop_collection() - self.assertEqual(0, Group.objects.count()) - - group = Group.objects.first() - self.assertEqual("hello - default", group.name) - - # Totally contrived now - perform a delete - # Only works as they have the same object_id - group.switch_db('testdb-1') - group.delete() - - with SwitchDB(Group, 'testdb-1') as Group: - self.assertEqual(0, Group.objects.count()) - - group = Group.objects.first() - self.assertEqual("hello - default", group.name) - if __name__ == '__main__': unittest.main() From 9797d7a7fb9c21684f360d03b06800b99b8093c4 Mon Sep 17 00:00:00 2001 From: Ross Lawley Date: Wed, 23 Jan 2013 21:19:21 +0000 Subject: [PATCH 082/189] Added switch_collection context manager and method (#220) --- mongoengine/context_managers.py | 45 +++++++++- mongoengine/document.py | 27 +++++- tests/test_connection.py | 24 ----- tests/test_context_managers.py | 153 ++++++++++++++++++++++++++++++++ tests/test_dereference.py | 71 --------------- 5 files changed, 223 insertions(+), 97 deletions(-) create mode 100644 tests/test_context_managers.py diff --git a/mongoengine/context_managers.py b/mongoengine/context_managers.py index 7255d51c..e73d4a2a 100644 --- a/mongoengine/context_managers.py +++ b/mongoengine/context_managers.py @@ -2,7 +2,7 @@ from mongoengine.common import _import_class from mongoengine.connection import DEFAULT_CONNECTION_NAME, get_db from mongoengine.queryset import OperationError, QuerySet -__all__ = ("switch_db", "no_dereference", "query_counter") +__all__ = ("switch_db", "switch_collection", "no_dereference", "query_counter") class switch_db(object): @@ -47,6 +47,49 @@ class switch_db(object): self.cls._collection = self.collection +class switch_collection(object): + """ switch_collection alias context manager. + + Example :: + + class Group(Document): + name = StringField() + + Group(name="test").save() # Saves in the default db + + with switch_collection(Group, 'group1') as Group: + Group(name="hello testdb!").save() # Saves in group1 collection + + """ + + def __init__(self, cls, collection_name): + """ Construct the switch_collection context manager + + :param cls: the class to change the registered db + :param collection_name: the name of the collection to use + """ + self.cls = cls + self.ori_collection = cls._get_collection() + self.ori_get_collection_name = cls._get_collection_name + self.collection_name = collection_name + + def __enter__(self): + """ change the _get_collection_name and clear the cached collection """ + + @classmethod + def _get_collection_name(cls): + return self.collection_name + + self.cls._get_collection_name = _get_collection_name + self.cls._collection = None + return self.cls + + def __exit__(self, t, value, traceback): + """ Reset the collection """ + self.cls._collection = self.ori_collection + self.cls._get_collection_name = self.ori_get_collection_name + + class no_dereference(object): """ no_dereference context manager. diff --git a/mongoengine/document.py b/mongoengine/document.py index 9d4a1e6e..75873b4b 100644 --- a/mongoengine/document.py +++ b/mongoengine/document.py @@ -11,7 +11,7 @@ from mongoengine.base import (DocumentMetaclass, TopLevelDocumentMetaclass, ALLOW_INHERITANCE, get_document) from mongoengine.queryset import OperationError, NotUniqueError from mongoengine.connection import get_db, DEFAULT_CONNECTION_NAME -from mongoengine.context_managers import switch_db +from mongoengine.context_managers import switch_db, switch_collection __all__ = ('Document', 'EmbeddedDocument', 'DynamicDocument', 'DynamicEmbeddedDocument', 'OperationError', @@ -398,6 +398,31 @@ class Document(BaseDocument): self._objects._collection_obj = collection return self + def switch_collection(self, collection_name): + """ + Temporarily switch the collection for a document instance. + + Only really useful for archiving off data and calling `save()`:: + + user = User.objects.get(id=user_id) + user.switch_collection('old-users') + user.save() + + If you need to read from another database see + :class:`~mongoengine.context_managers.switch_collection` + + :param collection_name: The database alias to use for saving the + document + """ + with switch_collection(self.__class__, collection_name) as cls: + collection = cls._get_collection() + self._get_collection = lambda: collection + self._collection = collection + self._created = True + self._objects = self.__class__.objects + self._objects._collection_obj = collection + return self + def select_related(self, max_depth=1): """Handles dereferencing of :class:`~bson.dbref.DBRef` objects to a maximum depth in order to cut down the number queries to mongodb. diff --git a/tests/test_connection.py b/tests/test_connection.py index 2a216fef..c32d231f 100644 --- a/tests/test_connection.py +++ b/tests/test_connection.py @@ -95,30 +95,6 @@ class ConnectionTest(unittest.TestCase): date_doc = DateDoc.objects.first() self.assertEqual(d, date_doc.the_date) - def test_switch_db_context_manager(self): - connect('mongoenginetest') - register_connection('testdb-1', 'mongoenginetest2') - - class Group(Document): - name = StringField() - - Group.drop_collection() - - Group(name="hello - default").save() - self.assertEqual(1, Group.objects.count()) - - with switch_db(Group, 'testdb-1') as Group: - - self.assertEqual(0, Group.objects.count()) - - Group(name="hello").save() - - self.assertEqual(1, Group.objects.count()) - - Group.drop_collection() - self.assertEqual(0, Group.objects.count()) - - self.assertEqual(1, Group.objects.count()) if __name__ == '__main__': unittest.main() diff --git a/tests/test_context_managers.py b/tests/test_context_managers.py new file mode 100644 index 00000000..10fe7b8e --- /dev/null +++ b/tests/test_context_managers.py @@ -0,0 +1,153 @@ +from __future__ import with_statement +import unittest + +from mongoengine import * +from mongoengine.connection import get_db +from mongoengine.context_managers import (switch_db, switch_collection, + no_dereference, query_counter) + + +class ContextManagersTest(unittest.TestCase): + + def test_switch_db_context_manager(self): + connect('mongoenginetest') + register_connection('testdb-1', 'mongoenginetest2') + + class Group(Document): + name = StringField() + + Group.drop_collection() + + Group(name="hello - default").save() + self.assertEqual(1, Group.objects.count()) + + with switch_db(Group, 'testdb-1') as Group: + + self.assertEqual(0, Group.objects.count()) + + Group(name="hello").save() + + self.assertEqual(1, Group.objects.count()) + + Group.drop_collection() + self.assertEqual(0, Group.objects.count()) + + self.assertEqual(1, Group.objects.count()) + + def test_switch_collection_context_manager(self): + connect('mongoenginetest') + register_connection('testdb-1', 'mongoenginetest2') + + class Group(Document): + name = StringField() + + Group.drop_collection() + with switch_collection(Group, 'group1') as Group: + Group.drop_collection() + + Group(name="hello - group").save() + self.assertEqual(1, Group.objects.count()) + + with switch_collection(Group, 'group1') as Group: + + self.assertEqual(0, Group.objects.count()) + + Group(name="hello - group1").save() + + self.assertEqual(1, Group.objects.count()) + + Group.drop_collection() + self.assertEqual(0, Group.objects.count()) + + self.assertEqual(1, Group.objects.count()) + + def test_no_dereference_context_manager_object_id(self): + """Ensure that DBRef items in ListFields aren't dereferenced. + """ + connect('mongoenginetest') + + class User(Document): + name = StringField() + + class Group(Document): + ref = ReferenceField(User, dbref=False) + generic = GenericReferenceField() + members = ListField(ReferenceField(User, dbref=False)) + + User.drop_collection() + Group.drop_collection() + + for i in xrange(1, 51): + User(name='user %s' % i).save() + + user = User.objects.first() + Group(ref=user, members=User.objects, generic=user).save() + + with no_dereference(Group) as NoDeRefGroup: + self.assertTrue(Group._fields['members']._auto_dereference) + self.assertFalse(NoDeRefGroup._fields['members']._auto_dereference) + + with no_dereference(Group) as Group: + group = Group.objects.first() + self.assertTrue(all([not isinstance(m, User) + for m in group.members])) + self.assertFalse(isinstance(group.ref, User)) + self.assertFalse(isinstance(group.generic, User)) + + self.assertTrue(all([isinstance(m, User) + for m in group.members])) + self.assertTrue(isinstance(group.ref, User)) + self.assertTrue(isinstance(group.generic, User)) + + def test_no_dereference_context_manager_dbref(self): + """Ensure that DBRef items in ListFields aren't dereferenced. + """ + connect('mongoenginetest') + + class User(Document): + name = StringField() + + class Group(Document): + ref = ReferenceField(User, dbref=True) + generic = GenericReferenceField() + members = ListField(ReferenceField(User, dbref=True)) + + User.drop_collection() + Group.drop_collection() + + for i in xrange(1, 51): + User(name='user %s' % i).save() + + user = User.objects.first() + Group(ref=user, members=User.objects, generic=user).save() + + with no_dereference(Group) as NoDeRefGroup: + self.assertTrue(Group._fields['members']._auto_dereference) + self.assertFalse(NoDeRefGroup._fields['members']._auto_dereference) + + with no_dereference(Group) as Group: + group = Group.objects.first() + self.assertTrue(all([not isinstance(m, User) + for m in group.members])) + self.assertFalse(isinstance(group.ref, User)) + self.assertFalse(isinstance(group.generic, User)) + + self.assertTrue(all([isinstance(m, User) + for m in group.members])) + self.assertTrue(isinstance(group.ref, User)) + self.assertTrue(isinstance(group.generic, User)) + + def test_query_counter(self): + connect('mongoenginetest') + db = get_db() + + with query_counter() as q: + self.assertEqual(0, q) + + for i in xrange(1, 51): + db.test.find({}).count() + + self.assertEqual(50, q) + +if __name__ == '__main__': + unittest.main() diff --git a/tests/test_dereference.py b/tests/test_dereference.py index 8e4ffdd8..adbc5192 100644 --- a/tests/test_dereference.py +++ b/tests/test_dereference.py @@ -1121,77 +1121,6 @@ class FieldTest(unittest.TestCase): self.assertEqual(q, 2) - def test_no_dereference_context_manager_object_id(self): - """Ensure that DBRef items in ListFields aren't dereferenced. - """ - class User(Document): - name = StringField() - - class Group(Document): - ref = ReferenceField(User, dbref=False) - generic = GenericReferenceField() - members = ListField(ReferenceField(User, dbref=False)) - - User.drop_collection() - Group.drop_collection() - - for i in xrange(1, 51): - User(name='user %s' % i).save() - - user = User.objects.first() - Group(ref=user, members=User.objects, generic=user).save() - - with no_dereference(Group) as NoDeRefGroup: - self.assertTrue(Group._fields['members']._auto_dereference) - self.assertFalse(NoDeRefGroup._fields['members']._auto_dereference) - - with no_dereference(Group) as Group: - group = Group.objects.first() - self.assertTrue(all([not isinstance(m, User) - for m in group.members])) - self.assertFalse(isinstance(group.ref, User)) - self.assertFalse(isinstance(group.generic, User)) - - self.assertTrue(all([isinstance(m, User) - for m in group.members])) - self.assertTrue(isinstance(group.ref, User)) - self.assertTrue(isinstance(group.generic, User)) - - def test_no_dereference_context_manager_dbref(self): - """Ensure that DBRef items in ListFields aren't dereferenced. - """ - class User(Document): - name = StringField() - - class Group(Document): - ref = ReferenceField(User, dbref=True) - generic = GenericReferenceField() - members = ListField(ReferenceField(User, dbref=True)) - - User.drop_collection() - Group.drop_collection() - - for i in xrange(1, 51): - User(name='user %s' % i).save() - - user = User.objects.first() - Group(ref=user, members=User.objects, generic=user).save() - - with no_dereference(Group) as NoDeRefGroup: - self.assertTrue(Group._fields['members']._auto_dereference) - self.assertFalse(NoDeRefGroup._fields['members']._auto_dereference) - - with no_dereference(Group) as Group: - group = Group.objects.first() - self.assertTrue(all([not isinstance(m, User) - for m in group.members])) - self.assertFalse(isinstance(group.ref, User)) - self.assertFalse(isinstance(group.generic, User)) - - self.assertTrue(all([isinstance(m, User) - for m in group.members])) - self.assertTrue(isinstance(group.ref, User)) - self.assertTrue(isinstance(group.generic, User)) if __name__ == '__main__': unittest.main() From d58f594c173fa57bb0b16e77c18724ea63dbf536 Mon Sep 17 00:00:00 2001 From: Ross Lawley Date: Wed, 23 Jan 2013 21:21:46 +0000 Subject: [PATCH 083/189] Updated changelog --- docs/changelog.rst | 2 ++ 1 file changed, 2 insertions(+) diff --git a/docs/changelog.rst b/docs/changelog.rst index bead6935..53dbeb9b 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -35,6 +35,8 @@ Changes in 0.8.X - Added switch_db context manager (#106) - Added switch_db method to document instances (#106) - Added no_dereference context manager (#82) +- Added switch_collection context manager (#220) +- Added switch_collection method to document instances (#220) Changes in 0.7.9 ================ From fff27f9b8744a747eb4abd85061fb60bbff5071e Mon Sep 17 00:00:00 2001 From: Ross Lawley Date: Thu, 24 Jan 2013 10:37:54 +0000 Subject: [PATCH 084/189] Added support for compound primary keys (#149) --- AUTHORS | 3 ++- docs/changelog.rst | 1 + docs/django.rst | 4 ++-- docs/guide/defining-documents.rst | 22 ++++++++++++-------- tests/document/indexes.py | 34 +++++++++++++++++++++++++++++++ 5 files changed, 52 insertions(+), 12 deletions(-) diff --git a/AUTHORS b/AUTHORS index aa7f833d..3f80dcae 100644 --- a/AUTHORS +++ b/AUTHORS @@ -135,4 +135,5 @@ that much better: * Marcelo Anton * Aleksey Porfirov * Nicolas Trippar - * Manuel Hermann \ No newline at end of file + * Manuel Hermann + * Gustavo Gawryszewski \ No newline at end of file diff --git a/docs/changelog.rst b/docs/changelog.rst index 53dbeb9b..7486ae9e 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -37,6 +37,7 @@ Changes in 0.8.X - Added no_dereference context manager (#82) - Added switch_collection context manager (#220) - Added switch_collection method to document instances (#220) +- Added support for compound primary keys (#149) Changes in 0.7.9 ================ diff --git a/docs/django.rst b/docs/django.rst index a4f05602..ba934324 100644 --- a/docs/django.rst +++ b/docs/django.rst @@ -2,7 +2,7 @@ Using MongoEngine with Django ============================= -.. note :: Updated to support Django 1.4 +.. note:: Updated to support Django 1.4 Connecting ========== @@ -10,7 +10,7 @@ In your **settings.py** file, ignore the standard database settings (unless you also plan to use the ORM in your project), and instead call :func:`~mongoengine.connect` somewhere in the settings module. -.. note :: If getting an ``ImproperlyConfigured: settings.DATABASES is +.. note:: If getting an ``ImproperlyConfigured: settings.DATABASES is improperly configured`` error you may need to remove ``django.contrib.sites`` from ``INSTALLED_APPS`` in settings.py. diff --git a/docs/guide/defining-documents.rst b/docs/guide/defining-documents.rst index 9abea9ba..c6982850 100644 --- a/docs/guide/defining-documents.rst +++ b/docs/guide/defining-documents.rst @@ -135,7 +135,8 @@ arguments can be set on all fields: field, will not have two documents in the collection with the same value. :attr:`primary_key` (Default: False) - When True, use this field as a primary key for the collection. + When True, use this field as a primary key for the collection. `DictField` + and `EmbeddedDocuments` both support being the primary key for a document. :attr:`choices` (Default: None) An iterable (e.g. a list or tuple) of choices to which the value of this @@ -441,6 +442,7 @@ The following example shows a :class:`Log` document that will be limited to Indexes ======= + You can specify indexes on collections to make querying faster. This is done by creating a list of index specifications called :attr:`indexes` in the :attr:`~mongoengine.Document.meta` dictionary, where an index specification may @@ -473,20 +475,22 @@ If a dictionary is passed then the following options are available: :attr:`unique` (Default: False) Whether the index should be unique. -.. note :: +.. note:: - To index embedded files / dictionary fields use 'dot' notation eg: - `rank.title` + Inheritance adds extra fields indices see: :ref:`document-inheritance`. -.. warning:: +Compound Indexes and Indexing sub documents +------------------------------------------- - Inheritance adds extra indices. - If don't need inheritance for a document turn inheritance off - - see :ref:`document-inheritance`. +Compound indexes can be created by adding the Embedded field or dictionary +field name to the index definition. +Sometimes its more efficient to index parts of Embeedded / dictionary fields, +in this case use 'dot' notation to identify the value to index eg: `rank.title` Geospatial indexes ---------------------------- +------------------ + Geospatial indexes will be automatically created for all :class:`~mongoengine.GeoPointField`\ s diff --git a/tests/document/indexes.py b/tests/document/indexes.py index fb278aa7..c059590b 100644 --- a/tests/document/indexes.py +++ b/tests/document/indexes.py @@ -693,5 +693,39 @@ class IndexesTest(unittest.TestCase): index_item = [('_id', 1), ('comments.comment_id', 1)] self.assertTrue(index_item in info) + def test_compound_key_embedded(self): + + class CompoundKey(EmbeddedDocument): + name = StringField(required=True) + term = StringField(required=True) + + class Report(Document): + key = EmbeddedDocumentField(CompoundKey, primary_key=True) + text = StringField() + + Report.drop_collection() + + my_key = CompoundKey(name="n", term="ok") + report = Report(text="OK", key=my_key).save() + + self.assertEqual({'text': 'OK', '_id': {'term': 'ok', 'name': 'n'}}, + report.to_mongo()) + self.assertEqual(report, Report.objects.get(pk=my_key)) + + def test_compound_key_dictfield(self): + + class Report(Document): + key = DictField(primary_key=True) + text = StringField() + + Report.drop_collection() + + my_key = {"name": "n", "term": "ok"} + report = Report(text="OK", key=my_key).save() + + self.assertEqual({'text': 'OK', '_id': {'term': 'ok', 'name': 'n'}}, + report.to_mongo()) + self.assertEqual(report, Report.objects.get(pk=my_key)) + if __name__ == '__main__': unittest.main() From e7ba5eb160e8f05385d8849eef031984ad916b04 Mon Sep 17 00:00:00 2001 From: Ross Lawley Date: Thu, 24 Jan 2013 10:41:01 +0000 Subject: [PATCH 085/189] Added #121 to changelog --- docs/changelog.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/changelog.rst b/docs/changelog.rst index 7486ae9e..219e935a 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -37,7 +37,7 @@ Changes in 0.8.X - Added no_dereference context manager (#82) - Added switch_collection context manager (#220) - Added switch_collection method to document instances (#220) -- Added support for compound primary keys (#149) +- Added support for compound primary keys (#149) (#121) Changes in 0.7.9 ================ From e38bf63be0c51c6243010f255bd45e9e2a8ddcec Mon Sep 17 00:00:00 2001 From: Ross Lawley Date: Thu, 24 Jan 2013 11:29:51 +0000 Subject: [PATCH 086/189] Fixed overriding objects with custom manager (#58) --- AUTHORS | 3 ++- docs/changelog.rst | 3 ++- mongoengine/document.py | 17 ++++++++--------- tests/queryset/queryset.py | 26 ++++++++++++++++++++++++++ 4 files changed, 38 insertions(+), 11 deletions(-) diff --git a/AUTHORS b/AUTHORS index 3f80dcae..c32ab9f9 100644 --- a/AUTHORS +++ b/AUTHORS @@ -136,4 +136,5 @@ that much better: * Aleksey Porfirov * Nicolas Trippar * Manuel Hermann - * Gustavo Gawryszewski \ No newline at end of file + * Gustavo Gawryszewski + * Max Countryman \ No newline at end of file diff --git a/docs/changelog.rst b/docs/changelog.rst index 219e935a..0d164b51 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -34,10 +34,11 @@ Changes in 0.8.X - Fixed validation for GenericReferences which havent been dereferenced - Added switch_db context manager (#106) - Added switch_db method to document instances (#106) -- Added no_dereference context manager (#82) +- Added no_dereference context manager (#82) (#61) - Added switch_collection context manager (#220) - Added switch_collection method to document instances (#220) - Added support for compound primary keys (#149) (#121) +- Fixed overriding objects with custom manager (#58) Changes in 0.7.9 ================ diff --git a/mongoengine/document.py b/mongoengine/document.py index 75873b4b..edc819cb 100644 --- a/mongoengine/document.py +++ b/mongoengine/document.py @@ -9,7 +9,7 @@ from mongoengine import signals from mongoengine.base import (DocumentMetaclass, TopLevelDocumentMetaclass, BaseDocument, BaseDict, BaseList, ALLOW_INHERITANCE, get_document) -from mongoengine.queryset import OperationError, NotUniqueError +from mongoengine.queryset import OperationError, NotUniqueError, QuerySet from mongoengine.connection import get_db, DEFAULT_CONNECTION_NAME from mongoengine.context_managers import switch_db, switch_collection @@ -328,10 +328,9 @@ class Document(BaseDocument): """ Returns the queryset to use for updating / reloading / deletions """ - qs = self.__class__.objects - if hasattr(self, '_objects'): - qs = self._objects - return qs + if not hasattr(self, '__objects'): + self.__objects = QuerySet(self, self._get_collection()) + return self.__objects @property def _object_key(self): @@ -394,8 +393,8 @@ class Document(BaseDocument): self._get_db = lambda: db self._collection = collection self._created = True - self._objects = self.__class__.objects - self._objects._collection_obj = collection + self.__objects = self._qs + self.__objects._collection_obj = collection return self def switch_collection(self, collection_name): @@ -419,8 +418,8 @@ class Document(BaseDocument): self._get_collection = lambda: collection self._collection = collection self._created = True - self._objects = self.__class__.objects - self._objects._collection_obj = collection + self.__objects = self._qs + self.__objects._collection_obj = collection return self def select_related(self, max_depth=1): diff --git a/tests/queryset/queryset.py b/tests/queryset/queryset.py index 35940447..0ad30927 100644 --- a/tests/queryset/queryset.py +++ b/tests/queryset/queryset.py @@ -2074,6 +2074,32 @@ class QuerySetTest(unittest.TestCase): BlogPost.drop_collection() + def test_custom_manager_overriding_objects_works(self): + + class Foo(Document): + bar = StringField(default='bar') + active = BooleanField(default=False) + + @queryset_manager + def objects(doc_cls, queryset): + return queryset(active=True) + + @queryset_manager + def with_inactive(doc_cls, queryset): + return queryset(active=False) + + Foo.drop_collection() + + Foo(active=True).save() + Foo(active=False).save() + + self.assertEqual(1, Foo.objects.count()) + self.assertEqual(1, Foo.with_inactive.count()) + + Foo.with_inactive.first().delete() + self.assertEqual(0, Foo.with_inactive.count()) + self.assertEqual(1, Foo.objects.count()) + def test_query_value_conversion(self): """Ensure that query values are properly converted when necessary. From eefbd3f5974e6fd8eb3810d01de168a2a280b0da Mon Sep 17 00:00:00 2001 From: Ross Lawley Date: Thu, 24 Jan 2013 12:52:16 +0000 Subject: [PATCH 087/189] Updated wobbly python 3.3 test --- tests/document/instance.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/tests/document/instance.py b/tests/document/instance.py index 4c67046a..247f6275 100644 --- a/tests/document/instance.py +++ b/tests/document/instance.py @@ -2026,8 +2026,6 @@ class ValidatorErrorTest(unittest.TestCase): try: User().validate() except ValidationError, e: - expected_error_message = """ValidationError(Field is required: ['username', 'name'])""" - self.assertEqual(e.message, expected_error_message) self.assertEqual(e.to_dict(), { 'username': 'Field is required', 'name': 'Field is required'}) From ed2ea24b75ffe70258882d1cf53a86e90b6ec1e4 Mon Sep 17 00:00:00 2001 From: Ross Lawley Date: Thu, 24 Jan 2013 13:10:51 +0000 Subject: [PATCH 088/189] More test edge case fixing --- tests/test_context_managers.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/test_context_managers.py b/tests/test_context_managers.py index 10fe7b8e..c9efe8b8 100644 --- a/tests/test_context_managers.py +++ b/tests/test_context_managers.py @@ -140,6 +140,7 @@ class ContextManagersTest(unittest.TestCase): def test_query_counter(self): connect('mongoenginetest') db = get_db() + db.test.find({}) with query_counter() as q: self.assertEqual(0, q) From ba48dfb4bf283c5b3f20d5a9f47d69ec09e6f2f7 Mon Sep 17 00:00:00 2001 From: Ross Lawley Date: Thu, 24 Jan 2013 17:33:10 +0000 Subject: [PATCH 089/189] Added no_dereference method for querysets (#82) (#61) --- docs/changelog.rst | 1 + docs/guide/querying.rst | 13 +++++++++---- mongoengine/base/document.py | 13 +++++++++++-- mongoengine/base/fields.py | 1 + mongoengine/context_managers.py | 14 +++----------- mongoengine/fields.py | 5 +++-- mongoengine/queryset/queryset.py | 15 ++++++++++++--- tests/queryset/queryset.py | 21 +++++++++++++++++++++ 8 files changed, 61 insertions(+), 22 deletions(-) diff --git a/docs/changelog.rst b/docs/changelog.rst index 0d164b51..e24eaf4f 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -39,6 +39,7 @@ Changes in 0.8.X - Added switch_collection method to document instances (#220) - Added support for compound primary keys (#149) (#121) - Fixed overriding objects with custom manager (#58) +- Added no_dereference method for querysets (#82) (#61) Changes in 0.7.9 ================ diff --git a/docs/guide/querying.rst b/docs/guide/querying.rst index 7ccf1432..32798531 100644 --- a/docs/guide/querying.rst +++ b/docs/guide/querying.rst @@ -373,17 +373,22 @@ Turning off dereferencing ------------------------- Sometimes for performance reasons you don't want to automatically dereference -data . To turn off all dereferencing you can use the +data. To turn off dereferencing of the results of a query use +:func:`~mongoengine.queryset.QuerySet.no_dereference` on the queryset like so:: + + post = Post.objects.no_dereference().first() + assert(isinstance(post.author, ObjectId)) + +You can also turn off all dereferencing for a fixed period by using the :class:`~mongoengine.context_managers.no_dereference` context manager:: with no_dereference(Post) as Post: post = Post.objects.first() assert(isinstance(post.author, ObjectId)) -.. note:: + # Outside the context manager dereferencing occurs. + assert(isinstance(post.author, User)) - :class:`~mongoengine.context_managers.no_dereference` only works on the - Default QuerySet manager. Advanced queries ================ diff --git a/mongoengine/base/document.py b/mongoengine/base/document.py index 9f400618..7c1597e2 100644 --- a/mongoengine/base/document.py +++ b/mongoengine/base/document.py @@ -1,3 +1,4 @@ +import copy import operator from functools import partial @@ -461,9 +462,10 @@ class BaseDocument(object): return cls._meta.get('collection', None) @classmethod - def _from_son(cls, son): + def _from_son(cls, son, _auto_dereference=True): """Create an instance of a Document (subclass) from a PyMongo SON. """ + # get the class name from the document, falling back to the given # class if unavailable class_name = son.get('_cls', cls._class_name) @@ -480,7 +482,12 @@ class BaseDocument(object): changed_fields = [] errors_dict = {} - for field_name, field in cls._fields.iteritems(): + fields = cls._fields + if not _auto_dereference: + fields = copy.copy(fields) + + for field_name, field in fields.iteritems(): + field._auto_dereference = _auto_dereference if field.db_field in data: value = data[field.db_field] try: @@ -507,6 +514,8 @@ class BaseDocument(object): obj = cls(__auto_convert=False, **data) obj._changed_fields = changed_fields obj._created = False + if not _auto_dereference: + obj._fields = fields return obj @classmethod diff --git a/mongoengine/base/fields.py b/mongoengine/base/fields.py index 82981e25..25f86afd 100644 --- a/mongoengine/base/fields.py +++ b/mongoengine/base/fields.py @@ -168,6 +168,7 @@ class ComplexBaseField(BaseField): (self.field is None or isinstance(self.field, (GenericReferenceField, ReferenceField)))) + self._auto_dereference = instance._fields[self.name]._auto_dereference if not self.__dereference and instance._initialised and dereference: instance._data[self.name] = self._dereference( instance._data.get(self.name), max_depth=1, instance=instance, diff --git a/mongoengine/context_managers.py b/mongoengine/context_managers.py index e73d4a2a..76d5fbfa 100644 --- a/mongoengine/context_managers.py +++ b/mongoengine/context_managers.py @@ -93,7 +93,8 @@ class switch_collection(object): class no_dereference(object): """ no_dereference context manager. - Turns off all dereferencing in Documents:: + Turns off all dereferencing in Documents for the duration of the context + manager:: with no_dereference(Group) as Group: Group.objects.find() @@ -118,21 +119,12 @@ class no_dereference(object): def __enter__(self): """ change the objects default and _auto_dereference values""" - if 'queryset_class' in self.cls._meta: - raise OperationError("no_dereference context manager only works on" - " default queryset classes") - objects = self.cls.__dict__['objects'] - objects.default = QuerySetNoDeRef - self.cls.objects = objects for field in self.deref_fields: self.cls._fields[field]._auto_dereference = False return self.cls def __exit__(self, t, value, traceback): """ Reset the default and _auto_dereference values""" - objects = self.cls.__dict__['objects'] - objects.default = QuerySet - self.cls.objects = objects for field in self.deref_fields: self.cls._fields[field]._auto_dereference = True return self.cls @@ -145,7 +137,7 @@ class QuerySetNoDeRef(QuerySet): class query_counter(object): - """ Query_counter contextmanager to get the number of queries. """ + """ Query_counter context manager to get the number of queries. """ def __init__(self): """ Construct the query_counter. """ diff --git a/mongoengine/fields.py b/mongoengine/fields.py index 1ccdb650..11e9d3f3 100644 --- a/mongoengine/fields.py +++ b/mongoengine/fields.py @@ -777,7 +777,7 @@ class ReferenceField(BaseField): # Get value from document instance if available value = instance._data.get(self.name) - + self._auto_dereference = instance._fields[self.name]._auto_dereference # Dereference DBRefs if self._auto_dereference and isinstance(value, DBRef): value = self.document_type._get_db().dereference(value) @@ -859,7 +859,8 @@ class GenericReferenceField(BaseField): return self value = instance._data.get(self.name) - if isinstance(value, (dict, SON)): + self._auto_dereference = instance._fields[self.name]._auto_dereference + if self._auto_dereference and isinstance(value, (dict, SON)): instance._data[self.name] = self.dereference(value) return super(GenericReferenceField, self).__get__(instance, owner) diff --git a/mongoengine/queryset/queryset.py b/mongoengine/queryset/queryset.py index a9ff6e73..f73b0e7c 100644 --- a/mongoengine/queryset/queryset.py +++ b/mongoengine/queryset/queryset.py @@ -42,6 +42,7 @@ class QuerySet(object): providing :class:`~mongoengine.Document` objects as the results. """ __dereference = False + _auto_dereference = True def __init__(self, document, collection): self._document = document @@ -145,10 +146,12 @@ class QuerySet(object): elif isinstance(key, int): if queryset._scalar: return queryset._get_scalar( - queryset._document._from_son(queryset._cursor[key])) + queryset._document._from_son(queryset._cursor[key], + _auto_dereference=self._auto_dereference)) if queryset._as_pymongo: return queryset._get_as_pymongo(queryset._cursor.next()) - return queryset._document._from_son(queryset._cursor[key]) + return queryset._document._from_son(queryset._cursor[key], + _auto_dereference=self._auto_dereference) raise AttributeError def __repr__(self): @@ -515,7 +518,7 @@ class QuerySet(object): '_where_clause', '_loaded_fields', '_ordering', '_snapshot', '_timeout', '_class_check', '_slave_okay', '_read_preference', '_iter', '_scalar', '_as_pymongo', '_as_pymongo_coerce', - '_limit', '_skip', '_hint') + '_limit', '_skip', '_hint', '_auto_dereference') for prop in copy_props: val = getattr(self, prop) @@ -1135,6 +1138,12 @@ class QuerySet(object): self.__dereference = _import_class('DeReference')() return self.__dereference + def no_dereference(self): + """Turn off any dereferencing.""" + queryset = self.clone() + queryset._auto_dereference = False + return queryset + # Helper Functions def _item_frequencies_map_reduce(self, field, normalize=False): diff --git a/tests/queryset/queryset.py b/tests/queryset/queryset.py index 0ad30927..c6b7c0e0 100644 --- a/tests/queryset/queryset.py +++ b/tests/queryset/queryset.py @@ -3103,5 +3103,26 @@ class QuerySetTest(unittest.TestCase): self.assertEqual(results[1]['name'], 'Barack Obama') self.assertEqual(results[1]['price'], Decimal('2.22')) + def test_no_dereference(self): + + class Organization(Document): + name = StringField() + + class User(Document): + name = StringField() + organization = ReferenceField(Organization) + + User.drop_collection() + Organization.drop_collection() + + whitehouse = Organization(name="White House").save() + User(name="Bob Dole", organization=whitehouse).save() + + qs = User.objects() + self.assertTrue(isinstance(qs.first().organization, Organization)) + self.assertFalse(isinstance(qs.no_dereference().first().organization, + Organization)) + self.assertTrue(isinstance(qs.first().organization, Organization)) + if __name__ == '__main__': unittest.main() From 9f551121fbbe852bd639ce4999ad0835db3fd8a6 Mon Sep 17 00:00:00 2001 From: Ross Lawley Date: Thu, 24 Jan 2013 17:41:21 +0000 Subject: [PATCH 090/189] Added docs for no_dereference and scalar (#68) --- mongoengine/queryset/queryset.py | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/mongoengine/queryset/queryset.py b/mongoengine/queryset/queryset.py index f73b0e7c..b5e33515 100644 --- a/mongoengine/queryset/queryset.py +++ b/mongoengine/queryset/queryset.py @@ -748,8 +748,12 @@ class QuerySet(object): """Instead of returning Document instances, return either a specific value or a tuple of values in order. - This effects all results and can be unset by calling ``scalar`` - without arguments. Calls ``only`` automatically. + Can be used along with + :func:`~mongoengine.queryset.QuerySet.no_dereference` to turn off + dereferencing. + + .. note:: This effects all results and can be unset by calling + ``scalar`` without arguments. Calls ``only`` automatically. :param fields: One or more fields to return instead of a Document. """ @@ -1139,7 +1143,8 @@ class QuerySet(object): return self.__dereference def no_dereference(self): - """Turn off any dereferencing.""" + """Turn off any dereferencing for the results of this queryset. + """ queryset = self.clone() queryset._auto_dereference = False return queryset From 83da08ef7dda6b40f0e288e98b6560107279e3c7 Mon Sep 17 00:00:00 2001 From: Ross Lawley Date: Thu, 24 Jan 2013 17:43:57 +0000 Subject: [PATCH 091/189] Documentation fixes --- docs/guide/defining-documents.rst | 2 +- docs/tutorial.rst | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/guide/defining-documents.rst b/docs/guide/defining-documents.rst index c6982850..3fdb9a66 100644 --- a/docs/guide/defining-documents.rst +++ b/docs/guide/defining-documents.rst @@ -47,7 +47,7 @@ be saved :: >>> Page.objects(tags='mongoengine').count() >>> 1 -..note:: +.. note:: There is one caveat on Dynamic Documents: fields cannot start with `_` diff --git a/docs/tutorial.rst b/docs/tutorial.rst index c2fb5b91..c4b69c49 100644 --- a/docs/tutorial.rst +++ b/docs/tutorial.rst @@ -170,7 +170,7 @@ To delete all the posts if a user is deleted set the rule:: See :class:`~mongoengine.ReferenceField` for more information. -..note:: +.. note:: MapFields and DictFields currently don't support automatic handling of deleted references From 621b2b3f72e142356e06e06845faea8caa8e9279 Mon Sep 17 00:00:00 2001 From: Ross Lawley Date: Fri, 25 Jan 2013 11:28:20 +0000 Subject: [PATCH 092/189] Undefined data should not override instance methods (#49) --- docs/changelog.rst | 1 + docs/guide/defining-documents.rst | 10 +- mongoengine/base/document.py | 16 ++- tests/document/__init__.py | 1 + tests/document/instance.py | 193 +++-------------------------- tests/document/validation.py | 195 ++++++++++++++++++++++++++++++ 6 files changed, 230 insertions(+), 186 deletions(-) create mode 100644 tests/document/validation.py diff --git a/docs/changelog.rst b/docs/changelog.rst index e24eaf4f..5bdd0f85 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -40,6 +40,7 @@ Changes in 0.8.X - Added support for compound primary keys (#149) (#121) - Fixed overriding objects with custom manager (#58) - Added no_dereference method for querysets (#82) (#61) +- Undefined data should not override instance methods (#49) Changes in 0.7.9 ================ diff --git a/docs/guide/defining-documents.rst b/docs/guide/defining-documents.rst index 3fdb9a66..350ba678 100644 --- a/docs/guide/defining-documents.rst +++ b/docs/guide/defining-documents.rst @@ -600,8 +600,7 @@ Working with existing data -------------------------- As MongoEngine no longer defaults to needing :attr:`_cls` you can quickly and easily get working with existing data. Just define the document to match -the expected schema in your database. If you have wildly varying schemas then -a :class:`~mongoengine.DynamicDocument` might be more appropriate. :: +the expected schema in your database :: # Will work with data in an existing collection named 'cmsPage' class Page(Document): @@ -609,3 +608,10 @@ a :class:`~mongoengine.DynamicDocument` might be more appropriate. :: meta = { 'collection': 'cmsPage' } + +If you have wildly varying schemas then using a +:class:`~mongoengine.DynamicDocument` might be more appropriate, instead of +defining all possible field types. + +If you use :class:`~mongoengine.Document` and the database contains data that +isn't defined then that data will be stored in the `document._data` dictionary. diff --git a/mongoengine/base/document.py b/mongoengine/base/document.py index 7c1597e2..a88a38bf 100644 --- a/mongoengine/base/document.py +++ b/mongoengine/base/document.py @@ -60,13 +60,17 @@ class BaseDocument(object): else: FileField = _import_class('FileField') for key, value in values.iteritems(): + if key == '__auto_convert': + continue key = self._reverse_db_field_map.get(key, key) - if (value is not None and __auto_convert and - key in self._fields): - field = self._fields.get(key) - if not isinstance(field, FileField): - value = field.to_python(value) - setattr(self, key, value) + if key in self._fields or key in ('id', 'pk', '_cls'): + if __auto_convert and value is not None: + field = self._fields.get(key) + if field and not isinstance(field, FileField): + value = field.to_python(value) + setattr(self, key, value) + else: + self._data[key] = value # Set any get_fieldname_display methods self.__set_field_display() diff --git a/tests/document/__init__.py b/tests/document/__init__.py index 7774ee19..1acc9f4b 100644 --- a/tests/document/__init__.py +++ b/tests/document/__init__.py @@ -9,6 +9,7 @@ from indexes import * from inheritance import * from instance import * from json_serialisation import * +from validation import * if __name__ == '__main__': unittest.main() diff --git a/tests/document/instance.py b/tests/document/instance.py index 247f6275..99e4edb0 100644 --- a/tests/document/instance.py +++ b/tests/document/instance.py @@ -23,7 +23,7 @@ from mongoengine.context_managers import switch_db TEST_IMAGE_PATH = os.path.join(os.path.dirname(__file__), '../fields/mongoengine.png') -__all__ = ("InstanceTest", "ValidatorErrorTest") +__all__ = ("InstanceTest",) class InstanceTest(unittest.TestCase): @@ -1977,192 +1977,29 @@ class InstanceTest(unittest.TestCase): group = Group.objects.first() self.assertEqual("hello - default", group.name) - -class ValidatorErrorTest(unittest.TestCase): - - def test_to_dict(self): - """Ensure a ValidationError handles error to_dict correctly. - """ - error = ValidationError('root') - self.assertEqual(error.to_dict(), {}) - - # 1st level error schema - error.errors = {'1st': ValidationError('bad 1st'), } - self.assertTrue('1st' in error.to_dict()) - self.assertEqual(error.to_dict()['1st'], 'bad 1st') - - # 2nd level error schema - error.errors = {'1st': ValidationError('bad 1st', errors={ - '2nd': ValidationError('bad 2nd'), - })} - self.assertTrue('1st' in error.to_dict()) - self.assertTrue(isinstance(error.to_dict()['1st'], dict)) - self.assertTrue('2nd' in error.to_dict()['1st']) - self.assertEqual(error.to_dict()['1st']['2nd'], 'bad 2nd') - - # moar levels - error.errors = {'1st': ValidationError('bad 1st', errors={ - '2nd': ValidationError('bad 2nd', errors={ - '3rd': ValidationError('bad 3rd', errors={ - '4th': ValidationError('Inception'), - }), - }), - })} - self.assertTrue('1st' in error.to_dict()) - self.assertTrue('2nd' in error.to_dict()['1st']) - self.assertTrue('3rd' in error.to_dict()['1st']['2nd']) - self.assertTrue('4th' in error.to_dict()['1st']['2nd']['3rd']) - self.assertEqual(error.to_dict()['1st']['2nd']['3rd']['4th'], - 'Inception') - - self.assertEqual(error.message, "root(2nd.3rd.4th.Inception: ['1st'])") - - def test_model_validation(self): + def test_no_overwritting_no_data_loss(self): class User(Document): username = StringField(primary_key=True) - name = StringField(required=True) - - try: - User().validate() - except ValidationError, e: - self.assertEqual(e.to_dict(), { - 'username': 'Field is required', - 'name': 'Field is required'}) - - def test_spaces_in_keys(self): - - class Embedded(DynamicEmbeddedDocument): - pass - - class Doc(DynamicDocument): - pass - - Doc.drop_collection() - doc = Doc() - setattr(doc, 'hello world', 1) - doc.save() - - one = Doc.objects.filter(**{'hello world': 1}).count() - self.assertEqual(1, one) - - def test_fields_rewrite(self): - class BasePerson(Document): - name = StringField() - age = IntField() - meta = {'abstract': True} - - class Person(BasePerson): - name = StringField(required=True) - - p = Person(age=15) - self.assertRaises(ValidationError, p.validate) - - def test_cascaded_save_wrong_reference(self): - - class ADocument(Document): - val = IntField() - - class BDocument(Document): - a = ReferenceField(ADocument) - - ADocument.drop_collection() - BDocument.drop_collection() - - a = ADocument() - a.val = 15 - a.save() - - b = BDocument() - b.a = a - b.save() - - a.delete() - - b = BDocument.objects.first() - b.save(cascade=True) - - def test_shard_key(self): - class LogEntry(Document): - machine = StringField() - log = StringField() - - meta = { - 'shard_key': ('machine',) - } - - LogEntry.drop_collection() - - log = LogEntry() - log.machine = "Localhost" - log.save() - - log.log = "Saving" - log.save() - - def change_shard_key(): - log.machine = "127.0.0.1" - - self.assertRaises(OperationError, change_shard_key) - - def test_shard_key_primary(self): - class LogEntry(Document): - machine = StringField(primary_key=True) - log = StringField() - - meta = { - 'shard_key': ('machine',) - } - - LogEntry.drop_collection() - - log = LogEntry() - log.machine = "Localhost" - log.save() - - log.log = "Saving" - log.save() - - def change_shard_key(): - log.machine = "127.0.0.1" - - self.assertRaises(OperationError, change_shard_key) - - def test_kwargs_simple(self): - - class Embedded(EmbeddedDocument): name = StringField() - class Doc(Document): - doc_name = StringField() - doc = EmbeddedDocumentField(Embedded) + @property + def foo(self): + return True - classic_doc = Doc(doc_name="my doc", doc=Embedded(name="embedded doc")) - dict_doc = Doc(**{"doc_name": "my doc", - "doc": {"name": "embedded doc"}}) + User.drop_collection() - self.assertEqual(classic_doc, dict_doc) - self.assertEqual(classic_doc._data, dict_doc._data) + user = User(username="Ross", foo="bar") + self.assertTrue(user.foo) - def test_kwargs_complex(self): - - class Embedded(EmbeddedDocument): - name = StringField() - - class Doc(Document): - doc_name = StringField() - docs = ListField(EmbeddedDocumentField(Embedded)) - - classic_doc = Doc(doc_name="my doc", docs=[ - Embedded(name="embedded doc1"), - Embedded(name="embedded doc2")]) - dict_doc = Doc(**{"doc_name": "my doc", - "docs": [{"name": "embedded doc1"}, - {"name": "embedded doc2"}]}) - - self.assertEqual(classic_doc, dict_doc) - self.assertEqual(classic_doc._data, dict_doc._data) + User._get_collection().save({"_id": "Ross", "foo": "Bar", + "data": [1, 2, 3]}) + user = User.objects.first() + self.assertEqual("Ross", user.username) + self.assertEqual(True, user.foo) + self.assertEqual("Bar", user._data["foo"]) + self.assertEqual([1, 2, 3], user._data["data"]) if __name__ == '__main__': unittest.main() diff --git a/tests/document/validation.py b/tests/document/validation.py new file mode 100644 index 00000000..dafb3a39 --- /dev/null +++ b/tests/document/validation.py @@ -0,0 +1,195 @@ +# -*- coding: utf-8 -*- +import sys +sys.path[0:0] = [""] + +import unittest + +from mongoengine import * + +__all__ = ("ValidatorErrorTest",) + + +class ValidatorErrorTest(unittest.TestCase): + + def test_to_dict(self): + """Ensure a ValidationError handles error to_dict correctly. + """ + error = ValidationError('root') + self.assertEqual(error.to_dict(), {}) + + # 1st level error schema + error.errors = {'1st': ValidationError('bad 1st'), } + self.assertTrue('1st' in error.to_dict()) + self.assertEqual(error.to_dict()['1st'], 'bad 1st') + + # 2nd level error schema + error.errors = {'1st': ValidationError('bad 1st', errors={ + '2nd': ValidationError('bad 2nd'), + })} + self.assertTrue('1st' in error.to_dict()) + self.assertTrue(isinstance(error.to_dict()['1st'], dict)) + self.assertTrue('2nd' in error.to_dict()['1st']) + self.assertEqual(error.to_dict()['1st']['2nd'], 'bad 2nd') + + # moar levels + error.errors = {'1st': ValidationError('bad 1st', errors={ + '2nd': ValidationError('bad 2nd', errors={ + '3rd': ValidationError('bad 3rd', errors={ + '4th': ValidationError('Inception'), + }), + }), + })} + self.assertTrue('1st' in error.to_dict()) + self.assertTrue('2nd' in error.to_dict()['1st']) + self.assertTrue('3rd' in error.to_dict()['1st']['2nd']) + self.assertTrue('4th' in error.to_dict()['1st']['2nd']['3rd']) + self.assertEqual(error.to_dict()['1st']['2nd']['3rd']['4th'], + 'Inception') + + self.assertEqual(error.message, "root(2nd.3rd.4th.Inception: ['1st'])") + + def test_model_validation(self): + + class User(Document): + username = StringField(primary_key=True) + name = StringField(required=True) + + try: + User().validate() + except ValidationError, e: + self.assertEqual(e.to_dict(), { + 'username': 'Field is required', + 'name': 'Field is required'}) + + def test_spaces_in_keys(self): + + class Embedded(DynamicEmbeddedDocument): + pass + + class Doc(DynamicDocument): + pass + + Doc.drop_collection() + doc = Doc() + setattr(doc, 'hello world', 1) + doc.save() + + one = Doc.objects.filter(**{'hello world': 1}).count() + self.assertEqual(1, one) + + def test_fields_rewrite(self): + class BasePerson(Document): + name = StringField() + age = IntField() + meta = {'abstract': True} + + class Person(BasePerson): + name = StringField(required=True) + + p = Person(age=15) + self.assertRaises(ValidationError, p.validate) + + def test_cascaded_save_wrong_reference(self): + + class ADocument(Document): + val = IntField() + + class BDocument(Document): + a = ReferenceField(ADocument) + + ADocument.drop_collection() + BDocument.drop_collection() + + a = ADocument() + a.val = 15 + a.save() + + b = BDocument() + b.a = a + b.save() + + a.delete() + + b = BDocument.objects.first() + b.save(cascade=True) + + def test_shard_key(self): + class LogEntry(Document): + machine = StringField() + log = StringField() + + meta = { + 'shard_key': ('machine',) + } + + LogEntry.drop_collection() + + log = LogEntry() + log.machine = "Localhost" + log.save() + + log.log = "Saving" + log.save() + + def change_shard_key(): + log.machine = "127.0.0.1" + + self.assertRaises(OperationError, change_shard_key) + + def test_shard_key_primary(self): + class LogEntry(Document): + machine = StringField(primary_key=True) + log = StringField() + + meta = { + 'shard_key': ('machine',) + } + + LogEntry.drop_collection() + + log = LogEntry() + log.machine = "Localhost" + log.save() + + log.log = "Saving" + log.save() + + def change_shard_key(): + log.machine = "127.0.0.1" + + self.assertRaises(OperationError, change_shard_key) + + def test_kwargs_simple(self): + + class Embedded(EmbeddedDocument): + name = StringField() + + class Doc(Document): + doc_name = StringField() + doc = EmbeddedDocumentField(Embedded) + + classic_doc = Doc(doc_name="my doc", doc=Embedded(name="embedded doc")) + dict_doc = Doc(**{"doc_name": "my doc", + "doc": {"name": "embedded doc"}}) + + self.assertEqual(classic_doc, dict_doc) + self.assertEqual(classic_doc._data, dict_doc._data) + + def test_kwargs_complex(self): + + class Embedded(EmbeddedDocument): + name = StringField() + + class Doc(Document): + doc_name = StringField() + docs = ListField(EmbeddedDocumentField(Embedded)) + + classic_doc = Doc(doc_name="my doc", docs=[ + Embedded(name="embedded doc1"), + Embedded(name="embedded doc2")]) + dict_doc = Doc(**{"doc_name": "my doc", + "docs": [{"name": "embedded doc1"}, + {"name": "embedded doc2"}]}) + + self.assertEqual(classic_doc, dict_doc) + self.assertEqual(classic_doc._data, dict_doc._data) From eb1b6e34c71abe45d96cbf356f1253a3ca5c51a6 Mon Sep 17 00:00:00 2001 From: Ross Lawley Date: Fri, 25 Jan 2013 11:51:58 +0000 Subject: [PATCH 093/189] Updated upgrade docs (#49) --- docs/upgrade.rst | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/docs/upgrade.rst b/docs/upgrade.rst index 9c6c9a9d..d3282487 100644 --- a/docs/upgrade.rst +++ b/docs/upgrade.rst @@ -56,6 +56,26 @@ you will need to declare :attr:`allow_inheritance` in the meta data like so: :: meta = {'allow_inheritance': True} +Previously, if you had data the database that wasn't defined in the Document +definition, it would set it as an attribute on the document. This is no longer +the case and the data is set only in the ``document._data`` dictionary: :: + + >>> from mongoengine import * + >>> class Animal(Document): + ... name = StringField() + ... + >>> cat = Animal(name="kit", size="small") + + # 0.7 + >>> cat.size + u'small' + + # 0.8 + >>> cat.size + Traceback (most recent call last): + File "", line 1, in + AttributeError: 'Animal' object has no attribute 'size' + Querysets ~~~~~~~~~ From 0ea363c7fc760e210bee5b83b3ab83f657a6e0ae Mon Sep 17 00:00:00 2001 From: Ross Lawley Date: Fri, 25 Jan 2013 12:13:46 +0000 Subject: [PATCH 094/189] Updated authors and changelof (#142) --- AUTHORS | 3 ++- docs/changelog.rst | 1 + 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/AUTHORS b/AUTHORS index c32ab9f9..ac811dd1 100644 --- a/AUTHORS +++ b/AUTHORS @@ -137,4 +137,5 @@ that much better: * Nicolas Trippar * Manuel Hermann * Gustavo Gawryszewski - * Max Countryman \ No newline at end of file + * Max Countryman + * caitifbrito \ No newline at end of file diff --git a/docs/changelog.rst b/docs/changelog.rst index 5bdd0f85..19bc4464 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -41,6 +41,7 @@ Changes in 0.8.X - Fixed overriding objects with custom manager (#58) - Added no_dereference method for querysets (#82) (#61) - Undefined data should not override instance methods (#49) +- Added Django Group and Permission (#142) Changes in 0.7.9 ================ From 9d9a4afee9aef2e11beb37207c914a8a76329370 Mon Sep 17 00:00:00 2001 From: Ross Lawley Date: Mon, 28 Jan 2013 12:05:09 +0000 Subject: [PATCH 095/189] Added Doc class and pk to Validation messages (#69) --- docs/changelog.rst | 1 + mongoengine/base/document.py | 8 +- tests/document/instance.py | 137 +++++++++++++++++++++--------- tests/document/validation.py | 159 ++++++++++++----------------------- 4 files changed, 161 insertions(+), 144 deletions(-) diff --git a/docs/changelog.rst b/docs/changelog.rst index 19bc4464..601c2dbc 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -42,6 +42,7 @@ Changes in 0.8.X - Added no_dereference method for querysets (#82) (#61) - Undefined data should not override instance methods (#49) - Added Django Group and Permission (#142) +- Added Doc class and pk to Validation messages (#69) Changes in 0.7.9 ================ diff --git a/mongoengine/base/document.py b/mongoengine/base/document.py index a88a38bf..4f5a87e5 100644 --- a/mongoengine/base/document.py +++ b/mongoengine/base/document.py @@ -274,7 +274,13 @@ class BaseDocument(object): field_name=field.name) if errors: - raise ValidationError('ValidationError', errors=errors) + pk = "None" + if hasattr(self, 'pk'): + pk = self.pk + elif self._instance: + pk = self._instance.pk + message = "ValidationError (%s:%s) " % (self._class_name, pk) + raise ValidationError(message, errors=errors) def to_json(self): """Converts a document to JSON""" diff --git a/tests/document/instance.py b/tests/document/instance.py index 99e4edb0..3d4e8a97 100644 --- a/tests/document/instance.py +++ b/tests/document/instance.py @@ -466,45 +466,6 @@ class InstanceTest(unittest.TestCase): doc = Doc.objects.get() self.assertEqual(doc, doc.embedded_field[0]._instance) - def test_embedded_document_validation(self): - """Ensure that embedded documents may be validated. - """ - class Comment(EmbeddedDocument): - date = DateTimeField() - content = StringField(required=True) - - comment = Comment() - self.assertRaises(ValidationError, comment.validate) - - comment.content = 'test' - comment.validate() - - comment.date = 4 - self.assertRaises(ValidationError, comment.validate) - - comment.date = datetime.now() - comment.validate() - self.assertEqual(comment._instance, None) - - def test_embedded_db_field_validate(self): - - class SubDoc(EmbeddedDocument): - val = IntField() - - class Doc(Document): - e = EmbeddedDocumentField(SubDoc, db_field='eb') - - Doc.drop_collection() - - Doc(e=SubDoc(val=15)).save() - - doc = Doc.objects.first() - doc.validate() - keys = doc._data.keys() - self.assertEqual(2, len(keys)) - self.assertTrue('id' in keys) - self.assertTrue('e' in keys) - def test_document_clean(self): class TestDocument(Document): status = StringField() @@ -2001,5 +1962,103 @@ class InstanceTest(unittest.TestCase): self.assertEqual("Bar", user._data["foo"]) self.assertEqual([1, 2, 3], user._data["data"]) + + def test_spaces_in_keys(self): + + class Embedded(DynamicEmbeddedDocument): + pass + + class Doc(DynamicDocument): + pass + + Doc.drop_collection() + doc = Doc() + setattr(doc, 'hello world', 1) + doc.save() + + one = Doc.objects.filter(**{'hello world': 1}).count() + self.assertEqual(1, one) + + def test_shard_key(self): + class LogEntry(Document): + machine = StringField() + log = StringField() + + meta = { + 'shard_key': ('machine',) + } + + LogEntry.drop_collection() + + log = LogEntry() + log.machine = "Localhost" + log.save() + + log.log = "Saving" + log.save() + + def change_shard_key(): + log.machine = "127.0.0.1" + + self.assertRaises(OperationError, change_shard_key) + + def test_shard_key_primary(self): + class LogEntry(Document): + machine = StringField(primary_key=True) + log = StringField() + + meta = { + 'shard_key': ('machine',) + } + + LogEntry.drop_collection() + + log = LogEntry() + log.machine = "Localhost" + log.save() + + log.log = "Saving" + log.save() + + def change_shard_key(): + log.machine = "127.0.0.1" + + self.assertRaises(OperationError, change_shard_key) + + def test_kwargs_simple(self): + + class Embedded(EmbeddedDocument): + name = StringField() + + class Doc(Document): + doc_name = StringField() + doc = EmbeddedDocumentField(Embedded) + + classic_doc = Doc(doc_name="my doc", doc=Embedded(name="embedded doc")) + dict_doc = Doc(**{"doc_name": "my doc", + "doc": {"name": "embedded doc"}}) + + self.assertEqual(classic_doc, dict_doc) + self.assertEqual(classic_doc._data, dict_doc._data) + + def test_kwargs_complex(self): + + class Embedded(EmbeddedDocument): + name = StringField() + + class Doc(Document): + doc_name = StringField() + docs = ListField(EmbeddedDocumentField(Embedded)) + + classic_doc = Doc(doc_name="my doc", docs=[ + Embedded(name="embedded doc1"), + Embedded(name="embedded doc2")]) + dict_doc = Doc(**{"doc_name": "my doc", + "docs": [{"name": "embedded doc1"}, + {"name": "embedded doc2"}]}) + + self.assertEqual(classic_doc, dict_doc) + self.assertEqual(classic_doc._data, dict_doc._data) + if __name__ == '__main__': unittest.main() diff --git a/tests/document/validation.py b/tests/document/validation.py index dafb3a39..aaf6b0cc 100644 --- a/tests/document/validation.py +++ b/tests/document/validation.py @@ -3,6 +3,7 @@ import sys sys.path[0:0] = [""] import unittest +from datetime import datetime from mongoengine import * @@ -11,6 +12,9 @@ __all__ = ("ValidatorErrorTest",) class ValidatorErrorTest(unittest.TestCase): + def setUp(self): + connect(db='mongoenginetest') + def test_to_dict(self): """Ensure a ValidationError handles error to_dict correctly. """ @@ -57,25 +61,19 @@ class ValidatorErrorTest(unittest.TestCase): try: User().validate() except ValidationError, e: + self.assertTrue("User:None" in e.message) self.assertEqual(e.to_dict(), { 'username': 'Field is required', 'name': 'Field is required'}) - def test_spaces_in_keys(self): - - class Embedded(DynamicEmbeddedDocument): - pass - - class Doc(DynamicDocument): - pass - - Doc.drop_collection() - doc = Doc() - setattr(doc, 'hello world', 1) - doc.save() - - one = Doc.objects.filter(**{'hello world': 1}).count() - self.assertEqual(1, one) + user = User(username="RossC0", name="Ross").save() + user.name = None + try: + user.save() + except ValidationError, e: + self.assertTrue("User:RossC0" in e.message) + self.assertEqual(e.to_dict(), { + 'name': 'Field is required'}) def test_fields_rewrite(self): class BasePerson(Document): @@ -89,107 +87,60 @@ class ValidatorErrorTest(unittest.TestCase): p = Person(age=15) self.assertRaises(ValidationError, p.validate) - def test_cascaded_save_wrong_reference(self): + def test_embedded_document_validation(self): + """Ensure that embedded documents may be validated. + """ + class Comment(EmbeddedDocument): + date = DateTimeField() + content = StringField(required=True) - class ADocument(Document): - val = IntField() + comment = Comment() + self.assertRaises(ValidationError, comment.validate) - class BDocument(Document): - a = ReferenceField(ADocument) + comment.content = 'test' + comment.validate() - ADocument.drop_collection() - BDocument.drop_collection() + comment.date = 4 + self.assertRaises(ValidationError, comment.validate) - a = ADocument() - a.val = 15 - a.save() + comment.date = datetime.now() + comment.validate() + self.assertEqual(comment._instance, None) - b = BDocument() - b.a = a - b.save() + def test_embedded_db_field_validate(self): - a.delete() - - b = BDocument.objects.first() - b.save(cascade=True) - - def test_shard_key(self): - class LogEntry(Document): - machine = StringField() - log = StringField() - - meta = { - 'shard_key': ('machine',) - } - - LogEntry.drop_collection() - - log = LogEntry() - log.machine = "Localhost" - log.save() - - log.log = "Saving" - log.save() - - def change_shard_key(): - log.machine = "127.0.0.1" - - self.assertRaises(OperationError, change_shard_key) - - def test_shard_key_primary(self): - class LogEntry(Document): - machine = StringField(primary_key=True) - log = StringField() - - meta = { - 'shard_key': ('machine',) - } - - LogEntry.drop_collection() - - log = LogEntry() - log.machine = "Localhost" - log.save() - - log.log = "Saving" - log.save() - - def change_shard_key(): - log.machine = "127.0.0.1" - - self.assertRaises(OperationError, change_shard_key) - - def test_kwargs_simple(self): - - class Embedded(EmbeddedDocument): - name = StringField() + class SubDoc(EmbeddedDocument): + val = IntField(required=True) class Doc(Document): - doc_name = StringField() - doc = EmbeddedDocumentField(Embedded) + id = StringField(primary_key=True) + e = EmbeddedDocumentField(SubDoc, db_field='eb') - classic_doc = Doc(doc_name="my doc", doc=Embedded(name="embedded doc")) - dict_doc = Doc(**{"doc_name": "my doc", - "doc": {"name": "embedded doc"}}) + try: + Doc(id="bad").validate() + except ValidationError, e: + self.assertTrue("SubDoc:None" in e.message) + self.assertEqual(e.to_dict(), { + 'e.val': 'Field is required'}) - self.assertEqual(classic_doc, dict_doc) - self.assertEqual(classic_doc._data, dict_doc._data) + Doc.drop_collection() - def test_kwargs_complex(self): + Doc(id="test", e=SubDoc(val=15)).save() - class Embedded(EmbeddedDocument): - name = StringField() + doc = Doc.objects.first() + keys = doc._data.keys() + self.assertEqual(2, len(keys)) + self.assertTrue('id' in keys) + self.assertTrue('e' in keys) - class Doc(Document): - doc_name = StringField() - docs = ListField(EmbeddedDocumentField(Embedded)) + doc.e.val = "OK" + try: + doc.save() + except ValidationError, e: + self.assertTrue("SubDoc:test" in e.message) + self.assertEqual(e.to_dict(), { + 'e.val': 'Field is required'}) - classic_doc = Doc(doc_name="my doc", docs=[ - Embedded(name="embedded doc1"), - Embedded(name="embedded doc2")]) - dict_doc = Doc(**{"doc_name": "my doc", - "docs": [{"name": "embedded doc1"}, - {"name": "embedded doc2"}]}) - self.assertEqual(classic_doc, dict_doc) - self.assertEqual(classic_doc._data, dict_doc._data) +if __name__ == '__main__': + unittest.main() From de2f774e8533ac18c161182cd7c37754e8d68844 Mon Sep 17 00:00:00 2001 From: Ross Lawley Date: Mon, 28 Jan 2013 13:29:44 +0000 Subject: [PATCH 096/189] Fix validation test --- tests/document/validation.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/document/validation.py b/tests/document/validation.py index aaf6b0cc..0f67f50a 100644 --- a/tests/document/validation.py +++ b/tests/document/validation.py @@ -137,7 +137,7 @@ class ValidatorErrorTest(unittest.TestCase): try: doc.save() except ValidationError, e: - self.assertTrue("SubDoc:test" in e.message) + self.assertTrue("Doc:test" in e.message) self.assertEqual(e.to_dict(), { 'e.val': 'Field is required'}) From f182daa85eae94ccfffaa40e04f6370c007d10c4 Mon Sep 17 00:00:00 2001 From: Ross Lawley Date: Mon, 28 Jan 2013 13:32:21 +0000 Subject: [PATCH 097/189] Fixed Documents deleted via a queryset don't call any signals (#105) --- docs/changelog.rst | 1 + mongoengine/queryset/queryset.py | 11 ++++++--- tests/document/instance.py | 40 ++++++++++++++++++++++++++++++-- 3 files changed, 47 insertions(+), 5 deletions(-) diff --git a/docs/changelog.rst b/docs/changelog.rst index 601c2dbc..8b41d6ee 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -43,6 +43,7 @@ Changes in 0.8.X - Undefined data should not override instance methods (#49) - Added Django Group and Permission (#142) - Added Doc class and pk to Validation messages (#69) +- Fixed Documents deleted via a queryset don't call any signals (#105) Changes in 0.7.9 ================ diff --git a/mongoengine/queryset/queryset.py b/mongoengine/queryset/queryset.py index b5e33515..65703c32 100644 --- a/mongoengine/queryset/queryset.py +++ b/mongoengine/queryset/queryset.py @@ -364,10 +364,15 @@ class QuerySet(object): queryset = self.clone() doc = queryset._document - # Handle deletes where skips or limits have been applied - if queryset._skip or queryset._limit: + has_delete_signal = ( + signals.pre_delete.has_receivers_for(self._document) or + signals.post_delete.has_receivers_for(self._document)) + + # Handle deletes where skips or limits have been applied or has a + # delete signal + if queryset._skip or queryset._limit or has_delete_signal: for doc in queryset: - doc.delete() + doc.delete(safe=safe) return delete_rules = doc._meta.get('delete_rules') or {} diff --git a/tests/document/instance.py b/tests/document/instance.py index 3d4e8a97..172f0ccf 100644 --- a/tests/document/instance.py +++ b/tests/document/instance.py @@ -19,6 +19,7 @@ from mongoengine.queryset import NULLIFY, Q from mongoengine.connection import get_db from mongoengine.base import get_document from mongoengine.context_managers import switch_db +from mongoengine import signals TEST_IMAGE_PATH = os.path.join(os.path.dirname(__file__), '../fields/mongoengine.png') @@ -1375,7 +1376,6 @@ class InstanceTest(unittest.TestCase): author.delete() self.assertEqual(len(BlogPost.objects), 0) - def test_reverse_delete_rule_cascade_and_nullify_complex_field(self): """Ensure that a referenced document is also deleted upon deletion for complex fields. @@ -1410,6 +1410,43 @@ class InstanceTest(unittest.TestCase): author.delete() self.assertEqual(len(BlogPost.objects), 0) + + def test_reverse_delete_rule_cascade_triggers_pre_delete_signal(self): + ''' ensure the pre_delete signal is triggered upon a cascading deletion + setup a blog post with content, an author and editor + delete the author which triggers deletion of blogpost via cascade + blog post's pre_delete signal alters an editor attribute + ''' + class Editor(self.Person): + review_queue = IntField(default=0) + + class BlogPost(Document): + content = StringField() + author = ReferenceField(self.Person, reverse_delete_rule=CASCADE) + editor = ReferenceField(Editor) + + @classmethod + def pre_delete(cls, sender, document, **kwargs): + # decrement the docs-to-review count + document.editor.update(dec__review_queue=1) + + signals.pre_delete.connect(BlogPost.pre_delete, sender=BlogPost) + + self.Person.drop_collection() + BlogPost.drop_collection() + Editor.drop_collection() + + author = self.Person(name='Will S.').save() + editor = Editor(name='Max P.', review_queue=1).save() + BlogPost(content='wrote some books', author=author, + editor=editor).save() + + # delete the author, the post is also deleted due to the CASCADE rule + author.delete() + # the pre-delete signal should have decremented the editor's queue + editor = Editor.objects(name='Max P.').get() + self.assertEqual(editor.review_queue, 0) + def test_two_way_reverse_delete_rule(self): """Ensure that Bi-Directional relationships work with reverse_delete_rule @@ -1426,7 +1463,6 @@ class InstanceTest(unittest.TestCase): Bar.register_delete_rule(Foo, 'bar', NULLIFY) Foo.register_delete_rule(Bar, 'foo', NULLIFY) - Bar.drop_collection() Foo.drop_collection() From 0cbd3663e47d677aa4921070888d32ae4823a9f1 Mon Sep 17 00:00:00 2001 From: Ross Lawley Date: Mon, 28 Jan 2013 13:40:28 +0000 Subject: [PATCH 098/189] Updated tests --- tests/all_warnings/__init__.py | 6 +++++- tests/document/class_methods.py | 2 ++ tests/document/dynamic.py | 1 - tests/document/indexes.py | 1 - tests/document/inheritance.py | 2 ++ tests/document/validation.py | 4 ++-- tests/test_connection.py | 9 +++++---- tests/test_context_managers.py | 2 ++ tests/test_dereference.py | 1 - tests/test_django.py | 5 +++++ tests/test_replicaset_connection.py | 2 ++ tests/test_signals.py | 15 ++++++++++----- 12 files changed, 35 insertions(+), 15 deletions(-) diff --git a/tests/all_warnings/__init__.py b/tests/all_warnings/__init__.py index 220b0bbf..8cbe22df 100644 --- a/tests/all_warnings/__init__.py +++ b/tests/all_warnings/__init__.py @@ -3,7 +3,8 @@ This test has been put into a module. This is because it tests warnings that only get triggered on first hit. This way we can ensure its imported into the top level and called first by the test suite. """ - +import sys +sys.path[0:0] = [""] import unittest import warnings @@ -88,3 +89,6 @@ class AllWarnings(unittest.TestCase): self.assertEqual(SyntaxWarning, warning["category"]) self.assertEqual('non_abstract_base', InheritedDocumentFailTest._get_collection_name()) + +import sys +sys.path[0:0] = [""] \ No newline at end of file diff --git a/tests/document/class_methods.py b/tests/document/class_methods.py index 8e9a8776..83e68ff8 100644 --- a/tests/document/class_methods.py +++ b/tests/document/class_methods.py @@ -1,5 +1,7 @@ # -*- coding: utf-8 -*- from __future__ import with_statement +import sys +sys.path[0:0] = [""] import unittest from mongoengine import * diff --git a/tests/document/dynamic.py b/tests/document/dynamic.py index 4848b8fa..5881cd07 100644 --- a/tests/document/dynamic.py +++ b/tests/document/dynamic.py @@ -1,6 +1,5 @@ import unittest import sys - sys.path[0:0] = [""] from mongoengine import * diff --git a/tests/document/indexes.py b/tests/document/indexes.py index c059590b..ff08ef1a 100644 --- a/tests/document/indexes.py +++ b/tests/document/indexes.py @@ -2,7 +2,6 @@ from __future__ import with_statement import unittest import sys - sys.path[0:0] = [""] import os diff --git a/tests/document/inheritance.py b/tests/document/inheritance.py index c5e1860b..3b550f1a 100644 --- a/tests/document/inheritance.py +++ b/tests/document/inheritance.py @@ -1,4 +1,6 @@ # -*- coding: utf-8 -*- +import sys +sys.path[0:0] = [""] import unittest import warnings diff --git a/tests/document/validation.py b/tests/document/validation.py index 0f67f50a..24ffed65 100644 --- a/tests/document/validation.py +++ b/tests/document/validation.py @@ -121,7 +121,7 @@ class ValidatorErrorTest(unittest.TestCase): except ValidationError, e: self.assertTrue("SubDoc:None" in e.message) self.assertEqual(e.to_dict(), { - 'e.val': 'Field is required'}) + "e": {'val': 'OK could not be converted to int'}}) Doc.drop_collection() @@ -139,7 +139,7 @@ class ValidatorErrorTest(unittest.TestCase): except ValidationError, e: self.assertTrue("Doc:test" in e.message) self.assertEqual(e.to_dict(), { - 'e.val': 'Field is required'}) + "e": {'val': 'OK could not be converted to int'}}) if __name__ == '__main__': diff --git a/tests/test_connection.py b/tests/test_connection.py index c32d231f..5b9743d6 100644 --- a/tests/test_connection.py +++ b/tests/test_connection.py @@ -1,13 +1,14 @@ from __future__ import with_statement -import datetime -import pymongo +import sys +sys.path[0:0] = [""] import unittest +import datetime -import mongoengine.connection - +import pymongo from bson.tz_util import utc from mongoengine import * +import mongoengine.connection from mongoengine.connection import get_db, get_connection, ConnectionError from mongoengine.context_managers import switch_db diff --git a/tests/test_context_managers.py b/tests/test_context_managers.py index c9efe8b8..eef63bee 100644 --- a/tests/test_context_managers.py +++ b/tests/test_context_managers.py @@ -1,4 +1,6 @@ from __future__ import with_statement +import sys +sys.path[0:0] = [""] import unittest from mongoengine import * diff --git a/tests/test_dereference.py b/tests/test_dereference.py index adbc5192..4198f3c4 100644 --- a/tests/test_dereference.py +++ b/tests/test_dereference.py @@ -1,7 +1,6 @@ from __future__ import with_statement import sys sys.path[0:0] = [""] - import unittest from bson import DBRef, ObjectId diff --git a/tests/test_django.py b/tests/test_django.py index 3b0b04f8..563f4074 100644 --- a/tests/test_django.py +++ b/tests/test_django.py @@ -1,4 +1,6 @@ from __future__ import with_statement +import sys +sys.path[0:0] = [""] import unittest from nose.plugins.skip import SkipTest from mongoengine.python_support import PY3 @@ -163,3 +165,6 @@ class MongoDBSessionTest(SessionTestsMixin, unittest.TestCase): key = session.session_key session = SessionStore(key) self.assertTrue('test_expire' in session, 'Session has expired before it is expected') + +if __name__ == '__main__': + unittest.main() diff --git a/tests/test_replicaset_connection.py b/tests/test_replicaset_connection.py index 3118c5a4..d27960f7 100644 --- a/tests/test_replicaset_connection.py +++ b/tests/test_replicaset_connection.py @@ -1,3 +1,5 @@ +import sys +sys.path[0:0] = [""] import unittest import pymongo diff --git a/tests/test_signals.py b/tests/test_signals.py index 2ca820da..fc638cfc 100644 --- a/tests/test_signals.py +++ b/tests/test_signals.py @@ -1,4 +1,6 @@ # -*- coding: utf-8 -*- +import sys +sys.path[0:0] = [""] import unittest from mongoengine import * @@ -21,6 +23,7 @@ class SignalTests(unittest.TestCase): def setUp(self): connect(db='mongoenginetest') + class Author(Document): name = StringField() @@ -70,7 +73,6 @@ class SignalTests(unittest.TestCase): signal_output.append('Not loaded') self.Author = Author - class Another(Document): name = StringField() @@ -122,8 +124,8 @@ class SignalTests(unittest.TestCase): self.ExplicitId = ExplicitId self.ExplicitId.objects.delete() - # Save up the number of connected signals so that we can check at the end - # that all the signals we register get properly unregistered + # Save up the number of connected signals so that we can check at the + # end that all the signals we register get properly unregistered self.pre_signals = ( len(signals.pre_init.receivers), len(signals.post_init.receivers), @@ -192,7 +194,7 @@ class SignalTests(unittest.TestCase): """ Model saves should throw some signals. """ def create_author(): - a1 = self.Author(name='Bill Shakespeare') + self.Author(name='Bill Shakespeare') def bulk_create_author_with_load(): a1 = self.Author(name='Bill Shakespeare') @@ -216,7 +218,7 @@ class SignalTests(unittest.TestCase): ]) a1.reload() - a1.name='William Shakespeare' + a1.name = 'William Shakespeare' self.assertEqual(self.get_signal_output(a1.save), [ "pre_save signal, William Shakespeare", "post_save signal, William Shakespeare", @@ -257,3 +259,6 @@ class SignalTests(unittest.TestCase): self.assertEqual(self.get_signal_output(ei.save), ['Is created']) # second time, it must be an update self.assertEqual(self.get_signal_output(ei.save), ['Is updated']) + +if __name__ == '__main__': + unittest.main() From 8c1f8e54cdb97a6bb4bf8523c05ec1b783e7a283 Mon Sep 17 00:00:00 2001 From: Ross Lawley Date: Mon, 28 Jan 2013 14:12:47 +0000 Subject: [PATCH 099/189] Added the "get_decoded" method to the MongoSession class (#216) --- AUTHORS | 3 ++- docs/changelog.rst | 1 + docs/upgrade.rst | 2 +- mongoengine/django/sessions.py | 3 ++- 4 files changed, 6 insertions(+), 3 deletions(-) diff --git a/AUTHORS b/AUTHORS index ac811dd1..64f0b0a2 100644 --- a/AUTHORS +++ b/AUTHORS @@ -138,4 +138,5 @@ that much better: * Manuel Hermann * Gustavo Gawryszewski * Max Countryman - * caitifbrito \ No newline at end of file + * caitifbrito + * lcya86 刘春洋 \ No newline at end of file diff --git a/docs/changelog.rst b/docs/changelog.rst index 8b41d6ee..65561544 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -44,6 +44,7 @@ Changes in 0.8.X - Added Django Group and Permission (#142) - Added Doc class and pk to Validation messages (#69) - Fixed Documents deleted via a queryset don't call any signals (#105) +- Added the "get_decoded" method to the MongoSession class (#216) Changes in 0.7.9 ================ diff --git a/docs/upgrade.rst b/docs/upgrade.rst index fcd5f714..8724503d 100644 --- a/docs/upgrade.rst +++ b/docs/upgrade.rst @@ -298,5 +298,5 @@ Alternatively, you can rename your collections eg :: mongodb 1.8 > 2.0 + =================== -Its been reported that indexes may need to be recreated to the newer version of indexes. +Its been reported that indexes may need to be recreated to the newer version of indexes. To do this drop indexes and call ``ensure_indexes`` on each model. diff --git a/mongoengine/django/sessions.py b/mongoengine/django/sessions.py index 17cae8ad..1c9288ed 100644 --- a/mongoengine/django/sessions.py +++ b/mongoengine/django/sessions.py @@ -25,6 +25,7 @@ MONGOENGINE_SESSION_DATA_ENCODE = getattr( settings, 'MONGOENGINE_SESSION_DATA_ENCODE', True) + class MongoSession(Document): session_key = fields.StringField(primary_key=True, max_length=40) session_data = fields.StringField() if MONGOENGINE_SESSION_DATA_ENCODE \ @@ -34,7 +35,7 @@ class MongoSession(Document): meta = {'collection': MONGOENGINE_SESSION_COLLECTION, 'db_alias': MONGOENGINE_SESSION_DB_ALIAS, 'allow_inheritance': False} - + def get_decoded(self): return SessionStore().decode(self.session_data) From 5b161b7445eb3d53b067ecab6b25a92a6001930f Mon Sep 17 00:00:00 2001 From: Ross Lawley Date: Mon, 28 Jan 2013 14:17:54 +0000 Subject: [PATCH 100/189] ReadPreference that overrides slave_okay (#218) --- mongoengine/queryset/queryset.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/mongoengine/queryset/queryset.py b/mongoengine/queryset/queryset.py index 65703c32..4c664614 100644 --- a/mongoengine/queryset/queryset.py +++ b/mongoengine/queryset/queryset.py @@ -1095,11 +1095,12 @@ class QuerySet(object): def _cursor_args(self): cursor_args = { 'snapshot': self._snapshot, - 'timeout': self._timeout, - 'slave_okay': self._slave_okay, + 'timeout': self._timeout } if self._read_preference is not None: cursor_args['read_preference'] = self._read_preference + else: + cursor_args['slave_okay'] = self._slave_okay if self._loaded_fields: cursor_args['fields'] = self._loaded_fields.as_dict() return cursor_args From 3208a7f15dbe8d99253c3a397c354812ba883953 Mon Sep 17 00:00:00 2001 From: Ross Lawley Date: Mon, 28 Jan 2013 14:28:40 +0000 Subject: [PATCH 101/189] Merge fix tests --- tests/fields/fields.py | 4 ---- 1 file changed, 4 deletions(-) diff --git a/tests/fields/fields.py b/tests/fields/fields.py index 8cf3bb5c..7c4e7853 100644 --- a/tests/fields/fields.py +++ b/tests/fields/fields.py @@ -1725,9 +1725,6 @@ class FieldTest(unittest.TestCase): Shirt.drop_collection() -<<<<<<< HEAD:tests/fields/fields.py -======= - def test_simple_choices_validation_invalid_value(self): """Ensure that error messages are correct. """ @@ -2060,7 +2057,6 @@ class FieldTest(unittest.TestCase): self.assertEqual(test_file.the_file.read(), b('Hello, World!')) ->>>>>>> de5fbfde2ca96b93490e0bc96e04f3aa4affcfb5:tests/test_fields.py def test_geo_indexes(self): """Ensure that indexes are created automatically for GeoPointFields. """ From 1ca098c40291ff205e64080dbb1877e8083c3c67 Mon Sep 17 00:00:00 2001 From: Ross Lawley Date: Mon, 28 Jan 2013 14:40:26 +0000 Subject: [PATCH 102/189] Fixed invalid choices error bubbling (#214) --- docs/changelog.rst | 1 + tests/fields/__init__.py | 2 +- tests/fields/{file.py => file_tests.py} | 0 3 files changed, 2 insertions(+), 1 deletion(-) rename tests/fields/{file.py => file_tests.py} (100%) diff --git a/docs/changelog.rst b/docs/changelog.rst index 65561544..f289b398 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -45,6 +45,7 @@ Changes in 0.8.X - Added Doc class and pk to Validation messages (#69) - Fixed Documents deleted via a queryset don't call any signals (#105) - Added the "get_decoded" method to the MongoSession class (#216) +- Fixed invalid choices error bubbling (#214) Changes in 0.7.9 ================ diff --git a/tests/fields/__init__.py b/tests/fields/__init__.py index 86dfa840..0731838b 100644 --- a/tests/fields/__init__.py +++ b/tests/fields/__init__.py @@ -1,2 +1,2 @@ from fields import * -from file import * \ No newline at end of file +from file_tests import * \ No newline at end of file diff --git a/tests/fields/file.py b/tests/fields/file_tests.py similarity index 100% rename from tests/fields/file.py rename to tests/fields/file_tests.py From 4177fc6df2800908c0b868bd577b9bce1bc85cfa Mon Sep 17 00:00:00 2001 From: Ross Lawley Date: Mon, 28 Jan 2013 15:57:33 +0000 Subject: [PATCH 103/189] Can call del Doc.attr to delete field value --- mongoengine/document.py | 11 ----------- 1 file changed, 11 deletions(-) diff --git a/mongoengine/document.py b/mongoengine/document.py index 66aa2632..525d9644 100644 --- a/mongoengine/document.py +++ b/mongoengine/document.py @@ -48,17 +48,6 @@ class EmbeddedDocument(BaseDocument): super(EmbeddedDocument, self).__init__(*args, **kwargs) self._changed_fields = [] - def __delattr__(self, *args, **kwargs): - """Handle deletions of fields""" - field_name = args[0] - if field_name in self._fields: - default = self._fields[field_name].default - if callable(default): - default = default() - setattr(self, field_name, default) - else: - super(EmbeddedDocument, self).__delattr__(*args, **kwargs) - def __eq__(self, other): if isinstance(other, self.__class__): return self._data == other._data From 9ca632d5184cac9ea5f7d660a63e79658881309e Mon Sep 17 00:00:00 2001 From: Ross Lawley Date: Mon, 28 Jan 2013 16:00:38 +0000 Subject: [PATCH 104/189] Updated Save so it calls $set and $unset in a single operation (#211) --- AUTHORS | 1 + docs/changelog.rst | 1 + 2 files changed, 2 insertions(+) diff --git a/AUTHORS b/AUTHORS index ff580242..fb895d7e 100644 --- a/AUTHORS +++ b/AUTHORS @@ -141,3 +141,4 @@ that much better: * caitifbrito * lcya86 刘春洋 * Martin Alderete (https://github.com/malderete) + * Nick Joyce diff --git a/docs/changelog.rst b/docs/changelog.rst index f289b398..213e6fba 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -46,6 +46,7 @@ Changes in 0.8.X - Fixed Documents deleted via a queryset don't call any signals (#105) - Added the "get_decoded" method to the MongoSession class (#216) - Fixed invalid choices error bubbling (#214) +- Updated Save so it calls $set and $unset in a single operation (#211) Changes in 0.7.9 ================ From 39dac7d4dbe809e7eacac190d348326718ec0c1b Mon Sep 17 00:00:00 2001 From: Ross Lawley Date: Mon, 28 Jan 2013 16:26:01 +0000 Subject: [PATCH 105/189] Fix file open rules --- tests/fields/fields.py | 8 ++++---- tests/fields/file_tests.py | 10 +++++----- 2 files changed, 9 insertions(+), 9 deletions(-) diff --git a/tests/fields/fields.py b/tests/fields/fields.py index 7c4e7853..124c9538 100644 --- a/tests/fields/fields.py +++ b/tests/fields/fields.py @@ -1950,7 +1950,7 @@ class FieldTest(unittest.TestCase): TestImage.drop_collection() t = TestImage() - t.image.put(open(TEST_IMAGE_PATH, 'r')) + t.image.put(open(TEST_IMAGE_PATH, 'rb')) t.save() t = TestImage.objects.first() @@ -1973,7 +1973,7 @@ class FieldTest(unittest.TestCase): TestImage.drop_collection() t = TestImage() - t.image.put(open(TEST_IMAGE_PATH, 'r')) + t.image.put(open(TEST_IMAGE_PATH, 'rb')) t.save() t = TestImage.objects.first() @@ -1996,7 +1996,7 @@ class FieldTest(unittest.TestCase): TestImage.drop_collection() t = TestImage() - t.image.put(open(TEST_IMAGE_PATH, 'r')) + t.image.put(open(TEST_IMAGE_PATH, 'rb')) t.save() t = TestImage.objects.first() @@ -2019,7 +2019,7 @@ class FieldTest(unittest.TestCase): TestImage.drop_collection() t = TestImage() - t.image.put(open(TEST_IMAGE_PATH, 'r')) + t.image.put(open(TEST_IMAGE_PATH, 'rb')) t.save() t = TestImage.objects.first() diff --git a/tests/fields/file_tests.py b/tests/fields/file_tests.py index a39dadbf..44d28626 100644 --- a/tests/fields/file_tests.py +++ b/tests/fields/file_tests.py @@ -208,7 +208,7 @@ class FileTest(unittest.TestCase): Animal.drop_collection() marmot = Animal(genus='Marmota', family='Sciuridae') - marmot_photo = open(TEST_IMAGE_PATH, 'r') # Retrieve a photo from disk + marmot_photo = open(TEST_IMAGE_PATH, 'rb') # Retrieve a photo from disk marmot.photo.put(marmot_photo, content_type='image/jpeg', foo='bar') marmot.photo.close() marmot.save() @@ -251,7 +251,7 @@ class FileTest(unittest.TestCase): TestImage.drop_collection() t = TestImage() - t.image.put(open(TEST_IMAGE_PATH, 'r')) + t.image.put(open(TEST_IMAGE_PATH, 'rb')) t.save() t = TestImage.objects.first() @@ -274,7 +274,7 @@ class FileTest(unittest.TestCase): TestImage.drop_collection() t = TestImage() - t.image.put(open(TEST_IMAGE_PATH, 'r')) + t.image.put(open(TEST_IMAGE_PATH, 'rb')) t.save() t = TestImage.objects.first() @@ -297,7 +297,7 @@ class FileTest(unittest.TestCase): TestImage.drop_collection() t = TestImage() - t.image.put(open(TEST_IMAGE_PATH, 'r')) + t.image.put(open(TEST_IMAGE_PATH, 'rb')) t.save() t = TestImage.objects.first() @@ -320,7 +320,7 @@ class FileTest(unittest.TestCase): TestImage.drop_collection() t = TestImage() - t.image.put(open(TEST_IMAGE_PATH, 'r')) + t.image.put(open(TEST_IMAGE_PATH, 'rb')) t.save() t = TestImage.objects.first() From 156ca44a135a0bc832645b0292ca3ff0e02a8148 Mon Sep 17 00:00:00 2001 From: Ross Lawley Date: Mon, 28 Jan 2013 16:49:34 +0000 Subject: [PATCH 106/189] Doc fix thanks to @jabapyth (#206) --- AUTHORS | 1 + mongoengine/queryset/queryset.py | 3 ++- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/AUTHORS b/AUTHORS index fb895d7e..7efd3ad2 100644 --- a/AUTHORS +++ b/AUTHORS @@ -142,3 +142,4 @@ that much better: * lcya86 刘春洋 * Martin Alderete (https://github.com/malderete) * Nick Joyce + * Jared Forsyth diff --git a/mongoengine/queryset/queryset.py b/mongoengine/queryset/queryset.py index 4c664614..d313740b 100644 --- a/mongoengine/queryset/queryset.py +++ b/mongoengine/queryset/queryset.py @@ -467,7 +467,8 @@ class QuerySet(object): def with_id(self, object_id): """Retrieve the object matching the id provided. Uses `object_id` only - and raises InvalidQueryError if a filter has been applied. + and raises InvalidQueryError if a filter has been applied. Returns + `None` if no document exists with that id. :param object_id: the value for the id of the document to look up From 025e17701bd6b403ea5aec41f484918d3c718e4e Mon Sep 17 00:00:00 2001 From: Ross Lawley Date: Tue, 29 Jan 2013 10:33:13 +0000 Subject: [PATCH 107/189] Fixed inner queryset looping (#204) --- AUTHORS | 1 + docs/changelog.rst | 1 + mongoengine/queryset/queryset.py | 10 ++++---- tests/queryset/queryset.py | 41 ++++++++++++++++++++++++++++++++ tests/test_django.py | 18 ++++++++++++++ 5 files changed, 67 insertions(+), 4 deletions(-) diff --git a/AUTHORS b/AUTHORS index 7efd3ad2..903542ad 100644 --- a/AUTHORS +++ b/AUTHORS @@ -143,3 +143,4 @@ that much better: * Martin Alderete (https://github.com/malderete) * Nick Joyce * Jared Forsyth + * Kenneth Falck diff --git a/docs/changelog.rst b/docs/changelog.rst index 213e6fba..e9783cd0 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -47,6 +47,7 @@ Changes in 0.8.X - Added the "get_decoded" method to the MongoSession class (#216) - Fixed invalid choices error bubbling (#214) - Updated Save so it calls $set and $unset in a single operation (#211) +- Fixed inner queryset looping (#204) Changes in 0.7.9 ================ diff --git a/mongoengine/queryset/queryset.py b/mongoengine/queryset/queryset.py index d313740b..ba6134fc 100644 --- a/mongoengine/queryset/queryset.py +++ b/mongoengine/queryset/queryset.py @@ -112,8 +112,11 @@ class QuerySet(object): def __iter__(self): """Support iterator protocol""" - self.rewind() - return self + queryset = self + if queryset._iter: + queryset = self.clone() + queryset.rewind() + return queryset def __len__(self): return self.count() @@ -159,7 +162,6 @@ class QuerySet(object): .. versionchanged:: 0.6.13 Now doesnt modify the cursor """ - if self._iter: return '.. queryset mid-iteration ..' @@ -537,7 +539,7 @@ class QuerySet(object): c._cursor_obj = self._cursor_obj.clone() if self._slice: - c._cursor_obj[self._slice] + c._cursor[self._slice] return c diff --git a/tests/queryset/queryset.py b/tests/queryset/queryset.py index 3d5c659c..d5e80be5 100644 --- a/tests/queryset/queryset.py +++ b/tests/queryset/queryset.py @@ -3125,5 +3125,46 @@ class QuerySetTest(unittest.TestCase): Organization)) self.assertTrue(isinstance(qs.first().organization, Organization)) + def test_nested_queryset_iterator(self): + # Try iterating the same queryset twice, nested. + names = ['Alice', 'Bob', 'Chuck', 'David', 'Eric', 'Francis', 'George'] + + class User(Document): + name = StringField() + + def __unicode__(self): + return self.name + + User.drop_collection() + + for name in names: + User(name=name).save() + + users = User.objects.all().order_by('name') + + outer_count = 0 + inner_count = 0 + inner_total_count = 0 + + self.assertEqual(len(users), 7) + + for i, outer_user in enumerate(users): + self.assertEqual(outer_user.name, names[i]) + outer_count += 1 + inner_count = 0 + + # Calling len might disrupt the inner loop if there are bugs + self.assertEqual(len(users), 7) + + for j, inner_user in enumerate(users): + self.assertEqual(inner_user.name, names[j]) + inner_count += 1 + inner_total_count += 1 + + self.assertEqual(inner_count, 7) # inner loop should always be executed seven times + + self.assertEqual(outer_count, 7) # outer loop should be executed seven times total + self.assertEqual(inner_total_count, 7 * 7) # inner loop should be executed fourtynine times total + if __name__ == '__main__': unittest.main() diff --git a/tests/test_django.py b/tests/test_django.py index 563f4074..dceeba27 100644 --- a/tests/test_django.py +++ b/tests/test_django.py @@ -136,7 +136,25 @@ class QuerySetTest(unittest.TestCase): start = end - 1 self.assertEqual(t.render(Context(d)), u'%d:%d:' % (start, end)) + def test_nested_queryset_template_iterator(self): + # Try iterating the same queryset twice, nested, in a Django template. + names = ['A', 'B', 'C', 'D'] + class User(Document): + name = StringField() + + def __unicode__(self): + return self.name + + User.drop_collection() + + for name in names: + User(name=name).save() + + users = User.objects.all().order_by('name') + template = Template("{% for user in users %}{{ user.name }}{% ifequal forloop.counter 2 %} {% for inner_user in users %}{{ inner_user.name }}{% endfor %} {% endifequal %}{% endfor %}") + rendered = template.render(Context({'users': users})) + self.assertEqual(rendered, 'AB ABCD CD') class MongoDBSessionTest(SessionTestsMixin, unittest.TestCase): backend = SessionStore From 2e18199eb2cad168f07caefd68f4cdd5f3078135 Mon Sep 17 00:00:00 2001 From: hellysmile Date: Fri, 1 Feb 2013 04:17:16 +0200 Subject: [PATCH 108/189] Django sessions TTL support --- docs/django.rst | 3 +++ mongoengine/django/sessions.py | 14 +++++++++++--- 2 files changed, 14 insertions(+), 3 deletions(-) diff --git a/docs/django.rst b/docs/django.rst index 144baab5..58eadc60 100644 --- a/docs/django.rst +++ b/docs/django.rst @@ -45,6 +45,9 @@ into you settings module:: SESSION_ENGINE = 'mongoengine.django.sessions' +Django provides session cookie, which expires after ```SESSION_COOKIE_AGE``` seconds, but doesnt delete cookie at sessions backend, so ```'mongoengine.django.sessions'``` supports `mongodb TTL +`_. + .. versionadded:: 0.2.1 Storage diff --git a/mongoengine/django/sessions.py b/mongoengine/django/sessions.py index 810b6265..20e6b62e 100644 --- a/mongoengine/django/sessions.py +++ b/mongoengine/django/sessions.py @@ -31,9 +31,17 @@ class MongoSession(Document): else fields.DictField() expire_date = fields.DateTimeField() - meta = {'collection': MONGOENGINE_SESSION_COLLECTION, - 'db_alias': MONGOENGINE_SESSION_DB_ALIAS, - 'allow_inheritance': False} + meta = { + 'collection': MONGOENGINE_SESSION_COLLECTION, + 'db_alias': MONGOENGINE_SESSION_DB_ALIAS, + 'allow_inheritance': False, + 'indexes': [ + { + 'fields': ['expire_date'], + 'expireAfterSeconds': settings.SESSION_COOKIE_AGE + } + ] + } class SessionStore(SessionBase): From d6b4ca7a985c1ba4c25692d9731b8b6cd12e7fb0 Mon Sep 17 00:00:00 2001 From: hellysmile Date: Fri, 1 Feb 2013 04:19:55 +0200 Subject: [PATCH 109/189] Fix docs quotes --- docs/django.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/django.rst b/docs/django.rst index 58eadc60..b2acee72 100644 --- a/docs/django.rst +++ b/docs/django.rst @@ -45,7 +45,7 @@ into you settings module:: SESSION_ENGINE = 'mongoengine.django.sessions' -Django provides session cookie, which expires after ```SESSION_COOKIE_AGE``` seconds, but doesnt delete cookie at sessions backend, so ```'mongoengine.django.sessions'``` supports `mongodb TTL +Django provides session cookie, which expires after ```SESSION_COOKIE_AGE``` seconds, but doesnt delete cookie at sessions backend, so ``'mongoengine.django.sessions'`` supports `mongodb TTL `_. .. versionadded:: 0.2.1 From 3477b0107a227b88a1936fb23fa24ad7b26524a6 Mon Sep 17 00:00:00 2001 From: Loic Raucy Date: Tue, 26 Feb 2013 11:12:37 +0100 Subject: [PATCH 110/189] Added regression test for numerical string keys. --- tests/test_fields.py | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/tests/test_fields.py b/tests/test_fields.py index 28af1b23..47669007 100644 --- a/tests/test_fields.py +++ b/tests/test_fields.py @@ -965,6 +965,24 @@ class FieldTest(unittest.TestCase): doc = self.db.test.find_one() self.assertEqual(doc['x']['DICTIONARY_KEY']['i'], 2) + def test_mapfield_numerical_index(self): + """Ensure that MapField accept numeric strings as indexes.""" + class Embedded(EmbeddedDocument): + name = StringField() + + class Test(Document): + my_map = MapField(EmbeddedDocumentField(Embedded)) + + Test.drop_collection() + + test = Test() + test.my_map['1'] = Embedded(name='test') + test.save() + test.my_map['1'].name = 'test updated' + test.save() + + Test.drop_collection() + def test_map_field_lookup(self): """Ensure MapField lookups succeed on Fields without a lookup method""" From d0245bb5ba3b0f4ca4ce654fd199c167ce8c5e96 Mon Sep 17 00:00:00 2001 From: Loic Raucy Date: Tue, 26 Feb 2013 11:14:47 +0100 Subject: [PATCH 111/189] Fixed #238: dictfields handle numerical strings indexes. --- mongoengine/base.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/mongoengine/base.py b/mongoengine/base.py index 013afe78..4f302a87 100644 --- a/mongoengine/base.py +++ b/mongoengine/base.py @@ -1193,7 +1193,7 @@ class BaseDocument(object): for p in parts: if isinstance(d, DBRef): break - elif p.isdigit(): + elif isinstance(d, list) and p.isdigit(): d = d[int(p)] elif hasattr(d, 'get'): d = d.get(p) @@ -1224,7 +1224,7 @@ class BaseDocument(object): parts = path.split('.') db_field_name = parts.pop() for p in parts: - if p.isdigit(): + if isinstance(d, list) and p.isdigit(): d = d[int(p)] elif (hasattr(d, '__getattribute__') and not isinstance(d, dict)): From 0d2e84b16b286fd7724676794a68aa0610285afc Mon Sep 17 00:00:00 2001 From: benoitlouy Date: Thu, 28 Feb 2013 00:37:34 -0500 Subject: [PATCH 112/189] Fix for issue #237: clearing changed fields recursively in EmbeddedDocuments after saving a Document --- mongoengine/base.py | 16 ++++++++++++++++ mongoengine/document.py | 2 +- 2 files changed, 17 insertions(+), 1 deletion(-) diff --git a/mongoengine/base.py b/mongoengine/base.py index 013afe78..521756cd 100644 --- a/mongoengine/base.py +++ b/mongoengine/base.py @@ -1123,6 +1123,22 @@ class BaseDocument(object): key not in self._changed_fields): self._changed_fields.append(key) + def _clear_changed_fields(self): + self._changed_fields = [] + EmbeddedDocumentField = _import_class("EmbeddedDocumentField") + for field_name, field in self._fields.iteritems(): + if (isinstance(field, ComplexBaseField) and + isinstance(field.field, EmbeddedDocumentField)): + field_value = getattr(self, field_name, None) + if field_value: + for idx in (field_value if isinstance(field_value, dict) + else xrange(len(field_value))): + field_value[idx]._clear_changed_fields() + elif isinstance(field, EmbeddedDocumentField): + field_value = getattr(self, field_name, None) + if field_value: + field_value._clear_changed_fields() + def _get_changed_fields(self, key='', inspected=None): """Returns a list of all fields that have explicitly been changed. """ diff --git a/mongoengine/document.py b/mongoengine/document.py index 7b3afafb..0cf07a9e 100644 --- a/mongoengine/document.py +++ b/mongoengine/document.py @@ -269,7 +269,7 @@ class Document(BaseDocument): if id_field not in self._meta.get('shard_key', []): self[id_field] = self._fields[id_field].to_python(object_id) - self._changed_fields = [] + self._clear_changed_fields() self._created = False signals.post_save.send(self.__class__, document=self, created=created) return self From 43327ea4e1ca2041de248b438d77b392bd59f6c3 Mon Sep 17 00:00:00 2001 From: benoitlouy Date: Fri, 1 Mar 2013 07:38:28 -0500 Subject: [PATCH 113/189] Add testcase for issue #237 --- tests/test_document.py | 34 ++++++++++++++++++++++++++++++++++ 1 file changed, 34 insertions(+) diff --git a/tests/test_document.py b/tests/test_document.py index 3e8d8134..134ba34f 100644 --- a/tests/test_document.py +++ b/tests/test_document.py @@ -3368,6 +3368,40 @@ class DocumentTest(unittest.TestCase): } ) ]), "1,2") + def test_complex_nesting_document_and_embeddeddocument(self): + class Macro(EmbeddedDocument): + value = DynamicField(default="UNDEFINED") + + class Parameter(EmbeddedDocument): + macros = MapField(EmbeddedDocumentField(Macro)) + + def expand(self): + self.macros["test"] = Macro() + + class Node(Document): + parameters = MapField(EmbeddedDocumentField(Parameter)) + + def expand(self): + self.flattened_parameter = {} + for parameter_name, parameter in self.parameters.iteritems(): + parameter.expand() + + class System(Document): + name = StringField(required=True) + nodes = MapField(ReferenceField(Node, dbref=False)) + + def save(self, *args, **kwargs): + for node_name, node in self.nodes.iteritems(): + node.expand() + node.save(*args, **kwargs) + super(System, self).save(*args, **kwargs) + + system = System(name="system") + system.save() + system.nodes["node"] = Node() + system.save() + system.nodes["node"].parameters["param"] = Parameter() + system.save() class ValidatorErrorTest(unittest.TestCase): From 2d6ae1691204b1036ee5b7595721303a8533a103 Mon Sep 17 00:00:00 2001 From: Jaepil Jeong Date: Thu, 14 Mar 2013 23:25:22 +0900 Subject: [PATCH 114/189] Added LongField to support 64-bit integer type. --- mongoengine/fields.py | 38 ++++++++++++++++++++++++++++++++++++-- 1 file changed, 36 insertions(+), 2 deletions(-) diff --git a/mongoengine/fields.py b/mongoengine/fields.py index de484a1d..c40491b6 100644 --- a/mongoengine/fields.py +++ b/mongoengine/fields.py @@ -27,7 +27,7 @@ except ImportError: Image = None ImageOps = None -__all__ = ['StringField', 'IntField', 'FloatField', 'BooleanField', +__all__ = ['StringField', 'IntField', 'LongField', 'FloatField', 'BooleanField', 'DateTimeField', 'EmbeddedDocumentField', 'ListField', 'DictField', 'ObjectIdField', 'ReferenceField', 'ValidationError', 'MapField', 'DecimalField', 'ComplexDateTimeField', 'URLField', 'DynamicField', @@ -153,7 +153,7 @@ class EmailField(StringField): class IntField(BaseField): - """An integer field. + """An 32-bit integer field. """ def __init__(self, min_value=None, max_value=None, **kwargs): @@ -186,6 +186,40 @@ class IntField(BaseField): return int(value) +class LongField(BaseField): + """An 64-bit integer field. + """ + + def __init__(self, min_value=None, max_value=None, **kwargs): + self.min_value, self.max_value = min_value, max_value + super(LongField, self).__init__(**kwargs) + + def to_python(self, value): + try: + value = long(value) + except ValueError: + pass + return value + + def validate(self, value): + try: + value = long(value) + except: + self.error('%s could not be converted to long' % value) + + if self.min_value is not None and value < self.min_value: + self.error('Long value is too small') + + if self.max_value is not None and value > self.max_value: + self.error('Long value is too large') + + def prepare_query_value(self, op, value): + if value is None: + return value + + return long(value) + + class FloatField(BaseField): """An floating point number field. """ From e9464e32db4bcd10b332fb4764b3f9189e096f36 Mon Sep 17 00:00:00 2001 From: Jaepil Jeong Date: Thu, 14 Mar 2013 23:59:50 +0900 Subject: [PATCH 115/189] Added test cases for LongField. --- tests/test_fields.py | 28 ++++++++++++++++++++++++++++ 1 file changed, 28 insertions(+) diff --git a/tests/test_fields.py b/tests/test_fields.py index 28af1b23..3ceff8d9 100644 --- a/tests/test_fields.py +++ b/tests/test_fields.py @@ -144,6 +144,17 @@ class FieldTest(unittest.TestCase): self.assertEqual(1, TestDocument.objects(int_fld__ne=None).count()) self.assertEqual(1, TestDocument.objects(float_fld__ne=None).count()) + def test_long_ne_operator(self): + class TestDocument(Document): + long_fld = LongField() + + TestDocument.drop_collection() + + TestDocument(long_fld=None).save() + TestDocument(long_fld=1).save() + + self.assertEqual(1, TestDocument.objects(long_fld__ne=None).count()) + def test_object_id_validation(self): """Ensure that invalid values cannot be assigned to string fields. """ @@ -217,6 +228,23 @@ class FieldTest(unittest.TestCase): person.age = 'ten' self.assertRaises(ValidationError, person.validate) + def test_long_validation(self): + """Ensure that invalid values cannot be assigned to long fields. + """ + class TestDocument(Document): + value = LongField(min_value=0, max_value=110) + + doc = TestDocument() + doc.value = 50 + doc.validate() + + doc.value = -1 + self.assertRaises(ValidationError, doc.validate) + doc.age = 120 + self.assertRaises(ValidationError, doc.validate) + doc.age = 'ten' + self.assertRaises(ValidationError, doc.validate) + def test_float_validation(self): """Ensure that invalid values cannot be assigned to float fields. """ From 67182713d96233d3d2feb0f67d39ddaf1789c692 Mon Sep 17 00:00:00 2001 From: Jaepil Jeong Date: Fri, 15 Mar 2013 00:12:48 +0900 Subject: [PATCH 116/189] Fixed potential overflow error. --- mongoengine/fields.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/mongoengine/fields.py b/mongoengine/fields.py index c40491b6..d7c7cf1f 100644 --- a/mongoengine/fields.py +++ b/mongoengine/fields.py @@ -213,6 +213,9 @@ class LongField(BaseField): if self.max_value is not None and value > self.max_value: self.error('Long value is too large') + if value > 0x7FFFFFFFFFFFFFFF: + self.error('Long value is too large') + def prepare_query_value(self, op, value): if value is None: return value From a762a10decef552ccf6fbee413819d18bede05ea Mon Sep 17 00:00:00 2001 From: Jaepil Jeong Date: Mon, 18 Mar 2013 19:30:04 +0900 Subject: [PATCH 117/189] Revert "Fixed potential overflow error." This reverts commit 67182713d96233d3d2feb0f67d39ddaf1789c692. --- mongoengine/fields.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/mongoengine/fields.py b/mongoengine/fields.py index d7c7cf1f..c40491b6 100644 --- a/mongoengine/fields.py +++ b/mongoengine/fields.py @@ -213,9 +213,6 @@ class LongField(BaseField): if self.max_value is not None and value > self.max_value: self.error('Long value is too large') - if value > 0x7FFFFFFFFFFFFFFF: - self.error('Long value is too large') - def prepare_query_value(self, op, value): if value is None: return value From faf840f924c6d8432c88c70ae949c8b14f0d72ed Mon Sep 17 00:00:00 2001 From: Paul Swartz Date: Mon, 25 Mar 2013 10:59:31 -0400 Subject: [PATCH 118/189] only mark a field as changed if the value has changed Prevents spurious changes from being recorded. --- AUTHORS | 1 + mongoengine/base.py | 14 ++++++-------- 2 files changed, 7 insertions(+), 8 deletions(-) diff --git a/AUTHORS b/AUTHORS index 82a1dfa4..3d05cc41 100644 --- a/AUTHORS +++ b/AUTHORS @@ -128,3 +128,4 @@ that much better: * Peter Teichman * Jakub Kot * Jorge Bastida + * Paul Swartz \ No newline at end of file diff --git a/mongoengine/base.py b/mongoengine/base.py index f73af4cc..0e46248f 100644 --- a/mongoengine/base.py +++ b/mongoengine/base.py @@ -205,8 +205,12 @@ class BaseField(object): def __set__(self, instance, value): """Descriptor for assigning a value to a field in a document. """ - instance._data[self.name] = value - if instance._initialised: + changed = False + if (self.name not in instance._data or + instance._data[self.name] != value): + changed = True + instance._data[self.name] = value + if changed and instance._initialised: instance._mark_as_changed(self.name) def error(self, message="", errors=None, field_name=None): @@ -317,12 +321,6 @@ class ComplexBaseField(BaseField): return value - def __set__(self, instance, value): - """Descriptor for assigning a value to a field in a document. - """ - instance._data[self.name] = value - instance._mark_as_changed(self.name) - def to_python(self, value): """Convert a MongoDB-compatible type to a Python type. """ From 20cb0285f004def84e83b501af35672d544afe0f Mon Sep 17 00:00:00 2001 From: Paul Swartz Date: Wed, 27 Mar 2013 14:53:47 -0400 Subject: [PATCH 119/189] explicitly check for Document instances when dereferencing In particular, `collections.namedtuple` instances also have a `_fields` attribute which confuses the dereferencing. --- mongoengine/dereference.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/mongoengine/dereference.py b/mongoengine/dereference.py index 386dbf4b..b227ed36 100644 --- a/mongoengine/dereference.py +++ b/mongoengine/dereference.py @@ -33,7 +33,7 @@ class DeReference(object): self.max_depth = max_depth doc_type = None - if instance and instance._fields: + if instance and isinstance(instance, Document): doc_type = instance._fields.get(name) if hasattr(doc_type, 'field'): doc_type = doc_type.field @@ -84,7 +84,7 @@ class DeReference(object): # Recursively find dbreferences depth += 1 for k, item in iterator: - if hasattr(item, '_fields'): + if isinstance(item, Document): for field_name, field in item._fields.iteritems(): v = item._data.get(field_name, None) if isinstance(v, (DBRef)): @@ -187,7 +187,7 @@ class DeReference(object): if k in self.object_map and not is_list: data[k] = self.object_map[k] - elif hasattr(v, '_fields'): + elif isinstance(v, Document): for field_name, field in v._fields.iteritems(): v = data[k]._data.get(field_name, None) if isinstance(v, (DBRef)): From 32d5c0c946f68a9328e23c688ad37b4472033e1c Mon Sep 17 00:00:00 2001 From: Alice Bevan-McGregor Date: Wed, 3 Apr 2013 15:00:34 -0400 Subject: [PATCH 120/189] Store ordered list of field names, and return the ordered list when iterating a document instance. --- mongoengine/base.py | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/mongoengine/base.py b/mongoengine/base.py index f73af4cc..2f956ccf 100644 --- a/mongoengine/base.py +++ b/mongoengine/base.py @@ -560,8 +560,11 @@ class DocumentMetaclass(type): # Set _fields and db_field maps attrs['_fields'] = doc_fields - attrs['_db_field_map'] = dict([(k, getattr(v, 'db_field', k)) - for k, v in doc_fields.iteritems()]) + attrs['_fields_ordered'] = tuple(i[1] + for i in sorted((v.creation_counter, v.name) + for v in doc_fields.itervalues())) + attrs['_db_field_map'] = dict((k, getattr(v, 'db_field', k)) + for k, v in doc_fields.iteritems()) attrs['_reverse_db_field_map'] = dict( (v, k) for k, v in attrs['_db_field_map'].iteritems()) @@ -1302,7 +1305,10 @@ class BaseDocument(object): return value def __iter__(self): - return iter(self._fields) + if 'id' in self._fields and 'id' not in self._fields_ordered: + return iter(('id', ) + self._fields_ordered) + + return iter(self._fields_ordered) def __getitem__(self, name): """Dictionary-style field access, return a field's value if present. From fc1ce6d39bd5dac86111276eae26de407a194ce6 Mon Sep 17 00:00:00 2001 From: Alice Bevan-McGregor Date: Wed, 3 Apr 2013 15:00:51 -0400 Subject: [PATCH 121/189] Allow construction of document instances using positional arguments. --- mongoengine/base.py | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/mongoengine/base.py b/mongoengine/base.py index 2f956ccf..36d7c295 100644 --- a/mongoengine/base.py +++ b/mongoengine/base.py @@ -907,7 +907,17 @@ class BaseDocument(object): _dynamic_lock = True _initialised = False - def __init__(self, **values): + def __init__(self, *args, **values): + if args: + # Combine positional arguments with named arguments. + # We only want named arguments. + field = iter(self._fields_ordered) + for value in args: + name = next(field) + if name in values: + raise TypeError("Multiple values for keyword argument '" + name + "'") + values[name] = value + signals.pre_init.send(self.__class__, document=self, values=values) self._data = {} From 07d3e52e6a461eeca1251911e5440486e0832c2a Mon Sep 17 00:00:00 2001 From: Alice Bevan-McGregor Date: Wed, 3 Apr 2013 15:03:33 -0400 Subject: [PATCH 122/189] Tests for construction using positional parameters. --- tests/test_document.py | 22 ++++++++++++++++++++++ 1 file changed, 22 insertions(+) diff --git a/tests/test_document.py b/tests/test_document.py index 3e8d8134..00059fa0 100644 --- a/tests/test_document.py +++ b/tests/test_document.py @@ -1386,6 +1386,28 @@ class DocumentTest(unittest.TestCase): person = self.Person(name="Test User", age=30) self.assertEqual(person.name, "Test User") self.assertEqual(person.age, 30) + + def test_positional_creation(self): + """Ensure that document may be created using positional arguments. + """ + person = self.Person("Test User", 42) + self.assertEqual(person.name, "Test User") + self.assertEqual(person.age, 42) + + def test_mixed_creation(self): + """Ensure that document may be created using mixed arguments. + """ + person = self.Person("Test User", age=42) + self.assertEqual(person.name, "Test User") + self.assertEqual(person.age, 42) + + def test_bad_mixed_creation(self): + """Ensure that document gives correct error when duplicating arguments + """ + def construct_bad_instance(): + return self.Person("Test User", 42, name="Bad User") + + self.assertRaises(TypeError, construct_bad_instance) def test_to_dbref(self): """Ensure that you can get a dbref of a document""" From d58341d7ae8e499f127dfc8ad0ce20be70b4c043 Mon Sep 17 00:00:00 2001 From: Ross Lawley Date: Thu, 11 Apr 2013 13:15:17 +0000 Subject: [PATCH 123/189] Fix doc generation path (#230) Add Lukaszb to Authors --- AUTHORS | 1 + docs/conf.py | 4 ++-- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/AUTHORS b/AUTHORS index 903542ad..c64aaece 100644 --- a/AUTHORS +++ b/AUTHORS @@ -144,3 +144,4 @@ that much better: * Nick Joyce * Jared Forsyth * Kenneth Falck + * Lukasz Balcerzak diff --git a/docs/conf.py b/docs/conf.py index 62fa1505..3cfcef5d 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -16,7 +16,7 @@ import sys, os # If extensions (or modules to document with autodoc) are in another directory, # add these directories to sys.path here. If the directory is relative to the # documentation root, use os.path.abspath to make it absolute, like shown here. -sys.path.append(os.path.abspath('..')) +sys.path.insert(0, os.path.abspath('..')) # -- General configuration ----------------------------------------------------- @@ -38,7 +38,7 @@ master_doc = 'index' # General information about the project. project = u'MongoEngine' -copyright = u'2009-2012, MongoEngine Authors' +copyright = u'2009, MongoEngine Authors' # The version info for the project you're documenting, acts as replacement for # |version| and |release|, also used in various other places throughout the From b6977a88ea02fba91809c19276852bb691e1c6ca Mon Sep 17 00:00:00 2001 From: Ross Lawley Date: Mon, 15 Apr 2013 07:32:04 +0000 Subject: [PATCH 124/189] Explicitly check for Document instances when dereferencing (#261) --- AUTHORS | 3 ++- docs/changelog.rst | 1 + 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/AUTHORS b/AUTHORS index aeb672cb..991b10ac 100644 --- a/AUTHORS +++ b/AUTHORS @@ -131,4 +131,5 @@ that much better: * Aleksandr Sorokoumov * Yohan Graterol * bool-dev - * Russ Weeks \ No newline at end of file + * Russ Weeks + * Paul Swartz \ No newline at end of file diff --git a/docs/changelog.rst b/docs/changelog.rst index 65a5aaf1..2ff2f6b4 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -4,6 +4,7 @@ Changelog Changes in 0.7.10 ================= +- Explicitly check for Document instances when dereferencing (#261) - Fixed order_by chaining issue (#265) - Added dereference support for tuples (#250) - Resolve field name to db field name when using distinct(#260, #264, #269) From da7a8939dfce47d866641cdaa7ab29fc444db2c9 Mon Sep 17 00:00:00 2001 From: Ross Lawley Date: Mon, 15 Apr 2013 07:41:04 +0000 Subject: [PATCH 125/189] Also check if a TopLevelMetaclass instance (#261) --- mongoengine/dereference.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mongoengine/dereference.py b/mongoengine/dereference.py index 4df1fe87..997b7858 100644 --- a/mongoengine/dereference.py +++ b/mongoengine/dereference.py @@ -33,7 +33,7 @@ class DeReference(object): self.max_depth = max_depth doc_type = None - if instance and isinstance(instance, Document): + if instance and isinstance(instance, (Document, TopLevelDocumentMetaclass)): doc_type = instance._fields.get(name) if hasattr(doc_type, 'field'): doc_type = doc_type.field From 97a98f004530c078a3a0ce5e687808b3d250f362 Mon Sep 17 00:00:00 2001 From: Ross Lawley Date: Mon, 15 Apr 2013 07:52:04 +0000 Subject: [PATCH 126/189] Only mark a field as changed if the value has changed (#258) --- tests/test_dereference.py | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/tests/test_dereference.py b/tests/test_dereference.py index d7438d2f..09001540 100644 --- a/tests/test_dereference.py +++ b/tests/test_dereference.py @@ -185,8 +185,9 @@ class FieldTest(unittest.TestCase): # Migrate the data for g in Group.objects(): - g.author = g.author - g.members = g.members + # Explicitly mark as changed so resets + g._mark_as_changed('author') + g._mark_as_changed('members') g.save() group = Group.objects.first() @@ -997,7 +998,7 @@ class FieldTest(unittest.TestCase): msg = Message.objects.get(id=1) self.assertEqual(0, msg.comments[0].id) self.assertEqual(1, msg.comments[1].id) - + def test_tuples_as_tuples(self): """ Ensure that tuples remain tuples when they are @@ -1007,16 +1008,16 @@ class FieldTest(unittest.TestCase): class EnumField(BaseField): def __init__(self, **kwargs): super(EnumField,self).__init__(**kwargs) - + def to_mongo(self, value): return value - + def to_python(self, value): return tuple(value) - + class TestDoc(Document): items = ListField(EnumField()) - + TestDoc.drop_collection() tuples = [(100,'Testing')] doc = TestDoc() From 757ff31661282d0296e1927b694a7951bfee6e9a Mon Sep 17 00:00:00 2001 From: Ross Lawley Date: Mon, 15 Apr 2013 07:53:57 +0000 Subject: [PATCH 127/189] Updated changelog --- docs/changelog.rst | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/changelog.rst b/docs/changelog.rst index 2ff2f6b4..45b2e09a 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -4,6 +4,7 @@ Changelog Changes in 0.7.10 ================= +- Only mark a field as changed if the value has changed (#258) - Explicitly check for Document instances when dereferencing (#261) - Fixed order_by chaining issue (#265) - Added dereference support for tuples (#250) From b451cc567d1d3d461a1d15dace37fb79f88a5279 Mon Sep 17 00:00:00 2001 From: Ross Lawley Date: Mon, 15 Apr 2013 07:59:24 +0000 Subject: [PATCH 128/189] Return '_id' as the key for document.id in _data dictionary * Re #146 Conflicts: mongoengine/base.py --- mongoengine/base.py | 6 +++--- tests/test_document.py | 20 ++++++++++++++++++-- 2 files changed, 21 insertions(+), 5 deletions(-) diff --git a/mongoengine/base.py b/mongoengine/base.py index 25c3bbd5..7e6d0aa7 100644 --- a/mongoengine/base.py +++ b/mongoengine/base.py @@ -192,7 +192,7 @@ class BaseField(object): return self # Get value from document instance if available, if not use default - value = instance._data.get(self.name) + value = instance._data.get(self.name or self.db_field) if value is None: value = self.default @@ -207,9 +207,9 @@ class BaseField(object): """ changed = False if (self.name not in instance._data or - instance._data[self.name] != value): + instance._data[self.name or self.db_field] != value): changed = True - instance._data[self.name] = value + instance._data[self.name or self.db_field] = value if changed and instance._initialised: instance._mark_as_changed(self.name) diff --git a/tests/test_document.py b/tests/test_document.py index 3e8d8134..b5542c25 100644 --- a/tests/test_document.py +++ b/tests/test_document.py @@ -1529,8 +1529,10 @@ class DocumentTest(unittest.TestCase): doc.validate() keys = doc._data.keys() self.assertEqual(2, len(keys)) - self.assertTrue(None in keys) self.assertTrue('e' in keys) + # Ensure that the _id field has the right id + self.assertTrue('_id' in keys) + self.assertEqual(doc._data.get('_id'), doc.id) def test_save(self): """Ensure that a document may be saved in the database. @@ -3368,6 +3370,21 @@ class DocumentTest(unittest.TestCase): } ) ]), "1,2") + def test_data_contains_idfield(self): + """Ensure that asking for _data returns 'id' + """ + class Person(Document): + name = StringField() + + Person.drop_collection() + person = Person() + person.name = "Harry Potter" + person.save(cascade=False) + + person = Person.objects.first() + self.assertTrue('_id' in person._data.keys()) + self.assertEqual(person._data.get('_id'), person.id) + class ValidatorErrorTest(unittest.TestCase): @@ -3521,6 +3538,5 @@ class ValidatorErrorTest(unittest.TestCase): self.assertRaises(OperationError, change_shard_key) - if __name__ == '__main__': unittest.main() From 6186691259898c8e5d9da8983a74792d440622cb Mon Sep 17 00:00:00 2001 From: Ross Lawley Date: Mon, 15 Apr 2013 08:01:24 +0000 Subject: [PATCH 129/189] Updated changelog and AUTHORS (#255) --- AUTHORS | 3 ++- docs/changelog.rst | 1 + 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/AUTHORS b/AUTHORS index 991b10ac..aa223ed1 100644 --- a/AUTHORS +++ b/AUTHORS @@ -132,4 +132,5 @@ that much better: * Yohan Graterol * bool-dev * Russ Weeks - * Paul Swartz \ No newline at end of file + * Paul Swartz + * Sundar Raman \ No newline at end of file diff --git a/docs/changelog.rst b/docs/changelog.rst index 45b2e09a..90d4d66a 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -4,6 +4,7 @@ Changelog Changes in 0.7.10 ================= +- Added "_id" to _data dictionary (#255) - Only mark a field as changed if the value has changed (#258) - Explicitly check for Document instances when dereferencing (#261) - Fixed order_by chaining issue (#265) From d80b1a774934ae1d79e8b43ded5fbff8e86c9d8a Mon Sep 17 00:00:00 2001 From: Ross Lawley Date: Mon, 15 Apr 2013 08:03:51 +0000 Subject: [PATCH 130/189] Test clean up (#255) --- tests/test_document.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/tests/test_document.py b/tests/test_document.py index b5542c25..051dc2a3 100644 --- a/tests/test_document.py +++ b/tests/test_document.py @@ -3377,9 +3377,7 @@ class DocumentTest(unittest.TestCase): name = StringField() Person.drop_collection() - person = Person() - person.name = "Harry Potter" - person.save(cascade=False) + Person(name="Harry Potter").save() person = Person.objects.first() self.assertTrue('_id' in person._data.keys()) From add0b463f5fc191161814a0eebf5d3e133ede6b0 Mon Sep 17 00:00:00 2001 From: Daniil Sharou Date: Tue, 16 Apr 2013 21:12:57 +0400 Subject: [PATCH 131/189] fix UnicodeEncodeError for dbref Fix "UnicodeEncodeError: 'ascii' codec can't encode character ..." error in case dbref contains non-ascii characters --- mongoengine/dereference.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mongoengine/dereference.py b/mongoengine/dereference.py index 997b7858..e1b0a038 100644 --- a/mongoengine/dereference.py +++ b/mongoengine/dereference.py @@ -115,7 +115,7 @@ class DeReference(object): object_map = {} for col, dbrefs in self.reference_map.iteritems(): keys = object_map.keys() - refs = list(set([dbref for dbref in dbrefs if str(dbref) not in keys])) + refs = list(set([dbref for dbref in dbrefs if dbref.encode('utf-8') not in keys])) if hasattr(col, 'objects'): # We have a document class for the refs references = col.objects.in_bulk(refs) for key, doc in references.iteritems(): From cc0a2cbc6f20756c5edac3c0785ac132be68df45 Mon Sep 17 00:00:00 2001 From: Daniil Sharou Date: Tue, 16 Apr 2013 22:34:33 +0400 Subject: [PATCH 132/189] fix UnicodeEncodeError for dbref Fix "UnicodeEncodeError: 'ascii' codec can't encode character ..." error in case dbref contains non-ascii characters --- mongoengine/dereference.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mongoengine/dereference.py b/mongoengine/dereference.py index e1b0a038..ed756157 100644 --- a/mongoengine/dereference.py +++ b/mongoengine/dereference.py @@ -115,7 +115,7 @@ class DeReference(object): object_map = {} for col, dbrefs in self.reference_map.iteritems(): keys = object_map.keys() - refs = list(set([dbref for dbref in dbrefs if dbref.encode('utf-8') not in keys])) + refs = list(set([dbref for dbref in dbrefs if unicode(dbref).encode('utf-8') not in keys])) if hasattr(col, 'objects'): # We have a document class for the refs references = col.objects.in_bulk(refs) for key, doc in references.iteritems(): From a5257643596836f2f58fccce55157e83dc4b4f27 Mon Sep 17 00:00:00 2001 From: Ross Lawley Date: Tue, 16 Apr 2013 20:12:01 +0000 Subject: [PATCH 133/189] Fixed clearing _changed_fields for complex nested embedded documents (#237, #239, #242) --- AUTHORS | 3 ++- docs/changelog.rst | 1 + 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/AUTHORS b/AUTHORS index aa223ed1..1c72b31a 100644 --- a/AUTHORS +++ b/AUTHORS @@ -133,4 +133,5 @@ that much better: * bool-dev * Russ Weeks * Paul Swartz - * Sundar Raman \ No newline at end of file + * Sundar Raman + * Benoit Louy \ No newline at end of file diff --git a/docs/changelog.rst b/docs/changelog.rst index 90d4d66a..99aa49aa 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -4,6 +4,7 @@ Changelog Changes in 0.7.10 ================= +- Fixed clearing _changed_fields for complex nested embedded documents (#237, #239, #242) - Added "_id" to _data dictionary (#255) - Only mark a field as changed if the value has changed (#258) - Explicitly check for Document instances when dereferencing (#261) From 6fe074fb13606ad82bb392f523acf412f65a66df Mon Sep 17 00:00:00 2001 From: Ross Lawley Date: Tue, 16 Apr 2013 20:21:11 +0000 Subject: [PATCH 134/189] Fixed issue with numerical keys in MapField(EmbeddedDocumentField()) (#240) --- AUTHORS | 3 ++- docs/changelog.rst | 1 + 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/AUTHORS b/AUTHORS index 1c72b31a..01f6f22e 100644 --- a/AUTHORS +++ b/AUTHORS @@ -134,4 +134,5 @@ that much better: * Russ Weeks * Paul Swartz * Sundar Raman - * Benoit Louy \ No newline at end of file + * Benoit Louy + * lraucy \ No newline at end of file diff --git a/docs/changelog.rst b/docs/changelog.rst index 99aa49aa..269a4228 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -4,6 +4,7 @@ Changelog Changes in 0.7.10 ================= +- Fixed issue with numerical keys in MapField(EmbeddedDocumentField()) (#240) - Fixed clearing _changed_fields for complex nested embedded documents (#237, #239, #242) - Added "_id" to _data dictionary (#255) - Only mark a field as changed if the value has changed (#258) From d02de0798f8ee0b04bbb33e2ca0c71651f32af09 Mon Sep 17 00:00:00 2001 From: Ross Lawley Date: Tue, 16 Apr 2013 20:26:23 +0000 Subject: [PATCH 135/189] Documentation fix explaining adding a dummy backend for django (#172) --- docs/django.rst | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/docs/django.rst b/docs/django.rst index 144baab5..58597469 100644 --- a/docs/django.rst +++ b/docs/django.rst @@ -10,6 +10,16 @@ In your **settings.py** file, ignore the standard database settings (unless you also plan to use the ORM in your project), and instead call :func:`~mongoengine.connect` somewhere in the settings module. +.. note :: + If you are not using another Database backend make sure you add a dummy + backend, by adding the following to ``settings.py``:: + + DATABASES = { + 'default': { + 'ENGINE': 'django.db.backends.dummy' + } + } + Authentication ============== MongoEngine includes a Django authentication backend, which uses MongoDB. The From 1f9ec0c888f6a5ea5a6e83267d361b917e7df104 Mon Sep 17 00:00:00 2001 From: Ross Lawley Date: Tue, 16 Apr 2013 20:30:40 +0000 Subject: [PATCH 136/189] Added Django sessions TTL support (#224) --- AUTHORS | 3 ++- docs/changelog.rst | 1 + 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/AUTHORS b/AUTHORS index 01f6f22e..a689ec6e 100644 --- a/AUTHORS +++ b/AUTHORS @@ -135,4 +135,5 @@ that much better: * Paul Swartz * Sundar Raman * Benoit Louy - * lraucy \ No newline at end of file + * lraucy + * hellysmile \ No newline at end of file diff --git a/docs/changelog.rst b/docs/changelog.rst index 269a4228..df15855c 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -4,6 +4,7 @@ Changelog Changes in 0.7.10 ================= +- Added Django sessions TTL support (#224) - Fixed issue with numerical keys in MapField(EmbeddedDocumentField()) (#240) - Fixed clearing _changed_fields for complex nested embedded documents (#237, #239, #242) - Added "_id" to _data dictionary (#255) From 3a85422e8f332484984a0b519f8cd614b7445148 Mon Sep 17 00:00:00 2001 From: Ross Lawley Date: Tue, 16 Apr 2013 20:35:29 +0000 Subject: [PATCH 137/189] Added 64-bit integer support (#251) --- AUTHORS | 3 ++- docs/changelog.rst | 1 + 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/AUTHORS b/AUTHORS index a689ec6e..370f082c 100644 --- a/AUTHORS +++ b/AUTHORS @@ -136,4 +136,5 @@ that much better: * Sundar Raman * Benoit Louy * lraucy - * hellysmile \ No newline at end of file + * hellysmile + * Jaepil Jeong \ No newline at end of file diff --git a/docs/changelog.rst b/docs/changelog.rst index df15855c..8ff95b56 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -4,6 +4,7 @@ Changelog Changes in 0.7.10 ================= +- Added 64-bit integer support (#251) - Added Django sessions TTL support (#224) - Fixed issue with numerical keys in MapField(EmbeddedDocumentField()) (#240) - Fixed clearing _changed_fields for complex nested embedded documents (#237, #239, #242) From b562e209d17afda19413c2971e69e2633f1f09e5 Mon Sep 17 00:00:00 2001 From: Ross Lawley Date: Tue, 16 Apr 2013 20:46:02 +0000 Subject: [PATCH 138/189] Updated EmailField length to support long domains (#243) --- docs/changelog.rst | 1 + mongoengine/fields.py | 2 +- tests/test_fields.py | 18 ++++++++++++++++-- 3 files changed, 18 insertions(+), 3 deletions(-) diff --git a/docs/changelog.rst b/docs/changelog.rst index 8ff95b56..91d4da04 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -4,6 +4,7 @@ Changelog Changes in 0.7.10 ================= +- Updated EmailField length to support long domains (#243) - Added 64-bit integer support (#251) - Added Django sessions TTL support (#224) - Fixed issue with numerical keys in MapField(EmbeddedDocumentField()) (#240) diff --git a/mongoengine/fields.py b/mongoengine/fields.py index c40491b6..2d1ee71c 100644 --- a/mongoengine/fields.py +++ b/mongoengine/fields.py @@ -143,7 +143,7 @@ class EmailField(StringField): EMAIL_REGEX = re.compile( r"(^[-!#$%&'*+/=?^_`{}|~0-9A-Z]+(\.[-!#$%&'*+/=?^_`{}|~0-9A-Z]+)*" # dot-atom r'|^"([\001-\010\013\014\016-\037!#-\[\]-\177]|\\[\001-011\013\014\016-\177])*"' # quoted-string - r')@(?:[A-Z0-9](?:[A-Z0-9-]{0,61}[A-Z0-9])?\.)+[A-Z]{2,6}\.?$', re.IGNORECASE # domain + r')@(?:[A-Z0-9](?:[A-Z0-9-]{0,253}[A-Z0-9])?\.)+[A-Z]{2,6}\.?$', re.IGNORECASE # domain ) def validate(self, value): diff --git a/tests/test_fields.py b/tests/test_fields.py index 1e693e8b..55ac6fb7 100644 --- a/tests/test_fields.py +++ b/tests/test_fields.py @@ -2371,12 +2371,26 @@ class FieldTest(unittest.TestCase): self.assertTrue(1 in error_dict['comments']) self.assertTrue('content' in error_dict['comments'][1]) self.assertEqual(error_dict['comments'][1]['content'], - u'Field is required') - + u'Field is required') post.comments[1].content = 'here we go' post.validate() + def test_email_field(self): + class User(Document): + email = EmailField() + + user = User(email="ross@example.com") + self.assertTrue(user.validate() is None) + + user = User(email=("Kofq@rhom0e4klgauOhpbpNdogawnyIKvQS0wk2mjqrgGQ5S" + "ucictfqpdkK9iS1zeFw8sg7s7cwAF7suIfUfeyueLpfosjn3" + "aJIazqqWkm7.net")) + self.assertTrue(user.validate() is None) + + user = User(email='me@localhost') + self.assertRaises(ValidationError, user.validate) + def test_email_field_honors_regex(self): class User(Document): email = EmailField(regex=r'\w+@example.com') From b4d87d91282247be21d46107935f31891a1802d6 Mon Sep 17 00:00:00 2001 From: Ross Lawley Date: Tue, 16 Apr 2013 20:50:34 +0000 Subject: [PATCH 139/189] Updated changelog --- docs/changelog.rst | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/changelog.rst b/docs/changelog.rst index 91d4da04..0443f74e 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -4,6 +4,7 @@ Changelog Changes in 0.7.10 ================= +- Allow construction using positional parameters (#268) - Updated EmailField length to support long domains (#243) - Added 64-bit integer support (#251) - Added Django sessions TTL support (#224) From c2d77f51bba6e75d427f11845ad61ccca85b9803 Mon Sep 17 00:00:00 2001 From: daniil Date: Wed, 17 Apr 2013 12:14:07 +0400 Subject: [PATCH 140/189] test for #278 issue --- tests/test_dereference.py | 24 ++++++++++++++++++++++++ 1 file changed, 24 insertions(+) diff --git a/tests/test_dereference.py b/tests/test_dereference.py index 09001540..8caefd39 100644 --- a/tests/test_dereference.py +++ b/tests/test_dereference.py @@ -1,3 +1,4 @@ +# -*- coding: utf-8 -*- from __future__ import with_statement import unittest @@ -1029,3 +1030,26 @@ class FieldTest(unittest.TestCase): self.assertTrue(tuple(x.items[0]) in tuples) self.assertTrue(x.items[0] in tuples) + def test_non_ascii_pk(self): + """ + Ensure that dbref conversion to string does not fail when + non-ascii characters are used in primary key + """ + class Brand(Document): + title = StringField(max_length=255, primary_key=True) + + class BrandGroup(Document): + title = StringField(max_length=255, primary_key=True) + brands = SortedListField(ReferenceField("Brand", dbref=True)) + + Brand.drop_collection() + BrandGroup.drop_collection() + + brand1 = Brand(title="Moschino").save() + brand2 = Brand(title=u"Денис Симачёв").save() + + BrandGroup(title="top_brands", brands=[brand1, brand2]).save() + brand_groups = BrandGroup.objects().all() + + self.assertEqual(2, len([brand for bg in brand_groups for brand in bg.brands])) + From 420376d036708a1077b535ca7dfaa9471c168496 Mon Sep 17 00:00:00 2001 From: Ross Lawley Date: Wed, 17 Apr 2013 14:27:33 +0000 Subject: [PATCH 141/189] Merge fixes --- mongoengine/base/document.py | 4 ++-- mongoengine/queryset/queryset.py | 8 ++++++++ tests/queryset/queryset.py | 20 ++++++++++++-------- 3 files changed, 22 insertions(+), 10 deletions(-) diff --git a/mongoengine/base/document.py b/mongoengine/base/document.py index ebb34101..7ec672fe 100644 --- a/mongoengine/base/document.py +++ b/mongoengine/base/document.py @@ -30,7 +30,7 @@ class BaseDocument(object): _dynamic_lock = True _initialised = False - def __init__(self, __auto_convert=True, *args, **values): + def __init__(self, *args, **values): """ Initialise a document or embedded document @@ -46,7 +46,7 @@ class BaseDocument(object): if name in values: raise TypeError("Multiple values for keyword argument '" + name + "'") values[name] = value - + __auto_convert = values.pop("__auto_convert", True) signals.pre_init.send(self.__class__, document=self, values=values) self._data = {} diff --git a/mongoengine/queryset/queryset.py b/mongoengine/queryset/queryset.py index c299190f..28a96189 100644 --- a/mongoengine/queryset/queryset.py +++ b/mongoengine/queryset/queryset.py @@ -1138,8 +1138,13 @@ class QuerySet(object): if self._hint != -1: self._cursor_obj.hint(self._hint) + return self._cursor_obj + def __deepcopy__(self, memo): + """Essential for chained queries with ReferenceFields involved""" + return self.clone() + @property def _query(self): if self._mongo_query is None: @@ -1302,6 +1307,9 @@ class QuerySet(object): except: pass key_list.append((key, direction)) + + if self._cursor_obj: + self._cursor_obj.sort(key_list) return key_list def _get_scalar(self, doc): diff --git a/tests/queryset/queryset.py b/tests/queryset/queryset.py index da0e89ab..1dccdb14 100644 --- a/tests/queryset/queryset.py +++ b/tests/queryset/queryset.py @@ -249,6 +249,10 @@ class QuerySetTest(unittest.TestCase): self.assertEqual(list(A.objects.none()), []) self.assertEqual(list(A.objects.none().all()), []) + def test_chaining(self): + class A(Document): + s = StringField() + class B(Document): ref = ReferenceField(A) boolfield = BooleanField(default=False) @@ -282,7 +286,7 @@ class QuerySetTest(unittest.TestCase): write_options = {"fsync": True} author, created = self.Person.objects.get_or_create( - name='Test User', write_options=write_options) + name='Test User', write_options=write_options) author.save(write_options=write_options) self.Person.objects.update(set__name='Ross', @@ -1475,7 +1479,6 @@ class QuerySetTest(unittest.TestCase): BlogPost.drop_collection() - def test_set_list_embedded_documents(self): class Author(EmbeddedDocument): @@ -1533,11 +1536,11 @@ class QuerySetTest(unittest.TestCase): BlogPost.drop_collection() blog_post_3 = BlogPost(title="Blog Post #3", - published_date=datetime(2010, 1, 6, 0, 0 ,0)) + published_date=datetime(2010, 1, 6, 0, 0, 0)) blog_post_2 = BlogPost(title="Blog Post #2", - published_date=datetime(2010, 1, 5, 0, 0 ,0)) + published_date=datetime(2010, 1, 5, 0, 0, 0)) blog_post_4 = BlogPost(title="Blog Post #4", - published_date=datetime(2010, 1, 7, 0, 0 ,0)) + published_date=datetime(2010, 1, 7, 0, 0, 0)) blog_post_1 = BlogPost(title="Blog Post #1", published_date=None) blog_post_3.save() @@ -1563,11 +1566,11 @@ class QuerySetTest(unittest.TestCase): BlogPost.drop_collection() blog_post_1 = BlogPost(title="A", - published_date=datetime(2010, 1, 6, 0, 0 ,0)) + published_date=datetime(2010, 1, 6, 0, 0, 0)) blog_post_2 = BlogPost(title="B", - published_date=datetime(2010, 1, 6, 0, 0 ,0)) + published_date=datetime(2010, 1, 6, 0, 0, 0)) blog_post_3 = BlogPost(title="C", - published_date=datetime(2010, 1, 7, 0, 0 ,0)) + published_date=datetime(2010, 1, 7, 0, 0, 0)) blog_post_2.save() blog_post_3.save() @@ -1604,6 +1607,7 @@ class QuerySetTest(unittest.TestCase): qs = self.Person.objects.all().limit(10) qs = qs.order_by('-age') + ages = [p.age for p in qs] self.assertEqual(ages, [40, 30, 20]) From ec639cd6e9acb10f43228487b634e74f11b54bcd Mon Sep 17 00:00:00 2001 From: Nicolas Cortot Date: Wed, 17 Apr 2013 16:19:53 +0200 Subject: [PATCH 142/189] Fix datetime call in UserManager --- mongoengine/django/auth.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mongoengine/django/auth.py b/mongoengine/django/auth.py index 024ae9c3..d22f0865 100644 --- a/mongoengine/django/auth.py +++ b/mongoengine/django/auth.py @@ -142,7 +142,7 @@ class UserManager(models.Manager): """ Creates and saves a User with the given username, e-mail and password. """ - now = datetime.datetime.now() + now = datetime_now() # Normalize the address by lowercasing the domain part of the email # address. From dcf3c86dce243962e9edc232c0ac4bea63f81ec5 Mon Sep 17 00:00:00 2001 From: Ross Lawley Date: Wed, 17 Apr 2013 15:07:57 +0000 Subject: [PATCH 143/189] Using "id" in data not "_id" as its a mapping of fieldnames (#255) --- docs/changelog.rst | 2 +- mongoengine/base.py | 13 +++++++------ tests/test_document.py | 16 ++++++++-------- 3 files changed, 16 insertions(+), 15 deletions(-) diff --git a/docs/changelog.rst b/docs/changelog.rst index 0443f74e..14964955 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -10,7 +10,7 @@ Changes in 0.7.10 - Added Django sessions TTL support (#224) - Fixed issue with numerical keys in MapField(EmbeddedDocumentField()) (#240) - Fixed clearing _changed_fields for complex nested embedded documents (#237, #239, #242) -- Added "_id" to _data dictionary (#255) +- Added "id" back to _data dictionary (#255) - Only mark a field as changed if the value has changed (#258) - Explicitly check for Document instances when dereferencing (#261) - Fixed order_by chaining issue (#265) diff --git a/mongoengine/base.py b/mongoengine/base.py index 0f136435..a7eb17b5 100644 --- a/mongoengine/base.py +++ b/mongoengine/base.py @@ -192,7 +192,7 @@ class BaseField(object): return self # Get value from document instance if available, if not use default - value = instance._data.get(self.name or self.db_field) + value = instance._data.get(self.name) if value is None: value = self.default @@ -207,9 +207,9 @@ class BaseField(object): """ changed = False if (self.name not in instance._data or - instance._data[self.name or self.db_field] != value): + instance._data[self.name] != value): changed = True - instance._data[self.name or self.db_field] = value + instance._data[self.name] = value if changed and instance._initialised: instance._mark_as_changed(self.name) @@ -825,6 +825,7 @@ class TopLevelDocumentMetaclass(DocumentMetaclass): if not new_class._meta.get('id_field'): new_class._meta['id_field'] = 'id' new_class._fields['id'] = ObjectIdField(db_field='_id') + new_class._fields['id'].name = 'id' new_class.id = new_class._fields['id'] # Merge in exceptions with parent hierarchy @@ -915,7 +916,7 @@ class BaseDocument(object): if name in values: raise TypeError("Multiple values for keyword argument '" + name + "'") values[name] = value - + signals.pre_init.send(self.__class__, document=self, values=values) self._data = {} @@ -1138,7 +1139,7 @@ class BaseDocument(object): self._changed_fields = [] EmbeddedDocumentField = _import_class("EmbeddedDocumentField") for field_name, field in self._fields.iteritems(): - if (isinstance(field, ComplexBaseField) and + if (isinstance(field, ComplexBaseField) and isinstance(field.field, EmbeddedDocumentField)): field_value = getattr(self, field_name, None) if field_value: @@ -1331,7 +1332,7 @@ class BaseDocument(object): def __iter__(self): if 'id' in self._fields and 'id' not in self._fields_ordered: return iter(('id', ) + self._fields_ordered) - + return iter(self._fields_ordered) def __getitem__(self, name): diff --git a/tests/test_document.py b/tests/test_document.py index ce007033..5b30a964 100644 --- a/tests/test_document.py +++ b/tests/test_document.py @@ -1386,27 +1386,27 @@ class DocumentTest(unittest.TestCase): person = self.Person(name="Test User", age=30) self.assertEqual(person.name, "Test User") self.assertEqual(person.age, 30) - + def test_positional_creation(self): """Ensure that document may be created using positional arguments. """ person = self.Person("Test User", 42) self.assertEqual(person.name, "Test User") self.assertEqual(person.age, 42) - + def test_mixed_creation(self): """Ensure that document may be created using mixed arguments. """ person = self.Person("Test User", age=42) self.assertEqual(person.name, "Test User") self.assertEqual(person.age, 42) - + def test_bad_mixed_creation(self): """Ensure that document gives correct error when duplicating arguments """ def construct_bad_instance(): return self.Person("Test User", 42, name="Bad User") - + self.assertRaises(TypeError, construct_bad_instance) def test_to_dbref(self): @@ -1553,8 +1553,8 @@ class DocumentTest(unittest.TestCase): self.assertEqual(2, len(keys)) self.assertTrue('e' in keys) # Ensure that the _id field has the right id - self.assertTrue('_id' in keys) - self.assertEqual(doc._data.get('_id'), doc.id) + self.assertTrue('id' in keys) + self.assertEqual(doc._data.get('id'), doc.id) def test_save(self): """Ensure that a document may be saved in the database. @@ -3402,8 +3402,8 @@ class DocumentTest(unittest.TestCase): Person(name="Harry Potter").save() person = Person.objects.first() - self.assertTrue('_id' in person._data.keys()) - self.assertEqual(person._data.get('_id'), person.id) + self.assertTrue('id' in person._data.keys()) + self.assertEqual(person._data.get('id'), person.id) def test_complex_nesting_document_and_embedded_document(self): From 03bfd018621233629467d131c8af52764c05e559 Mon Sep 17 00:00:00 2001 From: Ross Lawley Date: Wed, 17 Apr 2013 15:54:32 +0000 Subject: [PATCH 144/189] Updated field iteration for py2.5 --- mongoengine/base.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mongoengine/base.py b/mongoengine/base.py index a7eb17b5..1e1a0212 100644 --- a/mongoengine/base.py +++ b/mongoengine/base.py @@ -912,7 +912,7 @@ class BaseDocument(object): # We only want named arguments. field = iter(self._fields_ordered) for value in args: - name = next(field) + name = field.next() if name in values: raise TypeError("Multiple values for keyword argument '" + name + "'") values[name] = value From 073091a06e823b95234679d8f7a349714bf67bae Mon Sep 17 00:00:00 2001 From: Nicolas Cortot Date: Wed, 17 Apr 2013 21:45:54 +0200 Subject: [PATCH 145/189] Do not fail on delete() when blinker is not available --- mongoengine/queryset/queryset.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mongoengine/queryset/queryset.py b/mongoengine/queryset/queryset.py index 28a96189..15c8e634 100644 --- a/mongoengine/queryset/queryset.py +++ b/mongoengine/queryset/queryset.py @@ -366,7 +366,7 @@ class QuerySet(object): queryset = self.clone() doc = queryset._document - has_delete_signal = ( + has_delete_signal = signals.signals_available and ( signals.pre_delete.has_receivers_for(self._document) or signals.post_delete.has_receivers_for(self._document)) From 3f4992329831d063448c18d7a3005204e289eb65 Mon Sep 17 00:00:00 2001 From: Ross Lawley Date: Thu, 18 Apr 2013 13:21:36 +0000 Subject: [PATCH 146/189] Update AUTHORS and changelog (#278) --- AUTHORS | 3 ++- docs/changelog.rst | 1 + 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/AUTHORS b/AUTHORS index 370f082c..1262669d 100644 --- a/AUTHORS +++ b/AUTHORS @@ -137,4 +137,5 @@ that much better: * Benoit Louy * lraucy * hellysmile - * Jaepil Jeong \ No newline at end of file + * Jaepil Jeong + * Daniil Sharou diff --git a/docs/changelog.rst b/docs/changelog.rst index 14964955..386daa56 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -4,6 +4,7 @@ Changelog Changes in 0.7.10 ================= +- Fix UnicodeEncodeError for dbref (#278) - Allow construction using positional parameters (#268) - Updated EmailField length to support long domains (#243) - Added 64-bit integer support (#251) From 11085863033b71aa4026dcd3fe2c26adea54f8a4 Mon Sep 17 00:00:00 2001 From: Ross Lawley Date: Thu, 18 Apr 2013 13:26:35 +0000 Subject: [PATCH 147/189] Updated queryset --- tests/queryset/queryset.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tests/queryset/queryset.py b/tests/queryset/queryset.py index 1dccdb14..37670b0a 100644 --- a/tests/queryset/queryset.py +++ b/tests/queryset/queryset.py @@ -389,6 +389,8 @@ class QuerySetTest(unittest.TestCase): self.assertEqual(post.comments[1].by, 'jane') self.assertEqual(post.comments[1].votes, 8) + def test_update_using_positional_operator_matches_first(self): + # Currently the $ operator only applies to the first matched item in # the query From 5de4812477841e6458752562b0a936126c0ced6c Mon Sep 17 00:00:00 2001 From: Ross Lawley Date: Thu, 18 Apr 2013 13:38:36 +0000 Subject: [PATCH 148/189] Updating AUTHORS (#283) --- AUTHORS | 2 ++ 1 file changed, 2 insertions(+) diff --git a/AUTHORS b/AUTHORS index b00b0692..e388a04a 100644 --- a/AUTHORS +++ b/AUTHORS @@ -156,3 +156,5 @@ that much better: * Jared Forsyth * Kenneth Falck * Lukasz Balcerzak + * Nicolas Cortot + From 6dcd7006d05d7ce749786a9680496c5b221a85fe Mon Sep 17 00:00:00 2001 From: Ross Lawley Date: Fri, 19 Apr 2013 12:47:19 +0000 Subject: [PATCH 149/189] Fix test --- tests/test_dereference.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/test_dereference.py b/tests/test_dereference.py index 8caefd39..a3517d23 100644 --- a/tests/test_dereference.py +++ b/tests/test_dereference.py @@ -1020,7 +1020,7 @@ class FieldTest(unittest.TestCase): items = ListField(EnumField()) TestDoc.drop_collection() - tuples = [(100,'Testing')] + tuples = [(100, 'Testing')] doc = TestDoc() doc.items = tuples doc.save() @@ -1040,7 +1040,7 @@ class FieldTest(unittest.TestCase): class BrandGroup(Document): title = StringField(max_length=255, primary_key=True) - brands = SortedListField(ReferenceField("Brand", dbref=True)) + brands = ListField(ReferenceField("Brand", dbref=True)) Brand.drop_collection() BrandGroup.drop_collection() From e3600ef4de6ab13090f18a6155d5f515b7ae2e8d Mon Sep 17 00:00:00 2001 From: Ross Lawley Date: Fri, 19 Apr 2013 12:53:46 +0000 Subject: [PATCH 150/189] Updated version --- mongoengine/__init__.py | 2 +- python-mongoengine.spec | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/mongoengine/__init__.py b/mongoengine/__init__.py index b67512d7..64adb92e 100644 --- a/mongoengine/__init__.py +++ b/mongoengine/__init__.py @@ -12,7 +12,7 @@ from signals import * __all__ = (document.__all__ + fields.__all__ + connection.__all__ + queryset.__all__ + signals.__all__) -VERSION = (0, 7, 9) +VERSION = (0, 7, 10) def get_version(): diff --git a/python-mongoengine.spec b/python-mongoengine.spec index b1ec3361..ed4a8721 100644 --- a/python-mongoengine.spec +++ b/python-mongoengine.spec @@ -5,7 +5,7 @@ %define srcname mongoengine Name: python-%{srcname} -Version: 0.7.9 +Version: 0.7.10 Release: 1%{?dist} Summary: A Python Document-Object Mapper for working with MongoDB From 6affbbe86570a47d8110dd9c967ea2babe856bec Mon Sep 17 00:00:00 2001 From: Ross Lawley Date: Fri, 19 Apr 2013 13:08:46 +0000 Subject: [PATCH 151/189] Update changelog location --- python-mongoengine.spec | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/python-mongoengine.spec b/python-mongoengine.spec index ed4a8721..eaf478dc 100644 --- a/python-mongoengine.spec +++ b/python-mongoengine.spec @@ -51,4 +51,4 @@ rm -rf $RPM_BUILD_ROOT # %{python_sitearch}/* %changelog -* See: http://readthedocs.org/docs/mongoengine-odm/en/latest/changelog.html \ No newline at end of file +* See: http://docs.mongoengine.org/en/latest/changelog.html \ No newline at end of file From 485047f20b1f9b3a52ce0b11a426505ad03437eb Mon Sep 17 00:00:00 2001 From: Nicolas Cortot Date: Wed, 17 Apr 2013 21:38:11 +0200 Subject: [PATCH 152/189] Custom User Model for Django 1.5 --- docs/django.rst | 36 +++++++++ mongoengine/django/auth.py | 3 + mongoengine/django/mongo_auth/__init__.py | 0 mongoengine/django/mongo_auth/models.py | 90 +++++++++++++++++++++++ tests/test_django.py | 52 ++++++++++++- 5 files changed, 180 insertions(+), 1 deletion(-) create mode 100644 mongoengine/django/mongo_auth/__init__.py create mode 100644 mongoengine/django/mongo_auth/models.py diff --git a/docs/django.rst b/docs/django.rst index 6f27b902..2d58f95e 100644 --- a/docs/django.rst +++ b/docs/django.rst @@ -42,6 +42,42 @@ The :mod:`~mongoengine.django.auth` module also contains a .. versionadded:: 0.1.3 +Custom User model +================= +Django 1.5 introduced `Custom user Models +` +which can be used as an alternative the Mongoengine authentication backend. + +The main advantage of this option is that other components relying on +:mod:`django.contrib.auth` and supporting the new swappable user model are more +likely to work. For example, you can use the ``createsuperuser`` management +command as usual. + +To enable the custom User model in Django, add ``'mongoengine.django.mongo_auth'`` +in your ``INSTALLED_APPS`` and set ``'mongo_auth.MongoUser'`` as the custom user +user model to use. In your **settings.py** file you will have:: + + INSTALLED_APPS = ( + ... + 'django.contrib.auth', + 'mongoengine.django.mongo_auth', + ... + ) + + AUTH_USER_MODEL = 'mongo_auth.MongoUser' + +An additional ``MONGOENGINE_USER_DOCUMENT`` setting enables you to replace the +:class:`~mongoengine.django.auth.User` class with another class of your choice:: + + MONGOENGINE_USER_DOCUMENT = 'mongoengine.django.auth.User' + +The custom :class:`User` must be a :class:`~mongoengine.Document` class, but +otherwise has the same requirements as a standard custom user model, +as specified in the `Django Documentation +`. +In particular, the custom class must define :attr:`USERNAME_FIELD` and +:attr:`REQUIRED_FIELDS` attributes. + Sessions ======== Django allows the use of different backend stores for its sessions. MongoEngine diff --git a/mongoengine/django/auth.py b/mongoengine/django/auth.py index d22f0865..371f0e30 100644 --- a/mongoengine/django/auth.py +++ b/mongoengine/django/auth.py @@ -209,6 +209,9 @@ class User(Document): date_joined = DateTimeField(default=datetime_now, verbose_name=_('date joined')) + USERNAME_FIELD = 'username' + REQUIRED_FIELDS = ['email'] + meta = { 'allow_inheritance': True, 'indexes': [ diff --git a/mongoengine/django/mongo_auth/__init__.py b/mongoengine/django/mongo_auth/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/mongoengine/django/mongo_auth/models.py b/mongoengine/django/mongo_auth/models.py new file mode 100644 index 00000000..9629e644 --- /dev/null +++ b/mongoengine/django/mongo_auth/models.py @@ -0,0 +1,90 @@ +from importlib import import_module + +from django.conf import settings +from django.contrib.auth.models import UserManager +from django.core.exceptions import ImproperlyConfigured +from django.db import models +from django.utils.translation import ugettext_lazy as _ + + +MONGOENGINE_USER_DOCUMENT = getattr( + settings, 'MONGOENGINE_USER_DOCUMENT', 'mongoengine.django.auth.User') + + +class MongoUserManager(UserManager): + """A User manager wich allows the use of MongoEngine documents in Django. + + To use the manager, you must tell django.contrib.auth to use MongoUser as + the user model. In you settings.py, you need: + + INSTALLED_APPS = ( + ... + 'django.contrib.auth', + 'mongoengine.django.mongo_auth', + ... + ) + AUTH_USER_MODEL = 'mongo_auth.MongoUser' + + Django will use the model object to access the custom Manager, which will + replace the original queryset with MongoEngine querysets. + + By default, mongoengine.django.auth.User will be used to store users. You + can specify another document class in MONGOENGINE_USER_DOCUMENT in your + settings.py. + + The User Document class has the same requirements as a standard custom user + model: https://docs.djangoproject.com/en/dev/topics/auth/customizing/ + + In particular, the User Document class must define USERNAME_FIELD and + REQUIRED_FIELDS. + + `AUTH_USER_MODEL` has been added in Django 1.5. + + """ + + def contribute_to_class(self, model, name): + super(MongoUserManager, self).contribute_to_class(model, name) + self.dj_model = self.model + self.model = self._get_user_document() + + self.dj_model.USERNAME_FIELD = self.model.USERNAME_FIELD + username = models.CharField(_('username'), max_length=30, unique=True) + username.contribute_to_class(self.dj_model, self.dj_model.USERNAME_FIELD) + + self.dj_model.REQUIRED_FIELDS = self.model.REQUIRED_FIELDS + for name in self.dj_model.REQUIRED_FIELDS: + field = models.CharField(_(name), max_length=30) + field.contribute_to_class(self.dj_model, name) + + def _get_user_document(self): + try: + name = MONGOENGINE_USER_DOCUMENT + dot = name.rindex('.') + module = import_module(name[:dot]) + return getattr(module, name[dot + 1:]) + except ImportError: + raise ImproperlyConfigured("Error importing %s, please check " + "settings.MONGOENGINE_USER_DOCUMENT" + % name) + + def get(self, *args, **kwargs): + try: + return self.get_query_set().get(*args, **kwargs) + except self.model.DoesNotExist: + # ModelBackend expects this exception + raise self.dj_model.DoesNotExist + + @property + def db(self): + raise NotImplementedError + + def get_empty_query_set(self): + return self.model.objects.none() + + def get_query_set(self): + return self.model.objects + + +class MongoUser(models.Model): + objects = MongoUserManager() + diff --git a/tests/test_django.py b/tests/test_django.py index dceeba27..6f4b6eaa 100644 --- a/tests/test_django.py +++ b/tests/test_django.py @@ -14,9 +14,16 @@ try: from django.conf import settings from django.core.paginator import Paginator - settings.configure(USE_TZ=True) + settings.configure( + USE_TZ=True, + INSTALLED_APPS=('django.contrib.auth', 'mongoengine.django.mongo_auth'), + AUTH_USER_MODEL=('mongo_auth.MongoUser'), + ) + from django.contrib.auth import authenticate, get_user_model from django.contrib.sessions.tests import SessionTestsMixin + from mongoengine.django.auth import User + from mongoengine.django.mongo_auth.models import MongoUser, MongoUserManager from mongoengine.django.sessions import SessionStore, MongoSession except Exception, err: if PY3: @@ -156,6 +163,7 @@ class QuerySetTest(unittest.TestCase): rendered = template.render(Context({'users': users})) self.assertEqual(rendered, 'AB ABCD CD') + class MongoDBSessionTest(SessionTestsMixin, unittest.TestCase): backend = SessionStore @@ -184,5 +192,47 @@ class MongoDBSessionTest(SessionTestsMixin, unittest.TestCase): session = SessionStore(key) self.assertTrue('test_expire' in session, 'Session has expired before it is expected') + +class MongoAuthTest(unittest.TestCase): + user_data = { + 'username': 'user', + 'email': 'user@example.com', + 'password': 'test', + } + + def setUp(self): + if PY3: + raise SkipTest('django does not have Python 3 support') + connect(db='mongoenginetest') + User.drop_collection() + super(MongoAuthTest, self).setUp() + + def test_user_model(self): + self.assertEqual(get_user_model(), MongoUser) + + def test_user_manager(self): + manager = get_user_model()._default_manager + self.assertIsInstance(manager, MongoUserManager) + + def test_user_manager_exception(self): + manager = get_user_model()._default_manager + self.assertRaises(MongoUser.DoesNotExist, manager.get, + username='not found') + + def test_create_user(self): + manager = get_user_model()._default_manager + user = manager.create_user(**self.user_data) + self.assertIsInstance(user, User) + db_user = User.objects.get(username='user') + self.assertEqual(user.id, db_user.id) + + def test_authenticate(self): + get_user_model()._default_manager.create_user(**self.user_data) + user = authenticate(username='user', password='fail') + self.assertIsNone(user) + user = authenticate(username='user', password='test') + db_user = User.objects.get(username='user') + self.assertEqual(user.id, db_user.id) + if __name__ == '__main__': unittest.main() From dff44ef74e8e03bc82b82fa86582af45848ceabf Mon Sep 17 00:00:00 2001 From: Nicolas Cortot Date: Fri, 19 Apr 2013 17:50:15 +0200 Subject: [PATCH 153/189] Fixing warning which prevented tests from succeeding Now that we're importing the auth classes in the tests, no warning can be raised or test_dbref_reference_field_future_warning will fail. --- mongoengine/django/auth.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/mongoengine/django/auth.py b/mongoengine/django/auth.py index 371f0e30..0839b145 100644 --- a/mongoengine/django/auth.py +++ b/mongoengine/django/auth.py @@ -96,7 +96,7 @@ class Permission(Document): Three basic permissions -- add, change and delete -- are automatically created for each Django model. """ name = StringField(max_length=50, verbose_name=_('username')) - content_type = ReferenceField(ContentType) + content_type = ReferenceField(ContentType, dbref=True) codename = StringField(max_length=100, verbose_name=_('codename')) # FIXME: don't access field of the other class # unique_with=['content_type__app_label', 'content_type__model']) @@ -128,7 +128,7 @@ class Group(Document): """ name = StringField(max_length=80, unique=True, verbose_name=_('name')) # permissions = models.ManyToManyField(Permission, verbose_name=_('permissions'), blank=True) - permissions = ListField(ReferenceField(Permission, verbose_name=_('permissions'), required=False)) + permissions = ListField(ReferenceField(Permission, verbose_name=_('permissions'), required=False, dbref=True)) class Meta: verbose_name = _('group') From d39d10b9fb09c0a69dc00eb226db0429048e6b42 Mon Sep 17 00:00:00 2001 From: Nicolas Cortot Date: Fri, 19 Apr 2013 18:28:45 +0200 Subject: [PATCH 154/189] Tests should not require Django 1.5 to run --- tests/test_django.py | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/tests/test_django.py b/tests/test_django.py index 6f4b6eaa..01a105a3 100644 --- a/tests/test_django.py +++ b/tests/test_django.py @@ -20,10 +20,14 @@ try: AUTH_USER_MODEL=('mongo_auth.MongoUser'), ) - from django.contrib.auth import authenticate, get_user_model + try: + from django.contrib.auth import authenticate, get_user_model + from mongoengine.django.auth import User + from mongoengine.django.mongo_auth.models import MongoUser, MongoUserManager + DJ15 = True + except Exception: + DJ15 = False from django.contrib.sessions.tests import SessionTestsMixin - from mongoengine.django.auth import User - from mongoengine.django.mongo_auth.models import MongoUser, MongoUserManager from mongoengine.django.sessions import SessionStore, MongoSession except Exception, err: if PY3: @@ -203,6 +207,8 @@ class MongoAuthTest(unittest.TestCase): def setUp(self): if PY3: raise SkipTest('django does not have Python 3 support') + if not DJ15: + raise SkipTest('mongo_auth requires Django 1.5') connect(db='mongoenginetest') User.drop_collection() super(MongoAuthTest, self).setUp() From 681b74a41cd040e5e6b7e7a9cf1fb505964289f7 Mon Sep 17 00:00:00 2001 From: Nicolas Cortot Date: Fri, 19 Apr 2013 18:53:42 +0200 Subject: [PATCH 155/189] Travis: adding Django-1.5.1 to env --- .travis.yml | 15 ++++++++++++--- setup.py | 2 +- 2 files changed, 13 insertions(+), 4 deletions(-) diff --git a/.travis.yml b/.travis.yml index 1b9f5b7b..ad2678fe 100644 --- a/.travis.yml +++ b/.travis.yml @@ -7,12 +7,21 @@ python: - "3.2" - "3.3" env: - - PYMONGO=dev - - PYMONGO=2.5 - - PYMONGO=2.4.2 + - PYMONGO=dev DJANGO=1.5.1 + - PYMONGO=dev DJANGO=1.4.2 + - PYMONGO=2.5 DJANGO=1.5.1 + - PYMONGO=2.5 DJANGO=1.4.2 + - PYMONGO=2.4.2 DJANGO=1.4.2 +matrix: + exclude: + - python: "2.6" + env: PYMONGO=dev DJANGO=1.5.1 + - python: "2.6" + env: PYMONGO=2.5 DJANGO=1.5.1 install: - if [[ $TRAVIS_PYTHON_VERSION == '2.'* ]]; then cp /usr/lib/*/libz.so $VIRTUAL_ENV/lib/; fi - if [[ $TRAVIS_PYTHON_VERSION == '2.'* ]]; then pip install pil --use-mirrors ; true; fi + - if [[ $TRAVIS_PYTHON_VERSION == '2.'* ]]; then pip install django==$DJANGO --use-mirrors ; true; fi - if [[ $PYMONGO == 'dev' ]]; then pip install https://github.com/mongodb/mongo-python-driver/tarball/master; true; fi - if [[ $PYMONGO != 'dev' ]]; then pip install pymongo==$PYMONGO --use-mirrors; true; fi - python setup.py install diff --git a/setup.py b/setup.py index ba538fab..c6270d9a 100644 --- a/setup.py +++ b/setup.py @@ -58,7 +58,7 @@ if sys.version_info[0] == 3: extra_opts['packages'].append("tests") extra_opts['package_data'] = {"tests": ["fields/mongoengine.png"]} else: - extra_opts['tests_require'] = ['nose', 'coverage', 'blinker', 'django==1.4.2', 'PIL'] + extra_opts['tests_require'] = ['nose', 'coverage', 'blinker', 'django>=1.4.2', 'PIL'] extra_opts['packages'] = find_packages(exclude=('tests',)) setup(name='mongoengine', From 80db9e771671135e9260d088d4139e01b04b678c Mon Sep 17 00:00:00 2001 From: Ross Lawley Date: Mon, 22 Apr 2013 13:06:29 +0000 Subject: [PATCH 156/189] Updated travis --- .travis.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index 1b9f5b7b..0e47cb7a 100644 --- a/.travis.yml +++ b/.travis.yml @@ -23,4 +23,3 @@ notifications: branches: only: - master - - "0.8" From c16e6d74e6fe1271b628af476c4f8db2027aff66 Mon Sep 17 00:00:00 2001 From: Ross Lawley Date: Mon, 22 Apr 2013 15:07:15 +0000 Subject: [PATCH 157/189] Updated connection to use MongoClient (#262, #274) --- docs/changelog.rst | 1 + docs/guide/connecting.rst | 2 +- docs/upgrade.rst | 79 ++++++++++++++++------ mongoengine/connection.py | 8 +-- mongoengine/django/sessions.py | 2 +- mongoengine/document.py | 47 ++++++------- mongoengine/queryset/queryset.py | 111 ++++++++++++++++--------------- tests/document/indexes.py | 14 +++- tests/queryset/queryset.py | 33 +++++---- tests/test_connection.py | 9 ++- 10 files changed, 181 insertions(+), 125 deletions(-) diff --git a/docs/changelog.rst b/docs/changelog.rst index 45470000..d7d010cb 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -4,6 +4,7 @@ Changelog Changes in 0.8.X ================ +- Updated connection to use MongoClient (#262, #274) - Fixed db_alias and inherited Documents (#143) - Documentation update for document errors (#124) - Deprecated `get_or_create` (#35) diff --git a/docs/guide/connecting.rst b/docs/guide/connecting.rst index ebd61a97..de6794cd 100644 --- a/docs/guide/connecting.rst +++ b/docs/guide/connecting.rst @@ -29,7 +29,7 @@ name - just supply the uri as the :attr:`host` to ReplicaSets =========== -MongoEngine now supports :func:`~pymongo.replica_set_connection.ReplicaSetConnection` +MongoEngine supports :class:`~pymongo.mongo_replica_set_client.MongoReplicaSetClient` to use them please use a URI style connection and provide the `replicaSet` name in the connection kwargs. diff --git a/docs/upgrade.rst b/docs/upgrade.rst index 8724503d..356f5109 100644 --- a/docs/upgrade.rst +++ b/docs/upgrade.rst @@ -1,15 +1,15 @@ -========= +######### Upgrading -========= +######### 0.7 to 0.8 -========== +********** Inheritance ------------ +=========== Data Model -~~~~~~~~~~ +---------- The inheritance model has changed, we no longer need to store an array of :attr:`types` with the model we can just use the classname in :attr:`_cls`. @@ -44,7 +44,7 @@ inherited classes like so: :: Document Definition -~~~~~~~~~~~~~~~~~~~ +------------------- The default for inheritance has changed - its now off by default and :attr:`_cls` will not be stored automatically with the class. So if you extend @@ -77,7 +77,7 @@ the case and the data is set only in the ``document._data`` dictionary: :: AttributeError: 'Animal' object has no attribute 'size' Querysets -~~~~~~~~~ +========= Querysets now return clones and should no longer be considered editable in place. This brings us in line with how Django's querysets work and removes a @@ -98,8 +98,47 @@ update your code like so: :: mammals = Animal.objects(type="mammal").filter(order="Carnivora") # The final queryset is assgined to mammals [m for m in mammals] # This will return all carnivores +Client +====== +PyMongo 2.4 came with a new connection client; MongoClient_ and started the +depreciation of the old :class:`~pymongo.connection.Connection`. MongoEngine +now uses the latest `MongoClient` for connections. By default operations were +`safe` but if you turned them off or used the connection directly this will +impact your queries. + +Querysets +--------- + +Safe +^^^^ + +`safe` has been depreciated in the new MongoClient connection. Please use +`write_concern` instead. As `safe` always defaulted as `True` normally no code +change is required. To disable confirmation of the write just pass `{"w": 0}` +eg: :: + + # Old + Animal(name="Dinasour").save(safe=False) + + # new code: + Animal(name="Dinasour").save(write_concern={"w": 0}) + +Write Concern +^^^^^^^^^^^^^ + +`write_options` has been replaced with `write_concern` to bring it inline with +pymongo. To upgrade simply rename any instances where you used the `write_option` +keyword to `write_concern` like so:: + + # Old code: + Animal(name="Dinasour").save(write_options={"w": 2}) + + # new code: + Animal(name="Dinasour").save(write_concern={"w": 2}) + + Indexes -------- +======= Index methods are no longer tied to querysets but rather to the document class. Although `QuerySet._ensure_indexes` and `QuerySet.ensure_index` still exist. @@ -107,17 +146,19 @@ They should be replaced with :func:`~mongoengine.Document.ensure_indexes` / :func:`~mongoengine.Document.ensure_index`. SequenceFields --------------- +============== :class:`~mongoengine.fields.SequenceField` now inherits from `BaseField` to allow flexible storage of the calculated value. As such MIN and MAX settings are no longer handled. +.. _MongoClient: http://blog.mongodb.org/post/36666163412/introducing-mongoclient + 0.6 to 0.7 -========== +********** Cascade saves -------------- +============= Saves will raise a `FutureWarning` if they cascade and cascade hasn't been set to True. This is because in 0.8 it will default to False. If you require @@ -135,7 +176,7 @@ via `save` eg :: Remember: cascading saves **do not** cascade through lists. ReferenceFields ---------------- +=============== ReferenceFields now can store references as ObjectId strings instead of DBRefs. This will become the default in 0.8 and if `dbref` is not set a `FutureWarning` @@ -164,7 +205,7 @@ migrate :: item_frequencies ----------------- +================ In the 0.6 series we added support for null / zero / false values in item_frequencies. A side effect was to return keys in the value they are @@ -173,14 +214,14 @@ updated to handle native types rather than strings keys for the results of item frequency queries. BinaryFields ------------- +============ Binary fields have been updated so that they are native binary types. If you previously were doing `str` comparisons with binary field values you will have to update and wrap the value in a `str`. 0.5 to 0.6 -========== +********** Embedded Documents - if you had a `pk` field you will have to rename it from `_id` to `pk` as pk is no longer a property of Embedded Documents. @@ -200,13 +241,13 @@ don't define :attr:`allow_inheritance` in their meta. You may need to update pyMongo to 2.0 for use with Sharding. 0.4 to 0.5 -=========== +********** There have been the following backwards incompatibilities from 0.4 to 0.5. The main areas of changed are: choices in fields, map_reduce and collection names. Choice options: ---------------- +=============== Are now expected to be an iterable of tuples, with the first element in each tuple being the actual value to be stored. The second element is the @@ -214,7 +255,7 @@ human-readable name for the option. PyMongo / MongoDB ------------------ +================= map reduce now requires pymongo 1.11+- The pymongo `merge_output` and `reduce_output` parameters, have been depreciated. @@ -228,7 +269,7 @@ such the following have been changed: Default collection naming -------------------------- +========================= Previously it was just lowercase, its now much more pythonic and readable as its lowercase and underscores, previously :: diff --git a/mongoengine/connection.py b/mongoengine/connection.py index a47be446..3c53ea3c 100644 --- a/mongoengine/connection.py +++ b/mongoengine/connection.py @@ -1,5 +1,5 @@ import pymongo -from pymongo import Connection, ReplicaSetConnection, uri_parser +from pymongo import MongoClient, MongoReplicaSetClient, uri_parser __all__ = ['ConnectionError', 'connect', 'register_connection', @@ -112,15 +112,15 @@ def get_connection(alias=DEFAULT_CONNECTION_NAME, reconnect=False): conn_settings['slaves'] = slaves conn_settings.pop('read_preference', None) - connection_class = Connection + connection_class = MongoClient if 'replicaSet' in conn_settings: conn_settings['hosts_or_uri'] = conn_settings.pop('host', None) - # Discard port since it can't be used on ReplicaSetConnection + # Discard port since it can't be used on MongoReplicaSetClient conn_settings.pop('port', None) # Discard replicaSet if not base string if not isinstance(conn_settings['replicaSet'], basestring): conn_settings.pop('replicaSet', None) - connection_class = ReplicaSetConnection + connection_class = MongoReplicaSetClient try: _connections[alias] = connection_class(**conn_settings) diff --git a/mongoengine/django/sessions.py b/mongoengine/django/sessions.py index 0d199a6c..29583f5c 100644 --- a/mongoengine/django/sessions.py +++ b/mongoengine/django/sessions.py @@ -88,7 +88,7 @@ class SessionStore(SessionBase): s.session_data = self._get_session(no_load=must_create) s.expire_date = self.get_expiry_date() try: - s.save(force_insert=must_create, safe=True) + s.save(force_insert=must_create) except OperationError: if must_create: raise CreateError diff --git a/mongoengine/document.py b/mongoengine/document.py index 9057075e..54b55df9 100644 --- a/mongoengine/document.py +++ b/mongoengine/document.py @@ -142,7 +142,7 @@ class Document(BaseDocument): options.get('size') != max_size: msg = (('Cannot create collection "%s" as a capped ' 'collection as it already exists') - % cls._collection) + % cls._collection) raise InvalidCollectionError(msg) else: # Create the collection as a capped collection @@ -158,28 +158,24 @@ class Document(BaseDocument): cls.ensure_indexes() return cls._collection - def save(self, safe=True, force_insert=False, validate=True, clean=True, - write_options=None, cascade=None, cascade_kwargs=None, + def save(self, force_insert=False, validate=True, clean=True, + write_concern=None, cascade=None, cascade_kwargs=None, _refs=None, **kwargs): """Save the :class:`~mongoengine.Document` to the database. If the document already exists, it will be updated, otherwise it will be created. - If ``safe=True`` and the operation is unsuccessful, an - :class:`~mongoengine.OperationError` will be raised. - - :param safe: check if the operation succeeded before returning :param force_insert: only try to create a new document, don't allow updates of existing documents :param validate: validates the document; set to ``False`` to skip. :param clean: call the document clean method, requires `validate` to be True. - :param write_options: Extra keyword arguments are passed down to + :param write_concern: Extra keyword arguments are passed down to :meth:`~pymongo.collection.Collection.save` OR :meth:`~pymongo.collection.Collection.insert` which will be used as options for the resultant ``getLastError`` command. For example, - ``save(..., write_options={w: 2, fsync: True}, ...)`` will + ``save(..., write_concern={w: 2, fsync: True}, ...)`` will wait until at least two servers have recorded the write and will force an fsync on the primary server. :param cascade: Sets the flag for cascading saves. You can set a @@ -205,8 +201,8 @@ class Document(BaseDocument): if validate: self.validate(clean=clean) - if not write_options: - write_options = {} + if not write_concern: + write_concern = {} doc = self.to_mongo() @@ -216,11 +212,9 @@ class Document(BaseDocument): collection = self._get_collection() if created: if force_insert: - object_id = collection.insert(doc, safe=safe, - **write_options) + object_id = collection.insert(doc, **write_concern) else: - object_id = collection.save(doc, safe=safe, - **write_options) + object_id = collection.save(doc, **write_concern) else: object_id = doc['_id'] updates, removals = self._delta() @@ -247,7 +241,7 @@ class Document(BaseDocument): update_query["$unset"] = removals if updates or removals: last_error = collection.update(select_dict, update_query, - upsert=upsert, safe=safe, **write_options) + upsert=upsert, **write_concern) created = is_new_object(last_error) warn_cascade = not cascade and 'cascade' not in self._meta @@ -255,10 +249,9 @@ class Document(BaseDocument): if cascade is None else cascade) if cascade: kwargs = { - "safe": safe, "force_insert": force_insert, "validate": validate, - "write_options": write_options, + "write_concern": write_concern, "cascade": cascade } if cascade_kwargs: # Allow granular control over cascades @@ -305,7 +298,7 @@ class Document(BaseDocument): if ref and ref_id not in _refs: if warn_cascade: msg = ("Cascading saves will default to off in 0.8, " - "please explicitly set `.save(cascade=True)`") + "please explicitly set `.save(cascade=True)`") warnings.warn(msg, FutureWarning) _refs.append(ref_id) kwargs["_refs"] = _refs @@ -344,16 +337,21 @@ class Document(BaseDocument): # Need to add shard key to query, or you get an error return self._qs.filter(**self._object_key).update_one(**kwargs) - def delete(self, safe=False): + def delete(self, **write_concern): """Delete the :class:`~mongoengine.Document` from the database. This will only take effect if the document has been previously saved. - :param safe: check if the operation succeeded before returning + :param write_concern: Extra keyword arguments are passed down which + will be used as options for the resultant + ``getLastError`` command. For example, + ``save(..., write_concern={w: 2, fsync: True}, ...)`` will + wait until at least two servers have recorded the write and + will force an fsync on the primary server. """ signals.pre_delete.send(self.__class__, document=self) try: - self._qs.filter(**self._object_key).delete(safe=safe) + self._qs.filter(**self._object_key).delete(write_concern=write_concern) except pymongo.errors.OperationFailure, err: message = u'Could not delete document (%s)' % err.message raise OperationError(message) @@ -428,9 +426,8 @@ class Document(BaseDocument): .. versionchanged:: 0.6 Now chainable """ id_field = self._meta['id_field'] - obj = self._qs.filter( - **{id_field: self[id_field]} - ).limit(1).select_related(max_depth=max_depth) + obj = self._qs.filter(**{id_field: self[id_field]} + ).limit(1).select_related(max_depth=max_depth) if obj: obj = obj[0] else: diff --git a/mongoengine/queryset/queryset.py b/mongoengine/queryset/queryset.py index 15c8e634..71332b92 100644 --- a/mongoengine/queryset/queryset.py +++ b/mongoengine/queryset/queryset.py @@ -221,7 +221,7 @@ class QuerySet(object): """ return self._document(**kwargs).save() - def get_or_create(self, write_options=None, auto_save=True, + def get_or_create(self, write_concern=None, auto_save=True, *q_objs, **query): """Retrieve unique object or create, if it doesn't exist. Returns a tuple of ``(object, created)``, where ``object`` is the retrieved or @@ -239,9 +239,9 @@ class QuerySet(object): don't accidently duplicate data when using this method. This is now scheduled to be removed before 1.0 - :param write_options: optional extra keyword arguments used if we + :param write_concern: optional extra keyword arguments used if we have to create a new document. - Passes any write_options onto :meth:`~mongoengine.Document.save` + Passes any write_concern onto :meth:`~mongoengine.Document.save` :param auto_save: if the object is to be saved automatically if not found. @@ -266,7 +266,7 @@ class QuerySet(object): doc = self._document(**query) if auto_save: - doc.save(write_options=write_options) + doc.save(write_concern=write_concern) return doc, True def first(self): @@ -279,18 +279,13 @@ class QuerySet(object): result = None return result - def insert(self, doc_or_docs, load_bulk=True, safe=False, - write_options=None): + def insert(self, doc_or_docs, load_bulk=True, write_concern=None): """bulk insert documents - If ``safe=True`` and the operation is unsuccessful, an - :class:`~mongoengine.OperationError` will be raised. - :param docs_or_doc: a document or list of documents to be inserted :param load_bulk (optional): If True returns the list of document instances - :param safe: check if the operation succeeded before returning - :param write_options: Extra keyword arguments are passed down to + :param write_concern: Extra keyword arguments are passed down to :meth:`~pymongo.collection.Collection.insert` which will be used as options for the resultant ``getLastError`` command. For example, @@ -305,9 +300,8 @@ class QuerySet(object): """ Document = _import_class('Document') - if not write_options: - write_options = {} - write_options.update({'safe': safe}) + if not write_concern: + write_concern = {} docs = doc_or_docs return_one = False @@ -319,7 +313,7 @@ class QuerySet(object): for doc in docs: if not isinstance(doc, self._document): msg = ("Some documents inserted aren't instances of %s" - % str(self._document)) + % str(self._document)) raise OperationError(msg) if doc.pk and not doc._created: msg = "Some documents have ObjectIds use doc.update() instead" @@ -328,7 +322,7 @@ class QuerySet(object): signals.pre_bulk_insert.send(self._document, documents=docs) try: - ids = self._collection.insert(raw, **write_options) + ids = self._collection.insert(raw, **write_concern) except pymongo.errors.OperationFailure, err: message = 'Could not save document (%s)' if re.match('^E1100[01] duplicate key', unicode(err)): @@ -340,7 +334,7 @@ class QuerySet(object): if not load_bulk: signals.post_bulk_insert.send( - self._document, documents=docs, loaded=False) + self._document, documents=docs, loaded=False) return return_one and ids[0] or ids documents = self.in_bulk(ids) @@ -348,7 +342,7 @@ class QuerySet(object): for obj_id in ids: results.append(documents.get(obj_id)) signals.post_bulk_insert.send( - self._document, documents=results, loaded=True) + self._document, documents=results, loaded=True) return return_one and results[0] or results def count(self): @@ -358,10 +352,15 @@ class QuerySet(object): return 0 return self._cursor.count(with_limit_and_skip=True) - def delete(self, safe=False): + def delete(self, write_concern=None): """Delete the documents matched by the query. - :param safe: check if the operation succeeded before returning + :param write_concern: Extra keyword arguments are passed down which + will be used as options for the resultant + ``getLastError`` command. For example, + ``save(..., write_concern={w: 2, fsync: True}, ...)`` will + wait until at least two servers have recorded the write and + will force an fsync on the primary server. """ queryset = self.clone() doc = queryset._document @@ -370,11 +369,14 @@ class QuerySet(object): signals.pre_delete.has_receivers_for(self._document) or signals.post_delete.has_receivers_for(self._document)) + if not write_concern: + write_concern = {} + # Handle deletes where skips or limits have been applied or has a # delete signal if queryset._skip or queryset._limit or has_delete_signal: for doc in queryset: - doc.delete(safe=safe) + doc.delete(write_concern=write_concern) return delete_rules = doc._meta.get('delete_rules') or {} @@ -386,7 +388,7 @@ class QuerySet(object): if rule == DENY and document_cls.objects( **{field_name + '__in': self}).count() > 0: msg = ("Could not delete document (%s.%s refers to it)" - % (document_cls.__name__, field_name)) + % (document_cls.__name__, field_name)) raise OperationError(msg) for rule_entry in delete_rules: @@ -396,36 +398,38 @@ class QuerySet(object): ref_q = document_cls.objects(**{field_name + '__in': self}) ref_q_count = ref_q.count() if (doc != document_cls and ref_q_count > 0 - or (doc == document_cls and ref_q_count > 0)): - ref_q.delete(safe=safe) + or (doc == document_cls and ref_q_count > 0)): + ref_q.delete(write_concern=write_concern) elif rule == NULLIFY: document_cls.objects(**{field_name + '__in': self}).update( - safe_update=safe, - **{'unset__%s' % field_name: 1}) + write_concern=write_concern, **{'unset__%s' % field_name: 1}) elif rule == PULL: document_cls.objects(**{field_name + '__in': self}).update( - safe_update=safe, - **{'pull_all__%s' % field_name: self}) + write_concern=write_concern, + **{'pull_all__%s' % field_name: self}) - queryset._collection.remove(queryset._query, safe=safe) + queryset._collection.remove(queryset._query, write_concern=write_concern) - def update(self, safe_update=True, upsert=False, multi=True, - write_options=None, **update): - """Perform an atomic update on the fields matched by the query. When - ``safe_update`` is used, the number of affected documents is returned. + def update(self, upsert=False, multi=True, write_concern=None, **update): + """Perform an atomic update on the fields matched by the query. - :param safe_update: check if the operation succeeded before returning :param upsert: Any existing document with that "_id" is overwritten. - :param write_options: extra keyword arguments for - :meth:`~pymongo.collection.Collection.update` + :param multi: Update multiple documents. + :param write_concern: Extra keyword arguments are passed down which + will be used as options for the resultant + ``getLastError`` command. For example, + ``save(..., write_concern={w: 2, fsync: True}, ...)`` will + wait until at least two servers have recorded the write and + will force an fsync on the primary server. + :param update: Django-style update keyword arguments .. versionadded:: 0.2 """ if not update: raise OperationError("No update parameters, would remove data") - if not write_options: - write_options = {} + if not write_concern: + write_concern = {} queryset = self.clone() query = queryset._query @@ -441,8 +445,7 @@ class QuerySet(object): try: ret = queryset._collection.update(query, update, multi=multi, - upsert=upsert, safe=safe_update, - **write_options) + upsert=upsert, **write_concern) if ret is not None and 'n' in ret: return ret['n'] except pymongo.errors.OperationFailure, err: @@ -451,21 +454,21 @@ class QuerySet(object): raise OperationError(message) raise OperationError(u'Update failed (%s)' % unicode(err)) - def update_one(self, safe_update=True, upsert=False, write_options=None, - **update): - """Perform an atomic update on first field matched by the query. When - ``safe_update`` is used, the number of affected documents is returned. + def update_one(self, upsert=False, write_concern=None, **update): + """Perform an atomic update on first field matched by the query. - :param safe_update: check if the operation succeeded before returning :param upsert: Any existing document with that "_id" is overwritten. - :param write_options: extra keyword arguments for - :meth:`~pymongo.collection.Collection.update` + :param write_concern: Extra keyword arguments are passed down which + will be used as options for the resultant + ``getLastError`` command. For example, + ``save(..., write_concern={w: 2, fsync: True}, ...)`` will + wait until at least two servers have recorded the write and + will force an fsync on the primary server. :param update: Django-style update keyword arguments .. versionadded:: 0.2 """ - return self.update(safe_update=True, upsert=upsert, multi=False, - write_options=None, **update) + return self.update(upsert=upsert, multi=False, write_concern=None, **update) def with_id(self, object_id): """Retrieve the object matching the id provided. Uses `object_id` only @@ -498,7 +501,7 @@ class QuerySet(object): if self._scalar: for doc in docs: doc_map[doc['_id']] = self._get_scalar( - self._document._from_son(doc)) + self._document._from_son(doc)) elif self._as_pymongo: for doc in docs: doc_map[doc['_id']] = self._get_as_pymongo(doc) @@ -523,10 +526,10 @@ class QuerySet(object): c = self.__class__(self._document, self._collection_obj) copy_props = ('_mongo_query', '_initial_query', '_none', '_query_obj', - '_where_clause', '_loaded_fields', '_ordering', '_snapshot', - '_timeout', '_class_check', '_slave_okay', '_read_preference', - '_iter', '_scalar', '_as_pymongo', '_as_pymongo_coerce', - '_limit', '_skip', '_hint', '_auto_dereference') + '_where_clause', '_loaded_fields', '_ordering', '_snapshot', + '_timeout', '_class_check', '_slave_okay', '_read_preference', + '_iter', '_scalar', '_as_pymongo', '_as_pymongo_coerce', + '_limit', '_skip', '_hint', '_auto_dereference') for prop in copy_props: val = getattr(self, prop) diff --git a/tests/document/indexes.py b/tests/document/indexes.py index ff08ef1a..fea63a51 100644 --- a/tests/document/indexes.py +++ b/tests/document/indexes.py @@ -314,19 +314,27 @@ class IndexesTest(unittest.TestCase): """ class User(Document): meta = { + 'allow_inheritance': True, 'indexes': ['user_guid'], 'auto_create_index': False } user_guid = StringField(required=True) + class MongoUser(User): + pass + User.drop_collection() - u = User(user_guid='123') - u.save() + User(user_guid='123').save() + MongoUser(user_guid='123').save() - self.assertEqual(1, User.objects.count()) + self.assertEqual(2, User.objects.count()) info = User.objects._collection.index_information() self.assertEqual(info.keys(), ['_id_']) + + User.ensure_indexes() + info = User.objects._collection.index_information() + self.assertEqual(info.keys(), ['_cls_1_user_guid_1', '_id_']) User.drop_collection() def test_embedded_document_index(self): diff --git a/tests/queryset/queryset.py b/tests/queryset/queryset.py index 37670b0a..42e98ae7 100644 --- a/tests/queryset/queryset.py +++ b/tests/queryset/queryset.py @@ -278,24 +278,24 @@ class QuerySetTest(unittest.TestCase): query = query.filter(boolfield=True) self.assertEquals(query.count(), 1) - def test_update_write_options(self): - """Test that passing write_options works""" + def test_update_write_concern(self): + """Test that passing write_concern works""" self.Person.drop_collection() - write_options = {"fsync": True} + write_concern = {"fsync": True} author, created = self.Person.objects.get_or_create( - name='Test User', write_options=write_options) - author.save(write_options=write_options) + name='Test User', write_concern=write_concern) + author.save(write_concern=write_concern) self.Person.objects.update(set__name='Ross', - write_options=write_options) + write_concern=write_concern) author = self.Person.objects.first() self.assertEqual(author.name, 'Ross') - self.Person.objects.update_one(set__name='Test User', write_options=write_options) + self.Person.objects.update_one(set__name='Test User', write_concern=write_concern) author = self.Person.objects.first() self.assertEqual(author.name, 'Test User') @@ -592,10 +592,17 @@ class QuerySetTest(unittest.TestCase): blogs.append(Blog(title="post %s" % i, posts=[post1, post2])) Blog.objects.insert(blogs, load_bulk=False) - self.assertEqual(q, 1) # 1 for the insert + self.assertEqual(q, 1) # 1 for the insert + + Blog.drop_collection() + with query_counter() as q: + self.assertEqual(q, 0) + + Blog.ensure_indexes() + self.assertEqual(q, 1) Blog.objects.insert(blogs) - self.assertEqual(q, 3) # 1 for insert, and 1 for in bulk fetch (3 in total) + self.assertEqual(q, 3) # 1 for insert, and 1 for in bulk fetch (3 in total) Blog.drop_collection() @@ -619,7 +626,7 @@ class QuerySetTest(unittest.TestCase): self.assertRaises(OperationError, throw_operation_error) # Test can insert new doc - new_post = Blog(title="code", id=ObjectId()) + new_post = Blog(title="code123", id=ObjectId()) Blog.objects.insert(new_post) # test handles other classes being inserted @@ -655,13 +662,13 @@ class QuerySetTest(unittest.TestCase): Blog.objects.insert([blog1, blog2]) def throw_operation_error_not_unique(): - Blog.objects.insert([blog2, blog3], safe=True) + Blog.objects.insert([blog2, blog3]) self.assertRaises(NotUniqueError, throw_operation_error_not_unique) self.assertEqual(Blog.objects.count(), 2) - Blog.objects.insert([blog2, blog3], write_options={ - 'continue_on_error': True}) + Blog.objects.insert([blog2, blog3], write_concern={"w": 0, + 'continue_on_error': True}) self.assertEqual(Blog.objects.count(), 3) def test_get_changed_fields_query_count(self): diff --git a/tests/test_connection.py b/tests/test_connection.py index 5b9743d6..4b8a3d11 100644 --- a/tests/test_connection.py +++ b/tests/test_connection.py @@ -10,7 +10,6 @@ from bson.tz_util import utc from mongoengine import * import mongoengine.connection from mongoengine.connection import get_db, get_connection, ConnectionError -from mongoengine.context_managers import switch_db class ConnectionTest(unittest.TestCase): @@ -26,7 +25,7 @@ class ConnectionTest(unittest.TestCase): connect('mongoenginetest') conn = get_connection() - self.assertTrue(isinstance(conn, pymongo.connection.Connection)) + self.assertTrue(isinstance(conn, pymongo.mongo_client.MongoClient)) db = get_db() self.assertTrue(isinstance(db, pymongo.database.Database)) @@ -34,7 +33,7 @@ class ConnectionTest(unittest.TestCase): connect('mongoenginetest2', alias='testdb') conn = get_connection('testdb') - self.assertTrue(isinstance(conn, pymongo.connection.Connection)) + self.assertTrue(isinstance(conn, pymongo.mongo_client.MongoClient)) def test_connect_uri(self): """Ensure that the connect() method works properly with uri's @@ -52,7 +51,7 @@ class ConnectionTest(unittest.TestCase): connect("testdb_uri", host='mongodb://username:password@localhost/mongoenginetest') conn = get_connection() - self.assertTrue(isinstance(conn, pymongo.connection.Connection)) + self.assertTrue(isinstance(conn, pymongo.mongo_client.MongoClient)) db = get_db() self.assertTrue(isinstance(db, pymongo.database.Database)) @@ -65,7 +64,7 @@ class ConnectionTest(unittest.TestCase): self.assertRaises(ConnectionError, get_connection) conn = get_connection('testdb') - self.assertTrue(isinstance(conn, pymongo.connection.Connection)) + self.assertTrue(isinstance(conn, pymongo.mongo_client.MongoClient)) db = get_db('testdb') self.assertTrue(isinstance(db, pymongo.database.Database)) From efad628a87e3bb3e4ec55f1ddcaab68853917e36 Mon Sep 17 00:00:00 2001 From: Ross Lawley Date: Mon, 22 Apr 2013 15:32:11 +0000 Subject: [PATCH 158/189] Objects queryset manager now inherited (#256) --- docs/changelog.rst | 1 + mongoengine/base/metaclasses.py | 10 ++++----- tests/queryset/queryset.py | 36 +++++++++++++++++++++++++++++++++ 3 files changed, 42 insertions(+), 5 deletions(-) diff --git a/docs/changelog.rst b/docs/changelog.rst index d7d010cb..01f5a549 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -4,6 +4,7 @@ Changelog Changes in 0.8.X ================ +- Objects manager now inherited (#256) - Updated connection to use MongoClient (#262, #274) - Fixed db_alias and inherited Documents (#143) - Documentation update for document errors (#124) diff --git a/mongoengine/base/metaclasses.py b/mongoengine/base/metaclasses.py index a53744db..27040119 100644 --- a/mongoengine/base/metaclasses.py +++ b/mongoengine/base/metaclasses.py @@ -315,8 +315,8 @@ class TopLevelDocumentMetaclass(DocumentMetaclass): # may set allow_inheritance to False simple_class = all([b._meta.get('abstract') for b in flattened_bases if hasattr(b, '_meta')]) - if (not simple_class and meta['allow_inheritance'] == False and - not meta['abstract']): + if (not simple_class and meta['allow_inheritance'] is False and + not meta['abstract']): raise ValueError('Only direct subclasses of Document may set ' '"allow_inheritance" to False') @@ -339,9 +339,9 @@ class TopLevelDocumentMetaclass(DocumentMetaclass): if callable(collection): new_class._meta['collection'] = collection(new_class) - # Provide a default queryset unless one has been set - manager = attrs.get('objects', QuerySetManager()) - new_class.objects = manager + # Provide a default queryset unless exists or one has been set + if not hasattr(new_class, 'objects'): + new_class.objects = QuerySetManager() # Validate the fields and set primary key if needed for field_name, field in new_class._fields.iteritems(): diff --git a/tests/queryset/queryset.py b/tests/queryset/queryset.py index 42e98ae7..0d3ebf3d 100644 --- a/tests/queryset/queryset.py +++ b/tests/queryset/queryset.py @@ -2233,6 +2233,42 @@ class QuerySetTest(unittest.TestCase): self.assertEqual(0, Foo.with_inactive.count()) self.assertEqual(1, Foo.objects.count()) + def test_inherit_objects(self): + + class Foo(Document): + meta = {'allow_inheritance': True} + active = BooleanField(default=True) + + @queryset_manager + def objects(klass, queryset): + return queryset(active=True) + + class Bar(Foo): + pass + + Bar.drop_collection() + Bar.objects.create(active=False) + self.assertEqual(0, Bar.objects.count()) + + def test_inherit_objects_override(self): + + class Foo(Document): + meta = {'allow_inheritance': True} + active = BooleanField(default=True) + + @queryset_manager + def objects(klass, queryset): + return queryset(active=True) + + class Bar(Foo): + @queryset_manager + def objects(klass, queryset): + return queryset(active=False) + + Bar.drop_collection() + Bar.objects.create(active=False) + self.assertEqual(0, Foo.objects.count()) + self.assertEqual(1, Bar.objects.count()) def test_query_value_conversion(self): """Ensure that query values are properly converted when necessary. From 0d0befe23e7ec141e4c81d7f01a3ba7e57c43865 Mon Sep 17 00:00:00 2001 From: Ross Lawley Date: Mon, 22 Apr 2013 16:19:55 +0000 Subject: [PATCH 159/189] Removed __len__ from queryset (#247) --- docs/changelog.rst | 3 +- docs/code/tumblelog.py | 2 +- docs/upgrade.rst | 16 +++++++ mongoengine/queryset/queryset.py | 9 ++-- tests/document/instance.py | 32 +++++++------- tests/queryset/queryset.py | 74 ++++++++++++++++---------------- tests/queryset/transform.py | 8 ++-- tests/queryset/visitor.py | 8 ++-- 8 files changed, 82 insertions(+), 70 deletions(-) diff --git a/docs/changelog.rst b/docs/changelog.rst index 01f5a549..da1424eb 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -4,7 +4,8 @@ Changelog Changes in 0.8.X ================ -- Objects manager now inherited (#256) +- Removed __len__ from queryset (#247) +- Objects queryset manager now inherited (#256) - Updated connection to use MongoClient (#262, #274) - Fixed db_alias and inherited Documents (#143) - Documentation update for document errors (#124) diff --git a/docs/code/tumblelog.py b/docs/code/tumblelog.py index 6ba1eee2..0e40e899 100644 --- a/docs/code/tumblelog.py +++ b/docs/code/tumblelog.py @@ -45,7 +45,7 @@ print 'ALL POSTS' print for post in Post.objects: print post.title - print '=' * len(post.title) + print '=' * post.title.count() if isinstance(post, TextPost): print post.content diff --git a/docs/upgrade.rst b/docs/upgrade.rst index 356f5109..4f549d61 100644 --- a/docs/upgrade.rst +++ b/docs/upgrade.rst @@ -79,6 +79,9 @@ the case and the data is set only in the ``document._data`` dictionary: :: Querysets ========= +Attack of the clones +-------------------- + Querysets now return clones and should no longer be considered editable in place. This brings us in line with how Django's querysets work and removes a long running gotcha. If you edit your querysets inplace you will have to @@ -98,6 +101,19 @@ update your code like so: :: mammals = Animal.objects(type="mammal").filter(order="Carnivora") # The final queryset is assgined to mammals [m for m in mammals] # This will return all carnivores +No more len +----------- + +If you ever did len(queryset) it previously did a count() under the covers, this +caused some unusual issues - so now it has been removed in favour of the +explicit `queryset.count()` to update:: + + # Old code + len(Animal.objects(type="mammal")) + + # New code + Animal.objects(type="mammal").count()) + Client ====== PyMongo 2.4 came with a new connection client; MongoClient_ and started the diff --git a/mongoengine/queryset/queryset.py b/mongoengine/queryset/queryset.py index 71332b92..cc0b70f5 100644 --- a/mongoengine/queryset/queryset.py +++ b/mongoengine/queryset/queryset.py @@ -118,9 +118,6 @@ class QuerySet(object): queryset.rewind() return queryset - def __len__(self): - return self.count() - def __getitem__(self, key): """Support skip and limit using getitem and slicing syntax. """ @@ -149,12 +146,12 @@ class QuerySet(object): elif isinstance(key, int): if queryset._scalar: return queryset._get_scalar( - queryset._document._from_son(queryset._cursor[key], - _auto_dereference=self._auto_dereference)) + queryset._document._from_son(queryset._cursor[key], + _auto_dereference=self._auto_dereference)) if queryset._as_pymongo: return queryset._get_as_pymongo(queryset._cursor.next()) return queryset._document._from_son(queryset._cursor[key], - _auto_dereference=self._auto_dereference) + _auto_dereference=self._auto_dereference) raise AttributeError def __repr__(self): diff --git a/tests/document/instance.py b/tests/document/instance.py index 07991c1b..1adc1406 100644 --- a/tests/document/instance.py +++ b/tests/document/instance.py @@ -65,11 +65,11 @@ class InstanceTest(unittest.TestCase): for _ in range(10): Log().save() - self.assertEqual(len(Log.objects), 10) + self.assertEqual(Log.objects.count(), 10) # Check that extra documents don't increase the size Log().save() - self.assertEqual(len(Log.objects), 10) + self.assertEqual(Log.objects.count(), 10) options = Log.objects._collection.options() self.assertEqual(options['capped'], True) @@ -1040,9 +1040,9 @@ class InstanceTest(unittest.TestCase): """ person = self.Person(name="Test User", age=30) person.save() - self.assertEqual(len(self.Person.objects), 1) + self.assertEqual(self.Person.objects.count(), 1) person.delete() - self.assertEqual(len(self.Person.objects), 0) + self.assertEqual(self.Person.objects.count(), 0) def test_save_custom_id(self): """Ensure that a document may be saved with a custom _id. @@ -1356,12 +1356,12 @@ class InstanceTest(unittest.TestCase): post.save() reviewer.delete() - self.assertEqual(len(BlogPost.objects), 1) # No effect on the BlogPost + self.assertEqual(BlogPost.objects.count(), 1) # No effect on the BlogPost self.assertEqual(BlogPost.objects.get().reviewer, None) # Delete the Person, which should lead to deletion of the BlogPost, too author.delete() - self.assertEqual(len(BlogPost.objects), 0) + self.assertEqual(BlogPost.objects.count(), 0) def test_reverse_delete_rule_with_document_inheritance(self): """Ensure that a referenced document is also deleted upon deletion @@ -1391,12 +1391,12 @@ class InstanceTest(unittest.TestCase): post.save() reviewer.delete() - self.assertEqual(len(BlogPost.objects), 1) + self.assertEqual(BlogPost.objects.count(), 1) self.assertEqual(BlogPost.objects.get().reviewer, None) # Delete the Writer should lead to deletion of the BlogPost author.delete() - self.assertEqual(len(BlogPost.objects), 0) + self.assertEqual(BlogPost.objects.count(), 0) def test_reverse_delete_rule_cascade_and_nullify_complex_field(self): """Ensure that a referenced document is also deleted upon deletion for @@ -1425,12 +1425,12 @@ class InstanceTest(unittest.TestCase): # Deleting the reviewer should have no effect on the BlogPost reviewer.delete() - self.assertEqual(len(BlogPost.objects), 1) + self.assertEqual(BlogPost.objects.count(), 1) self.assertEqual(BlogPost.objects.get().reviewers, []) # Delete the Person, which should lead to deletion of the BlogPost, too author.delete() - self.assertEqual(len(BlogPost.objects), 0) + self.assertEqual(BlogPost.objects.count(), 0) def test_reverse_delete_rule_cascade_triggers_pre_delete_signal(self): ''' ensure the pre_delete signal is triggered upon a cascading deletion @@ -1498,7 +1498,7 @@ class InstanceTest(unittest.TestCase): f.delete() - self.assertEqual(len(Bar.objects), 1) # No effect on the BlogPost + self.assertEqual(Bar.objects.count(), 1) # No effect on the BlogPost self.assertEqual(Bar.objects.get().foo, None) def test_invalid_reverse_delete_rules_raise_errors(self): @@ -1549,7 +1549,7 @@ class InstanceTest(unittest.TestCase): # Delete the Person, which should lead to deletion of the BlogPost, and, # recursively to the Comment, too author.delete() - self.assertEqual(len(Comment.objects), 0) + self.assertEqual(Comment.objects.count(), 0) self.Person.drop_collection() BlogPost.drop_collection() @@ -1576,16 +1576,16 @@ class InstanceTest(unittest.TestCase): # Delete the Person should be denied self.assertRaises(OperationError, author.delete) # Should raise denied error - self.assertEqual(len(BlogPost.objects), 1) # No objects may have been deleted - self.assertEqual(len(self.Person.objects), 1) + self.assertEqual(BlogPost.objects.count(), 1) # No objects may have been deleted + self.assertEqual(self.Person.objects.count(), 1) # Other users, that don't have BlogPosts must be removable, like normal author = self.Person(name='Another User') author.save() - self.assertEqual(len(self.Person.objects), 2) + self.assertEqual(self.Person.objects.count(), 2) author.delete() - self.assertEqual(len(self.Person.objects), 1) + self.assertEqual(self.Person.objects.count(), 1) self.Person.drop_collection() BlogPost.drop_collection() diff --git a/tests/queryset/queryset.py b/tests/queryset/queryset.py index 0d3ebf3d..f5aec7ed 100644 --- a/tests/queryset/queryset.py +++ b/tests/queryset/queryset.py @@ -72,7 +72,7 @@ class QuerySetTest(unittest.TestCase): # Find all people in the collection people = self.Person.objects - self.assertEqual(len(people), 2) + self.assertEqual(people.count(), 2) results = list(people) self.assertTrue(isinstance(results[0], self.Person)) self.assertTrue(isinstance(results[0].id, (ObjectId, str, unicode))) @@ -83,7 +83,7 @@ class QuerySetTest(unittest.TestCase): # Use a query to filter the people found to just person1 people = self.Person.objects(age=20) - self.assertEqual(len(people), 1) + self.assertEqual(people.count(), 1) person = people.next() self.assertEqual(person.name, "User A") self.assertEqual(person.age, 20) @@ -130,7 +130,7 @@ class QuerySetTest(unittest.TestCase): for i in xrange(55): self.Person(name='A%s' % i, age=i).save() - self.assertEqual(len(self.Person.objects), 55) + self.assertEqual(self.Person.objects.count(), 55) self.assertEqual("Person object", "%s" % self.Person.objects[0]) self.assertEqual("[, ]", "%s" % self.Person.objects[1:3]) self.assertEqual("[, ]", "%s" % self.Person.objects[51:53]) @@ -211,10 +211,10 @@ class QuerySetTest(unittest.TestCase): Blog.drop_collection() Blog.objects.create(tags=['a', 'b']) - self.assertEqual(len(Blog.objects(tags__0='a')), 1) - self.assertEqual(len(Blog.objects(tags__0='b')), 0) - self.assertEqual(len(Blog.objects(tags__1='a')), 0) - self.assertEqual(len(Blog.objects(tags__1='b')), 1) + self.assertEqual(Blog.objects(tags__0='a').count(), 1) + self.assertEqual(Blog.objects(tags__0='b').count(), 0) + self.assertEqual(Blog.objects(tags__1='a').count(), 0) + self.assertEqual(Blog.objects(tags__1='b').count(), 1) Blog.drop_collection() @@ -229,13 +229,13 @@ class QuerySetTest(unittest.TestCase): self.assertEqual(blog, blog1) query = Blog.objects(posts__1__comments__1__name='testb') - self.assertEqual(len(query), 2) + self.assertEqual(query.count(), 2) query = Blog.objects(posts__1__comments__1__name='testa') - self.assertEqual(len(query), 0) + self.assertEqual(query.count(), 0) query = Blog.objects(posts__0__comments__1__name='testa') - self.assertEqual(len(query), 0) + self.assertEqual(query.count(), 0) Blog.drop_collection() @@ -344,7 +344,7 @@ class QuerySetTest(unittest.TestCase): # Update all of the first comments of second posts of all blogs blog = Blog.objects().update(set__posts__1__comments__0__name="testc") testc_blogs = Blog.objects(posts__1__comments__0__name="testc") - self.assertEqual(len(testc_blogs), 2) + self.assertEqual(testc_blogs.count(), 2) Blog.drop_collection() @@ -355,7 +355,7 @@ class QuerySetTest(unittest.TestCase): blog = Blog.objects().update_one( set__posts__1__comments__1__name="testc") testc_blogs = Blog.objects(posts__1__comments__1__name="testc") - self.assertEqual(len(testc_blogs), 1) + self.assertEqual(testc_blogs.count(), 1) # Check that using this indexing syntax on a non-list fails def non_list_indexing(): @@ -793,7 +793,7 @@ class QuerySetTest(unittest.TestCase): number = IntField() def __repr__(self): - return "" % self.number + return "" % self.number Doc.drop_collection() @@ -803,20 +803,17 @@ class QuerySetTest(unittest.TestCase): docs = Doc.objects.order_by('number') self.assertEqual(docs.count(), 1000) - self.assertEqual(len(docs), 1000) docs_string = "%s" % docs self.assertTrue("Doc: 0" in docs_string) self.assertEqual(docs.count(), 1000) - self.assertEqual(len(docs), 1000) # Limit and skip docs = docs[1:4] self.assertEqual('[, , ]', "%s" % docs) self.assertEqual(docs.count(), 3) - self.assertEqual(len(docs), 3) for doc in docs: self.assertEqual('.. queryset mid-iteration ..', repr(docs)) @@ -945,8 +942,10 @@ class QuerySetTest(unittest.TestCase): Blog.drop_collection() def assertSequence(self, qs, expected): + qs = list(qs) + expected = list(expected) self.assertEqual(len(qs), len(expected)) - for i in range(len(qs)): + for i in xrange(len(qs)): self.assertEqual(qs[i], expected[i]) def test_ordering(self): @@ -1124,13 +1123,13 @@ class QuerySetTest(unittest.TestCase): self.Person(name="User B", age=30).save() self.Person(name="User C", age=40).save() - self.assertEqual(len(self.Person.objects), 3) + self.assertEqual(self.Person.objects.count(), 3) self.Person.objects(age__lt=30).delete() - self.assertEqual(len(self.Person.objects), 2) + self.assertEqual(self.Person.objects.count(), 2) self.Person.objects.delete() - self.assertEqual(len(self.Person.objects), 0) + self.assertEqual(self.Person.objects.count(), 0) def test_reverse_delete_rule_cascade(self): """Ensure cascading deletion of referring documents from the database. @@ -2332,8 +2331,8 @@ class QuerySetTest(unittest.TestCase): t = Test(testdict={'f': 'Value'}) t.save() - self.assertEqual(len(Test.objects(testdict__f__startswith='Val')), 1) - self.assertEqual(len(Test.objects(testdict__f='Value')), 1) + self.assertEqual(Test.objects(testdict__f__startswith='Val').count(), 1) + self.assertEqual(Test.objects(testdict__f='Value').count(), 1) Test.drop_collection() class Test(Document): @@ -2342,8 +2341,8 @@ class QuerySetTest(unittest.TestCase): t = Test(testdict={'f': 'Value'}) t.save() - self.assertEqual(len(Test.objects(testdict__f='Value')), 1) - self.assertEqual(len(Test.objects(testdict__f__startswith='Val')), 1) + self.assertEqual(Test.objects(testdict__f='Value').count(), 1) + self.assertEqual(Test.objects(testdict__f__startswith='Val').count(), 1) Test.drop_collection() def test_bulk(self): @@ -2539,8 +2538,7 @@ class QuerySetTest(unittest.TestCase): # Finds only one point because only the first point is within 60km of # the reference point to the south. points = Point.objects( - location__within_spherical_distance=[[-122, 36.5], 60/earth_radius] - ); + location__within_spherical_distance=[[-122, 36.5], 60/earth_radius]) self.assertEqual(points.count(), 1) self.assertEqual(points[0].id, south_point.id) @@ -2551,7 +2549,7 @@ class QuerySetTest(unittest.TestCase): """ class CustomQuerySet(QuerySet): def not_empty(self): - return len(self) > 0 + return self.count() > 0 class Post(Document): meta = {'queryset_class': CustomQuerySet} @@ -2572,7 +2570,7 @@ class QuerySetTest(unittest.TestCase): class CustomQuerySet(QuerySet): def not_empty(self): - return len(self) > 0 + return self.count() > 0 class CustomQuerySetManager(QuerySetManager): queryset_class = CustomQuerySet @@ -2619,7 +2617,7 @@ class QuerySetTest(unittest.TestCase): class CustomQuerySet(QuerySet): def not_empty(self): - return len(self) > 0 + return self.count() > 0 class Base(Document): meta = {'abstract': True, 'queryset_class': CustomQuerySet} @@ -2642,7 +2640,7 @@ class QuerySetTest(unittest.TestCase): class CustomQuerySet(QuerySet): def not_empty(self): - return len(self) > 0 + return self.count() > 0 class CustomQuerySetManager(QuerySetManager): queryset_class = CustomQuerySet @@ -3044,14 +3042,14 @@ class QuerySetTest(unittest.TestCase): # Find all people in the collection people = self.Person.objects.scalar('name') - self.assertEqual(len(people), 2) + self.assertEqual(people.count(), 2) results = list(people) self.assertEqual(results[0], "User A") self.assertEqual(results[1], "User B") # Use a query to filter the people found to just person1 people = self.Person.objects(age=20).scalar('name') - self.assertEqual(len(people), 1) + self.assertEqual(people.count(), 1) person = people.next() self.assertEqual(person, "User A") @@ -3097,7 +3095,7 @@ class QuerySetTest(unittest.TestCase): for i in xrange(55): self.Person(name='A%s' % i, age=i).save() - self.assertEqual(len(self.Person.objects.scalar('name')), 55) + self.assertEqual(self.Person.objects.scalar('name').count(), 55) self.assertEqual("A0", "%s" % self.Person.objects.order_by('name').scalar('name').first()) self.assertEqual("A0", "%s" % self.Person.objects.scalar('name').order_by('name')[0]) if PY3: @@ -3314,7 +3312,7 @@ class QuerySetTest(unittest.TestCase): inner_count = 0 inner_total_count = 0 - self.assertEqual(len(users), 7) + self.assertEqual(users.count(), 7) for i, outer_user in enumerate(users): self.assertEqual(outer_user.name, names[i]) @@ -3322,17 +3320,17 @@ class QuerySetTest(unittest.TestCase): inner_count = 0 # Calling len might disrupt the inner loop if there are bugs - self.assertEqual(len(users), 7) + self.assertEqual(users.count(), 7) for j, inner_user in enumerate(users): self.assertEqual(inner_user.name, names[j]) inner_count += 1 inner_total_count += 1 - self.assertEqual(inner_count, 7) # inner loop should always be executed seven times + self.assertEqual(inner_count, 7) # inner loop should always be executed seven times - self.assertEqual(outer_count, 7) # outer loop should be executed seven times total - self.assertEqual(inner_total_count, 7 * 7) # inner loop should be executed fourtynine times total + self.assertEqual(outer_count, 7) # outer loop should be executed seven times total + self.assertEqual(inner_total_count, 7 * 7) # inner loop should be executed fourtynine times total if __name__ == '__main__': unittest.main() diff --git a/tests/queryset/transform.py b/tests/queryset/transform.py index d38cbfd5..bde4b6f1 100644 --- a/tests/queryset/transform.py +++ b/tests/queryset/transform.py @@ -53,14 +53,14 @@ class TransformTest(unittest.TestCase): BlogPost.objects(title=data['title'])._query) self.assertFalse('title' in BlogPost.objects(title=data['title'])._query) - self.assertEqual(len(BlogPost.objects(title=data['title'])), 1) + self.assertEqual(BlogPost.objects(title=data['title']).count(), 1) self.assertTrue('_id' in BlogPost.objects(pk=post.id)._query) - self.assertEqual(len(BlogPost.objects(pk=post.id)), 1) + self.assertEqual(BlogPost.objects(pk=post.id).count(), 1) self.assertTrue('postComments.commentContent' in BlogPost.objects(comments__content='test')._query) - self.assertEqual(len(BlogPost.objects(comments__content='test')), 1) + self.assertEqual(BlogPost.objects(comments__content='test').count(), 1) BlogPost.drop_collection() @@ -79,7 +79,7 @@ class TransformTest(unittest.TestCase): self.assertTrue('_id' in BlogPost.objects(pk=data['title'])._query) self.assertTrue('_id' in BlogPost.objects(title=data['title'])._query) - self.assertEqual(len(BlogPost.objects(pk=data['title'])), 1) + self.assertEqual(BlogPost.objects(pk=data['title']).count(), 1) BlogPost.drop_collection() diff --git a/tests/queryset/visitor.py b/tests/queryset/visitor.py index 98815dbc..bd81a654 100644 --- a/tests/queryset/visitor.py +++ b/tests/queryset/visitor.py @@ -268,8 +268,8 @@ class QTest(unittest.TestCase): self.Person(name='user3', age=30).save() self.Person(name='user4', age=40).save() - self.assertEqual(len(self.Person.objects(Q(age__in=[20]))), 2) - self.assertEqual(len(self.Person.objects(Q(age__in=[20, 30]))), 3) + self.assertEqual(self.Person.objects(Q(age__in=[20])).count(), 2) + self.assertEqual(self.Person.objects(Q(age__in=[20, 30])).count(), 3) # Test invalid query objs def wrong_query_objs(): @@ -311,8 +311,8 @@ class QTest(unittest.TestCase): BlogPost(tags=['python', 'mongo']).save() BlogPost(tags=['python']).save() - self.assertEqual(len(BlogPost.objects(Q(tags='mongo'))), 1) - self.assertEqual(len(BlogPost.objects(Q(tags='python'))), 2) + self.assertEqual(BlogPost.objects(Q(tags='mongo')).count(), 1) + self.assertEqual(BlogPost.objects(Q(tags='python')).count(), 2) BlogPost.drop_collection() From 14b6c471cf519df6df5c737faaff512a338ef918 Mon Sep 17 00:00:00 2001 From: Ross Lawley Date: Mon, 22 Apr 2013 16:37:09 +0000 Subject: [PATCH 160/189] Fix PY3 hasattr connecting to the db at define time --- mongoengine/base/metaclasses.py | 2 +- tests/document/instance.py | 2 ++ 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/mongoengine/base/metaclasses.py b/mongoengine/base/metaclasses.py index 27040119..def8a055 100644 --- a/mongoengine/base/metaclasses.py +++ b/mongoengine/base/metaclasses.py @@ -340,7 +340,7 @@ class TopLevelDocumentMetaclass(DocumentMetaclass): new_class._meta['collection'] = collection(new_class) # Provide a default queryset unless exists or one has been set - if not hasattr(new_class, 'objects'): + if 'objects' not in dir(new_class): new_class.objects = QuerySetManager() # Validate the fields and set primary key if needed diff --git a/tests/document/instance.py b/tests/document/instance.py index 1adc1406..5513ed8d 100644 --- a/tests/document/instance.py +++ b/tests/document/instance.py @@ -1857,6 +1857,8 @@ class InstanceTest(unittest.TestCase): def test_db_alias_propagates(self): """db_alias propagates? """ + register_connection('testdb-1', 'mongoenginetest2') + class A(Document): name = StringField() meta = {"db_alias": "testdb-1", "allow_inheritance": True} From e4f38b5665a3281eb26e365e1b5c8b050820efa1 Mon Sep 17 00:00:00 2001 From: Ross Lawley Date: Mon, 22 Apr 2013 16:46:59 +0000 Subject: [PATCH 161/189] Fragile test fix --- tests/document/indexes.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/document/indexes.py b/tests/document/indexes.py index fea63a51..61e3c0e7 100644 --- a/tests/document/indexes.py +++ b/tests/document/indexes.py @@ -334,7 +334,7 @@ class IndexesTest(unittest.TestCase): User.ensure_indexes() info = User.objects._collection.index_information() - self.assertEqual(info.keys(), ['_cls_1_user_guid_1', '_id_']) + self.assertEqual(sorted(info.keys()), ['_cls_1_user_guid_1', '_id_']) User.drop_collection() def test_embedded_document_index(self): From 81c7007f80a1147e8577ac2682f802f209e3338a Mon Sep 17 00:00:00 2001 From: Ross Lawley Date: Tue, 23 Apr 2013 10:38:32 +0000 Subject: [PATCH 162/189] Added with_limit_and_skip support to count() (#235) --- docs/changelog.rst | 1 + mongoengine/queryset/queryset.py | 8 +++-- tests/queryset/queryset.py | 52 ++++++++++++++++++-------------- 3 files changed, 36 insertions(+), 25 deletions(-) diff --git a/docs/changelog.rst b/docs/changelog.rst index da1424eb..4c0da7fa 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -4,6 +4,7 @@ Changelog Changes in 0.8.X ================ +- Added with_limit_and_skip support to count() (#235) - Removed __len__ from queryset (#247) - Objects queryset manager now inherited (#256) - Updated connection to use MongoClient (#262, #274) diff --git a/mongoengine/queryset/queryset.py b/mongoengine/queryset/queryset.py index cc0b70f5..37d07a8d 100644 --- a/mongoengine/queryset/queryset.py +++ b/mongoengine/queryset/queryset.py @@ -342,12 +342,16 @@ class QuerySet(object): self._document, documents=results, loaded=True) return return_one and results[0] or results - def count(self): + def count(self, with_limit_and_skip=True): """Count the selected elements in the query. + + :param with_limit_and_skip (optional): take any :meth:`limit` or + :meth:`skip` that has been applied to this cursor into account when + getting the count """ if self._limit == 0: return 0 - return self._cursor.count(with_limit_and_skip=True) + return self._cursor.count(with_limit_and_skip=with_limit_and_skip) def delete(self, write_concern=None): """Delete the documents matched by the query. diff --git a/tests/queryset/queryset.py b/tests/queryset/queryset.py index f5aec7ed..c7c4c7ce 100644 --- a/tests/queryset/queryset.py +++ b/tests/queryset/queryset.py @@ -20,7 +20,7 @@ from mongoengine.python_support import PY3 from mongoengine.context_managers import query_counter from mongoengine.queryset import (QuerySet, QuerySetManager, MultipleObjectsReturned, DoesNotExist, - QueryFieldList, queryset_manager) + queryset_manager) from mongoengine.errors import InvalidQueryError __all__ = ("QuerySetTest",) @@ -65,10 +65,8 @@ class QuerySetTest(unittest.TestCase): def test_find(self): """Ensure that a query returns a valid set of results. """ - person1 = self.Person(name="User A", age=20) - person1.save() - person2 = self.Person(name="User B", age=30) - person2.save() + self.Person(name="User A", age=20).save() + self.Person(name="User B", age=30).save() # Find all people in the collection people = self.Person.objects @@ -338,21 +336,20 @@ class QuerySetTest(unittest.TestCase): comment2 = Comment(name='testb') post1 = Post(comments=[comment1, comment2]) post2 = Post(comments=[comment2, comment2]) - blog1 = Blog.objects.create(posts=[post1, post2]) - blog2 = Blog.objects.create(posts=[post2, post1]) + Blog.objects.create(posts=[post1, post2]) + Blog.objects.create(posts=[post2, post1]) # Update all of the first comments of second posts of all blogs - blog = Blog.objects().update(set__posts__1__comments__0__name="testc") + Blog.objects().update(set__posts__1__comments__0__name="testc") testc_blogs = Blog.objects(posts__1__comments__0__name="testc") self.assertEqual(testc_blogs.count(), 2) Blog.drop_collection() - - blog1 = Blog.objects.create(posts=[post1, post2]) - blog2 = Blog.objects.create(posts=[post2, post1]) + Blog.objects.create(posts=[post1, post2]) + Blog.objects.create(posts=[post2, post1]) # Update only the first blog returned by the query - blog = Blog.objects().update_one( + Blog.objects().update_one( set__posts__1__comments__1__name="testc") testc_blogs = Blog.objects(posts__1__comments__1__name="testc") self.assertEqual(testc_blogs.count(), 1) @@ -2661,6 +2658,19 @@ class QuerySetTest(unittest.TestCase): Post.drop_collection() + def test_count_limit_and_skip(self): + class Post(Document): + title = StringField() + + Post.drop_collection() + + for i in xrange(10): + Post(title="Post %s" % i).save() + + self.assertEqual(5, Post.objects.limit(5).skip(5).count()) + + self.assertEqual(10, Post.objects.limit(5).skip(5).count(with_limit_and_skip=False)) + def test_call_after_limits_set(self): """Ensure that re-filtering after slicing works """ @@ -2669,10 +2679,8 @@ class QuerySetTest(unittest.TestCase): Post.drop_collection() - post1 = Post(title="Post 1") - post1.save() - post2 = Post(title="Post 2") - post2.save() + Post(title="Post 1").save() + Post(title="Post 2").save() posts = Post.objects.all()[0:1] self.assertEqual(len(list(posts())), 1) @@ -3205,20 +3213,18 @@ class QuerySetTest(unittest.TestCase): float_field = FloatField(default=1.1) boolean_field = BooleanField(default=True) datetime_field = DateTimeField(default=datetime.now) - embedded_document_field = EmbeddedDocumentField(EmbeddedDoc, - default=lambda: EmbeddedDoc()) + embedded_document_field = EmbeddedDocumentField( + EmbeddedDoc, default=lambda: EmbeddedDoc()) list_field = ListField(default=lambda: [1, 2, 3]) dict_field = DictField(default=lambda: {"hello": "world"}) objectid_field = ObjectIdField(default=ObjectId) - reference_field = ReferenceField(Simple, default=lambda: - Simple().save()) + reference_field = ReferenceField(Simple, default=lambda: Simple().save()) map_field = MapField(IntField(), default=lambda: {"simple": 1}) decimal_field = DecimalField(default=1.0) complex_datetime_field = ComplexDateTimeField(default=datetime.now) url_field = URLField(default="http://mongoengine.org") dynamic_field = DynamicField(default=1) - generic_reference_field = GenericReferenceField( - default=lambda: Simple().save()) + generic_reference_field = GenericReferenceField(default=lambda: Simple().save()) sorted_list_field = SortedListField(IntField(), default=lambda: [1, 2, 3]) email_field = EmailField(default="ross@example.com") @@ -3226,7 +3232,7 @@ class QuerySetTest(unittest.TestCase): sequence_field = SequenceField() uuid_field = UUIDField(default=uuid.uuid4) generic_embedded_document_field = GenericEmbeddedDocumentField( - default=lambda: EmbeddedDoc()) + default=lambda: EmbeddedDoc()) Simple.drop_collection() Doc.drop_collection() From e2f3406e897fe0a033ae340665427c2b15ad9ce1 Mon Sep 17 00:00:00 2001 From: Ross Lawley Date: Tue, 23 Apr 2013 14:06:29 +0000 Subject: [PATCH 163/189] Updated .only() behaviour - now like exclude it is chainable (#202) --- docs/changelog.rst | 1 + docs/upgrade.rst | 15 +++++++++++++ mongoengine/queryset/field_list.py | 23 +++++++++++++++---- mongoengine/queryset/queryset.py | 23 ++++++++++++++++--- tests/queryset/field_list.py | 36 +++++++++++++++--------------- 5 files changed, 73 insertions(+), 25 deletions(-) diff --git a/docs/changelog.rst b/docs/changelog.rst index 4c0da7fa..fb9c35da 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -4,6 +4,7 @@ Changelog Changes in 0.8.X ================ +- Updated .only() behaviour - now like exclude it is chainable (#202) - Added with_limit_and_skip support to count() (#235) - Removed __len__ from queryset (#247) - Objects queryset manager now inherited (#256) diff --git a/docs/upgrade.rst b/docs/upgrade.rst index 4f549d61..c4273f01 100644 --- a/docs/upgrade.rst +++ b/docs/upgrade.rst @@ -114,6 +114,21 @@ explicit `queryset.count()` to update:: # New code Animal.objects(type="mammal").count()) + +.only() now inline with .exclude() +---------------------------------- + +The behaviour of `.only()` was highly ambious, now it works in the mirror fashion +to `.exclude()`. Chaining `.only()` calls will increase the fields required:: + + # Old code + Animal.objects().only(['type', 'name']).only('name', 'order') # Would have returned just `name` + + # New code + Animal.objects().only('name') + Animal.objects().only(['name']).only('order') # Would return `name` and `order` + + Client ====== PyMongo 2.4 came with a new connection client; MongoClient_ and started the diff --git a/mongoengine/queryset/field_list.py b/mongoengine/queryset/field_list.py index 7b2b0cb5..73d3cc24 100644 --- a/mongoengine/queryset/field_list.py +++ b/mongoengine/queryset/field_list.py @@ -7,11 +7,20 @@ class QueryFieldList(object): ONLY = 1 EXCLUDE = 0 - def __init__(self, fields=[], value=ONLY, always_include=[]): + def __init__(self, fields=None, value=ONLY, always_include=None, _only_called=False): + """The QueryFieldList builder + + :param fields: A list of fields used in `.only()` or `.exclude()` + :param value: How to handle the fields; either `ONLY` or `EXCLUDE` + :param always_include: Any fields to always_include eg `_cls` + :param _only_called: Has `.only()` been called? If so its a set of fields + otherwise it performs a union. + """ self.value = value - self.fields = set(fields) - self.always_include = set(always_include) + self.fields = set(fields or []) + self.always_include = set(always_include or []) self._id = None + self._only_called = _only_called self.slice = {} def __add__(self, f): @@ -26,7 +35,10 @@ class QueryFieldList(object): self.slice = {} elif self.value is self.ONLY and f.value is self.ONLY: self._clean_slice() - self.fields = self.fields.intersection(f.fields) + if self._only_called: + self.fields = self.fields.union(f.fields) + else: + self.fields = f.fields elif self.value is self.EXCLUDE and f.value is self.EXCLUDE: self.fields = self.fields.union(f.fields) self._clean_slice() @@ -46,6 +58,9 @@ class QueryFieldList(object): self.fields = self.fields.union(self.always_include) else: self.fields -= self.always_include + + if getattr(f, '_only_called', False): + self._only_called = True return self def __nonzero__(self): diff --git a/mongoengine/queryset/queryset.py b/mongoengine/queryset/queryset.py index 37d07a8d..dcfb2402 100644 --- a/mongoengine/queryset/queryset.py +++ b/mongoengine/queryset/queryset.py @@ -624,19 +624,35 @@ class QuerySet(object): post = BlogPost.objects(...).only("title", "author.name") + .. note :: `only()` is chainable and will perform a union :: + So with the following it will fetch both: `title` and `author.name`:: + + post = BlogPost.objects.only("title").only("author.name") + + :func:`~mongoengine.queryset.QuerySet.all_fields` will reset any + field filters. + :param fields: fields to include .. versionadded:: 0.3 .. versionchanged:: 0.5 - Added subfield support """ fields = dict([(f, QueryFieldList.ONLY) for f in fields]) - return self.fields(**fields) + return self.fields(True, **fields) def exclude(self, *fields): """Opposite to .only(), exclude some document's fields. :: post = BlogPost.objects(...).exclude("comments") + .. note :: `exclude()` is chainable and will perform a union :: + So with the following it will exclude both: `title` and `author.name`:: + + post = BlogPost.objects.exclude("title").exclude("author.name") + + :func:`~mongoengine.queryset.QuerySet.all_fields` will reset any + field filters. + :param fields: fields to exclude .. versionadded:: 0.5 @@ -644,7 +660,7 @@ class QuerySet(object): fields = dict([(f, QueryFieldList.EXCLUDE) for f in fields]) return self.fields(**fields) - def fields(self, **kwargs): + def fields(self, _only_called=False, **kwargs): """Manipulate how you load this document's fields. Used by `.only()` and `.exclude()` to manipulate which fields to retrieve. Fields also allows for a greater level of control for example: @@ -678,7 +694,8 @@ class QuerySet(object): for value, group in itertools.groupby(fields, lambda x: x[1]): fields = [field for field, value in group] fields = queryset._fields_to_dbfields(fields) - queryset._loaded_fields += QueryFieldList(fields, value=value) + queryset._loaded_fields += QueryFieldList(fields, value=value, _only_called=_only_called) + return queryset def all_fields(self): diff --git a/tests/queryset/field_list.py b/tests/queryset/field_list.py index 4a8a72b3..2bdfce1f 100644 --- a/tests/queryset/field_list.py +++ b/tests/queryset/field_list.py @@ -20,47 +20,47 @@ class QueryFieldListTest(unittest.TestCase): def test_include_include(self): q = QueryFieldList() - q += QueryFieldList(fields=['a', 'b'], value=QueryFieldList.ONLY) - self.assertEqual(q.as_dict(), {'a': True, 'b': True}) + q += QueryFieldList(fields=['a', 'b'], value=QueryFieldList.ONLY, _only_called=True) + self.assertEqual(q.as_dict(), {'a': 1, 'b': 1}) q += QueryFieldList(fields=['b', 'c'], value=QueryFieldList.ONLY) - self.assertEqual(q.as_dict(), {'b': True}) + self.assertEqual(q.as_dict(), {'a': 1, 'b': 1, 'c': 1}) def test_include_exclude(self): q = QueryFieldList() q += QueryFieldList(fields=['a', 'b'], value=QueryFieldList.ONLY) - self.assertEqual(q.as_dict(), {'a': True, 'b': True}) + self.assertEqual(q.as_dict(), {'a': 1, 'b': 1}) q += QueryFieldList(fields=['b', 'c'], value=QueryFieldList.EXCLUDE) - self.assertEqual(q.as_dict(), {'a': True}) + self.assertEqual(q.as_dict(), {'a': 1}) def test_exclude_exclude(self): q = QueryFieldList() q += QueryFieldList(fields=['a', 'b'], value=QueryFieldList.EXCLUDE) - self.assertEqual(q.as_dict(), {'a': False, 'b': False}) + self.assertEqual(q.as_dict(), {'a': 0, 'b': 0}) q += QueryFieldList(fields=['b', 'c'], value=QueryFieldList.EXCLUDE) - self.assertEqual(q.as_dict(), {'a': False, 'b': False, 'c': False}) + self.assertEqual(q.as_dict(), {'a': 0, 'b': 0, 'c': 0}) def test_exclude_include(self): q = QueryFieldList() q += QueryFieldList(fields=['a', 'b'], value=QueryFieldList.EXCLUDE) - self.assertEqual(q.as_dict(), {'a': False, 'b': False}) + self.assertEqual(q.as_dict(), {'a': 0, 'b': 0}) q += QueryFieldList(fields=['b', 'c'], value=QueryFieldList.ONLY) - self.assertEqual(q.as_dict(), {'c': True}) + self.assertEqual(q.as_dict(), {'c': 1}) def test_always_include(self): q = QueryFieldList(always_include=['x', 'y']) q += QueryFieldList(fields=['a', 'b', 'x'], value=QueryFieldList.EXCLUDE) q += QueryFieldList(fields=['b', 'c'], value=QueryFieldList.ONLY) - self.assertEqual(q.as_dict(), {'x': True, 'y': True, 'c': True}) + self.assertEqual(q.as_dict(), {'x': 1, 'y': 1, 'c': 1}) def test_reset(self): q = QueryFieldList(always_include=['x', 'y']) q += QueryFieldList(fields=['a', 'b', 'x'], value=QueryFieldList.EXCLUDE) q += QueryFieldList(fields=['b', 'c'], value=QueryFieldList.ONLY) - self.assertEqual(q.as_dict(), {'x': True, 'y': True, 'c': True}) + self.assertEqual(q.as_dict(), {'x': 1, 'y': 1, 'c': 1}) q.reset() self.assertFalse(q) q += QueryFieldList(fields=['b', 'c'], value=QueryFieldList.ONLY) - self.assertEqual(q.as_dict(), {'x': True, 'y': True, 'b': True, 'c': True}) + self.assertEqual(q.as_dict(), {'x': 1, 'y': 1, 'b': 1, 'c': 1}) def test_using_a_slice(self): q = QueryFieldList() @@ -97,7 +97,7 @@ class OnlyExcludeAllTest(unittest.TestCase): qs = MyDoc.objects.fields(**dict(((i, 1) for i in include))) self.assertEqual(qs._loaded_fields.as_dict(), - {'a': 1, 'b': 1, 'c': 1, 'd': 1, 'e': 1}) + {'a': 1, 'b': 1, 'c': 1, 'd': 1, 'e': 1}) qs = qs.only(*only) self.assertEqual(qs._loaded_fields.as_dict(), {'b': 1, 'c': 1}) qs = qs.exclude(*exclude) @@ -134,15 +134,15 @@ class OnlyExcludeAllTest(unittest.TestCase): qs = qs.only(*only) qs = qs.fields(slice__b=5) self.assertEqual(qs._loaded_fields.as_dict(), - {'b': {'$slice': 5}, 'c': 1}) + {'b': {'$slice': 5}, 'c': 1}) qs = qs.fields(slice__c=[5, 1]) self.assertEqual(qs._loaded_fields.as_dict(), - {'b': {'$slice': 5}, 'c': {'$slice': [5, 1]}}) + {'b': {'$slice': 5}, 'c': {'$slice': [5, 1]}}) qs = qs.exclude('c') self.assertEqual(qs._loaded_fields.as_dict(), - {'b': {'$slice': 5}}) + {'b': {'$slice': 5}}) def test_only(self): """Ensure that QuerySet.only only returns the requested fields. @@ -328,7 +328,7 @@ class OnlyExcludeAllTest(unittest.TestCase): Numbers.drop_collection() - numbers = Numbers(n=[0,1,2,3,4,5,-5,-4,-3,-2,-1]) + numbers = Numbers(n=[0, 1, 2, 3, 4, 5, -5, -4, -3, -2, -1]) numbers.save() # first three @@ -368,7 +368,7 @@ class OnlyExcludeAllTest(unittest.TestCase): Numbers.drop_collection() numbers = Numbers() - numbers.embedded = EmbeddedNumber(n=[0,1,2,3,4,5,-5,-4,-3,-2,-1]) + numbers.embedded = EmbeddedNumber(n=[0, 1, 2, 3, 4, 5, -5, -4, -3, -2, -1]) numbers.save() # first three From a692316293c1cdea8c8d10089e0651e7478dcc28 Mon Sep 17 00:00:00 2001 From: Ross Lawley Date: Tue, 23 Apr 2013 14:09:41 +0000 Subject: [PATCH 164/189] Update to upgrade docs --- docs/upgrade.rst | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/docs/upgrade.rst b/docs/upgrade.rst index c4273f01..5490757d 100644 --- a/docs/upgrade.rst +++ b/docs/upgrade.rst @@ -126,7 +126,9 @@ to `.exclude()`. Chaining `.only()` calls will increase the fields required:: # New code Animal.objects().only('name') - Animal.objects().only(['name']).only('order') # Would return `name` and `order` + + # Note: + Animal.objects().only(['name']).only('order') # Now returns `name` *and* `order` Client From 94d1e566c018bc26875b1cbc585ba2215d88f80d Mon Sep 17 00:00:00 2001 From: Ross Lawley Date: Tue, 23 Apr 2013 14:44:17 +0000 Subject: [PATCH 165/189] Added SequenceField.set_next_value(value) helper (#159) --- docs/changelog.rst | 1 + mongoengine/fields.py | 19 +++++++++++++++---- tests/fields/fields.py | 40 +++++++++++++++++++++++++--------------- 3 files changed, 41 insertions(+), 19 deletions(-) diff --git a/docs/changelog.rst b/docs/changelog.rst index fb9c35da..d22fc600 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -4,6 +4,7 @@ Changelog Changes in 0.8.X ================ +- Added SequenceField.set_next_value(value) helper (#159) - Updated .only() behaviour - now like exclude it is chainable (#202) - Added with_limit_and_skip support to count() (#235) - Removed __len__ from queryset (#247) diff --git a/mongoengine/fields.py b/mongoengine/fields.py index 690e7ace..4fc65c7e 100644 --- a/mongoengine/fields.py +++ b/mongoengine/fields.py @@ -1408,13 +1408,13 @@ class SequenceField(BaseField): COLLECTION_NAME = 'mongoengine.counters' VALUE_DECORATOR = int - def __init__(self, collection_name=None, db_alias=None, - sequence_name=None, value_decorator=None, *args, **kwargs): + def __init__(self, collection_name=None, db_alias=None, sequence_name=None, + value_decorator=None, *args, **kwargs): self.collection_name = collection_name or self.COLLECTION_NAME self.db_alias = db_alias or DEFAULT_CONNECTION_NAME self.sequence_name = sequence_name self.value_decorator = (callable(value_decorator) and - value_decorator or self.VALUE_DECORATOR) + value_decorator or self.VALUE_DECORATOR) return super(SequenceField, self).__init__(*args, **kwargs) def generate(self): @@ -1430,6 +1430,17 @@ class SequenceField(BaseField): upsert=True) return self.value_decorator(counter['next']) + def set_next_value(self, value): + """Helper method to set the next sequence value""" + sequence_name = self.get_sequence_name() + sequence_id = "%s.%s" % (sequence_name, self.name) + collection = get_db(alias=self.db_alias)[self.collection_name] + counter = collection.find_and_modify(query={"_id": sequence_id}, + update={"$set": {"next": value}}, + new=True, + upsert=True) + return self.value_decorator(counter['next']) + def get_sequence_name(self): if self.sequence_name: return self.sequence_name @@ -1438,7 +1449,7 @@ class SequenceField(BaseField): return owner._get_collection_name() else: return ''.join('_%s' % c if c.isupper() else c - for c in owner._class_name).strip('_').lower() + for c in owner._class_name).strip('_').lower() def __get__(self, instance, owner): value = super(SequenceField, self).__get__(instance, owner) diff --git a/tests/fields/fields.py b/tests/fields/fields.py index 9a7b82f7..ade44b8d 100644 --- a/tests/fields/fields.py +++ b/tests/fields/fields.py @@ -2164,18 +2164,21 @@ class FieldTest(unittest.TestCase): Person.drop_collection() for x in xrange(10): - p = Person(name="Person %s" % x) - p.save() + Person(name="Person %s" % x).save() c = self.db['mongoengine.counters'].find_one({'_id': 'person.id'}) self.assertEqual(c['next'], 10) ids = [i.id for i in Person.objects] - self.assertEqual(ids, range(1, 11)) + self.assertEqual(ids, xrange(1, 11)) c = self.db['mongoengine.counters'].find_one({'_id': 'person.id'}) self.assertEqual(c['next'], 10) + Person.id.set_next_value(1000) + c = self.db['mongoengine.counters'].find_one({'_id': 'person.id'}) + self.assertEqual(c['next'], 1000) + def test_sequence_field_sequence_name(self): class Person(Document): id = SequenceField(primary_key=True, sequence_name='jelly') @@ -2185,8 +2188,7 @@ class FieldTest(unittest.TestCase): Person.drop_collection() for x in xrange(10): - p = Person(name="Person %s" % x) - p.save() + Person(name="Person %s" % x).save() c = self.db['mongoengine.counters'].find_one({'_id': 'jelly.id'}) self.assertEqual(c['next'], 10) @@ -2197,6 +2199,10 @@ class FieldTest(unittest.TestCase): c = self.db['mongoengine.counters'].find_one({'_id': 'jelly.id'}) self.assertEqual(c['next'], 10) + Person.id.set_next_value(1000) + c = self.db['mongoengine.counters'].find_one({'_id': 'jelly.id'}) + self.assertEqual(c['next'], 1000) + def test_multiple_sequence_fields(self): class Person(Document): id = SequenceField(primary_key=True) @@ -2207,21 +2213,28 @@ class FieldTest(unittest.TestCase): Person.drop_collection() for x in xrange(10): - p = Person(name="Person %s" % x) - p.save() + Person(name="Person %s" % x).save() c = self.db['mongoengine.counters'].find_one({'_id': 'person.id'}) self.assertEqual(c['next'], 10) ids = [i.id for i in Person.objects] - self.assertEqual(ids, range(1, 11)) + self.assertEqual(ids, xrange(1, 11)) counters = [i.counter for i in Person.objects] - self.assertEqual(counters, range(1, 11)) + self.assertEqual(counters, xrange(1, 11)) c = self.db['mongoengine.counters'].find_one({'_id': 'person.id'}) self.assertEqual(c['next'], 10) + Person.id.set_next_value(1000) + c = self.db['mongoengine.counters'].find_one({'_id': 'person.id'}) + self.assertEqual(c['next'], 1000) + + Person.counter.set_next_value(999) + c = self.db['mongoengine.counters'].find_one({'_id': 'person.counter'}) + self.assertEqual(c['next'], 999) + def test_sequence_fields_reload(self): class Animal(Document): counter = SequenceField() @@ -2230,8 +2243,7 @@ class FieldTest(unittest.TestCase): self.db['mongoengine.counters'].drop() Animal.drop_collection() - a = Animal(name="Boi") - a.save() + a = Animal(name="Boi").save() self.assertEqual(a.counter, 1) a.reload() @@ -2261,10 +2273,8 @@ class FieldTest(unittest.TestCase): Person.drop_collection() for x in xrange(10): - a = Animal(name="Animal %s" % x) - a.save() - p = Person(name="Person %s" % x) - p.save() + Animal(name="Animal %s" % x).save() + Person(name="Person %s" % x).save() c = self.db['mongoengine.counters'].find_one({'_id': 'person.id'}) self.assertEqual(c['next'], 10) From 3653981416146bdceb1ff6106c9c3d35d9c7db0e Mon Sep 17 00:00:00 2001 From: Ross Lawley Date: Tue, 23 Apr 2013 15:12:57 +0000 Subject: [PATCH 166/189] Added ImageField support for inline replacements (#86) --- docs/changelog.rst | 1 + mongoengine/fields.py | 24 ++++++++++-------------- tests/fields/file_tests.py | 30 ++++++++++++++++++++++++++++++ tests/fields/mongodb_leaf.png | Bin 0 -> 4971 bytes 4 files changed, 41 insertions(+), 14 deletions(-) create mode 100644 tests/fields/mongodb_leaf.png diff --git a/docs/changelog.rst b/docs/changelog.rst index d22fc600..476753da 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -4,6 +4,7 @@ Changelog Changes in 0.8.X ================ +- ImageFields now support inline replacements (#86) - Added SequenceField.set_next_value(value) helper (#159) - Updated .only() behaviour - now like exclude it is chainable (#202) - Added with_limit_and_skip support to count() (#235) diff --git a/mongoengine/fields.py b/mongoengine/fields.py index 4fc65c7e..45304291 100644 --- a/mongoengine/fields.py +++ b/mongoengine/fields.py @@ -1165,13 +1165,11 @@ class FileField(BaseField): grid_file.delete() except: pass - # Create a new file with the new data - grid_file.put(value) - else: - # Create a new proxy object as we don't already have one - instance._data[key] = self.proxy_class(key=key, instance=instance, - collection_name=self.collection_name) - instance._data[key].put(value) + + # Create a new proxy object as we don't already have one + instance._data[key] = self.proxy_class(key=key, instance=instance, + collection_name=self.collection_name) + instance._data[key].put(value) else: instance._data[key] = value @@ -1208,6 +1206,8 @@ class ImageGridFsProxy(GridFSProxy): Insert a image in database applying field properties (size, thumbnail_size) """ + if not self.instance: + import ipdb; ipdb.set_trace(); field = self.instance._fields[self.key] try: @@ -1235,10 +1235,7 @@ class ImageGridFsProxy(GridFSProxy): size = field.thumbnail_size if size['force']: - thumbnail = ImageOps.fit(img, - (size['width'], - size['height']), - Image.ANTIALIAS) + thumbnail = ImageOps.fit(img, (size['width'], size['height']), Image.ANTIALIAS) else: thumbnail = img.copy() thumbnail.thumbnail((size['width'], @@ -1246,8 +1243,7 @@ class ImageGridFsProxy(GridFSProxy): Image.ANTIALIAS) if thumbnail: - thumb_id = self._put_thumbnail(thumbnail, - img_format) + thumb_id = self._put_thumbnail(thumbnail, img_format) else: thumb_id = None @@ -1350,7 +1346,7 @@ class ImageField(FileField): if isinstance(att, (tuple, list)): if PY3: value = dict(itertools.zip_longest(params_size, att, - fillvalue=None)) + fillvalue=None)) else: value = dict(map(None, params_size, att)) diff --git a/tests/fields/file_tests.py b/tests/fields/file_tests.py index 44d28626..c5842d81 100644 --- a/tests/fields/file_tests.py +++ b/tests/fields/file_tests.py @@ -16,6 +16,7 @@ from mongoengine.connection import get_db from mongoengine.python_support import PY3, b, StringIO TEST_IMAGE_PATH = os.path.join(os.path.dirname(__file__), 'mongoengine.png') +TEST_IMAGE2_PATH = os.path.join(os.path.dirname(__file__), 'mongodb_leaf.png') class FileTest(unittest.TestCase): @@ -217,6 +218,19 @@ class FileTest(unittest.TestCase): self.assertEqual(marmot.photo.content_type, 'image/jpeg') self.assertEqual(marmot.photo.foo, 'bar') + def test_file_reassigning(self): + class TestFile(Document): + the_file = FileField() + TestFile.drop_collection() + + test_file = TestFile(the_file=open(TEST_IMAGE_PATH, 'rb')).save() + self.assertEqual(test_file.the_file.get().length, 8313) + + test_file = TestFile.objects.first() + test_file.the_file = open(TEST_IMAGE2_PATH, 'rb') + test_file.save() + self.assertEqual(test_file.the_file.get().length, 4971) + def test_file_boolean(self): """Ensure that a boolean test of a FileField indicates its presence """ @@ -264,6 +278,22 @@ class FileTest(unittest.TestCase): t.image.delete() + def test_image_field_reassigning(self): + if PY3: + raise SkipTest('PIL does not have Python 3 support') + + class TestFile(Document): + the_file = ImageField() + TestFile.drop_collection() + + test_file = TestFile(the_file=open(TEST_IMAGE_PATH, 'rb')).save() + self.assertEqual(test_file.the_file.size, (371, 76)) + + test_file = TestFile.objects.first() + test_file.the_file = open(TEST_IMAGE2_PATH, 'rb') + test_file.save() + self.assertEqual(test_file.the_file.size, (45, 101)) + def test_image_field_resize(self): if PY3: raise SkipTest('PIL does not have Python 3 support') diff --git a/tests/fields/mongodb_leaf.png b/tests/fields/mongodb_leaf.png new file mode 100644 index 0000000000000000000000000000000000000000..36661cefc824b1707a9a6063d133cefe3566e1a9 GIT binary patch literal 4971 zcmV-x6O`4Tx05}naRo`#hR1`jmZ&IWdKOk5~hl<6oRa0BJ8yc;~21%2p?MfD<>DVeH z9(p*dx19w`~g7O0}n_%Aq@s%d)fBDv`JHkDym6Hd+5XuAtvnwRpGmK zVkc9?T=n|PIo~X-eVh__(Z?q}P9Z-Dj?gOW6|D%o20XmjW-qs4UjrD(li^iv8@eK9k+ZFm zVRFymFOPAzG5-%Pn|1W;U4vNroTa&AxDScmEA~{ri9gr1^c?U@uwSpaNnw8l_>cP1 zd;)kMQS_;jeRSUEM_*s96y65j1$)tOrwdK{YIQMt92l|D^(E_=$Rjw{b!QT@q!)ni zR`|5oW9X5n$Wv+HVc@|^eX5yXnsHX8PF3UX~a6)MwxDE0HaPjyrlI!;jX{6Kvuh*8ej?;85ekN$?5uuCiS zBTvvVG+XTxAO{m@bvM#Jr)z6J><&E22D|vq?Y?Vkbo_DijopiF$2PET#mZ8eu=y$(ArYkv7@Ex`GL?QCc!_*KFrd&;n1r7 zqW-CFs9&fT)ZaU5gc&=gBz-DaCw(vdOp0__x+47~U6sC(E(JNe@4cTT*n6*E zVH4eoU1-&7pEV~_PRe`a7v+@vy!^5}8?Y3)UmlaER00009a7bBm000XU000XU0RWnu7ytkc?MXyIRCoc^ zTx*OTRT(~W`M$liv?9=Q-7ZBg?UscIL7{S~{J>Oftboy=iB=P-L@Ce^2(~VzNNU(d z)MEHiLIP?a0#-wiU<~39ei#f=%GRb7soO493zb{Tx8EF}=RN1lH#6US*PTt^Y-i5x zy*%eV&zv)t-72lMD*CwU@vA&UCcAY(6%|IIH?>+sr&yCzP1E$5~jDW6IrmiiO zN*^82*KGUJ@~YMsmV)rug&T)g&qTBfjr7XLSDqDV^+bg0q+AM=^8C9#)Sy>9wqm)` z>P1iU0*qAP1GG|e7T&dFOVPj!iS(6^T=`uU>E8ovNdvV4`oMRlz3Ya4G;%?az@6o; zZ7%=y@*gTs{|q=m2fzhC5NH*K5^1sPMf#Zqca98o?Ifc(Ezno|>WX#Ri`Fx;S^%R2 z@TiHKhyyUiS-z)VICpz$ z z8eMW9)jCaWgI@Z;#cN>vQobA<&|M0 zRQ}W-^ZFXKwFG_s{TEyh!@~|JZ-!)*=#(Oj5U^dVQ%~J6{L^M4$-btbhqqidOZjTE z0EVQllF@WxV~ng3MTn%`M9D~d*AH%3yto~mrl8B=yPIJ!q#+tk!f=r{ZKgL#5$=fD zJ34kcmH{(0OhxcQ~=6-gaOwUMd;0fav*DKcODdq>(hts zSn~b+-dbxz(3PsU$v8=HxGjo69w_&R|$9Xg> z**tMYzOD36L^jgYCc`V4R6*HkJq1FJ5nZQ&|f1 z8C%XhAHDPFj7dwxk&6d_l?gyEw3=-+GB814?8IjpZgk|V>$p?AF4j#HQlRmYWt9O2 z+hUwRQRspq1{`%0KogGJDtda};OL^uv$84Bi0s#l0BOM=`m}lM9EEl$lJP|K&J(6~ z%DXkI7lS_SzOz3GF#Q4&9-NbIDI&A&jKFIhhqFj*U02~77qC093N6cu#-OVebrrcv z;ub=Y9F&PM;F_oba{!s=jc&oZVXSDi6oW?73L32$HpDP(`4q9l&s{~##Rw)Pqn0e^ z45|hzThghg8v*T&UWWH?>MXfSBVYLgv~B>Th}6Xq`jB(klgF&Y?WzordMt=cZs4+r z>grZqf=-uskWRf;6VRu8?BFFv6XFdhBD(~ELIh_S5RLPQlb|&%Mk4NMSRkn6L4$U| zGQPx#R)gynU*n7jXsxQ3h!gS|J;p6X3_R0jfjR*KbXzv$k{eAP;3P1X&3+9JFq1sk zl}us_3J{AaVs7xYx=`jELzIgGS{m;UF)h+eBQZtnF!AaG{g4lkb2mdy@1~vdX3_+l zaY{3_B*<%3qhadQO(v;p{VCTb5LD!y$2sAMML9tcdS^hTvyF|$7=bNoO6H#BIB|(! zfA+_~7QI+DS@4c^uhV?Ko`U)ub{j`&iqN!i;#?s&O{UO*B2GFBO=c{KC0mYdC;b?3 z)yk2tQrti3s3nUf8W47oE^HOa0!r2s;A~ttPDx}A4^Fr%+x${A;rYa`o+V_h8ARC4_U?p*&m;n^m(UnPtQfM4v2R~4vU?@Q&wCg2S5w^Jyn5pH2^CFZnnm&w( zdX(ef_{LOjUnzrd85<(dQ$GO12d>v`;9q>qy~oTxB5o$aDA96yxzWunF>&6r@#(9QYWU?09GRG!hvJ_#n3?^x} z9-Of%!tp7V;*Cqo7Wlph)3c=)4ge;-`Jc^J?qzbLnEYXFG-4vREGZHP& z;$^Bj*9VB%hFEl0i-^<`tBA2^U~yokWCmg?XyMXf!BNIcN3G4+= zia02UBJN%!0(F8w_#^JxJsEIDTGsfON2ZNct{#Wwt$ku*If%Sfq$NcdR2UV@{e9K2DCd_5! zVrwEIJV%om4BX00V1=-45>S~L^dWb0)FVjIM^3Bma5w&DEj4DA^(%IW&Ro z$$wqMoR0AxM~BTs%3R8^Fo9Zt>`Mw*ibx|n+|da*@smE$#on>K?}vDU^onfbG|1M1 zR*|&3&T0a~s<51h|C?;~y|DVCL`e{{+olrIsR{P^ zlQ-;q)y%Ec65LdG@NM29MI5IckZdjv+=+m)0<6cwFrBzzc#W;RyqNyRaqwlM@RU~@ zlMTE9fIe{5e8Hle9gJNfr7mi`7lAHi#i>{_r(`&jAT!`!&Fq#kq-)+;`aeroZ;v~7Y-WMp(TIa=)` zi8CnVX@lBpdp2#KbRscmHsID~OH6@|C}Lh6$V{2h$Lw{3u6roA3-DYKGK!xX+x*}4gKfEJpCk6W7H|*FC-&hNqQ-si3Z8T=(m4lh!-x1tD*iA4@MUPaOr!S6g#C{Cgh*AfK_PHYND0;O3`?qsnj z-Vg${{Pa_2OZ3Ml z?%w&A6c$wKKvMv4@EhjSqe#`FN(Pd)IY=JyfY_K!_WnfPxlAekT3ubr1wDE5_#OoJ z7j~ZDfo`lrMx7p|38w}W5za%d6}Wg5w8z6LfE+LH8yip0IJ%i~L9?U(o%h$^A@4nw zCAC2jBVs5LZ{ZG1HqRLiMNC3TYy0)swbk;5p!clb!&eL6XtvR0R}q>O$GnK##!~au zvXmId;}iGnitjbDiVZk$ntJ1N^Ke+beK+m+GZv5^z`eBa zOu%xbOA}7CWH%aq2%TH)G4~&J;H=S12FUyOy}om0ra8CT7HDb$_?^J?JDE$=1-Q)x ztSj4mo*IA*e_ADxs)T;!68#8&H=%WE8#GN#p0wkO2;z1z+k_$v?tE`RXi^Jw?WvY~ zsl0@TF>V2Vqa?_;1_58jExUnp(SW*?s5(72y}Oyf*|m+5`jnY|H)Ck{Sp~s@E?W-cvD? z3~~I8`^I)|&UCe`b_|*p=!02UHkpW#+6@0v2(1&kOkgq@Fj_}TX_l)1( z5@C)&SDMTumMUrlv;RhN7102DbhS@zE6<*FJnrVwx2V*`$INw;WBn%L=BxhDtk)86ObH}b?Z0PyLw9hE*0ey= z#NIXA5B7S8&jF~X<-$Vf2l16pzPAAQwS%8V7dfO3dfNXC?kToT2QYQ9@-Rfjq``Yf z?wX@}Ze8`ki1K)E0J(@x=_X%x}jE%YdrR4EL0v#rQ9BcQ`l@OKtz=z1(O0vdo1;sR8qatYDy z>l$1m#Tfxz_QJRDe<1*LkLv6Dh=BIWex(Fzl(8fB^z`*MideVpGZN{Zp3 Date: Tue, 23 Apr 2013 15:32:10 +0000 Subject: [PATCH 167/189] Update build assets for PY3 --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index ba538fab..e545ba23 100644 --- a/setup.py +++ b/setup.py @@ -56,7 +56,7 @@ if sys.version_info[0] == 3: extra_opts['packages'] = find_packages(exclude=('tests',)) if "test" in sys.argv or "nosetests" in sys.argv: extra_opts['packages'].append("tests") - extra_opts['package_data'] = {"tests": ["fields/mongoengine.png"]} + extra_opts['package_data'] = {"tests": ["fields/mongoengine.png", "fields/mongodb_leaf.png"]} else: extra_opts['tests_require'] = ['nose', 'coverage', 'blinker', 'django==1.4.2', 'PIL'] extra_opts['packages'] = find_packages(exclude=('tests',)) From 1e1e48732a92cc5e355301e7178b2337f2e48626 Mon Sep 17 00:00:00 2001 From: Ross Lawley Date: Tue, 23 Apr 2013 15:33:38 +0000 Subject: [PATCH 168/189] Update setup.py python versions / langauge in changelog --- docs/changelog.rst | 2 +- setup.py | 1 - 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/docs/changelog.rst b/docs/changelog.rst index 476753da..49ee0ec3 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -4,7 +4,7 @@ Changelog Changes in 0.8.X ================ -- ImageFields now support inline replacements (#86) +- Added ImageField support for inline replacements (#86) - Added SequenceField.set_next_value(value) helper (#159) - Updated .only() behaviour - now like exclude it is chainable (#202) - Added with_limit_and_skip support to count() (#235) diff --git a/setup.py b/setup.py index e545ba23..13c11a90 100644 --- a/setup.py +++ b/setup.py @@ -38,7 +38,6 @@ CLASSIFIERS = [ 'Operating System :: OS Independent', 'Programming Language :: Python', "Programming Language :: Python :: 2", - "Programming Language :: Python :: 2.5", "Programming Language :: Python :: 2.6", "Programming Language :: Python :: 2.7", "Programming Language :: Python :: 3", From 88f96b08386fcf9607600a79f6d5768e81c65a67 Mon Sep 17 00:00:00 2001 From: Ross Lawley Date: Tue, 23 Apr 2013 15:59:23 +0000 Subject: [PATCH 169/189] Ensure all field params are documented (#97) --- docs/apireference.rst | 49 +++++++++++++++++++++++------------------ docs/django.rst | 2 +- docs/guide/querying.rst | 4 ++-- docs/upgrade.rst | 2 +- mongoengine/__init__.py | 4 +++- mongoengine/errors.py | 4 +++- mongoengine/fields.py | 23 ++++++++++--------- 7 files changed, 50 insertions(+), 38 deletions(-) diff --git a/docs/apireference.rst b/docs/apireference.rst index 049cc303..0040f45e 100644 --- a/docs/apireference.rst +++ b/docs/apireference.rst @@ -54,28 +54,33 @@ Querying Fields ====== -.. autoclass:: mongoengine.BinaryField -.. autoclass:: mongoengine.BooleanField -.. autoclass:: mongoengine.ComplexDateTimeField -.. autoclass:: mongoengine.DateTimeField -.. autoclass:: mongoengine.DecimalField -.. autoclass:: mongoengine.DictField -.. autoclass:: mongoengine.DynamicField -.. autoclass:: mongoengine.EmailField -.. autoclass:: mongoengine.EmbeddedDocumentField -.. autoclass:: mongoengine.FileField -.. autoclass:: mongoengine.FloatField -.. autoclass:: mongoengine.GenericEmbeddedDocumentField -.. autoclass:: mongoengine.GenericReferenceField -.. autoclass:: mongoengine.GeoPointField -.. autoclass:: mongoengine.ImageField -.. autoclass:: mongoengine.IntField -.. autoclass:: mongoengine.ListField -.. autoclass:: mongoengine.MapField -.. autoclass:: mongoengine.ObjectIdField -.. autoclass:: mongoengine.ReferenceField -.. autoclass:: mongoengine.SequenceField -.. autoclass:: mongoengine.SortedListField .. autoclass:: mongoengine.StringField .. autoclass:: mongoengine.URLField +.. autoclass:: mongoengine.EmailField +.. autoclass:: mongoengine.IntField +.. autoclass:: mongoengine.LongField +.. autoclass:: mongoengine.FloatField +.. autoclass:: mongoengine.DecimalField +.. autoclass:: mongoengine.BooleanField +.. autoclass:: mongoengine.DateTimeField +.. autoclass:: mongoengine.ComplexDateTimeField +.. autoclass:: mongoengine.EmbeddedDocumentField +.. autoclass:: mongoengine.GenericEmbeddedDocumentField +.. autoclass:: mongoengine.DynamicField +.. autoclass:: mongoengine.ListField +.. autoclass:: mongoengine.SortedListField +.. autoclass:: mongoengine.DictField +.. autoclass:: mongoengine.MapField +.. autoclass:: mongoengine.ReferenceField +.. autoclass:: mongoengine.GenericReferenceField +.. autoclass:: mongoengine.BinaryField +.. autoclass:: mongoengine.FileField +.. autoclass:: mongoengine.ImageField +.. autoclass:: mongoengine.GeoPointField +.. autoclass:: mongoengine.SequenceField +.. autoclass:: mongoengine.ObjectIdField .. autoclass:: mongoengine.UUIDField +.. autoclass:: mongoengine.GridFSError +.. autoclass:: mongoengine.GridFSProxy +.. autoclass:: mongoengine.ImageGridFsProxy +.. autoclass:: mongoengine.ImproperlyConfigured diff --git a/docs/django.rst b/docs/django.rst index 6f27b902..5c9e7bfb 100644 --- a/docs/django.rst +++ b/docs/django.rst @@ -10,7 +10,7 @@ In your **settings.py** file, ignore the standard database settings (unless you also plan to use the ORM in your project), and instead call :func:`~mongoengine.connect` somewhere in the settings module. -.. note :: +.. note:: If you are not using another Database backend you may need to add a dummy database backend to ``settings.py`` eg:: diff --git a/docs/guide/querying.rst b/docs/guide/querying.rst index 32798531..5e250cea 100644 --- a/docs/guide/querying.rst +++ b/docs/guide/querying.rst @@ -450,7 +450,7 @@ modifier comes before the field, not after it:: >>> post.tags ['database', 'nosql'] -.. note :: +.. note:: In version 0.5 the :meth:`~mongoengine.Document.save` runs atomic updates on changed documents by tracking changes to that document. @@ -466,7 +466,7 @@ cannot use the `$` syntax in keyword arguments it has been mapped to `S`:: >>> post.tags ['database', 'mongodb'] -.. note :: +.. note:: Currently only top level lists are handled, future versions of mongodb / pymongo plan to support nested positional operators. See `The $ positional operator `_. diff --git a/docs/upgrade.rst b/docs/upgrade.rst index 5490757d..564b7f6d 100644 --- a/docs/upgrade.rst +++ b/docs/upgrade.rst @@ -205,7 +205,7 @@ via `save` eg :: # Or in code: my_document.save(cascade=True) -.. note :: +.. note:: Remember: cascading saves **do not** cascade through lists. ReferenceFields diff --git a/mongoengine/__init__.py b/mongoengine/__init__.py index da72e53b..6fe6d088 100644 --- a/mongoengine/__init__.py +++ b/mongoengine/__init__.py @@ -8,10 +8,12 @@ import queryset from queryset import * import signals from signals import * +from errors import * +import errors import django __all__ = (list(document.__all__) + fields.__all__ + connection.__all__ + - list(queryset.__all__) + signals.__all__) + list(queryset.__all__) + signals.__all__ + list(errors.__all__)) VERSION = (0, 8, 0, '+') diff --git a/mongoengine/errors.py b/mongoengine/errors.py index 9cfcd1d2..4b6b562c 100644 --- a/mongoengine/errors.py +++ b/mongoengine/errors.py @@ -3,7 +3,9 @@ from collections import defaultdict from mongoengine.python_support import txt_type -__all__ = ('NotRegistered', 'InvalidDocumentError', 'ValidationError') +__all__ = ('NotRegistered', 'InvalidDocumentError', 'LookUpError', + 'DoesNotExist', 'MultipleObjectsReturned', 'InvalidQueryError', + 'OperationError', 'NotUniqueError', 'ValidationError') class NotRegistered(Exception): diff --git a/mongoengine/fields.py b/mongoengine/fields.py index 45304291..e23b90a6 100644 --- a/mongoengine/fields.py +++ b/mongoengine/fields.py @@ -4,7 +4,6 @@ import itertools import re import time import urllib2 -import urlparse import uuid import warnings from operator import itemgetter @@ -16,7 +15,7 @@ from mongoengine.errors import ValidationError from mongoengine.python_support import (PY3, bin_type, txt_type, str_types, StringIO) from base import (BaseField, ComplexBaseField, ObjectIdField, - get_document, BaseDocument, ALLOW_INHERITANCE) + get_document, BaseDocument) from queryset import DO_NOTHING, QuerySet from document import Document, EmbeddedDocument from connection import get_db, DEFAULT_CONNECTION_NAME @@ -27,13 +26,17 @@ except ImportError: Image = None ImageOps = None -__all__ = ['StringField', 'IntField', 'LongField', 'FloatField', 'BooleanField', - 'DateTimeField', 'EmbeddedDocumentField', 'ListField', 'DictField', - 'ObjectIdField', 'ReferenceField', 'ValidationError', 'MapField', - 'DecimalField', 'ComplexDateTimeField', 'URLField', 'DynamicField', - 'GenericReferenceField', 'FileField', 'BinaryField', - 'SortedListField', 'EmailField', 'GeoPointField', 'ImageField', - 'SequenceField', 'UUIDField', 'GenericEmbeddedDocumentField'] +__all__ = ['StringField', 'URLField', 'EmailField', 'IntField', 'LongField', + 'FloatField', 'DecimalField', 'BooleanField', 'DateTimeField', + 'ComplexDateTimeField', 'EmbeddedDocumentField', 'ObjectIdField', + 'GenericEmbeddedDocumentField', 'DynamicField', 'ListField', + 'SortedListField', 'DictField', 'MapField', 'ReferenceField', + 'GenericReferenceField', 'BinaryField', 'GridFSError', + 'GridFSProxy', 'FileField', 'ImageGridFsProxy', + 'ImproperlyConfigured', 'ImageField', 'GeoPointField', + 'SequenceField', 'UUIDField'] + + RECURSIVE_REFERENCE_CONSTANT = 'self' @@ -351,7 +354,7 @@ class DateTimeField(BaseField): kwargs = {'microsecond': usecs} try: # Seconds are optional, so try converting seconds first. return datetime.datetime(*time.strptime(value, - '%Y-%m-%d %H:%M:%S')[:6], **kwargs) + '%Y-%m-%d %H:%M:%S')[:6], **kwargs) except ValueError: try: # Try without seconds. return datetime.datetime(*time.strptime(value, From 8a7b619b77417bd2fa4e0dbc1fe14a25320018c0 Mon Sep 17 00:00:00 2001 From: Ross Lawley Date: Tue, 23 Apr 2013 20:37:05 +0000 Subject: [PATCH 170/189] Tutorial updates --- docs/guide/connecting.rst | 11 +++++- docs/index.rst | 2 +- docs/tutorial.rst | 81 +++++++++++++++++++++++++-------------- 3 files changed, 63 insertions(+), 31 deletions(-) diff --git a/docs/guide/connecting.rst b/docs/guide/connecting.rst index de6794cd..dea6a3dc 100644 --- a/docs/guide/connecting.rst +++ b/docs/guide/connecting.rst @@ -74,9 +74,13 @@ to point across databases and collections. Below is an example schema, using Switch Database Context Manager =============================== -Sometimes you might want to switch the database to query against for a class. +Sometimes you may want to switch the database to query against for a class, +for example, you archive older data into a separate database for performance +reasons. + The :class:`~mongoengine.context_managers.switch_db` context manager allows -you to change the database alias for a class eg :: +you to change the database alias for a given class allowing quick and easy +access to the same User document across databases.eg :: from mongoengine.context_managers import switch_db @@ -87,3 +91,6 @@ you to change the database alias for a class eg :: with switch_db(User, 'archive-user-db') as User: User(name="Ross").save() # Saves the 'archive-user-db' + +.. note:: Make sure any aliases have been registered with + :func:`~mongoengine.register_connection` before using the context manager. diff --git a/docs/index.rst b/docs/index.rst index f6d44b51..4d0d2116 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -7,7 +7,7 @@ MongoDB. To install it, simply run .. code-block:: console - # pip install -U mongoengine + $ pip install -U mongoengine :doc:`tutorial` Start here for a quick overview. diff --git a/docs/tutorial.rst b/docs/tutorial.rst index c4b69c49..423df9b0 100644 --- a/docs/tutorial.rst +++ b/docs/tutorial.rst @@ -1,6 +1,7 @@ ======== Tutorial ======== + This tutorial introduces **MongoEngine** by means of example --- we will walk through how to create a simple **Tumblelog** application. A Tumblelog is a type of blog where posts are not constrained to being conventional text-based posts. @@ -12,23 +13,29 @@ interface. Getting started =============== + Before we start, make sure that a copy of MongoDB is running in an accessible location --- running it locally will be easier, but if that is not an option -then it may be run on a remote server. +then it may be run on a remote server. If you haven't installed mongoengine, +simply use pip to install it like so:: + + $ pip install mongoengine Before we can start using MongoEngine, we need to tell it how to connect to our instance of :program:`mongod`. For this we use the :func:`~mongoengine.connect` -function. The only argument we need to provide is the name of the MongoDB -database to use:: +function. If running locally the only argument we need to provide is the name +of the MongoDB database to use:: from mongoengine import * connect('tumblelog') -For more information about connecting to MongoDB see :ref:`guide-connecting`. +There are lots of options for connecting to MongoDB, for more information about +them see the :ref:`guide-connecting` guide. Defining our documents ====================== + MongoDB is *schemaless*, which means that no schema is enforced by the database --- we may add and remove fields however we want and MongoDB won't complain. This makes life a lot easier in many regards, especially when there is a change @@ -39,17 +46,19 @@ define utility methods on our documents in the same way that traditional In our Tumblelog application we need to store several different types of information. We will need to have a collection of **users**, so that we may -link posts to an individual. We also need to store our different types -**posts** (text, image and link) in the database. To aid navigation of our +link posts to an individual. We also need to store our different types of +**posts** (eg: text, image and link) in the database. To aid navigation of our Tumblelog, posts may have **tags** associated with them, so that the list of posts shown to the user may be limited to posts that have been assigned a -specified tag. Finally, it would be nice if **comments** could be added to -posts. We'll start with **users**, as the others are slightly more involved. +specific tag. Finally, it would be nice if **comments** could be added to +posts. We'll start with **users**, as the other document models are slightly +more involved. Users ----- + Just as if we were using a relational database with an ORM, we need to define -which fields a :class:`User` may have, and what their types will be:: +which fields a :class:`User` may have, and what types of data they might store:: class User(Document): email = StringField(required=True) @@ -58,11 +67,13 @@ which fields a :class:`User` may have, and what their types will be:: This looks similar to how a the structure of a table would be defined in a regular ORM. The key difference is that this schema will never be passed on to -MongoDB --- this will only be enforced at the application level. Also, the User -documents will be stored in a MongoDB *collection* rather than a table. +MongoDB --- this will only be enforced at the application level, making future +changes easy to manage. Also, the User documents will be stored in a +MongoDB *collection* rather than a table. Posts, Comments and Tags ------------------------ + Now we'll think about how to store the rest of the information. If we were using a relational database, we would most likely have a table of **posts**, a table of **comments** and a table of **tags**. To associate the comments with @@ -75,16 +86,17 @@ of them stand out as particularly intuitive solutions. Posts ^^^^^ -But MongoDB *isn't* a relational database, so we're not going to do it that + +Happily mongoDB *isn't* a relational database, so we're not going to do it that way. As it turns out, we can use MongoDB's schemaless nature to provide us with -a much nicer solution. We will store all of the posts in *one collection* --- -each post type will just have the fields it needs. If we later want to add +a much nicer solution. We will store all of the posts in *one collection* and +each post type will only store the fields it needs. If we later want to add video posts, we don't have to modify the collection at all, we just *start using* the new fields we need to support video posts. This fits with the Object-Oriented principle of *inheritance* nicely. We can think of :class:`Post` as a base class, and :class:`TextPost`, :class:`ImagePost` and :class:`LinkPost` as subclasses of :class:`Post`. In fact, MongoEngine supports -this kind of modelling out of the box - all you need do is turn on inheritance +this kind of modelling out of the box --- all you need do is turn on inheritance by setting :attr:`allow_inheritance` to True in the :attr:`meta`:: class Post(Document): @@ -109,6 +121,7 @@ when they are saved, and dereferenced when they are loaded. Tags ^^^^ + Now that we have our Post models figured out, how will we attach tags to them? MongoDB allows us to store lists of items natively, so rather than having a link table, we can just store a list of tags in each post. So, for both @@ -126,11 +139,14 @@ size of our database. So let's take a look that the code our modified The :class:`~mongoengine.ListField` object that is used to define a Post's tags takes a field object as its first argument --- this means that you can have -lists of any type of field (including lists). Note that we don't need to -modify the specialised post types as they all inherit from :class:`Post`. +lists of any type of field (including lists). + +.. note:: We don't need to modify the specialised post types as they all + inherit from :class:`Post`. Comments ^^^^^^^^ + A comment is typically associated with *one* post. In a relational database, to display a post with its comments, we would have to retrieve the post from the database, then query the database again for the comments associated with the @@ -181,15 +197,15 @@ Now that we've defined how our documents will be structured, let's start adding some documents to the database. Firstly, we'll need to create a :class:`User` object:: - john = User(email='jdoe@example.com', first_name='John', last_name='Doe') - john.save() + ross = User(email='ross@example.com', first_name='Ross', last_name='Lawley').save() -Note that we could have also defined our user using attribute syntax:: +.. note:: + We could have also defined our user using attribute syntax:: - john = User(email='jdoe@example.com') - john.first_name = 'John' - john.last_name = 'Doe' - john.save() + ross = User(email='ross@example.com') + ross.first_name = 'Ross' + ross.last_name = 'Lawley' + ross.save() Now that we've got our user in the database, let's add a couple of posts:: @@ -198,16 +214,17 @@ Now that we've got our user in the database, let's add a couple of posts:: post1.tags = ['mongodb', 'mongoengine'] post1.save() - post2 = LinkPost(title='MongoEngine Documentation', author=john) - post2.link_url = 'http://tractiondigital.com/labs/mongoengine/docs' + post2 = LinkPost(title='MongoEngine Documentation', author=ross) + post2.link_url = 'http://docs.mongoengine.com/' post2.tags = ['mongoengine'] post2.save() -Note that if you change a field on a object that has already been saved, then -call :meth:`save` again, the document will be updated. +.. note:: If you change a field on a object that has already been saved, then + call :meth:`save` again, the document will be updated. Accessing our data ================== + So now we've got a couple of posts in our database, how do we display them? Each document class (i.e. any class that inherits either directly or indirectly from :class:`~mongoengine.Document`) has an :attr:`objects` attribute, which is @@ -219,6 +236,7 @@ class. So let's see how we can get our posts' titles:: Retrieving type-specific information ------------------------------------ + This will print the titles of our posts, one on each line. But What if we want to access the type-specific data (link_url, content, etc.)? One way is simply to use the :attr:`objects` attribute of a subclass of :class:`Post`:: @@ -257,6 +275,7 @@ text post, and "Link: " if it was a link post. Searching our posts by tag -------------------------- + The :attr:`objects` attribute of a :class:`~mongoengine.Document` is actually a :class:`~mongoengine.queryset.QuerySet` object. This lazily queries the database only when you need the data. It may also be filtered to narrow down @@ -275,3 +294,9 @@ used on :class:`~mongoengine.queryset.QuerySet` objects:: num_posts = Post.objects(tags='mongodb').count() print 'Found %d posts with tag "mongodb"' % num_posts +Learning more about mongoengine +------------------------------- + +If you got this far you've made a great start, so well done! The next step on +your mongoengine journey is the `full user guide `_, where you +can learn indepth about how to use mongoengine and mongodb. \ No newline at end of file From 5271f3b4a0a75753d6e984267b177b22f67c3b5e Mon Sep 17 00:00:00 2001 From: Ross Lawley Date: Tue, 23 Apr 2013 20:49:22 +0000 Subject: [PATCH 171/189] More doc updates --- docs/guide/installing.rst | 4 ++-- docs/index.rst | 45 ++++++++++++++++++++++----------------- 2 files changed, 28 insertions(+), 21 deletions(-) diff --git a/docs/guide/installing.rst b/docs/guide/installing.rst index f15d3dbb..e93f0485 100644 --- a/docs/guide/installing.rst +++ b/docs/guide/installing.rst @@ -22,10 +22,10 @@ Alternatively, if you don't have setuptools installed, `download it from PyPi $ python setup.py install To use the bleeding-edge version of MongoEngine, you can get the source from -`GitHub `_ and install it as above: +`GitHub `_ and install it as above: .. code-block:: console - $ git clone git://github.com/hmarr/mongoengine + $ git clone git://github.com/mongoengine/mongoengine $ cd mongoengine $ python setup.py install diff --git a/docs/index.rst b/docs/index.rst index 4d0d2116..4aca82da 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -10,13 +10,15 @@ MongoDB. To install it, simply run $ pip install -U mongoengine :doc:`tutorial` - Start here for a quick overview. + A quick tutorial building a tumblelog to get you up and running with + MongoEngine. :doc:`guide/index` - The Full guide to MongoEngine + The Full guide to MongoEngine - from modeling documents to storing files, + from querying for data to firing signals and *everything* between. :doc:`apireference` - The complete API documentation. + The complete API documentation --- the innards of documents, querysets and fields. :doc:`upgrade` How to upgrade MongoEngine. @@ -28,35 +30,40 @@ Community --------- To get help with using MongoEngine, use the `MongoEngine Users mailing list -`_ or come chat on the -`#mongoengine IRC channel `_. +`_ or the ever popular +`stackoverflow `_. Contributing ------------ -The source is available on `GitHub `_ and -contributions are always encouraged. Contributions can be as simple as -minor tweaks to this documentation. To contribute, fork the project on +**Yes please!** We are always looking for contributions, additions and improvements. + +The source is available on `GitHub `_ +and contributions are always encouraged. Contributions can be as simple as +minor tweaks to this documentation, the website or the core. + +To contribute, fork the project on `GitHub `_ and send a pull request. -Also, you can join the developers' `mailing list -`_. - Changes ------- + See the :doc:`changelog` for a full list of changes to MongoEngine and :doc:`upgrade` for upgrade information. -.. toctree:: - :hidden: +.. note:: Always read and test the `upgrade `_ documentation before + putting updates live in production **;)** - tutorial - guide/index - apireference - django - changelog - upgrade +.. toctree:: + :hidden: + + tutorial + guide/index + apireference + django + changelog + upgrade Indices and tables ------------------ From 9bd8b3e9a536f07020bfeb36624ac844c253f6e5 Mon Sep 17 00:00:00 2001 From: Ross Lawley Date: Tue, 23 Apr 2013 20:55:37 +0000 Subject: [PATCH 172/189] Connection doc updates --- docs/guide/connecting.rst | 19 +++++++++++-------- 1 file changed, 11 insertions(+), 8 deletions(-) diff --git a/docs/guide/connecting.rst b/docs/guide/connecting.rst index dea6a3dc..8674b5eb 100644 --- a/docs/guide/connecting.rst +++ b/docs/guide/connecting.rst @@ -6,20 +6,23 @@ Connecting to MongoDB To connect to a running instance of :program:`mongod`, use the :func:`~mongoengine.connect` function. The first argument is the name of the -database to connect to. If the database does not exist, it will be created. If -the database requires authentication, :attr:`username` and :attr:`password` -arguments may be provided:: +database to connect to:: from mongoengine import connect - connect('project1', username='webapp', password='pwd123') + connect('project1') By default, MongoEngine assumes that the :program:`mongod` instance is running -on **localhost** on port **27017**. If MongoDB is running elsewhere, you may -provide :attr:`host` and :attr:`port` arguments to +on **localhost** on port **27017**. If MongoDB is running elsewhere, you should +provide the :attr:`host` and :attr:`port` arguments to :func:`~mongoengine.connect`:: connect('project1', host='192.168.1.35', port=12345) +If the database requires authentication, :attr:`username` and :attr:`password` +arguments should be provided:: + + connect('project1', username='webapp', password='pwd123') + Uri style connections are also supported as long as you include the database name - just supply the uri as the :attr:`host` to :func:`~mongoengine.connect`:: @@ -74,8 +77,8 @@ to point across databases and collections. Below is an example schema, using Switch Database Context Manager =============================== -Sometimes you may want to switch the database to query against for a class, -for example, you archive older data into a separate database for performance +Sometimes you may want to switch the database to query against for a class +for example, archiving older data into a separate database for performance reasons. The :class:`~mongoengine.context_managers.switch_db` context manager allows From c59ea268451246a8693c4dd0987ec9bd39cdcd96 Mon Sep 17 00:00:00 2001 From: Ross Lawley Date: Wed, 24 Apr 2013 11:17:21 +0000 Subject: [PATCH 173/189] Updated docs to clarify or != | (##288) --- docs/guide/querying.rst | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/docs/guide/querying.rst b/docs/guide/querying.rst index 5e250cea..60702ec6 100644 --- a/docs/guide/querying.rst +++ b/docs/guide/querying.rst @@ -392,6 +392,7 @@ You can also turn off all dereferencing for a fixed period by using the Advanced queries ================ + Sometimes calling a :class:`~mongoengine.queryset.QuerySet` object with keyword arguments can't fully express the query you want to use -- for example if you need to combine a number of constraints using *and* and *or*. This is made @@ -410,6 +411,11 @@ calling it with keyword arguments:: # Get top posts Post.objects((Q(featured=True) & Q(hits__gte=1000)) | Q(hits__gte=5000)) +.. warning:: You have to use bitwise operators. You cannot use ``or``, ``and`` + to combine queries as ``Q(a=a) or Q(b=b)`` is not the same as + ``Q(a=a) | Q(b=b)``. As ``Q(a=a)`` equates to true ``Q(a=a) or Q(b=b)`` is + the same as ``Q(a=a)``. + .. _guide-atomic-updates: Atomic updates From c60ea4082807a35412e936d5cd6049cedae9e164 Mon Sep 17 00:00:00 2001 From: Ross Lawley Date: Wed, 24 Apr 2013 12:14:34 +0000 Subject: [PATCH 174/189] ReferenceField now store ObjectId's by default rather than DBRef (#290) --- docs/changelog.rst | 1 + docs/upgrade.rst | 29 +++++++++++ mongoengine/fields.py | 9 +--- tests/all_warnings/__init__.py | 22 -------- tests/migration/__init__.py | 4 +- ...py => convert_to_new_inheritance_model.py} | 2 +- .../refrencefield_dbref_to_object_id.py | 52 +++++++++++++++++++ tests/test_signals.py | 5 +- 8 files changed, 92 insertions(+), 32 deletions(-) rename tests/migration/{test_convert_to_new_inheritance_model.py => convert_to_new_inheritance_model.py} (97%) create mode 100644 tests/migration/refrencefield_dbref_to_object_id.py diff --git a/docs/changelog.rst b/docs/changelog.rst index 49ee0ec3..7b51a79b 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -4,6 +4,7 @@ Changelog Changes in 0.8.X ================ +- ReferenceField now store ObjectId's by default rather than DBRef (#290) - Added ImageField support for inline replacements (#86) - Added SequenceField.set_next_value(value) helper (#159) - Updated .only() behaviour - now like exclude it is chainable (#202) diff --git a/docs/upgrade.rst b/docs/upgrade.rst index 564b7f6d..eec9d626 100644 --- a/docs/upgrade.rst +++ b/docs/upgrade.rst @@ -76,6 +76,35 @@ the case and the data is set only in the ``document._data`` dictionary: :: File "", line 1, in AttributeError: 'Animal' object has no attribute 'size' +ReferenceField +-------------- + +ReferenceFields now store ObjectId's by default - this is more efficient than +DBRefs as we already know what Document types they reference. + + # Old code + class Animal(Document): + name = ReferenceField('self') + + # New code to keep dbrefs + class Animal(Document): + name = ReferenceField('self', dbref=True) + +To migrate all the references you need to touch each object and mark it as dirty +eg:: + + # Doc definition + class Person(Document): + name = StringField() + parent = ReferenceField('self') + friends = ListField(ReferenceField('self')) + + # Mark all ReferenceFields as dirty and save + for p in Person.objects: + p._mark_as_dirty('parent') + p._mark_as_dirty('friends') + p.save() + Querysets ========= diff --git a/mongoengine/fields.py b/mongoengine/fields.py index e23b90a6..979699ce 100644 --- a/mongoengine/fields.py +++ b/mongoengine/fields.py @@ -772,7 +772,7 @@ class ReferenceField(BaseField): .. versionchanged:: 0.5 added `reverse_delete_rule` """ - def __init__(self, document_type, dbref=None, + def __init__(self, document_type, dbref=False, reverse_delete_rule=DO_NOTHING, **kwargs): """Initialises the Reference Field. @@ -786,12 +786,7 @@ class ReferenceField(BaseField): self.error('Argument to ReferenceField constructor must be a ' 'document class or a string') - if dbref is None: - msg = ("ReferenceFields will default to using ObjectId " - "in 0.8, set DBRef=True if this isn't desired") - warnings.warn(msg, FutureWarning) - - self.dbref = dbref if dbref is not None else True # To change in 0.8 + self.dbref = dbref self.document_type_obj = document_type self.reverse_delete_rule = reverse_delete_rule super(ReferenceField, self).__init__(**kwargs) diff --git a/tests/all_warnings/__init__.py b/tests/all_warnings/__init__.py index 74533de2..d74d39e9 100644 --- a/tests/all_warnings/__init__.py +++ b/tests/all_warnings/__init__.py @@ -30,28 +30,6 @@ class AllWarnings(unittest.TestCase): # restore default handling of warnings warnings.showwarning = self.showwarning_default - def test_dbref_reference_field_future_warning(self): - - class Person(Document): - name = StringField() - parent = ReferenceField('self') - - Person.drop_collection() - - p1 = Person() - p1.parent = None - p1.save() - - p2 = Person(name="Wilson Jr") - p2.parent = p1 - p2.save(cascade=False) - - self.assertTrue(len(self.warning_list) > 0) - warning = self.warning_list[0] - self.assertEqual(FutureWarning, warning["category"]) - self.assertTrue("ReferenceFields will default to using ObjectId" - in str(warning["message"])) - def test_document_save_cascade_future_warning(self): class Person(Document): diff --git a/tests/migration/__init__.py b/tests/migration/__init__.py index 882e7370..f7ad6747 100644 --- a/tests/migration/__init__.py +++ b/tests/migration/__init__.py @@ -1,4 +1,6 @@ +from convert_to_new_inheritance_model import * +from refrencefield_dbref_to_object_id import * from turn_off_inheritance import * if __name__ == '__main__': - unittest.main() \ No newline at end of file + unittest.main() diff --git a/tests/migration/test_convert_to_new_inheritance_model.py b/tests/migration/convert_to_new_inheritance_model.py similarity index 97% rename from tests/migration/test_convert_to_new_inheritance_model.py rename to tests/migration/convert_to_new_inheritance_model.py index d4337bf3..89ee9e9d 100644 --- a/tests/migration/test_convert_to_new_inheritance_model.py +++ b/tests/migration/convert_to_new_inheritance_model.py @@ -38,7 +38,7 @@ class ConvertToNewInheritanceModel(unittest.TestCase): # 3. Confirm extra data is removed count = collection.find({'_types': {"$exists": True}}).count() - assert count == 0 + self.assertEqual(0, count) # 4. Remove indexes info = collection.index_information() diff --git a/tests/migration/refrencefield_dbref_to_object_id.py b/tests/migration/refrencefield_dbref_to_object_id.py new file mode 100644 index 00000000..d3acbe92 --- /dev/null +++ b/tests/migration/refrencefield_dbref_to_object_id.py @@ -0,0 +1,52 @@ +# -*- coding: utf-8 -*- +import unittest + +from mongoengine import Document, connect +from mongoengine.connection import get_db +from mongoengine.fields import StringField, ReferenceField, ListField + +__all__ = ('ConvertToObjectIdsModel', ) + + +class ConvertToObjectIdsModel(unittest.TestCase): + + def setUp(self): + connect(db='mongoenginetest') + self.db = get_db() + + def test_how_to_convert_to_object_id_reference_fields(self): + """Demonstrates migrating from 0.7 to 0.8 + """ + + # 1. Old definition - using dbrefs + class Person(Document): + name = StringField() + parent = ReferenceField('self', dbref=True) + friends = ListField(ReferenceField('self', dbref=True)) + + Person.drop_collection() + + p1 = Person(name="Wilson", parent=None).save() + f1 = Person(name="John", parent=None).save() + f2 = Person(name="Paul", parent=None).save() + f3 = Person(name="George", parent=None).save() + f4 = Person(name="Ringo", parent=None).save() + Person(name="Wilson Jr", parent=p1, friends=[f1, f2, f3, f4]).save() + + # 2. Start the migration by changing the schema + # Change ReferenceField as now dbref defaults to False + class Person(Document): + name = StringField() + parent = ReferenceField('self') + friends = ListField(ReferenceField('self')) + + # 3. Loop all the objects and mark parent as changed + for p in Person.objects: + p._mark_as_changed('parent') + p._mark_as_changed('friends') + p.save() + + # 4. Confirmation of the fix! + wilson = Person.objects(name="Wilson Jr").as_pymongo()[0] + self.assertEqual(p1.id, wilson['parent']) + self.assertEqual([f1.id, f2.id, f3.id, f4.id], wilson['friends']) diff --git a/tests/test_signals.py b/tests/test_signals.py index fc638cfc..32517ddf 100644 --- a/tests/test_signals.py +++ b/tests/test_signals.py @@ -72,6 +72,7 @@ class SignalTests(unittest.TestCase): else: signal_output.append('Not loaded') self.Author = Author + Author.drop_collection() class Another(Document): name = StringField() @@ -110,6 +111,7 @@ class SignalTests(unittest.TestCase): signal_output.append('post_delete Another signal, %s' % document) self.Another = Another + Another.drop_collection() class ExplicitId(Document): id = IntField(primary_key=True) @@ -123,7 +125,8 @@ class SignalTests(unittest.TestCase): signal_output.append('Is updated') self.ExplicitId = ExplicitId - self.ExplicitId.objects.delete() + ExplicitId.drop_collection() + # Save up the number of connected signals so that we can check at the # end that all the signals we register get properly unregistered self.pre_signals = ( From fe62c3aacb0f9a3e06f96f758bf1e3496f753d7d Mon Sep 17 00:00:00 2001 From: Ross Lawley Date: Thu, 25 Apr 2013 10:24:33 +0000 Subject: [PATCH 175/189] Cascading saves now default to off (#291) --- docs/changelog.rst | 1 + docs/upgrade.rst | 31 ++++++++++++++++++++++++++++--- mongoengine/document.py | 9 ++------- tests/all_warnings/__init__.py | 30 +----------------------------- tests/document/instance.py | 27 ++++++++++++++++++++++++++- 5 files changed, 58 insertions(+), 40 deletions(-) diff --git a/docs/changelog.rst b/docs/changelog.rst index 7b51a79b..6c19933e 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -4,6 +4,7 @@ Changelog Changes in 0.8.X ================ +- Cascading saves now default to off (#291) - ReferenceField now store ObjectId's by default rather than DBRef (#290) - Added ImageField support for inline replacements (#86) - Added SequenceField.set_next_value(value) helper (#159) diff --git a/docs/upgrade.rst b/docs/upgrade.rst index eec9d626..86d9f9d7 100644 --- a/docs/upgrade.rst +++ b/docs/upgrade.rst @@ -5,11 +5,21 @@ Upgrading 0.7 to 0.8 ********** -Inheritance -=========== +There have been numerous backwards breaking changes in 0.8. The reasons for +these are ensure that MongoEngine has sane defaults going forward and +performs the best it can out the box. Where possible there have been +FutureWarnings to help get you ready for the change, but that hasn't been +possible for the whole of the release. + +.. warning:: Breaking changes - test upgrading on a test system before putting +live. There maybe multiple manual steps in migrating and these are best honed +on a staging / test system. Data Model ----------- +========== + +Inheritance +----------- The inheritance model has changed, we no longer need to store an array of :attr:`types` with the model we can just use the classname in :attr:`_cls`. @@ -105,6 +115,21 @@ eg:: p._mark_as_dirty('friends') p.save() + +Cascading Saves +--------------- +To improve performance document saves will no longer automatically cascade. +Any changes to a Documents references will either have to be saved manually or +you will have to explicitly tell it to cascade on save:: + + # At the class level: + class Person(Document): + meta = {'cascade': True} + + # Or on save: + my_document.save(cascade=True) + + Querysets ========= diff --git a/mongoengine/document.py b/mongoengine/document.py index 54b55df9..d0cafa3e 100644 --- a/mongoengine/document.py +++ b/mongoengine/document.py @@ -244,7 +244,6 @@ class Document(BaseDocument): upsert=upsert, **write_concern) created = is_new_object(last_error) - warn_cascade = not cascade and 'cascade' not in self._meta cascade = (self._meta.get('cascade', True) if cascade is None else cascade) if cascade: @@ -257,7 +256,7 @@ class Document(BaseDocument): if cascade_kwargs: # Allow granular control over cascades kwargs.update(cascade_kwargs) kwargs['_refs'] = _refs - self.cascade_save(warn_cascade=warn_cascade, **kwargs) + self.cascade_save(**kwargs) except pymongo.errors.OperationFailure, err: message = 'Could not save document (%s)' @@ -276,7 +275,7 @@ class Document(BaseDocument): signals.post_save.send(self.__class__, document=self, created=created) return self - def cascade_save(self, warn_cascade=None, *args, **kwargs): + def cascade_save(self, *args, **kwargs): """Recursively saves any references / generic references on an objects""" import fields @@ -296,10 +295,6 @@ class Document(BaseDocument): ref_id = "%s,%s" % (ref.__class__.__name__, str(ref._data)) if ref and ref_id not in _refs: - if warn_cascade: - msg = ("Cascading saves will default to off in 0.8, " - "please explicitly set `.save(cascade=True)`") - warnings.warn(msg, FutureWarning) _refs.append(ref_id) kwargs["_refs"] = _refs ref.save(**kwargs) diff --git a/tests/all_warnings/__init__.py b/tests/all_warnings/__init__.py index d74d39e9..53ce638c 100644 --- a/tests/all_warnings/__init__.py +++ b/tests/all_warnings/__init__.py @@ -17,7 +17,7 @@ __all__ = ('AllWarnings', ) class AllWarnings(unittest.TestCase): def setUp(self): - conn = connect(db='mongoenginetest') + connect(db='mongoenginetest') self.warning_list = [] self.showwarning_default = warnings.showwarning warnings.showwarning = self.append_to_warning_list @@ -30,31 +30,6 @@ class AllWarnings(unittest.TestCase): # restore default handling of warnings warnings.showwarning = self.showwarning_default - def test_document_save_cascade_future_warning(self): - - class Person(Document): - name = StringField() - parent = ReferenceField('self') - - Person.drop_collection() - - p1 = Person(name="Wilson Snr") - p1.parent = None - p1.save() - - p2 = Person(name="Wilson Jr") - p2.parent = p1 - p2.parent.name = "Poppa Wilson" - p2.save() - - self.assertTrue(len(self.warning_list) > 0) - if len(self.warning_list) > 1: - print self.warning_list - warning = self.warning_list[0] - self.assertEqual(FutureWarning, warning["category"]) - self.assertTrue("Cascading saves will default to off in 0.8" - in str(warning["message"])) - def test_document_collection_syntax_warning(self): class NonAbstractBase(Document): @@ -67,6 +42,3 @@ class AllWarnings(unittest.TestCase): self.assertEqual(SyntaxWarning, warning["category"]) self.assertEqual('non_abstract_base', InheritedDocumentFailTest._get_collection_name()) - -import sys -sys.path[0:0] = [""] diff --git a/tests/document/instance.py b/tests/document/instance.py index 5513ed8d..a75cc4dc 100644 --- a/tests/document/instance.py +++ b/tests/document/instance.py @@ -678,7 +678,7 @@ class InstanceTest(unittest.TestCase): p1.reload() self.assertEqual(p1.name, p.parent.name) - def test_save_cascade_meta(self): + def test_save_cascade_meta_false(self): class Person(Document): name = StringField() @@ -707,6 +707,31 @@ class InstanceTest(unittest.TestCase): p1.reload() self.assertEqual(p1.name, p.parent.name) + def test_save_cascade_meta_true(self): + + class Person(Document): + name = StringField() + parent = ReferenceField('self') + + meta = {'cascade': False} + + Person.drop_collection() + + p1 = Person(name="Wilson Snr") + p1.parent = None + p1.save() + + p2 = Person(name="Wilson Jr") + p2.parent = p1 + p2.save(cascade=True) + + p = Person.objects(name="Wilson Jr").get() + p.parent.name = "Daddy Wilson" + p.save() + + p1.reload() + self.assertNotEqual(p1.name, p.parent.name) + def test_save_cascades_generically(self): class Person(Document): From cb9166aba41b045d6868bb381d75517b05db8758 Mon Sep 17 00:00:00 2001 From: Ross Lawley Date: Thu, 25 Apr 2013 11:04:33 +0000 Subject: [PATCH 176/189] Auth cleanups - removed duplicates --- mongoengine/django/auth.py | 162 ++++++++++++++++--------------------- 1 file changed, 68 insertions(+), 94 deletions(-) diff --git a/mongoengine/django/auth.py b/mongoengine/django/auth.py index d22f0865..65822441 100644 --- a/mongoengine/django/auth.py +++ b/mongoengine/django/auth.py @@ -1,8 +1,7 @@ from mongoengine import * from django.utils.encoding import smart_str -from django.contrib.auth.models import _user_get_all_permissions -from django.contrib.auth.models import _user_has_perm +from django.contrib.auth.models import _user_has_perm, _user_get_all_permissions, _user_has_module_perms from django.db import models from django.contrib.contenttypes.models import ContentTypeManager from django.contrib import auth @@ -38,11 +37,12 @@ from .utils import datetime_now REDIRECT_FIELD_NAME = 'next' + class ContentType(Document): name = StringField(max_length=100) app_label = StringField(max_length=100) model = StringField(max_length=100, verbose_name=_('python model class name'), - unique_with='app_label') + unique_with='app_label') objects = ContentTypeManager() class Meta: @@ -72,9 +72,11 @@ class ContentType(Document): def natural_key(self): return (self.app_label, self.model) + class SiteProfileNotAvailable(Exception): pass + class PermissionManager(models.Manager): def get_by_natural_key(self, codename, app_label, model): return self.get( @@ -82,18 +84,28 @@ class PermissionManager(models.Manager): content_type=ContentType.objects.get_by_natural_key(app_label, model) ) + class Permission(Document): - """The permissions system provides a way to assign permissions to specific users and groups of users. + """The permissions system provides a way to assign permissions to specific + users and groups of users. - The permission system is used by the Django admin site, but may also be useful in your own code. The Django admin site uses permissions as follows: + The permission system is used by the Django admin site, but may also be + useful in your own code. The Django admin site uses permissions as follows: - - The "add" permission limits the user's ability to view the "add" form and add an object. - - The "change" permission limits a user's ability to view the change list, view the "change" form and change an object. + - The "add" permission limits the user's ability to view the "add" + form and add an object. + - The "change" permission limits a user's ability to view the change + list, view the "change" form and change an object. - The "delete" permission limits the ability to delete an object. - Permissions are set globally per type of object, not per specific object instance. It is possible to say "Mary may change news stories," but it's not currently possible to say "Mary may change news stories, but only the ones she created herself" or "Mary may only change news stories that have a certain status or publication date." + Permissions are set globally per type of object, not per specific object + instance. It is possible to say "Mary may change news stories," but it's + not currently possible to say "Mary may change news stories, but only the + ones she created herself" or "Mary may only change news stories that have + a certain status or publication date." - Three basic permissions -- add, change and delete -- are automatically created for each Django model. + Three basic permissions -- add, change and delete -- are automatically + created for each Django model. """ name = StringField(max_length=50, verbose_name=_('username')) content_type = ReferenceField(ContentType) @@ -119,12 +131,22 @@ class Permission(Document): return (self.codename,) + self.content_type.natural_key() natural_key.dependencies = ['contenttypes.contenttype'] + class Group(Document): - """Groups are a generic way of categorizing users to apply permissions, or some other label, to those users. A user can belong to any number of groups. + """Groups are a generic way of categorizing users to apply permissions, + or some other label, to those users. A user can belong to any number of + groups. - A user in a group automatically has all the permissions granted to that group. For example, if the group Site editors has the permission can_edit_home_page, any user in that group will have that permission. + A user in a group automatically has all the permissions granted to that + group. For example, if the group Site editors has the permission + can_edit_home_page, any user in that group will have that permission. - Beyond permissions, groups are a convenient way to categorize users to apply some label, or extended functionality, to them. For example, you could create a group 'Special users', and you could write code that would do special things to those users -- such as giving them access to a members-only portion of your site, or sending them members-only e-mail messages. + Beyond permissions, groups are a convenient way to categorize users to + apply some label, or extended functionality, to them. For example, you + could create a group 'Special users', and you could write code that would + do special things to those users -- such as giving them access to a + members-only portion of your site, or sending them members-only + e-mail messages. """ name = StringField(max_length=80, unique=True, verbose_name=_('name')) # permissions = models.ManyToManyField(Permission, verbose_name=_('permissions'), blank=True) @@ -137,6 +159,7 @@ class Group(Document): def __unicode__(self): return self.name + class UserManager(models.Manager): def create_user(self, username, email, password=None): """ @@ -154,8 +177,8 @@ class UserManager(models.Manager): email = '@'.join([email_name, domain_part.lower()]) user = self.model(username=username, email=email, is_staff=False, - is_active=True, is_superuser=False, last_login=now, - date_joined=now) + is_active=True, is_superuser=False, last_login=now, + date_joined=now) user.set_password(password) user.save(using=self._db) @@ -177,7 +200,6 @@ class UserManager(models.Manager): return ''.join([choice(allowed_chars) for i in range(length)]) - class User(Document): """A User document that aims to mirror most of the API specified by Django at http://docs.djangoproject.com/en/dev/topics/auth/#users @@ -248,25 +270,6 @@ class User(Document): """ return check_password(raw_password, self.password) - def get_all_permissions(self, obj=None): - return _user_get_all_permissions(self, obj) - - def has_perm(self, perm, obj=None): - """ - Returns True if the user has the specified permission. This method - queries all available auth backends, but returns immediately if any - backend returns True. Thus, a user who has permission from a single - auth backend is assumed to have permission in general. If an object is - provided, permissions for this specific object are checked. - """ - - # Active superusers have all permissions. - if self.is_active and self.is_superuser: - return True - - # Otherwise we need to check the backends. - return _user_has_perm(self, perm, obj) - @classmethod def create_user(cls, username, password, email=None): """Create (and save) a new user with the given username, password and @@ -289,68 +292,47 @@ class User(Document): user.save() return user - def get_all_permissions(self, obj=None): + def get_group_permissions(self, obj=None): + """ + Returns a list of permission strings that this user has through his/her + groups. This method queries all available auth backends. If an object + is passed in, only permissions matching this object are returned. + """ permissions = set() - anon = self.is_anonymous() for backend in auth.get_backends(): - if not anon or backend.supports_anonymous_user: - if hasattr(backend, "get_all_permissions"): - if obj is not None: - if backend.supports_object_permissions: - permissions.update( - backend.get_all_permissions(user, obj) - ) - else: - permissions.update(backend.get_all_permissions(self)) + if hasattr(backend, "get_group_permissions"): + permissions.update(backend.get_group_permissions(self, obj)) return permissions - def get_and_delete_messages(self): - return [] + def get_all_permissions(self, obj=None): + return _user_get_all_permissions(self, obj) def has_perm(self, perm, obj=None): - anon = self.is_anonymous() - active = self.is_active - for backend in auth.get_backends(): - if (not active and not anon and backend.supports_inactive_user) or \ - (not anon or backend.supports_anonymous_user): - if hasattr(backend, "has_perm"): - if obj is not None: - if (backend.supports_object_permissions and - backend.has_perm(self, perm, obj)): - return True - else: - if backend.has_perm(self, perm): - return True - return False + """ + Returns True if the user has the specified permission. This method + queries all available auth backends, but returns immediately if any + backend returns True. Thus, a user who has permission from a single + auth backend is assumed to have permission in general. If an object is + provided, permissions for this specific object are checked. + """ - def has_perms(self, perm_list, obj=None): - """ - Returns True if the user has each of the specified permissions. - If object is passed, it checks if the user has all required perms - for this object. - """ - for perm in perm_list: - if not self.has_perm(perm, obj): - return False - return True + # Active superusers have all permissions. + if self.is_active and self.is_superuser: + return True + + # Otherwise we need to check the backends. + return _user_has_perm(self, perm, obj) def has_module_perms(self, app_label): - anon = self.is_anonymous() - active = self.is_active - for backend in auth.get_backends(): - if (not active and not anon and backend.supports_inactive_user) or \ - (not anon or backend.supports_anonymous_user): - if hasattr(backend, "has_module_perms"): - if backend.has_module_perms(self, app_label): - return True - return False + """ + Returns True if the user has any permissions in the given app label. + Uses pretty much the same logic as has_perm, above. + """ + # Active superusers have all permissions. + if self.is_active and self.is_superuser: + return True - def get_and_delete_messages(self): - messages = [] - for m in self.message_set.all(): - messages.append(m.message) - m.delete() - return messages + return _user_has_module_perms(self, app_label) def email_user(self, subject, message, from_email=None): "Sends an e-mail to this User." @@ -386,14 +368,6 @@ class User(Document): raise SiteProfileNotAvailable return self._profile_cache - def _get_message_set(self): - import warnings - warnings.warn('The user messaging API is deprecated. Please update' - ' your code to use the new messages framework.', - category=DeprecationWarning) - return self._message_set - message_set = property(_get_message_set) - class MongoEngineBackend(object): """Authenticate using MongoEngine and mongoengine.django.auth.User. From df4dc3492cf9cf8613eb13f8fedf78224fa70a37 Mon Sep 17 00:00:00 2001 From: Ross Lawley Date: Thu, 25 Apr 2013 11:41:01 +0000 Subject: [PATCH 177/189] Upgrade changelog, docs and django/auth.py --- AUTHORS | 1 - docs/changelog.rst | 1 + docs/upgrade.rst | 6 +++--- mongoengine/django/auth.py | 5 ++--- 4 files changed, 6 insertions(+), 7 deletions(-) diff --git a/AUTHORS b/AUTHORS index e388a04a..44e19bf6 100644 --- a/AUTHORS +++ b/AUTHORS @@ -157,4 +157,3 @@ that much better: * Kenneth Falck * Lukasz Balcerzak * Nicolas Cortot - diff --git a/docs/changelog.rst b/docs/changelog.rst index 6c19933e..bd3821fe 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -4,6 +4,7 @@ Changelog Changes in 0.8.X ================ +- Added Custom User Model for Django 1.5 (#285) - Cascading saves now default to off (#291) - ReferenceField now store ObjectId's by default rather than DBRef (#290) - Added ImageField support for inline replacements (#86) diff --git a/docs/upgrade.rst b/docs/upgrade.rst index 86d9f9d7..738a949b 100644 --- a/docs/upgrade.rst +++ b/docs/upgrade.rst @@ -12,8 +12,8 @@ FutureWarnings to help get you ready for the change, but that hasn't been possible for the whole of the release. .. warning:: Breaking changes - test upgrading on a test system before putting -live. There maybe multiple manual steps in migrating and these are best honed -on a staging / test system. + live. There maybe multiple manual steps in migrating and these are best honed + on a staging / test system. Data Model ========== @@ -90,7 +90,7 @@ ReferenceField -------------- ReferenceFields now store ObjectId's by default - this is more efficient than -DBRefs as we already know what Document types they reference. +DBRefs as we already know what Document types they reference:: # Old code class Animal(Document): diff --git a/mongoengine/django/auth.py b/mongoengine/django/auth.py index 8fcbca97..cff4b743 100644 --- a/mongoengine/django/auth.py +++ b/mongoengine/django/auth.py @@ -108,7 +108,7 @@ class Permission(Document): created for each Django model. """ name = StringField(max_length=50, verbose_name=_('username')) - content_type = ReferenceField(ContentType, dbref=True) + content_type = ReferenceField(ContentType) codename = StringField(max_length=100, verbose_name=_('codename')) # FIXME: don't access field of the other class # unique_with=['content_type__app_label', 'content_type__model']) @@ -149,8 +149,7 @@ class Group(Document): e-mail messages. """ name = StringField(max_length=80, unique=True, verbose_name=_('name')) - # permissions = models.ManyToManyField(Permission, verbose_name=_('permissions'), blank=True) - permissions = ListField(ReferenceField(Permission, verbose_name=_('permissions'), required=False, dbref=True)) + permissions = ListField(ReferenceField(Permission, verbose_name=_('permissions'), required=False)) class Meta: verbose_name = _('group') From 3fc5dc852335317ae024cac81ae448d985ef9764 Mon Sep 17 00:00:00 2001 From: Ross Lawley Date: Thu, 25 Apr 2013 11:46:18 +0000 Subject: [PATCH 178/189] Testing if travis 2.6 is >= 2.6.6 --- .travis.yml | 6 ------ 1 file changed, 6 deletions(-) diff --git a/.travis.yml b/.travis.yml index c5fe62ea..e78bda5a 100644 --- a/.travis.yml +++ b/.travis.yml @@ -12,12 +12,6 @@ env: - PYMONGO=2.5 DJANGO=1.5.1 - PYMONGO=2.5 DJANGO=1.4.2 - PYMONGO=2.4.2 DJANGO=1.4.2 -matrix: - exclude: - - python: "2.6" - env: PYMONGO=dev DJANGO=1.5.1 - - python: "2.6" - env: PYMONGO=2.5 DJANGO=1.5.1 install: - if [[ $TRAVIS_PYTHON_VERSION == '2.'* ]]; then cp /usr/lib/*/libz.so $VIRTUAL_ENV/lib/; fi - if [[ $TRAVIS_PYTHON_VERSION == '2.'* ]]; then pip install pil --use-mirrors ; true; fi From bafdf0381adbfa2f9a626d5d0ad4720366af3a03 Mon Sep 17 00:00:00 2001 From: Ross Lawley Date: Thu, 25 Apr 2013 11:59:56 +0000 Subject: [PATCH 179/189] Updates --- .travis.yml | 6 ++++++ docs/upgrade.rst | 5 +++++ 2 files changed, 11 insertions(+) diff --git a/.travis.yml b/.travis.yml index e78bda5a..c5fe62ea 100644 --- a/.travis.yml +++ b/.travis.yml @@ -12,6 +12,12 @@ env: - PYMONGO=2.5 DJANGO=1.5.1 - PYMONGO=2.5 DJANGO=1.4.2 - PYMONGO=2.4.2 DJANGO=1.4.2 +matrix: + exclude: + - python: "2.6" + env: PYMONGO=dev DJANGO=1.5.1 + - python: "2.6" + env: PYMONGO=2.5 DJANGO=1.5.1 install: - if [[ $TRAVIS_PYTHON_VERSION == '2.'* ]]; then cp /usr/lib/*/libz.so $VIRTUAL_ENV/lib/; fi - if [[ $TRAVIS_PYTHON_VERSION == '2.'* ]]; then pip install pil --use-mirrors ; true; fi diff --git a/docs/upgrade.rst b/docs/upgrade.rst index 738a949b..6138bb45 100644 --- a/docs/upgrade.rst +++ b/docs/upgrade.rst @@ -15,6 +15,11 @@ possible for the whole of the release. live. There maybe multiple manual steps in migrating and these are best honed on a staging / test system. +Python +======= + +Support for python 2.5 has been dropped. + Data Model ========== From f7bc58a767c80495ca46bd05a8b2f04aeaae462e Mon Sep 17 00:00:00 2001 From: Ross Lawley Date: Thu, 25 Apr 2013 12:03:44 +0000 Subject: [PATCH 180/189] Added assertIn / assertNotIn for python 2.6 --- .travis.yml | 6 ------ tests/test_django.py | 8 +++++++- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/.travis.yml b/.travis.yml index c5fe62ea..e78bda5a 100644 --- a/.travis.yml +++ b/.travis.yml @@ -12,12 +12,6 @@ env: - PYMONGO=2.5 DJANGO=1.5.1 - PYMONGO=2.5 DJANGO=1.4.2 - PYMONGO=2.4.2 DJANGO=1.4.2 -matrix: - exclude: - - python: "2.6" - env: PYMONGO=dev DJANGO=1.5.1 - - python: "2.6" - env: PYMONGO=2.5 DJANGO=1.5.1 install: - if [[ $TRAVIS_PYTHON_VERSION == '2.'* ]]; then cp /usr/lib/*/libz.so $VIRTUAL_ENV/lib/; fi - if [[ $TRAVIS_PYTHON_VERSION == '2.'* ]]; then pip install pil --use-mirrors ; true; fi diff --git a/tests/test_django.py b/tests/test_django.py index 01a105a3..573c0728 100644 --- a/tests/test_django.py +++ b/tests/test_django.py @@ -178,6 +178,12 @@ class MongoDBSessionTest(SessionTestsMixin, unittest.TestCase): MongoSession.drop_collection() super(MongoDBSessionTest, self).setUp() + def assertIn(self, first, second, msg=None): + self.assertTrue(first in second, msg) + + def assertNotIn(self, first, second, msg=None): + self.assertFalse(first in second, msg) + def test_first_save(self): session = SessionStore() session['test'] = True @@ -188,7 +194,7 @@ class MongoDBSessionTest(SessionTestsMixin, unittest.TestCase): activate_timezone(FixedOffset(60, 'UTC+1')) # create and save new session session = SessionStore() - session.set_expiry(600) # expire in 600 seconds + session.set_expiry(600) # expire in 600 seconds session['test_expire'] = True session.save() # reload session with key From d0d9c3ea26ba7f053969fd84518477a4e6544be9 Mon Sep 17 00:00:00 2001 From: Ross Lawley Date: Thu, 25 Apr 2013 12:21:25 +0000 Subject: [PATCH 181/189] Test to ensure that pickled complex fields work with save() (#228) --- tests/document/instance.py | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/tests/document/instance.py b/tests/document/instance.py index a75cc4dc..b800d90f 100644 --- a/tests/document/instance.py +++ b/tests/document/instance.py @@ -1694,11 +1694,19 @@ class InstanceTest(unittest.TestCase): self.assertEqual(resurrected, pickle_doc) + # Test pickling changed data + pickle_doc.lists.append("3") + pickled_doc = pickle.dumps(pickle_doc) + resurrected = pickle.loads(pickled_doc) + + self.assertEqual(resurrected, pickle_doc) resurrected.string = "Two" resurrected.save() - pickle_doc = pickle_doc.reload() + pickle_doc = PickleTest.objects.first() self.assertEqual(resurrected, pickle_doc) + self.assertEqual(pickle_doc.string, "Two") + self.assertEqual(pickle_doc.lists, ["1", "2", "3"]) def test_throw_invalid_document_error(self): From ac6e793bbe8e907b7f1469a18709e18d045306a2 Mon Sep 17 00:00:00 2001 From: Ross Lawley Date: Thu, 25 Apr 2013 13:43:56 +0000 Subject: [PATCH 182/189] UUIDField now stores as a binary by default (#292) --- docs/changelog.rst | 1 + docs/upgrade.rst | 24 ++ mongoengine/fields.py | 12 +- tests/__init__.py | 4 +- tests/document/delta.py | 47 ++-- tests/fields/fields.py | 305 +------------------------ tests/migration/__init__.py | 1 + tests/migration/uuidfield_to_binary.py | 48 ++++ 8 files changed, 109 insertions(+), 333 deletions(-) create mode 100644 tests/migration/uuidfield_to_binary.py diff --git a/docs/changelog.rst b/docs/changelog.rst index bd3821fe..9ea4ad54 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -4,6 +4,7 @@ Changelog Changes in 0.8.X ================ +- UUIDField now stores as a binary by default (#292) - Added Custom User Model for Django 1.5 (#285) - Cascading saves now default to off (#291) - ReferenceField now store ObjectId's by default rather than DBRef (#290) diff --git a/docs/upgrade.rst b/docs/upgrade.rst index 6138bb45..bcdc1101 100644 --- a/docs/upgrade.rst +++ b/docs/upgrade.rst @@ -120,6 +120,30 @@ eg:: p._mark_as_dirty('friends') p.save() +UUIDField +--------- + +UUIDFields now default to storing binary values:: + + # Old code + class Animal(Document): + uuid = UUIDField() + + # New code + class Animal(Document): + uuid = UUIDField(binary=False) + +To migrate all the uuid's you need to touch each object and mark it as dirty +eg:: + + # Doc definition + class Animal(Document): + uuid = UUIDField() + + # Mark all ReferenceFields as dirty and save + for a in Animal.objects: + a._mark_as_dirty('uuid') + a.save() Cascading Saves --------------- diff --git a/mongoengine/fields.py b/mongoengine/fields.py index 979699ce..bea827c4 100644 --- a/mongoengine/fields.py +++ b/mongoengine/fields.py @@ -1474,19 +1474,15 @@ class UUIDField(BaseField): """ _binary = None - def __init__(self, binary=None, **kwargs): + def __init__(self, binary=True, **kwargs): """ Store UUID data in the database - :param binary: (optional) boolean store as binary. + :param binary: if False store as a string. + .. versionchanged:: 0.8.0 .. versionchanged:: 0.6.19 """ - if binary is None: - binary = False - msg = ("UUIDFields will soon default to store as binary, please " - "configure binary=False if you wish to store as a string") - warnings.warn(msg, FutureWarning) self._binary = binary super(UUIDField, self).__init__(**kwargs) @@ -1504,6 +1500,8 @@ class UUIDField(BaseField): def to_mongo(self, value): if not self._binary: return unicode(value) + elif isinstance(value, basestring): + return uuid.UUID(value) return value def prepare_query_value(self, op, value): diff --git a/tests/__init__.py b/tests/__init__.py index 152a8ce0..b24df5d2 100644 --- a/tests/__init__.py +++ b/tests/__init__.py @@ -1,3 +1,5 @@ from all_warnings import AllWarnings from document import * -from queryset import * \ No newline at end of file +from queryset import * +from fields import * +from migration import * diff --git a/tests/document/delta.py b/tests/document/delta.py index c6191d9b..16ab609b 100644 --- a/tests/document/delta.py +++ b/tests/document/delta.py @@ -129,14 +129,14 @@ class DeltaTest(unittest.TestCase): } self.assertEqual(doc.embedded_field._delta(), (embedded_delta, {})) self.assertEqual(doc._delta(), - ({'embedded_field': embedded_delta}, {})) + ({'embedded_field': embedded_delta}, {})) doc.save() doc = doc.reload(10) doc.embedded_field.dict_field = {} self.assertEqual(doc._get_changed_fields(), - ['embedded_field.dict_field']) + ['embedded_field.dict_field']) self.assertEqual(doc.embedded_field._delta(), ({}, {'dict_field': 1})) self.assertEqual(doc._delta(), ({}, {'embedded_field.dict_field': 1})) doc.save() @@ -145,7 +145,7 @@ class DeltaTest(unittest.TestCase): doc.embedded_field.list_field = [] self.assertEqual(doc._get_changed_fields(), - ['embedded_field.list_field']) + ['embedded_field.list_field']) self.assertEqual(doc.embedded_field._delta(), ({}, {'list_field': 1})) self.assertEqual(doc._delta(), ({}, {'embedded_field.list_field': 1})) doc.save() @@ -160,7 +160,7 @@ class DeltaTest(unittest.TestCase): doc.embedded_field.list_field = ['1', 2, embedded_2] self.assertEqual(doc._get_changed_fields(), - ['embedded_field.list_field']) + ['embedded_field.list_field']) self.assertEqual(doc.embedded_field._delta(), ({ 'list_field': ['1', 2, { @@ -192,11 +192,11 @@ class DeltaTest(unittest.TestCase): doc.embedded_field.list_field[2].string_field = 'world' self.assertEqual(doc._get_changed_fields(), - ['embedded_field.list_field.2.string_field']) + ['embedded_field.list_field.2.string_field']) self.assertEqual(doc.embedded_field._delta(), - ({'list_field.2.string_field': 'world'}, {})) + ({'list_field.2.string_field': 'world'}, {})) self.assertEqual(doc._delta(), - ({'embedded_field.list_field.2.string_field': 'world'}, {})) + ({'embedded_field.list_field.2.string_field': 'world'}, {})) doc.save() doc = doc.reload(10) self.assertEqual(doc.embedded_field.list_field[2].string_field, @@ -206,7 +206,7 @@ class DeltaTest(unittest.TestCase): doc.embedded_field.list_field[2].string_field = 'hello world' doc.embedded_field.list_field[2] = doc.embedded_field.list_field[2] self.assertEqual(doc._get_changed_fields(), - ['embedded_field.list_field']) + ['embedded_field.list_field']) self.assertEqual(doc.embedded_field._delta(), ({ 'list_field': ['1', 2, { '_cls': 'Embedded', @@ -225,40 +225,40 @@ class DeltaTest(unittest.TestCase): doc.save() doc = doc.reload(10) self.assertEqual(doc.embedded_field.list_field[2].string_field, - 'hello world') + 'hello world') # Test list native methods doc.embedded_field.list_field[2].list_field.pop(0) self.assertEqual(doc._delta(), - ({'embedded_field.list_field.2.list_field': - [2, {'hello': 'world'}]}, {})) + ({'embedded_field.list_field.2.list_field': + [2, {'hello': 'world'}]}, {})) doc.save() doc = doc.reload(10) doc.embedded_field.list_field[2].list_field.append(1) self.assertEqual(doc._delta(), - ({'embedded_field.list_field.2.list_field': - [2, {'hello': 'world'}, 1]}, {})) + ({'embedded_field.list_field.2.list_field': + [2, {'hello': 'world'}, 1]}, {})) doc.save() doc = doc.reload(10) self.assertEqual(doc.embedded_field.list_field[2].list_field, - [2, {'hello': 'world'}, 1]) + [2, {'hello': 'world'}, 1]) doc.embedded_field.list_field[2].list_field.sort(key=str) doc.save() doc = doc.reload(10) self.assertEqual(doc.embedded_field.list_field[2].list_field, - [1, 2, {'hello': 'world'}]) + [1, 2, {'hello': 'world'}]) del(doc.embedded_field.list_field[2].list_field[2]['hello']) self.assertEqual(doc._delta(), - ({'embedded_field.list_field.2.list_field': [1, 2, {}]}, {})) + ({'embedded_field.list_field.2.list_field': [1, 2, {}]}, {})) doc.save() doc = doc.reload(10) del(doc.embedded_field.list_field[2].list_field) self.assertEqual(doc._delta(), - ({}, {'embedded_field.list_field.2.list_field': 1})) + ({}, {'embedded_field.list_field.2.list_field': 1})) doc.save() doc = doc.reload(10) @@ -269,9 +269,9 @@ class DeltaTest(unittest.TestCase): doc.dict_field['Embedded'].string_field = 'Hello World' self.assertEqual(doc._get_changed_fields(), - ['dict_field.Embedded.string_field']) + ['dict_field.Embedded.string_field']) self.assertEqual(doc._delta(), - ({'dict_field.Embedded.string_field': 'Hello World'}, {})) + ({'dict_field.Embedded.string_field': 'Hello World'}, {})) def test_circular_reference_deltas(self): self.circular_reference_deltas(Document, Document) @@ -289,10 +289,11 @@ class DeltaTest(unittest.TestCase): name = StringField() owner = ReferenceField('Person') - person = Person(name="owner") - person.save() - organization = Organization(name="company") - organization.save() + Person.drop_collection() + Organization.drop_collection() + + person = Person(name="owner").save() + organization = Organization(name="company").save() person.owns.append(organization) organization.owner = person diff --git a/tests/fields/fields.py b/tests/fields/fields.py index ade44b8d..7eae3f43 100644 --- a/tests/fields/fields.py +++ b/tests/fields/fields.py @@ -354,7 +354,6 @@ class FieldTest(unittest.TestCase): person.api_key = api_key self.assertRaises(ValidationError, person.validate) - def test_datetime_validation(self): """Ensure that invalid values cannot be assigned to datetime fields. """ @@ -1805,304 +1804,6 @@ class FieldTest(unittest.TestCase): Shirt.drop_collection() - def test_file_fields(self): - """Ensure that file fields can be written to and their data retrieved - """ - class PutFile(Document): - the_file = FileField() - - class StreamFile(Document): - the_file = FileField() - - class SetFile(Document): - the_file = FileField() - - text = b('Hello, World!') - more_text = b('Foo Bar') - content_type = 'text/plain' - - PutFile.drop_collection() - StreamFile.drop_collection() - SetFile.drop_collection() - - putfile = PutFile() - putfile.the_file.put(text, content_type=content_type) - putfile.save() - putfile.validate() - result = PutFile.objects.first() - self.assertTrue(putfile == result) - self.assertEqual(result.the_file.read(), text) - self.assertEqual(result.the_file.content_type, content_type) - result.the_file.delete() # Remove file from GridFS - PutFile.objects.delete() - - # Ensure file-like objects are stored - putfile = PutFile() - putstring = StringIO() - putstring.write(text) - putstring.seek(0) - putfile.the_file.put(putstring, content_type=content_type) - putfile.save() - putfile.validate() - result = PutFile.objects.first() - self.assertTrue(putfile == result) - self.assertEqual(result.the_file.read(), text) - self.assertEqual(result.the_file.content_type, content_type) - result.the_file.delete() - - streamfile = StreamFile() - streamfile.the_file.new_file(content_type=content_type) - streamfile.the_file.write(text) - streamfile.the_file.write(more_text) - streamfile.the_file.close() - streamfile.save() - streamfile.validate() - result = StreamFile.objects.first() - self.assertTrue(streamfile == result) - self.assertEqual(result.the_file.read(), text + more_text) - self.assertEqual(result.the_file.content_type, content_type) - result.the_file.seek(0) - self.assertEqual(result.the_file.tell(), 0) - self.assertEqual(result.the_file.read(len(text)), text) - self.assertEqual(result.the_file.tell(), len(text)) - self.assertEqual(result.the_file.read(len(more_text)), more_text) - self.assertEqual(result.the_file.tell(), len(text + more_text)) - result.the_file.delete() - - # Ensure deleted file returns None - self.assertTrue(result.the_file.read() == None) - - setfile = SetFile() - setfile.the_file = text - setfile.save() - setfile.validate() - result = SetFile.objects.first() - self.assertTrue(setfile == result) - self.assertEqual(result.the_file.read(), text) - - # Try replacing file with new one - result.the_file.replace(more_text) - result.save() - result.validate() - result = SetFile.objects.first() - self.assertTrue(setfile == result) - self.assertEqual(result.the_file.read(), more_text) - result.the_file.delete() - - PutFile.drop_collection() - StreamFile.drop_collection() - SetFile.drop_collection() - - # Make sure FileField is optional and not required - class DemoFile(Document): - the_file = FileField() - DemoFile.objects.create() - - - def test_file_field_no_default(self): - - class GridDocument(Document): - the_file = FileField() - - GridDocument.drop_collection() - - with tempfile.TemporaryFile() as f: - f.write(b("Hello World!")) - f.flush() - - # Test without default - doc_a = GridDocument() - doc_a.save() - - - doc_b = GridDocument.objects.with_id(doc_a.id) - doc_b.the_file.replace(f, filename='doc_b') - doc_b.save() - self.assertNotEqual(doc_b.the_file.grid_id, None) - - # Test it matches - doc_c = GridDocument.objects.with_id(doc_b.id) - self.assertEqual(doc_b.the_file.grid_id, doc_c.the_file.grid_id) - - # Test with default - doc_d = GridDocument(the_file=b('')) - doc_d.save() - - doc_e = GridDocument.objects.with_id(doc_d.id) - self.assertEqual(doc_d.the_file.grid_id, doc_e.the_file.grid_id) - - doc_e.the_file.replace(f, filename='doc_e') - doc_e.save() - - doc_f = GridDocument.objects.with_id(doc_e.id) - self.assertEqual(doc_e.the_file.grid_id, doc_f.the_file.grid_id) - - db = GridDocument._get_db() - grid_fs = gridfs.GridFS(db) - self.assertEqual(['doc_b', 'doc_e'], grid_fs.list()) - - def test_file_uniqueness(self): - """Ensure that each instance of a FileField is unique - """ - class TestFile(Document): - name = StringField() - the_file = FileField() - - # First instance - test_file = TestFile() - test_file.name = "Hello, World!" - test_file.the_file.put(b('Hello, World!')) - test_file.save() - - # Second instance - test_file_dupe = TestFile() - data = test_file_dupe.the_file.read() # Should be None - - self.assertTrue(test_file.name != test_file_dupe.name) - self.assertTrue(test_file.the_file.read() != data) - - TestFile.drop_collection() - - def test_file_boolean(self): - """Ensure that a boolean test of a FileField indicates its presence - """ - class TestFile(Document): - the_file = FileField() - - test_file = TestFile() - self.assertFalse(bool(test_file.the_file)) - test_file.the_file = b('Hello, World!') - test_file.the_file.content_type = 'text/plain' - test_file.save() - self.assertTrue(bool(test_file.the_file)) - - TestFile.drop_collection() - - def test_file_cmp(self): - """Test comparing against other types""" - class TestFile(Document): - the_file = FileField() - - test_file = TestFile() - self.assertFalse(test_file.the_file in [{"test": 1}]) - - def test_image_field(self): - if PY3: - raise SkipTest('PIL does not have Python 3 support') - - class TestImage(Document): - image = ImageField() - - TestImage.drop_collection() - - t = TestImage() - t.image.put(open(TEST_IMAGE_PATH, 'rb')) - t.save() - - t = TestImage.objects.first() - - self.assertEqual(t.image.format, 'PNG') - - w, h = t.image.size - self.assertEqual(w, 371) - self.assertEqual(h, 76) - - t.image.delete() - - def test_image_field_resize(self): - if PY3: - raise SkipTest('PIL does not have Python 3 support') - - class TestImage(Document): - image = ImageField(size=(185, 37)) - - TestImage.drop_collection() - - t = TestImage() - t.image.put(open(TEST_IMAGE_PATH, 'rb')) - t.save() - - t = TestImage.objects.first() - - self.assertEqual(t.image.format, 'PNG') - w, h = t.image.size - - self.assertEqual(w, 185) - self.assertEqual(h, 37) - - t.image.delete() - - def test_image_field_resize_force(self): - if PY3: - raise SkipTest('PIL does not have Python 3 support') - - class TestImage(Document): - image = ImageField(size=(185, 37, True)) - - TestImage.drop_collection() - - t = TestImage() - t.image.put(open(TEST_IMAGE_PATH, 'rb')) - t.save() - - t = TestImage.objects.first() - - self.assertEqual(t.image.format, 'PNG') - w, h = t.image.size - - self.assertEqual(w, 185) - self.assertEqual(h, 37) - - t.image.delete() - - def test_image_field_thumbnail(self): - if PY3: - raise SkipTest('PIL does not have Python 3 support') - - class TestImage(Document): - image = ImageField(thumbnail_size=(92, 18)) - - TestImage.drop_collection() - - t = TestImage() - t.image.put(open(TEST_IMAGE_PATH, 'rb')) - t.save() - - t = TestImage.objects.first() - - self.assertEqual(t.image.thumbnail.format, 'PNG') - self.assertEqual(t.image.thumbnail.width, 92) - self.assertEqual(t.image.thumbnail.height, 18) - - t.image.delete() - - def test_file_multidb(self): - register_connection('test_files', 'test_files') - class TestFile(Document): - name = StringField() - the_file = FileField(db_alias="test_files", - collection_name="macumba") - - TestFile.drop_collection() - - # delete old filesystem - get_db("test_files").macumba.files.drop() - get_db("test_files").macumba.chunks.drop() - - # First instance - test_file = TestFile() - test_file.name = "Hello, World!" - test_file.the_file.put(b('Hello, World!'), - name="hello.txt") - test_file.save() - - data = get_db("test_files").macumba.files.find_one() - self.assertEqual(data.get('name'), 'hello.txt') - - test_file = TestFile.objects.first() - self.assertEqual(test_file.the_file.read(), - b('Hello, World!')) - def test_geo_indexes(self): """Ensure that indexes are created automatically for GeoPointFields. """ @@ -2170,7 +1871,7 @@ class FieldTest(unittest.TestCase): self.assertEqual(c['next'], 10) ids = [i.id for i in Person.objects] - self.assertEqual(ids, xrange(1, 11)) + self.assertEqual(ids, range(1, 11)) c = self.db['mongoengine.counters'].find_one({'_id': 'person.id'}) self.assertEqual(c['next'], 10) @@ -2219,10 +1920,10 @@ class FieldTest(unittest.TestCase): self.assertEqual(c['next'], 10) ids = [i.id for i in Person.objects] - self.assertEqual(ids, xrange(1, 11)) + self.assertEqual(ids, range(1, 11)) counters = [i.counter for i in Person.objects] - self.assertEqual(counters, xrange(1, 11)) + self.assertEqual(counters, range(1, 11)) c = self.db['mongoengine.counters'].find_one({'_id': 'person.id'}) self.assertEqual(c['next'], 10) diff --git a/tests/migration/__init__.py b/tests/migration/__init__.py index f7ad6747..bff50c36 100644 --- a/tests/migration/__init__.py +++ b/tests/migration/__init__.py @@ -1,6 +1,7 @@ from convert_to_new_inheritance_model import * from refrencefield_dbref_to_object_id import * from turn_off_inheritance import * +from uuidfield_to_binary import * if __name__ == '__main__': unittest.main() diff --git a/tests/migration/uuidfield_to_binary.py b/tests/migration/uuidfield_to_binary.py new file mode 100644 index 00000000..a535e91f --- /dev/null +++ b/tests/migration/uuidfield_to_binary.py @@ -0,0 +1,48 @@ +# -*- coding: utf-8 -*- +import unittest +import uuid + +from mongoengine import Document, connect +from mongoengine.connection import get_db +from mongoengine.fields import StringField, UUIDField, ListField + +__all__ = ('ConvertToBinaryUUID', ) + + +class ConvertToBinaryUUID(unittest.TestCase): + + def setUp(self): + connect(db='mongoenginetest') + self.db = get_db() + + def test_how_to_convert_to_binary_uuid_fields(self): + """Demonstrates migrating from 0.7 to 0.8 + """ + + # 1. Old definition - using dbrefs + class Person(Document): + name = StringField() + uuid = UUIDField(binary=False) + uuids = ListField(UUIDField(binary=False)) + + Person.drop_collection() + Person(name="Wilson Jr", uuid=uuid.uuid4(), + uuids=[uuid.uuid4(), uuid.uuid4()]).save() + + # 2. Start the migration by changing the schema + # Change UUIDFIeld as now binary defaults to True + class Person(Document): + name = StringField() + uuid = UUIDField() + uuids = ListField(UUIDField()) + + # 3. Loop all the objects and mark parent as changed + for p in Person.objects: + p._mark_as_changed('uuid') + p._mark_as_changed('uuids') + p.save() + + # 4. Confirmation of the fix! + wilson = Person.objects(name="Wilson Jr").as_pymongo()[0] + self.assertTrue(isinstance(wilson['uuid'], uuid.UUID)) + self.assertTrue(all([isinstance(u, uuid.UUID) for u in wilson['uuids']])) From 5e94637adc2d0fcb89b5569cba5ffec13d511147 Mon Sep 17 00:00:00 2001 From: Ross Lawley Date: Thu, 25 Apr 2013 15:39:57 +0000 Subject: [PATCH 183/189] DecimalField now stores as float not string (#289) --- docs/apireference.rst | 60 ++++++++++++------------ docs/changelog.rst | 1 + docs/conf.py | 7 ++- docs/upgrade.rst | 29 ++++++++++++ mongoengine/fields.py | 54 ++++++++++++++++----- mongoengine/queryset/transform.py | 6 +-- tests/fields/fields.py | 45 ++++++++++++++++-- tests/migration/__init__.py | 1 + tests/migration/decimalfield_as_float.py | 50 ++++++++++++++++++++ tests/queryset/queryset.py | 4 +- 10 files changed, 204 insertions(+), 53 deletions(-) create mode 100644 tests/migration/decimalfield_as_float.py diff --git a/docs/apireference.rst b/docs/apireference.rst index 0040f45e..3a156299 100644 --- a/docs/apireference.rst +++ b/docs/apireference.rst @@ -54,33 +54,33 @@ Querying Fields ====== -.. autoclass:: mongoengine.StringField -.. autoclass:: mongoengine.URLField -.. autoclass:: mongoengine.EmailField -.. autoclass:: mongoengine.IntField -.. autoclass:: mongoengine.LongField -.. autoclass:: mongoengine.FloatField -.. autoclass:: mongoengine.DecimalField -.. autoclass:: mongoengine.BooleanField -.. autoclass:: mongoengine.DateTimeField -.. autoclass:: mongoengine.ComplexDateTimeField -.. autoclass:: mongoengine.EmbeddedDocumentField -.. autoclass:: mongoengine.GenericEmbeddedDocumentField -.. autoclass:: mongoengine.DynamicField -.. autoclass:: mongoengine.ListField -.. autoclass:: mongoengine.SortedListField -.. autoclass:: mongoengine.DictField -.. autoclass:: mongoengine.MapField -.. autoclass:: mongoengine.ReferenceField -.. autoclass:: mongoengine.GenericReferenceField -.. autoclass:: mongoengine.BinaryField -.. autoclass:: mongoengine.FileField -.. autoclass:: mongoengine.ImageField -.. autoclass:: mongoengine.GeoPointField -.. autoclass:: mongoengine.SequenceField -.. autoclass:: mongoengine.ObjectIdField -.. autoclass:: mongoengine.UUIDField -.. autoclass:: mongoengine.GridFSError -.. autoclass:: mongoengine.GridFSProxy -.. autoclass:: mongoengine.ImageGridFsProxy -.. autoclass:: mongoengine.ImproperlyConfigured +.. autoclass:: mongoengine.fields.StringField +.. autoclass:: mongoengine.fields.URLField +.. autoclass:: mongoengine.fields.EmailField +.. autoclass:: mongoengine.fields.IntField +.. autoclass:: mongoengine.fields.LongField +.. autoclass:: mongoengine.fields.FloatField +.. autoclass:: mongoengine.fields.DecimalField +.. autoclass:: mongoengine.fields.BooleanField +.. autoclass:: mongoengine.fields.DateTimeField +.. autoclass:: mongoengine.fields.ComplexDateTimeField +.. autoclass:: mongoengine.fields.EmbeddedDocumentField +.. autoclass:: mongoengine.fields.GenericEmbeddedDocumentField +.. autoclass:: mongoengine.fields.DynamicField +.. autoclass:: mongoengine.fields.ListField +.. autoclass:: mongoengine.fields.SortedListField +.. autoclass:: mongoengine.fields.DictField +.. autoclass:: mongoengine.fields.MapField +.. autoclass:: mongoengine.fields.ReferenceField +.. autoclass:: mongoengine.fields.GenericReferenceField +.. autoclass:: mongoengine.fields.BinaryField +.. autoclass:: mongoengine.fields.FileField +.. autoclass:: mongoengine.fields.ImageField +.. autoclass:: mongoengine.fields.GeoPointField +.. autoclass:: mongoengine.fields.SequenceField +.. autoclass:: mongoengine.fields.ObjectIdField +.. autoclass:: mongoengine.fields.UUIDField +.. autoclass:: mongoengine.fields.GridFSError +.. autoclass:: mongoengine.fields.GridFSProxy +.. autoclass:: mongoengine.fields.ImageGridFsProxy +.. autoclass:: mongoengine.fields.ImproperlyConfigured diff --git a/docs/changelog.rst b/docs/changelog.rst index 9ea4ad54..d0167c51 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -4,6 +4,7 @@ Changelog Changes in 0.8.X ================ +- DecimalField now stores as float not string (#289) - UUIDField now stores as a binary by default (#292) - Added Custom User Model for Django 1.5 (#285) - Cascading saves now default to off (#291) diff --git a/docs/conf.py b/docs/conf.py index 3cfcef5d..8bcb9ec9 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -173,8 +173,8 @@ latex_paper_size = 'a4' # Grouping the document tree into LaTeX files. List of tuples # (source start file, target name, title, author, documentclass [howto/manual]). latex_documents = [ - ('index', 'MongoEngine.tex', u'MongoEngine Documentation', - u'Harry Marr', 'manual'), + ('index', 'MongoEngine.tex', 'MongoEngine Documentation', + 'Ross Lawley', 'manual'), ] # The name of an image file (relative to this directory) to place at the top of @@ -193,3 +193,6 @@ latex_documents = [ # If false, no module index is generated. #latex_use_modindex = True + +autoclass_content = 'both' + diff --git a/docs/upgrade.rst b/docs/upgrade.rst index bcdc1101..dddce912 100644 --- a/docs/upgrade.rst +++ b/docs/upgrade.rst @@ -145,6 +145,35 @@ eg:: a._mark_as_dirty('uuid') a.save() +DecimalField +------------ + +DecimalField now store floats - previous it was storing strings and that +made it impossible to do comparisons when querying correctly.:: + + # Old code + class Person(Document): + balance = DecimalField() + + # New code + class Person(Document): + balance = DecimalField(force_string=True) + +To migrate all the uuid's you need to touch each object and mark it as dirty +eg:: + + # Doc definition + class Person(Document): + balance = DecimalField() + + # Mark all ReferenceFields as dirty and save + for p in Person.objects: + p._mark_as_dirty('balance') + p.save() + +.. note:: DecimalField's have also been improved with the addition of precision + and rounding. See :class:`~mongoengine.DecimalField` for more information. + Cascading Saves --------------- To improve performance document saves will no longer automatically cascade. diff --git a/mongoengine/fields.py b/mongoengine/fields.py index bea827c4..2e149330 100644 --- a/mongoengine/fields.py +++ b/mongoengine/fields.py @@ -260,30 +260,57 @@ class FloatField(BaseField): class DecimalField(BaseField): """A fixed-point decimal number field. + .. versionchanged:: 0.8 .. versionadded:: 0.3 """ - def __init__(self, min_value=None, max_value=None, **kwargs): - self.min_value, self.max_value = min_value, max_value + def __init__(self, min_value=None, max_value=None, force_string=False, + precision=2, rounding=decimal.ROUND_HALF_UP, **kwargs): + """ + :param min_value: Validation rule for the minimum acceptable value. + :param max_value: Validation rule for the maximum acceptable value. + :param force_string: Store as a string. + :param precision: Number of decimal places to store. + :param rounding: The rounding rule from the python decimal libary: + + - decimial.ROUND_CEILING (towards Infinity) + - decimial.ROUND_DOWN (towards zero) + - decimial.ROUND_FLOOR (towards -Infinity) + - decimial.ROUND_HALF_DOWN (to nearest with ties going towards zero) + - decimial.ROUND_HALF_EVEN (to nearest with ties going to nearest even integer) + - decimial.ROUND_HALF_UP (to nearest with ties going away from zero) + - decimial.ROUND_UP (away from zero) + - decimial.ROUND_05UP (away from zero if last digit after rounding towards zero would have been 0 or 5; otherwise towards zero) + + Defaults to: ``decimal.ROUND_HALF_UP`` + + """ + self.min_value = min_value + self.max_value = max_value + self.force_string = force_string + self.precision = decimal.Decimal(".%s" % ("0" * precision)) + self.rounding = rounding + super(DecimalField, self).__init__(**kwargs) def to_python(self, value): - original_value = value - if not isinstance(value, basestring): - value = unicode(value) - try: - value = decimal.Decimal(value) - except ValueError: - return original_value - return value + if value is None: + return value + + return decimal.Decimal(value).quantize(self.precision, + rounding=self.rounding) def to_mongo(self, value): - return unicode(value) + if value is None: + return value + if self.force_string: + return unicode(value) + return float(self.to_python(value)) def validate(self, value): if not isinstance(value, decimal.Decimal): if not isinstance(value, basestring): - value = str(value) + value = unicode(value) try: value = decimal.Decimal(value) except Exception, exc: @@ -295,6 +322,9 @@ class DecimalField(BaseField): if self.max_value is not None and value > self.max_value: self.error('Decimal value is too large') + def prepare_query_value(self, op, value): + return self.to_mongo(value) + class BooleanField(BaseField): """A boolean field type. diff --git a/mongoengine/queryset/transform.py b/mongoengine/queryset/transform.py index 71f12e37..3da26935 100644 --- a/mongoengine/queryset/transform.py +++ b/mongoengine/queryset/transform.py @@ -9,7 +9,7 @@ __all__ = ('query', 'update') COMPARISON_OPERATORS = ('ne', 'gt', 'gte', 'lt', 'lte', 'in', 'nin', 'mod', - 'all', 'size', 'exists', 'not') + 'all', 'size', 'exists', 'not') GEO_OPERATORS = ('within_distance', 'within_spherical_distance', 'within_box', 'within_polygon', 'near', 'near_sphere', 'max_distance') @@ -74,7 +74,7 @@ def query(_doc_cls=None, _field_operation=False, **query): if op in singular_ops: if isinstance(field, basestring): if (op in STRING_OPERATORS and - isinstance(value, basestring)): + isinstance(value, basestring)): StringField = _import_class('StringField') value = StringField.prepare_query_value(op, value) else: @@ -144,7 +144,7 @@ def query(_doc_cls=None, _field_operation=False, **query): merge_query[k].append(mongo_query[k]) del mongo_query[k] if isinstance(v, list): - value = [{k:val} for val in v] + value = [{k: val} for val in v] if '$and' in mongo_query.keys(): mongo_query['$and'].append(value) else: diff --git a/tests/fields/fields.py b/tests/fields/fields.py index 7eae3f43..4fa6989c 100644 --- a/tests/fields/fields.py +++ b/tests/fields/fields.py @@ -272,10 +272,8 @@ class FieldTest(unittest.TestCase): Person.drop_collection() - person = Person() - person.height = Decimal('1.89') - person.save() - person.reload() + Person(height=Decimal('1.89')).save() + person = Person.objects.first() self.assertEqual(person.height, Decimal('1.89')) person.height = '2.0' @@ -289,6 +287,45 @@ class FieldTest(unittest.TestCase): Person.drop_collection() + def test_decimal_comparison(self): + + class Person(Document): + money = DecimalField() + + Person.drop_collection() + + Person(money=6).save() + Person(money=8).save() + Person(money=10).save() + + self.assertEqual(2, Person.objects(money__gt=Decimal("7")).count()) + self.assertEqual(2, Person.objects(money__gt=7).count()) + self.assertEqual(2, Person.objects(money__gt="7").count()) + + def test_decimal_storage(self): + class Person(Document): + btc = DecimalField(precision=4) + + Person.drop_collection() + Person(btc=10).save() + Person(btc=10.1).save() + Person(btc=10.11).save() + Person(btc="10.111").save() + Person(btc=Decimal("10.1111")).save() + Person(btc=Decimal("10.11111")).save() + + # How its stored + expected = [{'btc': 10.0}, {'btc': 10.1}, {'btc': 10.11}, + {'btc': 10.111}, {'btc': 10.1111}, {'btc': 10.1111}] + actual = list(Person.objects.exclude('id').as_pymongo()) + self.assertEqual(expected, actual) + + # How it comes out locally + expected = [Decimal('10.0000'), Decimal('10.1000'), Decimal('10.1100'), + Decimal('10.1110'), Decimal('10.1111'), Decimal('10.1111')] + actual = list(Person.objects().scalar('btc')) + self.assertEqual(expected, actual) + def test_boolean_validation(self): """Ensure that invalid values cannot be assigned to boolean fields. """ diff --git a/tests/migration/__init__.py b/tests/migration/__init__.py index bff50c36..6fc83e02 100644 --- a/tests/migration/__init__.py +++ b/tests/migration/__init__.py @@ -1,4 +1,5 @@ from convert_to_new_inheritance_model import * +from decimalfield_as_float import * from refrencefield_dbref_to_object_id import * from turn_off_inheritance import * from uuidfield_to_binary import * diff --git a/tests/migration/decimalfield_as_float.py b/tests/migration/decimalfield_as_float.py new file mode 100644 index 00000000..3903c913 --- /dev/null +++ b/tests/migration/decimalfield_as_float.py @@ -0,0 +1,50 @@ + # -*- coding: utf-8 -*- +import unittest +import decimal +from decimal import Decimal + +from mongoengine import Document, connect +from mongoengine.connection import get_db +from mongoengine.fields import StringField, DecimalField, ListField + +__all__ = ('ConvertDecimalField', ) + + +class ConvertDecimalField(unittest.TestCase): + + def setUp(self): + connect(db='mongoenginetest') + self.db = get_db() + + def test_how_to_convert_decimal_fields(self): + """Demonstrates migrating from 0.7 to 0.8 + """ + + # 1. Old definition - using dbrefs + class Person(Document): + name = StringField() + money = DecimalField(force_string=True) + monies = ListField(DecimalField(force_string=True)) + + Person.drop_collection() + Person(name="Wilson Jr", money=Decimal("2.50"), + monies=[Decimal("2.10"), Decimal("5.00")]).save() + + # 2. Start the migration by changing the schema + # Change DecimalField - add precision and rounding settings + class Person(Document): + name = StringField() + money = DecimalField(precision=2, rounding=decimal.ROUND_HALF_UP) + monies = ListField(DecimalField(precision=2, + rounding=decimal.ROUND_HALF_UP)) + + # 3. Loop all the objects and mark parent as changed + for p in Person.objects: + p._mark_as_changed('money') + p._mark_as_changed('monies') + p.save() + + # 4. Confirmation of the fix! + wilson = Person.objects(name="Wilson Jr").as_pymongo()[0] + self.assertTrue(isinstance(wilson['money'], float)) + self.assertTrue(all([isinstance(m, float) for m in wilson['monies']])) diff --git a/tests/queryset/queryset.py b/tests/queryset/queryset.py index c7c4c7ce..5e403c4e 100644 --- a/tests/queryset/queryset.py +++ b/tests/queryset/queryset.py @@ -3262,9 +3262,9 @@ class QuerySetTest(unittest.TestCase): self.assertTrue(isinstance(results[0], dict)) self.assertTrue(isinstance(results[1], dict)) self.assertEqual(results[0]['name'], 'Bob Dole') - self.assertEqual(results[0]['price'], '1.11') + self.assertEqual(results[0]['price'], 1.11) self.assertEqual(results[1]['name'], 'Barack Obama') - self.assertEqual(results[1]['price'], '2.22') + self.assertEqual(results[1]['price'], 2.22) # Test coerce_types users = User.objects.only('name', 'price').as_pymongo(coerce_types=True) From 13d8dfdb5fc44ce92ed3373111d4e2e74cf2e66b Mon Sep 17 00:00:00 2001 From: Ross Lawley Date: Fri, 26 Apr 2013 08:43:38 +0000 Subject: [PATCH 184/189] Save py2.6 from Decimal Float fun --- mongoengine/fields.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/mongoengine/fields.py b/mongoengine/fields.py index 2e149330..a5dbf5d0 100644 --- a/mongoengine/fields.py +++ b/mongoengine/fields.py @@ -297,8 +297,9 @@ class DecimalField(BaseField): if value is None: return value - return decimal.Decimal(value).quantize(self.precision, - rounding=self.rounding) + # Convert to string for python 2.6 before casting to Decimal + value = decimal.Decimal("%s" % value) + return value.quantize(self.precision, rounding=self.rounding) def to_mongo(self, value): if value is None: From 7765f272ac74f1b25e9e057d13b54408031d1594 Mon Sep 17 00:00:00 2001 From: Ross Lawley Date: Fri, 26 Apr 2013 08:46:46 +0000 Subject: [PATCH 185/189] Documentation api and reference cleanups --- docs/django.rst | 2 +- docs/guide/defining-documents.rst | 82 +++++++++++++++---------------- docs/guide/document-instances.rst | 4 +- docs/guide/gridfs.rst | 6 +-- docs/guide/querying.rst | 16 +++--- docs/tutorial.rst | 8 +-- docs/upgrade.rst | 11 ++++- mongoengine/document.py | 2 +- mongoengine/fields.py | 2 +- mongoengine/queryset/queryset.py | 8 +-- 10 files changed, 75 insertions(+), 66 deletions(-) diff --git a/docs/django.rst b/docs/django.rst index e3a1c6b1..d60e55d9 100644 --- a/docs/django.rst +++ b/docs/django.rst @@ -98,7 +98,7 @@ Django provides session cookie, which expires after ```SESSION_COOKIE_AGE``` sec Storage ======= -With MongoEngine's support for GridFS via the :class:`~mongoengine.FileField`, +With MongoEngine's support for GridFS via the :class:`~mongoengine.fields.FileField`, it is useful to have a Django file storage backend that wraps this. The new storage module is called :class:`~mongoengine.django.storage.GridFSStorage`. Using it is very similar to using the default FileSystemStorage.:: diff --git a/docs/guide/defining-documents.rst b/docs/guide/defining-documents.rst index 350ba678..d18606ac 100644 --- a/docs/guide/defining-documents.rst +++ b/docs/guide/defining-documents.rst @@ -62,31 +62,31 @@ not provided. Default values may optionally be a callable, which will be called to retrieve the value (such as in the above example). The field types available are as follows: -* :class:`~mongoengine.BinaryField` -* :class:`~mongoengine.BooleanField` -* :class:`~mongoengine.ComplexDateTimeField` -* :class:`~mongoengine.DateTimeField` -* :class:`~mongoengine.DecimalField` -* :class:`~mongoengine.DictField` -* :class:`~mongoengine.DynamicField` -* :class:`~mongoengine.EmailField` -* :class:`~mongoengine.EmbeddedDocumentField` -* :class:`~mongoengine.FileField` -* :class:`~mongoengine.FloatField` -* :class:`~mongoengine.GenericEmbeddedDocumentField` -* :class:`~mongoengine.GenericReferenceField` -* :class:`~mongoengine.GeoPointField` -* :class:`~mongoengine.ImageField` -* :class:`~mongoengine.IntField` -* :class:`~mongoengine.ListField` -* :class:`~mongoengine.MapField` -* :class:`~mongoengine.ObjectIdField` -* :class:`~mongoengine.ReferenceField` -* :class:`~mongoengine.SequenceField` -* :class:`~mongoengine.SortedListField` -* :class:`~mongoengine.StringField` -* :class:`~mongoengine.URLField` -* :class:`~mongoengine.UUIDField` +* :class:`~mongoengine.fields.BinaryField` +* :class:`~mongoengine.fields.BooleanField` +* :class:`~mongoengine.fields.ComplexDateTimeField` +* :class:`~mongoengine.fields.DateTimeField` +* :class:`~mongoengine.fields.DecimalField` +* :class:`~mongoengine.fields.DictField` +* :class:`~mongoengine.fields.DynamicField` +* :class:`~mongoengine.fields.EmailField` +* :class:`~mongoengine.fields.EmbeddedDocumentField` +* :class:`~mongoengine.fields.FileField` +* :class:`~mongoengine.fields.FloatField` +* :class:`~mongoengine.fields.GenericEmbeddedDocumentField` +* :class:`~mongoengine.fields.GenericReferenceField` +* :class:`~mongoengine.fields.GeoPointField` +* :class:`~mongoengine.fields.ImageField` +* :class:`~mongoengine.fields.IntField` +* :class:`~mongoengine.fields.ListField` +* :class:`~mongoengine.fields.MapField` +* :class:`~mongoengine.fields.ObjectIdField` +* :class:`~mongoengine.fields.ReferenceField` +* :class:`~mongoengine.fields.SequenceField` +* :class:`~mongoengine.fields.SortedListField` +* :class:`~mongoengine.fields.StringField` +* :class:`~mongoengine.fields.URLField` +* :class:`~mongoengine.fields.UUIDField` Field arguments --------------- @@ -110,7 +110,7 @@ arguments can be set on all fields: The definion of default parameters follow `the general rules on Python `__, which means that some care should be taken when dealing with default mutable objects - (like in :class:`~mongoengine.ListField` or :class:`~mongoengine.DictField`):: + (like in :class:`~mongoengine.fields.ListField` or :class:`~mongoengine.fields.DictField`):: class ExampleFirst(Document): # Default an empty list @@ -172,8 +172,8 @@ arguments can be set on all fields: List fields ----------- MongoDB allows the storage of lists of items. To add a list of items to a -:class:`~mongoengine.Document`, use the :class:`~mongoengine.ListField` field -type. :class:`~mongoengine.ListField` takes another field object as its first +:class:`~mongoengine.Document`, use the :class:`~mongoengine.fields.ListField` field +type. :class:`~mongoengine.fields.ListField` takes another field object as its first argument, which specifies which type elements may be stored within the list:: class Page(Document): @@ -191,7 +191,7 @@ inherit from :class:`~mongoengine.EmbeddedDocument` rather than content = StringField() To embed the document within another document, use the -:class:`~mongoengine.EmbeddedDocumentField` field type, providing the embedded +:class:`~mongoengine.fields.EmbeddedDocumentField` field type, providing the embedded document class as the first argument:: class Page(Document): @@ -206,7 +206,7 @@ Dictionary Fields Often, an embedded document may be used instead of a dictionary -- generally this is recommended as dictionaries don't support validation or custom field types. However, sometimes you will not know the structure of what you want to -store; in this situation a :class:`~mongoengine.DictField` is appropriate:: +store; in this situation a :class:`~mongoengine.fields.DictField` is appropriate:: class SurveyResponse(Document): date = DateTimeField() @@ -224,7 +224,7 @@ other objects, so are the most flexible field type available. Reference fields ---------------- References may be stored to other documents in the database using the -:class:`~mongoengine.ReferenceField`. Pass in another document class as the +:class:`~mongoengine.fields.ReferenceField`. Pass in another document class as the first argument to the constructor, then simply assign document objects to the field:: @@ -245,9 +245,9 @@ field:: The :class:`User` object is automatically turned into a reference behind the scenes, and dereferenced when the :class:`Page` object is retrieved. -To add a :class:`~mongoengine.ReferenceField` that references the document +To add a :class:`~mongoengine.fields.ReferenceField` that references the document being defined, use the string ``'self'`` in place of the document class as the -argument to :class:`~mongoengine.ReferenceField`'s constructor. To reference a +argument to :class:`~mongoengine.fields.ReferenceField`'s constructor. To reference a document that has not yet been defined, use the name of the undefined document as the constructor's argument:: @@ -325,7 +325,7 @@ Its value can take any of the following constants: :const:`mongoengine.PULL` Removes the reference to the object (using MongoDB's "pull" operation) from any object's fields of - :class:`~mongoengine.ListField` (:class:`~mongoengine.ReferenceField`). + :class:`~mongoengine.fields.ListField` (:class:`~mongoengine.fields.ReferenceField`). .. warning:: @@ -352,7 +352,7 @@ Its value can take any of the following constants: Generic reference fields '''''''''''''''''''''''' A second kind of reference field also exists, -:class:`~mongoengine.GenericReferenceField`. This allows you to reference any +:class:`~mongoengine.fields.GenericReferenceField`. This allows you to reference any kind of :class:`~mongoengine.Document`, and hence doesn't take a :class:`~mongoengine.Document` subclass as a constructor argument:: @@ -376,15 +376,15 @@ kind of :class:`~mongoengine.Document`, and hence doesn't take a .. note:: - Using :class:`~mongoengine.GenericReferenceField`\ s is slightly less - efficient than the standard :class:`~mongoengine.ReferenceField`\ s, so if + Using :class:`~mongoengine.fields.GenericReferenceField`\ s is slightly less + efficient than the standard :class:`~mongoengine.fields.ReferenceField`\ s, so if you will only be referencing one document type, prefer the standard - :class:`~mongoengine.ReferenceField`. + :class:`~mongoengine.fields.ReferenceField`. Uniqueness constraints ---------------------- MongoEngine allows you to specify that a field should be unique across a -collection by providing ``unique=True`` to a :class:`~mongoengine.Field`\ 's +collection by providing ``unique=True`` to a :class:`~mongoengine.fields.Field`\ 's constructor. If you try to save a document that has the same value for a unique field as a document that is already in the database, a :class:`~mongoengine.OperationError` will be raised. You may also specify @@ -492,11 +492,11 @@ Geospatial indexes ------------------ Geospatial indexes will be automatically created for all -:class:`~mongoengine.GeoPointField`\ s +:class:`~mongoengine.fields.GeoPointField`\ s It is also possible to explicitly define geospatial indexes. This is useful if you need to define a geospatial index on a subfield of a -:class:`~mongoengine.DictField` or a custom field that contains a +:class:`~mongoengine.fields.DictField` or a custom field that contains a point. To create a geospatial index you must prefix the field with the ***** sign. :: diff --git a/docs/guide/document-instances.rst b/docs/guide/document-instances.rst index e8e7d63c..619f3e83 100644 --- a/docs/guide/document-instances.rst +++ b/docs/guide/document-instances.rst @@ -68,8 +68,8 @@ document values for example:: Cascading Saves --------------- -If your document contains :class:`~mongoengine.ReferenceField` or -:class:`~mongoengine.GenericReferenceField` objects, then by default the +If your document contains :class:`~mongoengine.fields.ReferenceField` or +:class:`~mongoengine.fields.GenericReferenceField` objects, then by default the :meth:`~mongoengine.Document.save` method will automatically save any changes to those objects as well. If this is not desired passing :attr:`cascade` as False to the save method turns this feature off. diff --git a/docs/guide/gridfs.rst b/docs/guide/gridfs.rst index 11259477..d81bb922 100644 --- a/docs/guide/gridfs.rst +++ b/docs/guide/gridfs.rst @@ -7,7 +7,7 @@ GridFS Writing ------- -GridFS support comes in the form of the :class:`~mongoengine.FileField` field +GridFS support comes in the form of the :class:`~mongoengine.fields.FileField` field object. This field acts as a file-like object and provides a couple of different ways of inserting and retrieving data. Arbitrary metadata such as content type can also be stored alongside the files. In the following example, @@ -27,7 +27,7 @@ a document is created to store details about animals, including a photo:: Retrieval --------- -So using the :class:`~mongoengine.FileField` is just like using any other +So using the :class:`~mongoengine.fields.FileField` is just like using any other field. The file can also be retrieved just as easily:: marmot = Animal.objects(genus='Marmota').first() @@ -37,7 +37,7 @@ field. The file can also be retrieved just as easily:: Streaming --------- -Streaming data into a :class:`~mongoengine.FileField` is achieved in a +Streaming data into a :class:`~mongoengine.fields.FileField` is achieved in a slightly different manner. First, a new file must be created by calling the :func:`new_file` method. Data can then be written using :func:`write`:: diff --git a/docs/guide/querying.rst b/docs/guide/querying.rst index 60702ec6..3a25c286 100644 --- a/docs/guide/querying.rst +++ b/docs/guide/querying.rst @@ -79,7 +79,7 @@ expressions: * ``match`` -- performs an $elemMatch so you can match an entire document within an array There are a few special operators for performing geographical queries, that -may used with :class:`~mongoengine.GeoPointField`\ s: +may used with :class:`~mongoengine.fields.GeoPointField`\ s: * ``within_distance`` -- provide a list containing a point and a maximum distance (e.g. [(41.342, -87.653), 5]) @@ -100,7 +100,7 @@ Querying lists -------------- On most fields, this syntax will look up documents where the field specified matches the given value exactly, but when the field refers to a -:class:`~mongoengine.ListField`, a single item may be provided, in which case +:class:`~mongoengine.fields.ListField`, a single item may be provided, in which case lists that contain that item will be matched:: class Page(Document): @@ -319,7 +319,7 @@ Retrieving a subset of fields Sometimes a subset of fields on a :class:`~mongoengine.Document` is required, and for efficiency only these should be retrieved from the database. This issue is especially important for MongoDB, as fields may often be extremely large -(e.g. a :class:`~mongoengine.ListField` of +(e.g. a :class:`~mongoengine.fields.ListField` of :class:`~mongoengine.EmbeddedDocument`\ s, which represent the comments on a blog post. To select only a subset of fields, use :meth:`~mongoengine.queryset.QuerySet.only`, specifying the fields you want to @@ -351,14 +351,14 @@ If you later need the missing fields, just call Getting related data -------------------- -When iterating the results of :class:`~mongoengine.ListField` or -:class:`~mongoengine.DictField` we automatically dereference any +When iterating the results of :class:`~mongoengine.fields.ListField` or +:class:`~mongoengine.fields.DictField` we automatically dereference any :class:`~pymongo.dbref.DBRef` objects as efficiently as possible, reducing the number the queries to mongo. There are times when that efficiency is not enough, documents that have -:class:`~mongoengine.ReferenceField` objects or -:class:`~mongoengine.GenericReferenceField` objects at the top level are +:class:`~mongoengine.fields.ReferenceField` objects or +:class:`~mongoengine.fields.GenericReferenceField` objects at the top level are expensive as the number of queries to MongoDB can quickly rise. To limit the number of queries use @@ -541,7 +541,7 @@ Javascript code. When accessing a field on a collection object, use square-bracket notation, and prefix the MongoEngine field name with a tilde. The field name that follows the tilde will be translated to the name used in the database. Note that when referring to fields on embedded documents, -the name of the :class:`~mongoengine.EmbeddedDocumentField`, followed by a dot, +the name of the :class:`~mongoengine.fields.EmbeddedDocumentField`, followed by a dot, should be used before the name of the field on the embedded document. The following example shows how the substitutions are made:: diff --git a/docs/tutorial.rst b/docs/tutorial.rst index 423df9b0..c2f481b9 100644 --- a/docs/tutorial.rst +++ b/docs/tutorial.rst @@ -115,7 +115,7 @@ by setting :attr:`allow_inheritance` to True in the :attr:`meta`:: link_url = StringField() We are storing a reference to the author of the posts using a -:class:`~mongoengine.ReferenceField` object. These are similar to foreign key +:class:`~mongoengine.fields.ReferenceField` object. These are similar to foreign key fields in traditional ORMs, and are automatically translated into references when they are saved, and dereferenced when they are loaded. @@ -137,7 +137,7 @@ size of our database. So let's take a look that the code our modified author = ReferenceField(User) tags = ListField(StringField(max_length=30)) -The :class:`~mongoengine.ListField` object that is used to define a Post's tags +The :class:`~mongoengine.fields.ListField` object that is used to define a Post's tags takes a field object as its first argument --- this means that you can have lists of any type of field (including lists). @@ -174,7 +174,7 @@ We can then store a list of comment documents in our post document:: Handling deletions of references ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -The :class:`~mongoengine.ReferenceField` object takes a keyword +The :class:`~mongoengine.fields.ReferenceField` object takes a keyword `reverse_delete_rule` for handling deletion rules if the reference is deleted. To delete all the posts if a user is deleted set the rule:: @@ -184,7 +184,7 @@ To delete all the posts if a user is deleted set the rule:: tags = ListField(StringField(max_length=30)) comments = ListField(EmbeddedDocumentField(Comment)) -See :class:`~mongoengine.ReferenceField` for more information. +See :class:`~mongoengine.fields.ReferenceField` for more information. .. note:: MapFields and DictFields currently don't support automatic handling of diff --git a/docs/upgrade.rst b/docs/upgrade.rst index dddce912..0ae65f32 100644 --- a/docs/upgrade.rst +++ b/docs/upgrade.rst @@ -120,6 +120,9 @@ eg:: p._mark_as_dirty('friends') p.save() +`An example test migration is available on github +`_. + UUIDField --------- @@ -145,6 +148,9 @@ eg:: a._mark_as_dirty('uuid') a.save() +`An example test migration is available on github +`_. + DecimalField ------------ @@ -172,7 +178,10 @@ eg:: p.save() .. note:: DecimalField's have also been improved with the addition of precision - and rounding. See :class:`~mongoengine.DecimalField` for more information. + and rounding. See :class:`~mongoengine.fields.DecimalField` for more information. + +`An example test migration is available on github +`_. Cascading Saves --------------- diff --git a/mongoengine/document.py b/mongoengine/document.py index d0cafa3e..c4542a2e 100644 --- a/mongoengine/document.py +++ b/mongoengine/document.py @@ -559,7 +559,7 @@ class DynamicDocument(Document): way as an ordinary document but has expando style properties. Any data passed or set against the :class:`~mongoengine.DynamicDocument` that is not a field is automatically converted into a - :class:`~mongoengine.DynamicField` and data can be attributed to that + :class:`~mongoengine.fields.DynamicField` and data can be attributed to that field. .. note:: diff --git a/mongoengine/fields.py b/mongoengine/fields.py index a5dbf5d0..cf2c802c 100644 --- a/mongoengine/fields.py +++ b/mongoengine/fields.py @@ -782,7 +782,7 @@ class ReferenceField(BaseField): * NULLIFY - Updates the reference to null. * CASCADE - Deletes the documents associated with the reference. * DENY - Prevent the deletion of the reference object. - * PULL - Pull the reference from a :class:`~mongoengine.ListField` + * PULL - Pull the reference from a :class:`~mongoengine.fields.ListField` of references Alternative syntax for registering delete rules (useful when implementing diff --git a/mongoengine/queryset/queryset.py b/mongoengine/queryset/queryset.py index dcfb2402..769cf68b 100644 --- a/mongoengine/queryset/queryset.py +++ b/mongoengine/queryset/queryset.py @@ -1049,7 +1049,7 @@ class QuerySet(object): """) for result in self.map_reduce(map_func, reduce_func, - finalize_f=finalize_func, output='inline'): + finalize_f=finalize_func, output='inline'): return result.value else: return 0 @@ -1062,11 +1062,11 @@ class QuerySet(object): .. note:: Can only do direct simple mappings and cannot map across - :class:`~mongoengine.ReferenceField` or - :class:`~mongoengine.GenericReferenceField` for more complex + :class:`~mongoengine.fields.ReferenceField` or + :class:`~mongoengine.fields.GenericReferenceField` for more complex counting a manual map reduce call would is required. - If the field is a :class:`~mongoengine.ListField`, the items within + If the field is a :class:`~mongoengine.fields.ListField`, the items within each list will be counted individually. :param field: the field to use From 2447349383ca86dd57ff83403ce6bf9aebef90f6 Mon Sep 17 00:00:00 2001 From: Ross Lawley Date: Fri, 26 Apr 2013 09:59:43 +0000 Subject: [PATCH 186/189] Added a note about distinct being a command --- mongoengine/queryset/queryset.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/mongoengine/queryset/queryset.py b/mongoengine/queryset/queryset.py index 769cf68b..5c7c7c89 100644 --- a/mongoengine/queryset/queryset.py +++ b/mongoengine/queryset/queryset.py @@ -608,6 +608,9 @@ class QuerySet(object): :param field: the field to select distinct values from + .. note:: This is a command and won't take ordering or limit into + account. + .. versionadded:: 0.4 .. versionchanged:: 0.5 - Fixed handling references .. versionchanged:: 0.6 - Improved db_field refrence handling From 36993097b4668e20e809a9bbb9a575b45b004939 Mon Sep 17 00:00:00 2001 From: Ross Lawley Date: Fri, 26 Apr 2013 11:38:45 +0000 Subject: [PATCH 187/189] Document serialization uses field order to ensure a strict order is set (#296) --- docs/changelog.rst | 1 + docs/guide/defining-documents.rst | 4 ++++ docs/guide/document-instances.rst | 14 ++++++++----- docs/upgrade.rst | 19 ++++++++++++++--- mongoengine/base/document.py | 34 ++++++++++++++++++++++--------- tests/document/dynamic.py | 3 ++- tests/document/inheritance.py | 27 ++++++++++++++++++++---- tests/document/instance.py | 15 ++++++++++++++ 8 files changed, 94 insertions(+), 23 deletions(-) diff --git a/docs/changelog.rst b/docs/changelog.rst index d0167c51..f786c1d6 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -4,6 +4,7 @@ Changelog Changes in 0.8.X ================ +- Document serialization uses field order to ensure a strict order is set (#296) - DecimalField now stores as float not string (#289) - UUIDField now stores as a binary by default (#292) - Added Custom User Model for Django 1.5 (#285) diff --git a/docs/guide/defining-documents.rst b/docs/guide/defining-documents.rst index d18606ac..36e0efea 100644 --- a/docs/guide/defining-documents.rst +++ b/docs/guide/defining-documents.rst @@ -24,6 +24,9 @@ objects** as class attributes to the document class:: title = StringField(max_length=200, required=True) date_modified = DateTimeField(default=datetime.datetime.now) +As BSON (the binary format for storing data in mongodb) is order dependent, +documents are serialized based on their field order. + Dynamic document schemas ======================== One of the benefits of MongoDb is dynamic schemas for a collection, whilst data @@ -51,6 +54,7 @@ be saved :: There is one caveat on Dynamic Documents: fields cannot start with `_` +Dynamic fields are stored in alphabetical order *after* any declared fields. Fields ====== diff --git a/docs/guide/document-instances.rst b/docs/guide/document-instances.rst index 619f3e83..f9a6610f 100644 --- a/docs/guide/document-instances.rst +++ b/docs/guide/document-instances.rst @@ -30,11 +30,14 @@ already exist, then any changes will be updated atomically. For example:: .. note:: - Changes to documents are tracked and on the whole perform `set` operations. + Changes to documents are tracked and on the whole perform ``set`` operations. - * ``list_field.pop(0)`` - *sets* the resulting list + * ``list_field.push(0)`` - *sets* the resulting list * ``del(list_field)`` - *unsets* whole list + With lists its preferable to use ``Doc.update(push__list_field=0)`` as + this stops the whole list being updated - stopping any race conditions. + .. seealso:: :ref:`guide-atomic-updates` @@ -70,9 +73,10 @@ Cascading Saves --------------- If your document contains :class:`~mongoengine.fields.ReferenceField` or :class:`~mongoengine.fields.GenericReferenceField` objects, then by default the -:meth:`~mongoengine.Document.save` method will automatically save any changes to -those objects as well. If this is not desired passing :attr:`cascade` as False -to the save method turns this feature off. +:meth:`~mongoengine.Document.save` method will not save any changes to +those objects. If you want all references to also be saved also, noting each +save is a separate query, then passing :attr:`cascade` as True +to the save method will cascade any saves. Deleting documents ------------------ diff --git a/docs/upgrade.rst b/docs/upgrade.rst index 0ae65f32..bb5705ca 100644 --- a/docs/upgrade.rst +++ b/docs/upgrade.rst @@ -120,7 +120,7 @@ eg:: p._mark_as_dirty('friends') p.save() -`An example test migration is available on github +`An example test migration for ReferenceFields is available on github `_. UUIDField @@ -148,7 +148,7 @@ eg:: a._mark_as_dirty('uuid') a.save() -`An example test migration is available on github +`An example test migration for UUIDFields is available on github `_. DecimalField @@ -180,7 +180,7 @@ eg:: .. note:: DecimalField's have also been improved with the addition of precision and rounding. See :class:`~mongoengine.fields.DecimalField` for more information. -`An example test migration is available on github +`An example test migration for DecimalFields is available on github `_. Cascading Saves @@ -196,6 +196,19 @@ you will have to explicitly tell it to cascade on save:: # Or on save: my_document.save(cascade=True) +Storage +------- + +Document and Embedded Documents are now serialized based on declared field order. +Previously, the data was passed to mongodb as a dictionary and which meant that +order wasn't guaranteed - so things like ``$addToSet`` operations on +:class:`~mongoengine.EmbeddedDocument` could potentially fail in unexpected +ways. + +If this impacts you, you may want to rewrite the objects using the +``doc.mark_as_dirty('field')`` pattern described above. If you are using a +compound primary key then you will need to ensure the order is fixed and match +your EmbeddedDocument to that order. Querysets ========= diff --git a/mongoengine/base/document.py b/mongoengine/base/document.py index 7ec672fe..53686b25 100644 --- a/mongoengine/base/document.py +++ b/mongoengine/base/document.py @@ -6,6 +6,7 @@ from functools import partial import pymongo from bson import json_util from bson.dbref import DBRef +from bson.son import SON from mongoengine import signals from mongoengine.common import _import_class @@ -228,11 +229,16 @@ class BaseDocument(object): pass def to_mongo(self): - """Return data dictionary ready for use with MongoDB. + """Return as SON data ready for use with MongoDB. """ - data = {} - for field_name, field in self._fields.iteritems(): + data = SON() + data["_id"] = None + data['_cls'] = self._class_name + + for field_name in self: value = self._data.get(field_name, None) + field = self._fields.get(field_name) + if value is not None: value = field.to_mongo(value) @@ -244,19 +250,27 @@ class BaseDocument(object): if value is not None: data[field.db_field] = value - # Only add _cls if allow_inheritance is True - if (hasattr(self, '_meta') and - self._meta.get('allow_inheritance', ALLOW_INHERITANCE) == True): - data['_cls'] = self._class_name + # If "_id" has not been set, then try and set it + if data["_id"] is None: + data["_id"] = self._data.get("id", None) - if '_id' in data and data['_id'] is None: - del data['_id'] + if data['_id'] is None: + data.pop('_id') + + # Only add _cls if allow_inheritance is True + if (not hasattr(self, '_meta') or + not self._meta.get('allow_inheritance', ALLOW_INHERITANCE)): + data.pop('_cls') if not self._dynamic: return data - for name, field in self._dynamic_fields.items(): + # Sort dynamic fields by key + dynamic_fields = sorted(self._dynamic_fields.iteritems(), + key=operator.itemgetter(0)) + for name, field in dynamic_fields: data[name] = field.to_mongo(self._data.get(name, None)) + return data def validate(self, clean=True): diff --git a/tests/document/dynamic.py b/tests/document/dynamic.py index 5881cd07..6263e68c 100644 --- a/tests/document/dynamic.py +++ b/tests/document/dynamic.py @@ -31,8 +31,9 @@ class DynamicTest(unittest.TestCase): self.assertEqual(p.to_mongo(), {"_cls": "Person", "name": "James", "age": 34}) - + self.assertEqual(p.to_mongo().keys(), ["_cls", "name", "age"]) p.save() + self.assertEqual(p.to_mongo().keys(), ["_id", "_cls", "name", "age"]) self.assertEqual(self.Person.objects.first().age, 34) diff --git a/tests/document/inheritance.py b/tests/document/inheritance.py index 3b550f1a..f0116311 100644 --- a/tests/document/inheritance.py +++ b/tests/document/inheritance.py @@ -143,7 +143,7 @@ class InheritanceTest(unittest.TestCase): self.assertEqual(Animal._superclasses, ()) self.assertEqual(Animal._subclasses, ('Animal', 'Animal.Fish', - 'Animal.Fish.Pike')) + 'Animal.Fish.Pike')) self.assertEqual(Fish._superclasses, ('Animal', )) self.assertEqual(Fish._subclasses, ('Animal.Fish', 'Animal.Fish.Pike')) @@ -168,6 +168,26 @@ class InheritanceTest(unittest.TestCase): self.assertEqual(Employee._get_collection_name(), Person._get_collection_name()) + def test_inheritance_to_mongo_keys(self): + """Ensure that document may inherit fields from a superclass document. + """ + class Person(Document): + name = StringField() + age = IntField() + + meta = {'allow_inheritance': True} + + class Employee(Person): + salary = IntField() + + self.assertEqual(['age', 'id', 'name', 'salary'], + sorted(Employee._fields.keys())) + self.assertEqual(Person(name="Bob", age=35).to_mongo().keys(), + ['_cls', 'name', 'age']) + self.assertEqual(Employee(name="Bob", age=35, salary=0).to_mongo().keys(), + ['_cls', 'name', 'age', 'salary']) + self.assertEqual(Employee._get_collection_name(), + Person._get_collection_name()) def test_polymorphic_queries(self): """Ensure that the correct subclasses are returned from a query @@ -197,7 +217,6 @@ class InheritanceTest(unittest.TestCase): classes = [obj.__class__ for obj in Human.objects] self.assertEqual(classes, [Human]) - def test_allow_inheritance(self): """Ensure that inheritance may be disabled on simple classes and that _cls and _subclasses will not be used. @@ -213,8 +232,8 @@ class InheritanceTest(unittest.TestCase): self.assertRaises(ValueError, create_dog_class) # Check that _cls etc aren't present on simple documents - dog = Animal(name='dog') - dog.save() + dog = Animal(name='dog').save() + self.assertEqual(dog.to_mongo().keys(), ['_id', 'name']) collection = self.db[Animal._get_collection_name()] obj = collection.find_one() diff --git a/tests/document/instance.py b/tests/document/instance.py index b800d90f..06744ab4 100644 --- a/tests/document/instance.py +++ b/tests/document/instance.py @@ -428,6 +428,21 @@ class InstanceTest(unittest.TestCase): self.assertFalse('age' in person) self.assertFalse('nationality' in person) + def test_embedded_document_to_mongo(self): + class Person(EmbeddedDocument): + name = StringField() + age = IntField() + + meta = {"allow_inheritance": True} + + class Employee(Person): + salary = IntField() + + self.assertEqual(Person(name="Bob", age=35).to_mongo().keys(), + ['_cls', 'name', 'age']) + self.assertEqual(Employee(name="Bob", age=35, salary=0).to_mongo().keys(), + ['_cls', 'name', 'age', 'salary']) + def test_embedded_document(self): """Ensure that embedded documents are set up correctly. """ From 5e65d278324c86bc615747108658957b4e9dff03 Mon Sep 17 00:00:00 2001 From: Ross Lawley Date: Fri, 26 Apr 2013 11:46:12 +0000 Subject: [PATCH 188/189] PEP8 x == True should be x is True --- mongoengine/base/fields.py | 2 +- mongoengine/document.py | 2 +- mongoengine/queryset/queryset.py | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/mongoengine/base/fields.py b/mongoengine/base/fields.py index 6ebba362..3929a3a5 100644 --- a/mongoengine/base/fields.py +++ b/mongoengine/base/fields.py @@ -295,7 +295,7 @@ class ComplexBaseField(BaseField): meta = getattr(v, '_meta', {}) allow_inheritance = ( meta.get('allow_inheritance', ALLOW_INHERITANCE) - == True) + is True) if not allow_inheritance and not self.field: value_dict[k] = GenericReferenceField().to_mongo(v) else: diff --git a/mongoengine/document.py b/mongoengine/document.py index c4542a2e..bd6ce191 100644 --- a/mongoengine/document.py +++ b/mongoengine/document.py @@ -548,7 +548,7 @@ class Document(BaseDocument): # If _cls is being used (for polymorphism), it needs an index, # only if another index doesn't begin with _cls if (index_cls and not cls_indexed and - cls._meta.get('allow_inheritance', ALLOW_INHERITANCE) == True): + cls._meta.get('allow_inheritance', ALLOW_INHERITANCE) is True): collection.ensure_index('_cls', background=background, **index_opts) diff --git a/mongoengine/queryset/queryset.py b/mongoengine/queryset/queryset.py index 5c7c7c89..65d6553f 100644 --- a/mongoengine/queryset/queryset.py +++ b/mongoengine/queryset/queryset.py @@ -66,7 +66,7 @@ class QuerySet(object): # If inheritance is allowed, only return instances and instances of # subclasses of the class being used - if document._meta.get('allow_inheritance') == True: + if document._meta.get('allow_inheritance') is True: self._initial_query = {"_cls": {"$in": self._document._subclasses}} self._loaded_fields = QueryFieldList(always_include=['_cls']) self._cursor_obj = None From 6e2d2f33deeaee2b69685a9c2389cacdfefefa38 Mon Sep 17 00:00:00 2001 From: Ross Lawley Date: Fri, 26 Apr 2013 14:33:40 +0000 Subject: [PATCH 189/189] Updated benchmarks for #27 --- benchmark.py | 143 ++++++++++++++++++++++++++++++++++++++++----------- 1 file changed, 113 insertions(+), 30 deletions(-) diff --git a/benchmark.py b/benchmark.py index 0197e1d7..16b2fd47 100644 --- a/benchmark.py +++ b/benchmark.py @@ -86,17 +86,43 @@ def main(): ---------------------------------------------------------------------------------------------------- Creating 10000 dictionaries - MongoEngine, force=True 8.36906409264 + 0.8.X + ---------------------------------------------------------------------------------------------------- + Creating 10000 dictionaries - Pymongo + 3.69964408875 + ---------------------------------------------------------------------------------------------------- + Creating 10000 dictionaries - Pymongo write_concern={"w": 0} + 3.5526599884 + ---------------------------------------------------------------------------------------------------- + Creating 10000 dictionaries - MongoEngine + 7.00959801674 + ---------------------------------------------------------------------------------------------------- + Creating 10000 dictionaries without continual assign - MongoEngine + 5.60943293571 + ---------------------------------------------------------------------------------------------------- + Creating 10000 dictionaries - MongoEngine - write_concern={"w": 0}, cascade=True + 6.715102911 + ---------------------------------------------------------------------------------------------------- + Creating 10000 dictionaries - MongoEngine, write_concern={"w": 0}, validate=False, cascade=True + 5.50644683838 + ---------------------------------------------------------------------------------------------------- + Creating 10000 dictionaries - MongoEngine, write_concern={"w": 0}, validate=False + 4.69851183891 + ---------------------------------------------------------------------------------------------------- + Creating 10000 dictionaries - MongoEngine, force_insert=True, write_concern={"w": 0}, validate=False + 4.68946313858 + ---------------------------------------------------------------------------------------------------- """ setup = """ -from pymongo import Connection -connection = Connection() +from pymongo import MongoClient +connection = MongoClient() connection.drop_database('timeit_test') """ stmt = """ -from pymongo import Connection -connection = Connection() +from pymongo import MongoClient +connection = MongoClient() db = connection.timeit_test noddy = db.noddy @@ -106,7 +132,7 @@ for i in xrange(10000): for j in range(20): example['fields']["key"+str(j)] = "value "+str(j) - noddy.insert(example) + noddy.save(example) myNoddys = noddy.find() [n for n in myNoddys] # iterate @@ -117,9 +143,32 @@ myNoddys = noddy.find() t = timeit.Timer(stmt=stmt, setup=setup) print t.timeit(1) + stmt = """ +from pymongo import MongoClient +connection = MongoClient() + +db = connection.timeit_test +noddy = db.noddy + +for i in xrange(10000): + example = {'fields': {}} + for j in range(20): + example['fields']["key"+str(j)] = "value "+str(j) + + noddy.save(example, write_concern={"w": 0}) + +myNoddys = noddy.find() +[n for n in myNoddys] # iterate +""" + + print "-" * 100 + print """Creating 10000 dictionaries - Pymongo write_concern={"w": 0}""" + t = timeit.Timer(stmt=stmt, setup=setup) + print t.timeit(1) + setup = """ -from pymongo import Connection -connection = Connection() +from pymongo import MongoClient +connection = MongoClient() connection.drop_database('timeit_test') connection.disconnect() @@ -149,33 +198,18 @@ myNoddys = Noddy.objects() stmt = """ for i in xrange(10000): noddy = Noddy() + fields = {} for j in range(20): - noddy.fields["key"+str(j)] = "value "+str(j) - noddy.save(safe=False, validate=False) + fields["key"+str(j)] = "value "+str(j) + noddy.fields = fields + noddy.save() myNoddys = Noddy.objects() [n for n in myNoddys] # iterate """ print "-" * 100 - print """Creating 10000 dictionaries - MongoEngine, safe=False, validate=False""" - t = timeit.Timer(stmt=stmt, setup=setup) - print t.timeit(1) - - - stmt = """ -for i in xrange(10000): - noddy = Noddy() - for j in range(20): - noddy.fields["key"+str(j)] = "value "+str(j) - noddy.save(safe=False, validate=False, cascade=False) - -myNoddys = Noddy.objects() -[n for n in myNoddys] # iterate -""" - - print "-" * 100 - print """Creating 10000 dictionaries - MongoEngine, safe=False, validate=False, cascade=False""" + print """Creating 10000 dictionaries without continual assign - MongoEngine""" t = timeit.Timer(stmt=stmt, setup=setup) print t.timeit(1) @@ -184,16 +218,65 @@ for i in xrange(10000): noddy = Noddy() for j in range(20): noddy.fields["key"+str(j)] = "value "+str(j) - noddy.save(force_insert=True, safe=False, validate=False, cascade=False) + noddy.save(write_concern={"w": 0}, cascade=True) myNoddys = Noddy.objects() [n for n in myNoddys] # iterate """ print "-" * 100 - print """Creating 10000 dictionaries - MongoEngine, force=True""" + print """Creating 10000 dictionaries - MongoEngine - write_concern={"w": 0}, cascade = True""" t = timeit.Timer(stmt=stmt, setup=setup) print t.timeit(1) + stmt = """ +for i in xrange(10000): + noddy = Noddy() + for j in range(20): + noddy.fields["key"+str(j)] = "value "+str(j) + noddy.save(write_concern={"w": 0}, validate=False, cascade=True) + +myNoddys = Noddy.objects() +[n for n in myNoddys] # iterate +""" + + print "-" * 100 + print """Creating 10000 dictionaries - MongoEngine, write_concern={"w": 0}, validate=False, cascade=True""" + t = timeit.Timer(stmt=stmt, setup=setup) + print t.timeit(1) + + stmt = """ +for i in xrange(10000): + noddy = Noddy() + for j in range(20): + noddy.fields["key"+str(j)] = "value "+str(j) + noddy.save(validate=False, write_concern={"w": 0}) + +myNoddys = Noddy.objects() +[n for n in myNoddys] # iterate +""" + + print "-" * 100 + print """Creating 10000 dictionaries - MongoEngine, write_concern={"w": 0}, validate=False""" + t = timeit.Timer(stmt=stmt, setup=setup) + print t.timeit(1) + + stmt = """ +for i in xrange(10000): + noddy = Noddy() + for j in range(20): + noddy.fields["key"+str(j)] = "value "+str(j) + noddy.save(force_insert=True, write_concern={"w": 0}, validate=False) + +myNoddys = Noddy.objects() +[n for n in myNoddys] # iterate +""" + + print "-" * 100 + print """Creating 10000 dictionaries - MongoEngine, force_insert=True, write_concern={"w": 0}, validate=False""" + t = timeit.Timer(stmt=stmt, setup=setup) + print t.timeit(1) + + if __name__ == "__main__": - main() + main() \ No newline at end of file