diff --git a/docs/changelog.rst b/docs/changelog.rst
index d5f79585..4b9a547f 100644
--- a/docs/changelog.rst
+++ b/docs/changelog.rst
@@ -2,25 +2,32 @@
Changelog
=========
+Changes in v0.2.2
+=================
+- Fixed bug that prevented indexes from being used on ``ListField``\ s
+- ``Document.filter()`` added as an alias to ``Document.__call__()``
+- ``validate()`` may now be used on ``EmbeddedDocument``\ s
+
Changes in v0.2.1
=================
- Added a MongoEngine backend for Django sessions
-- Added force_insert to Document.save()
-- Improved querying syntax for ListField and EmbeddedDocumentField
-- Added support for user-defined primary keys (_ids in MongoDB)
+- Added ``force_insert`` to ``Document.save()``
+- Improved querying syntax for ``ListField`` and ``EmbeddedDocumentField``
+- Added support for user-defined primary keys (``_id`` in MongoDB)
Changes in v0.2
===============
-- Added Q class for building advanced queries
-- Added QuerySet methods for atomic updates to documents
-- Fields may now specify ``unique=True`` to enforce uniqueness across a collection
+- Added ``Q`` class for building advanced queries
+- Added ``QuerySet`` methods for atomic updates to documents
+- Fields may now specify ``unique=True`` to enforce uniqueness across a
+ collection
- Added option for default document ordering
- Fixed bug in index definitions
Changes in v0.1.3
=================
- Added Django authentication backend
-- Added Document.meta support for indexes, which are ensured just before
+- Added ``Document.meta`` support for indexes, which are ensured just before
querying takes place
- A few minor bugfixes
@@ -30,8 +37,8 @@ Changes in v0.1.2
- Query values may be processed before before being used in queries
- Made connections lazy
- Fixed bug in Document dictionary-style access
-- Added BooleanField
-- Added Document.reload method
+- Added ``BooleanField``
+- Added ``Document.reload()`` method
Changes in v0.1.1
diff --git a/docs/guide/connecting.rst b/docs/guide/connecting.rst
new file mode 100644
index 00000000..64e7666a
--- /dev/null
+++ b/docs/guide/connecting.rst
@@ -0,0 +1,20 @@
+.. _guide-connecting:
+
+=====================
+Connecting to MongoDB
+=====================
+To connect to a running instance of :program:`mongod`, use the
+:func:`~mongoengine.connect` function. The first argument is the name of the
+database to connect to. If the database does not exist, it will be created. If
+the database requires authentication, :attr:`username` and :attr:`password`
+arguments may be provided::
+
+ from mongoengine import connect
+ connect('project1', username='webapp', password='pwd123')
+
+By default, MongoEngine assumes that the :program:`mongod` instance is running
+on **localhost** on port **27017**. If MongoDB is running elsewhere, you may
+provide :attr:`host` and :attr:`port` arguments to
+:func:`~mongoengine.connect`::
+
+ connect('project1', host='192.168.1.35', port=12345)
diff --git a/docs/guide/defining-documents.rst b/docs/guide/defining-documents.rst
new file mode 100644
index 00000000..7683387e
--- /dev/null
+++ b/docs/guide/defining-documents.rst
@@ -0,0 +1,238 @@
+==================
+Defining documents
+==================
+In MongoDB, a **document** is roughly equivalent to a **row** in an RDBMS. When
+working with relational databases, rows are stored in **tables**, which have a
+strict **schema** that the rows follow. MongoDB stores documents in
+**collections** rather than tables - the principle difference is that no schema
+is enforced at a database level.
+
+Defining a document's schema
+============================
+MongoEngine allows you to define schemata for documents as this helps to reduce
+coding errors, and allows for utility methods to be defined on fields which may
+be present.
+
+To define a schema for a document, create a class that inherits from
+:class:`~mongoengine.Document`. Fields are specified by adding **field
+objects** as class attributes to the document class::
+
+ from mongoengine import *
+ import datetime
+
+ class Page(Document):
+ title = StringField(max_length=200, required=True)
+ date_modified = DateTimeField(default=datetime.now)
+
+Fields
+======
+By default, fields are not required. To make a field mandatory, set the
+:attr:`required` keyword argument of a field to ``True``. Fields also may have
+validation constraints available (such as :attr:`max_length` in the example
+above). Fields may also take default values, which will be used if a value is
+not provided. Default values may optionally be a callable, which will be called
+to retrieve the value (such as in the above example). The field types available
+are as follows:
+
+* :class:`~mongoengine.StringField`
+* :class:`~mongoengine.IntField`
+* :class:`~mongoengine.FloatField`
+* :class:`~mongoengine.DateTimeField`
+* :class:`~mongoengine.ListField`
+* :class:`~mongoengine.ObjectIdField`
+* :class:`~mongoengine.EmbeddedDocumentField`
+* :class:`~mongoengine.ReferenceField`
+
+List fields
+^^^^^^^^^^^
+MongoDB allows the storage of lists of items. To add a list of items to a
+:class:`~mongoengine.Document`, use the :class:`~mongoengine.ListField` field
+type. :class:`~mongoengine.ListField` takes another field object as its first
+argument, which specifies which type elements may be stored within the list::
+
+ class Page(Document):
+ tags = ListField(StringField(max_length=50))
+
+Embedded documents
+^^^^^^^^^^^^^^^^^^
+MongoDB has the ability to embed documents within other documents. Schemata may
+be defined for these embedded documents, just as they may be for regular
+documents. To create an embedded document, just define a document as usual, but
+inherit from :class:`~mongoengine.EmbeddedDocument` rather than
+:class:`~mongoengine.Document`::
+
+ class Comment(EmbeddedDocument):
+ content = StringField()
+
+To embed the document within another document, use the
+:class:`~mongoengine.EmbeddedDocumentField` field type, providing the embedded
+document class as the first argument::
+
+ class Page(Document):
+ comments = ListField(EmbeddedDocumentField(Comment))
+
+ comment1 = Comment('Good work!')
+ comment2 = Comment('Nice article!')
+ page = Page(comments=[comment1, comment2])
+
+Reference fields
+^^^^^^^^^^^^^^^^
+References may be stored to other documents in the database using the
+:class:`~mongoengine.ReferenceField`. Pass in another document class as the
+first argument to the constructor, then simply assign document objects to the
+field::
+
+ class User(Document):
+ name = StringField()
+
+ class Page(Document):
+ content = StringField()
+ author = ReferenceField(User)
+
+ john = User(name="John Smith")
+ john.save()
+
+ post = Page(content="Test Page")
+ post.author = john
+ post.save()
+
+The :class:`User` object is automatically turned into a reference behind the
+scenes, and dereferenced when the :class:`Page` object is retrieved.
+
+Uniqueness constraints
+^^^^^^^^^^^^^^^^^^^^^^
+MongoEngine allows you to specify that a field should be unique across a
+collection by providing ``unique=True`` to a :class:`~mongoengine.Field`\ 's
+constructor. If you try to save a document that has the same value for a unique
+field as a document that is already in the database, a
+:class:`~mongoengine.OperationError` will be raised. You may also specify
+multi-field uniqueness constraints by using :attr:`unique_with`, which may be
+either a single field name, or a list or tuple of field names::
+
+ class User(Document):
+ username = StringField(unique=True)
+ first_name = StringField()
+ last_name = StringField(unique_with='last_name')
+
+Document collections
+====================
+Document classes that inherit **directly** from :class:`~mongoengine.Document`
+will have their own **collection** in the database. The name of the collection
+is by default the name of the class, coverted to lowercase (so in the example
+above, the collection would be called `page`). If you need to change the name
+of the collection (e.g. to use MongoEngine with an existing database), then
+create a class dictionary attribute called :attr:`meta` on your document, and
+set :attr:`collection` to the name of the collection that you want your
+document class to use::
+
+ class Page(Document):
+ title = StringField(max_length=200, required=True)
+ meta = {'collection': 'cmsPage'}
+
+Capped collections
+^^^^^^^^^^^^^^^^^^
+A :class:`~mongoengine.Document` may use a **Capped Collection** by specifying
+:attr:`max_documents` and :attr:`max_size` in the :attr:`meta` dictionary.
+:attr:`max_documents` is the maximum number of documents that is allowed to be
+stored in the collection, and :attr:`max_size` is the maximum size of the
+collection in bytes. If :attr:`max_size` is not specified and
+:attr:`max_documents` is, :attr:`max_size` defaults to 10000000 bytes (10MB).
+The following example shows a :class:`Log` document that will be limited to
+1000 entries and 2MB of disk space::
+
+ class Log(Document):
+ ip_address = StringField()
+ meta = {'max_documents': 1000, 'max_size': 2000000}
+
+Indexes
+=======
+You can specify indexes on collections to make querying faster. This is done
+by creating a list of index specifications called :attr:`indexes` in the
+:attr:`~mongoengine.Document.meta` dictionary, where an index specification may
+either be a single field name, or a tuple containing multiple field names. A
+direction may be specified on fields by prefixing the field name with a **+**
+or a **-** sign. Note that direction only matters on multi-field indexes. ::
+
+ class Page(Document):
+ title = StringField()
+ rating = StringField()
+ meta = {
+ 'indexes': ['title', ('title', '-rating')]
+ }
+
+Ordering
+========
+A default ordering can be specified for your
+:class:`~mongoengine.queryset.QuerySet` using the :attr:`ordering` attribute of
+:attr:`~mongoengine.Document.meta`. Ordering will be applied when the
+:class:`~mongoengine.queryset.QuerySet` is created, and can be overridden by
+subsequent calls to :meth:`~mongoengine.queryset.QuerySet.order_by`. ::
+
+ from datetime import datetime
+
+ class BlogPost(Document):
+ title = StringField()
+ published_date = DateTimeField()
+
+ meta = {
+ 'ordering': ['-published_date']
+ }
+
+ blog_post_1 = BlogPost(title="Blog Post #1")
+ blog_post_1.published_date = datetime(2010, 1, 5, 0, 0 ,0))
+
+ blog_post_2 = BlogPost(title="Blog Post #2")
+ blog_post_2.published_date = datetime(2010, 1, 6, 0, 0 ,0))
+
+ blog_post_3 = BlogPost(title="Blog Post #3")
+ blog_post_3.published_date = datetime(2010, 1, 7, 0, 0 ,0))
+
+ blog_post_1.save()
+ blog_post_2.save()
+ blog_post_3.save()
+
+ # get the "first" BlogPost using default ordering
+ # from BlogPost.meta.ordering
+ latest_post = BlogPost.objects.first()
+ self.assertEqual(latest_post.title, "Blog Post #3")
+
+ # override default ordering, order BlogPosts by "published_date"
+ first_post = BlogPost.objects.order_by("+published_date").first()
+ self.assertEqual(first_post.title, "Blog Post #1")
+
+Document inheritance
+====================
+To create a specialised type of a :class:`~mongoengine.Document` you have
+defined, you may subclass it and add any extra fields or methods you may need.
+As this is new class is not a direct subclass of
+:class:`~mongoengine.Document`, it will not be stored in its own collection; it
+will use the same collection as its superclass uses. This allows for more
+convenient and efficient retrieval of related documents::
+
+ # Stored in a collection named 'page'
+ class Page(Document):
+ title = StringField(max_length=200, required=True)
+
+ # Also stored in the collection named 'page'
+ class DatedPage(Page):
+ date = DateTimeField()
+
+Working with existing data
+^^^^^^^^^^^^^^^^^^^^^^^^^^
+To enable correct retrieval of documents involved in this kind of heirarchy,
+two extra attributes are stored on each document in the database: :attr:`_cls`
+and :attr:`_types`. These are hidden from the user through the MongoEngine
+interface, but may not be present if you are trying to use MongoEngine with
+an existing database. For this reason, you may disable this inheritance
+mechansim, removing the dependency of :attr:`_cls` and :attr:`_types`, enabling
+you to work with existing databases. To disable inheritance on a document
+class, set :attr:`allow_inheritance` to ``False`` in the :attr:`meta`
+dictionary::
+
+ # Will work with data in an existing collection named 'cmsPage'
+ class Page(Document):
+ title = StringField(max_length=200, required=True)
+ meta = {
+ 'collection': 'cmsPage',
+ 'allow_inheritance': False,
+ }
diff --git a/docs/guide/document-instances.rst b/docs/guide/document-instances.rst
new file mode 100644
index 00000000..756bc3d5
--- /dev/null
+++ b/docs/guide/document-instances.rst
@@ -0,0 +1,65 @@
+===================
+Documents instances
+===================
+To create a new document object, create an instance of the relevant document
+class, providing values for its fields as its constructor keyword arguments.
+You may provide values for any of the fields on the document::
+
+ >>> page = Page(title="Test Page")
+ >>> page.title
+ 'Test Page'
+
+You may also assign values to the document's fields using standard object
+attribute syntax::
+
+ >>> page.title = "Example Page"
+ >>> page.title
+ 'Example Page'
+
+Saving and deleting documents
+-----------------------------
+To save the document to the database, call the
+:meth:`~mongoengine.Document.save` method. If the document does not exist in
+the database, it will be created. If it does already exist, it will be
+updated.
+
+To delete a document, call the :meth:`~mongoengine.Document.delete` method.
+Note that this will only work if the document exists in the database and has a
+valide :attr:`id`.
+
+.. seealso::
+ :ref:`guide-atomic-updates`
+
+Document IDs
+------------
+Each document in the database has a unique id. This may be accessed through the
+:attr:`id` attribute on :class:`~mongoengine.Document` objects. Usually, the id
+will be generated automatically by the database server when the object is save,
+meaning that you may only access the :attr:`id` field once a document has been
+saved::
+
+ >>> page = Page(title="Test Page")
+ >>> page.id
+ >>> page.save()
+ >>> page.id
+ ObjectId('123456789abcdef000000000')
+
+Alternatively, you may define one of your own fields to be the document's
+"primary key" by providing ``primary_key=True`` as a keyword argument to a
+field's constructor. Under the hood, MongoEngine will use this field as the
+:attr:`id`; in fact :attr:`id` is actually aliased to your primary key field so
+you may still use :attr:`id` to access the primary key if you want::
+
+ >>> class User(Document):
+ ... email = StringField(primary_key=True)
+ ... name = StringField()
+ ...
+ >>> bob = User(email='bob@example.com', name='Bob')
+ >>> bob.save()
+ >>> bob.id == bob.email == 'bob@example.com'
+ True
+
+.. note::
+ If you define your own primary key field, the field implicitly becomes
+ required, so a :class:`ValidationError` will be thrown if you don't provide
+ it.
diff --git a/docs/guide/index.rst b/docs/guide/index.rst
new file mode 100644
index 00000000..7fdfe932
--- /dev/null
+++ b/docs/guide/index.rst
@@ -0,0 +1,12 @@
+==========
+User Guide
+==========
+
+.. toctree::
+ :maxdepth: 2
+
+ installing
+ connecting
+ defining-documents
+ document-instances
+ querying
diff --git a/docs/guide/installing.rst b/docs/guide/installing.rst
new file mode 100644
index 00000000..132f1079
--- /dev/null
+++ b/docs/guide/installing.rst
@@ -0,0 +1,31 @@
+======================
+Installing MongoEngine
+======================
+To use MongoEngine, you will need to download `MongoDB `_
+and ensure it is running in an accessible location. You will also need
+`PyMongo `_ to use MongoEngine, but if you
+install MongoEngine using setuptools, then the dependencies will be handled for
+you.
+
+MongoEngine is available on PyPI, so to use it you can use
+:program:`easy_install`:
+
+.. code-block:: console
+
+ # easy_install mongoengine
+
+Alternatively, if you don't have setuptools installed, `download it from PyPi
+`_ and run
+
+.. code-block:: console
+
+ # python setup.py install
+
+To use the bleeding-edge version of MongoEngine, you can get the source from
+`GitHub `_ and install it as above:
+
+.. code-block:: console
+
+ # git clone git://github.com/hmarr/mongoengine
+ # cd mongoengine
+ # python setup.py install
diff --git a/docs/guide/querying.rst b/docs/guide/querying.rst
new file mode 100644
index 00000000..84ee6acf
--- /dev/null
+++ b/docs/guide/querying.rst
@@ -0,0 +1,196 @@
+=====================
+Querying the database
+=====================
+:class:`~mongoengine.Document` classes have an :attr:`objects` attribute, which
+is used for accessing the objects in the database associated with the class.
+The :attr:`objects` attribute is actually a
+:class:`~mongoengine.queryset.QuerySetManager`, which creates and returns a new
+a new :class:`~mongoengine.queryset.QuerySet` object on access. The
+:class:`~mongoengine.queryset.QuerySet` object may may be iterated over to
+fetch documents from the database::
+
+ # Prints out the names of all the users in the database
+ for user in User.objects:
+ print user.name
+
+Filtering queries
+-----------------
+The query may be filtered by calling the
+:class:`~mongoengine.queryset.QuerySet` object with field lookup keyword
+arguments. The keys in the keyword arguments correspond to fields on the
+:class:`~mongoengine.Document` you are querying::
+
+ # This will return a QuerySet that will only iterate over users whose
+ # 'country' field is set to 'uk'
+ uk_users = User.objects(country='uk')
+
+Fields on embedded documents may also be referred to using field lookup syntax
+by using a double-underscore in place of the dot in object attribute access
+syntax::
+
+ # This will return a QuerySet that will only iterate over pages that have
+ # been written by a user whose 'country' field is set to 'uk'
+ uk_pages = Page.objects(author__country='uk')
+
+Querying lists
+^^^^^^^^^^^^^^
+On most fields, this syntax will look up documents where the field specified
+matches the given value exactly, but when the field refers to a
+:class:`~mongoengine.ListField`, a single item may be provided, in which case
+lists that contain that item will be matched::
+
+ class Page(Document):
+ tags = ListField(StringField())
+
+ # This will match all pages that have the word 'coding' as an item in the
+ # 'tags' list
+ Page.objects(tags='coding')
+
+Query operators
+---------------
+Operators other than equality may also be used in queries; just attach the
+operator name to a key with a double-underscore::
+
+ # Only find users whose age is 18 or less
+ young_users = Users.objects(age__lte=18)
+
+Available operators are as follows:
+
+* ``neq`` -- not equal to
+* ``lt`` -- less than
+* ``lte`` -- less than or equal to
+* ``gt`` -- greater than
+* ``gte`` -- greater than or equal to
+* ``in`` -- value is in list (a list of values should be provided)
+* ``nin`` -- value is not in list (a list of values should be provided)
+* ``mod`` -- ``value % x == y``, where ``x`` and ``y`` are two provided values
+* ``all`` -- every item in array is in list of values provided
+* ``size`` -- the size of the array is
+* ``exists`` -- value for field exists
+
+Limiting and skipping results
+-----------------------------
+Just as with traditional ORMs, you may limit the number of results returned, or
+skip a number or results in you query.
+:meth:`~mongoengine.queryset.QuerySet.limit` and
+:meth:`~mongoengine.queryset.QuerySet.skip` and methods are available on
+:class:`~mongoengine.queryset.QuerySet` objects, but the prefered syntax for
+achieving this is using array-slicing syntax::
+
+ # Only the first 5 people
+ users = User.objects[:5]
+
+ # All except for the first 5 people
+ users = User.objects[5:]
+
+ # 5 users, starting from the 10th user found
+ users = User.objects[10:15]
+
+Aggregation
+-----------
+MongoDB provides some aggregation methods out of the box, but there are not as
+many as you typically get with an RDBMS. MongoEngine provides a wrapper around
+the built-in methods and provides some of its own, which are implemented as
+Javascript code that is executed on the database server.
+
+Counting results
+^^^^^^^^^^^^^^^^
+Just as with limiting and skipping results, there is a method on
+:class:`~mongoengine.queryset.QuerySet` objects --
+:meth:`~mongoengine.queryset.QuerySet.count`, but there is also a more Pythonic
+way of achieving this::
+
+ num_users = len(User.objects)
+
+Further aggregation
+^^^^^^^^^^^^^^^^^^^
+You may sum over the values of a specific field on documents using
+:meth:`~mongoengine.queryset.QuerySet.sum`::
+
+ yearly_expense = Employee.objects.sum('salary')
+
+.. note::
+ If the field isn't present on a document, that document will be ignored from
+ the sum.
+
+To get the average (mean) of a field on a collection of documents, use
+:meth:`~mongoengine.queryset.QuerySet.average`::
+
+ mean_age = User.objects.average('age')
+
+As MongoDB provides native lists, MongoEngine provides a helper method to get a
+dictionary of the frequencies of items in lists across an entire collection --
+:meth:`~mongoengine.queryset.QuerySet.item_frequencies`. An example of its use
+would be generating "tag-clouds"::
+
+ class Article(Document):
+ tag = ListField(StringField())
+
+ # After adding some tagged articles...
+ tag_freqs = Article.objects.item_frequencies('tag', normalize=True)
+
+ from operator import itemgetter
+ top_tags = sorted(tag_freqs.items(), key=itemgetter(1), reverse=True)[:10]
+
+Advanced queries
+----------------
+Sometimes calling a :class:`~mongoengine.queryset.QuerySet` object with keyword
+arguments can't fully express the query you want to use -- for example if you
+need to combine a number of constraints using *and* and *or*. This is made
+possible in MongoEngine through the :class:`~mongoengine.queryset.Q` class.
+A :class:`~mongoengine.queryset.Q` object represents part of a query, and
+can be initialised using the same keyword-argument syntax you use to query
+documents. To build a complex query, you may combine
+:class:`~mongoengine.queryset.Q` objects using the ``&`` (and) and ``|`` (or)
+operators. To use :class:`~mongoengine.queryset.Q` objects, pass them in
+as positional arguments to :attr:`Document.objects` when you filter it by
+calling it with keyword arguments::
+
+ # Get published posts
+ Post.objects(Q(published=True) | Q(publish_date__lte=datetime.now()))
+
+ # Get top posts
+ Post.objects((Q(featured=True) & Q(hits__gte=1000)) | Q(hits__gte=5000))
+
+.. warning::
+ Only use these advanced queries if absolutely necessary as they will execute
+ significantly slower than regular queries. This is because they are not
+ natively supported by MongoDB -- they are compiled to Javascript and sent
+ to the server for execution.
+
+.. _guide-atomic-updates:
+
+Atomic updates
+--------------
+Documents may be updated atomically by using the
+:meth:`~mongoengine.queryset.QuerySet.update_one` and
+:meth:`~mongoengine.queryset.QuerySet.update` methods on a
+:meth:`~mongoengine.queryset.QuerySet`. There are several different "modifiers"
+that you may use with these methods:
+
+* ``set`` -- set a particular value
+* ``unset`` -- delete a particular value (since MongoDB v1.3+)
+* ``inc`` -- increment a value by a given amount
+* ``dec`` -- decrement a value by a given amount
+* ``push`` -- append a value to a list
+* ``push_all`` -- append several values to a list
+* ``pull`` -- remove a value from a list
+* ``pull_all`` -- remove several values from a list
+
+The syntax for atomic updates is similar to the querying syntax, but the
+modifier comes before the field, not after it::
+
+ >>> post = BlogPost(title='Test', page_views=0, tags=['database'])
+ >>> post.save()
+ >>> BlogPost.objects(id=post.id).update_one(inc__page_views=1)
+ >>> post.reload() # the document has been changed, so we need to reload it
+ >>> post.page_views
+ 1
+ >>> BlogPost.objects(id=post.id).update_one(set__title='Example Post')
+ >>> post.reload()
+ >>> post.title
+ 'Example Post'
+ >>> BlogPost.objects(id=post.id).update_one(push__tags='nosql')
+ >>> post.reload()
+ >>> post.tags
+ ['database', 'nosql']
diff --git a/docs/index.rst b/docs/index.rst
index 40672f49..1205e4b9 100644
--- a/docs/index.rst
+++ b/docs/index.rst
@@ -1,5 +1,6 @@
+==============================
MongoEngine User Documentation
-=======================================
+==============================
MongoEngine is an Object-Document Mapper, written in Python for working with
MongoDB. To install it, simply run
@@ -14,7 +15,7 @@ The source is available on `GitHub `_.
:maxdepth: 2
tutorial
- userguide
+ guide/index
apireference
django
changelog
diff --git a/docs/userguide.rst b/docs/userguide.rst
deleted file mode 100644
index e2ee72d0..00000000
--- a/docs/userguide.rst
+++ /dev/null
@@ -1,534 +0,0 @@
-==========
-User Guide
-==========
-
-Installing
-==========
-MongoEngine is available on PyPI, so to use it you can use
-:program:`easy_install`
-
-.. code-block:: console
-
- # easy_install mongoengine
-
-Alternatively, if you don't have setuptools installed, `download it from PyPi
-`_ and run
-
-.. code-block:: console
-
- # python setup.py install
-
-.. _guide-connecting:
-
-Connecting to MongoDB
-=====================
-To connect to a running instance of :program:`mongod`, use the
-:func:`~mongoengine.connect` function. The first argument is the name of the
-database to connect to. If the database does not exist, it will be created. If
-the database requires authentication, :attr:`username` and :attr:`password`
-arguments may be provided::
-
- from mongoengine import connect
- connect('project1', username='webapp', password='pwd123')
-
-By default, MongoEngine assumes that the :program:`mongod` instance is running
-on **localhost** on port **27017**. If MongoDB is running elsewhere, you may
-provide :attr:`host` and :attr:`port` arguments to
-:func:`~mongoengine.connect`::
-
- connect('project1', host='192.168.1.35', port=12345)
-
-Defining documents
-==================
-In MongoDB, a **document** is roughly equivalent to a **row** in an RDBMS. When
-working with relational databases, rows are stored in **tables**, which have a
-strict **schema** that the rows follow. MongoDB stores documents in
-**collections** rather than tables - the principle difference is that no schema
-is enforced at a database level.
-
-Defining a document's schema
-----------------------------
-MongoEngine allows you to define schemata for documents as this helps to reduce
-coding errors, and allows for utility methods to be defined on fields which may
-be present.
-
-To define a schema for a document, create a class that inherits from
-:class:`~mongoengine.Document`. Fields are specified by adding **field
-objects** as class attributes to the document class::
-
- from mongoengine import *
- import datetime
-
- class Page(Document):
- title = StringField(max_length=200, required=True)
- date_modified = DateTimeField(default=datetime.now)
-
-Fields
-------
-By default, fields are not required. To make a field mandatory, set the
-:attr:`required` keyword argument of a field to ``True``. Fields also may have
-validation constraints available (such as :attr:`max_length` in the example
-above). Fields may also take default values, which will be used if a value is
-not provided. Default values may optionally be a callable, which will be called
-to retrieve the value (such as in the above example). The field types available
-are as follows:
-
-* :class:`~mongoengine.StringField`
-* :class:`~mongoengine.IntField`
-* :class:`~mongoengine.FloatField`
-* :class:`~mongoengine.DateTimeField`
-* :class:`~mongoengine.ListField`
-* :class:`~mongoengine.ObjectIdField`
-* :class:`~mongoengine.EmbeddedDocumentField`
-* :class:`~mongoengine.ReferenceField`
-
-List fields
-^^^^^^^^^^^
-MongoDB allows the storage of lists of items. To add a list of items to a
-:class:`~mongoengine.Document`, use the :class:`~mongoengine.ListField` field
-type. :class:`~mongoengine.ListField` takes another field object as its first
-argument, which specifies which type elements may be stored within the list::
-
- class Page(Document):
- tags = ListField(StringField(max_length=50))
-
-Embedded documents
-^^^^^^^^^^^^^^^^^^
-MongoDB has the ability to embed documents within other documents. Schemata may
-be defined for these embedded documents, just as they may be for regular
-documents. To create an embedded document, just define a document as usual, but
-inherit from :class:`~mongoengine.EmbeddedDocument` rather than
-:class:`~mongoengine.Document`::
-
- class Comment(EmbeddedDocument):
- content = StringField()
-
-To embed the document within another document, use the
-:class:`~mongoengine.EmbeddedDocumentField` field type, providing the embedded
-document class as the first argument::
-
- class Page(Document):
- comments = ListField(EmbeddedDocumentField(Comment))
-
- comment1 = Comment('Good work!')
- comment2 = Comment('Nice article!')
- page = Page(comments=[comment1, comment2])
-
-Reference fields
-^^^^^^^^^^^^^^^^
-References may be stored to other documents in the database using the
-:class:`~mongoengine.ReferenceField`. Pass in another document class as the
-first argument to the constructor, then simply assign document objects to the
-field::
-
- class User(Document):
- name = StringField()
-
- class Page(Document):
- content = StringField()
- author = ReferenceField(User)
-
- john = User(name="John Smith")
- john.save()
-
- post = Page(content="Test Page")
- post.author = john
- post.save()
-
-The :class:`User` object is automatically turned into a reference behind the
-scenes, and dereferenced when the :class:`Page` object is retrieved.
-
-Uniqueness constraints
-^^^^^^^^^^^^^^^^^^^^^^
-MongoEngine allows you to specify that a field should be unique across a
-collection by providing ``unique=True`` to a :class:`~mongoengine.Field`\ 's
-constructor. If you try to save a document that has the same value for a unique
-field as a document that is already in the database, a
-:class:`~mongoengine.OperationError` will be raised. You may also specify
-multi-field uniqueness constraints by using :attr:`unique_with`, which may be
-either a single field name, or a list or tuple of field names::
-
- class User(Document):
- username = StringField(unique=True)
- first_name = StringField()
- last_name = StringField(unique_with='last_name')
-
-Document collections
---------------------
-Document classes that inherit **directly** from :class:`~mongoengine.Document`
-will have their own **collection** in the database. The name of the collection
-is by default the name of the class, coverted to lowercase (so in the example
-above, the collection would be called `page`). If you need to change the name
-of the collection (e.g. to use MongoEngine with an existing database), then
-create a class dictionary attribute called :attr:`meta` on your document, and
-set :attr:`collection` to the name of the collection that you want your
-document class to use::
-
- class Page(Document):
- title = StringField(max_length=200, required=True)
- meta = {'collection': 'cmsPage'}
-
-Capped collections
-^^^^^^^^^^^^^^^^^^
-A :class:`~mongoengine.Document` may use a **Capped Collection** by specifying
-:attr:`max_documents` and :attr:`max_size` in the :attr:`meta` dictionary.
-:attr:`max_documents` is the maximum number of documents that is allowed to be
-stored in the collection, and :attr:`max_size` is the maximum size of the
-collection in bytes. If :attr:`max_size` is not specified and
-:attr:`max_documents` is, :attr:`max_size` defaults to 10000000 bytes (10MB).
-The following example shows a :class:`Log` document that will be limited to
-1000 entries and 2MB of disk space::
-
- class Log(Document):
- ip_address = StringField()
- meta = {'max_documents': 1000, 'max_size': 2000000}
-
-Indexes
--------
-You can specify indexes on collections to make querying faster. This is done
-by creating a list of index specifications called :attr:`indexes` in the
-:attr:`~mongoengine.Document.meta` dictionary, where an index specification may
-either be a single field name, or a tuple containing multiple field names. A
-direction may be specified on fields by prefixing the field name with a **+**
-or a **-** sign. Note that direction only matters on multi-field indexes. ::
-
- class Page(Document):
- title = StringField()
- rating = StringField()
- meta = {
- 'indexes': ['title', ('title', '-rating')]
- }
-
-Ordering
---------
-A default ordering can be specified for your
-:class:`~mongoengine.queryset.QuerySet` using the :attr:`ordering` attribute of
-:attr:`~mongoengine.Document.meta`. Ordering will be applied when the
-:class:`~mongoengine.queryset.QuerySet` is created, and can be overridden by
-subsequent calls to :meth:`~mongoengine.queryset.QuerySet.order_by`. ::
-
- from datetime import datetime
-
- class BlogPost(Document):
- title = StringField()
- published_date = DateTimeField()
-
- meta = {
- 'ordering': ['-published_date']
- }
-
- blog_post_1 = BlogPost(title="Blog Post #1")
- blog_post_1.published_date = datetime(2010, 1, 5, 0, 0 ,0))
-
- blog_post_2 = BlogPost(title="Blog Post #2")
- blog_post_2.published_date = datetime(2010, 1, 6, 0, 0 ,0))
-
- blog_post_3 = BlogPost(title="Blog Post #3")
- blog_post_3.published_date = datetime(2010, 1, 7, 0, 0 ,0))
-
- blog_post_1.save()
- blog_post_2.save()
- blog_post_3.save()
-
- # get the "first" BlogPost using default ordering
- # from BlogPost.meta.ordering
- latest_post = BlogPost.objects.first()
- self.assertEqual(latest_post.title, "Blog Post #3")
-
- # override default ordering, order BlogPosts by "published_date"
- first_post = BlogPost.objects.order_by("+published_date").first()
- self.assertEqual(first_post.title, "Blog Post #1")
-
-Document inheritance
---------------------
-To create a specialised type of a :class:`~mongoengine.Document` you have
-defined, you may subclass it and add any extra fields or methods you may need.
-As this is new class is not a direct subclass of
-:class:`~mongoengine.Document`, it will not be stored in its own collection; it
-will use the same collection as its superclass uses. This allows for more
-convenient and efficient retrieval of related documents::
-
- # Stored in a collection named 'page'
- class Page(Document):
- title = StringField(max_length=200, required=True)
-
- # Also stored in the collection named 'page'
- class DatedPage(Page):
- date = DateTimeField()
-
-Working with existing data
-^^^^^^^^^^^^^^^^^^^^^^^^^^
-To enable correct retrieval of documents involved in this kind of heirarchy,
-two extra attributes are stored on each document in the database: :attr:`_cls`
-and :attr:`_types`. These are hidden from the user through the MongoEngine
-interface, but may not be present if you are trying to use MongoEngine with
-an existing database. For this reason, you may disable this inheritance
-mechansim, removing the dependency of :attr:`_cls` and :attr:`_types`, enabling
-you to work with existing databases. To disable inheritance on a document
-class, set :attr:`allow_inheritance` to ``False`` in the :attr:`meta`
-dictionary::
-
- # Will work with data in an existing collection named 'cmsPage'
- class Page(Document):
- title = StringField(max_length=200, required=True)
- meta = {
- 'collection': 'cmsPage',
- 'allow_inheritance': False,
- }
-
-Documents instances
-===================
-To create a new document object, create an instance of the relevant document
-class, providing values for its fields as its constructor keyword arguments.
-You may provide values for any of the fields on the document::
-
- >>> page = Page(title="Test Page")
- >>> page.title
- 'Test Page'
-
-You may also assign values to the document's fields using standard object
-attribute syntax::
-
- >>> page.title = "Example Page"
- >>> page.title
- 'Example Page'
-
-Saving and deleting documents
------------------------------
-To save the document to the database, call the
-:meth:`~mongoengine.Document.save` method. If the document does not exist in
-the database, it will be created. If it does already exist, it will be
-updated.
-
-To delete a document, call the :meth:`~mongoengine.Document.delete` method.
-Note that this will only work if the document exists in the database and has a
-valide :attr:`id`.
-
-Document IDs
-------------
-Each document in the database has a unique id. This may be accessed through the
-:attr:`id` attribute on :class:`~mongoengine.Document` objects. Usually, the id
-will be generated automatically by the database server when the object is save,
-meaning that you may only access the :attr:`id` field once a document has been
-saved::
-
- >>> page = Page(title="Test Page")
- >>> page.id
- >>> page.save()
- >>> page.id
- ObjectId('123456789abcdef000000000')
-
-Alternatively, you may define one of your own fields to be the document's
-"primary key" by providing ``primary_key=True`` as a keyword argument to a
-field's constructor. Under the hood, MongoEngine will use this field as the
-:attr:`id`; in fact :attr:`id` is actually aliased to your primary key field so
-you may still use :attr:`id` to access the primary key if you want::
-
- >>> class User(Document):
- ... email = StringField(primary_key=True)
- ... name = StringField()
- ...
- >>> bob = User(email='bob@example.com', name='Bob')
- >>> bob.save()
- >>> bob.id == bob.email == 'bob@example.com'
- True
-
-.. note::
- If you define your own primary key field, the field implicitly becomes
- required, so a :class:`ValidationError` will be thrown if you don't provide
- it.
-
-Querying the database
-=====================
-:class:`~mongoengine.Document` classes have an :attr:`objects` attribute, which
-is used for accessing the objects in the database associated with the class.
-The :attr:`objects` attribute is actually a
-:class:`~mongoengine.queryset.QuerySetManager`, which creates and returns a new
-a new :class:`~mongoengine.queryset.QuerySet` object on access. The
-:class:`~mongoengine.queryset.QuerySet` object may may be iterated over to
-fetch documents from the database::
-
- # Prints out the names of all the users in the database
- for user in User.objects:
- print user.name
-
-Filtering queries
------------------
-The query may be filtered by calling the
-:class:`~mongoengine.queryset.QuerySet` object with field lookup keyword
-arguments. The keys in the keyword arguments correspond to fields on the
-:class:`~mongoengine.Document` you are querying::
-
- # This will return a QuerySet that will only iterate over users whose
- # 'country' field is set to 'uk'
- uk_users = User.objects(country='uk')
-
-Fields on embedded documents may also be referred to using field lookup syntax
-by using a double-underscore in place of the dot in object attribute access
-syntax::
-
- # This will return a QuerySet that will only iterate over pages that have
- # been written by a user whose 'country' field is set to 'uk'
- uk_pages = Page.objects(author__country='uk')
-
-Querying lists
-^^^^^^^^^^^^^^
-On most fields, this syntax will look up documents where the field specified
-matches the given value exactly, but when the field refers to a
-:class:`~mongoengine.ListField`, a single item may be provided, in which case
-lists that contain that item will be matched::
-
- class Page(Document):
- tags = ListField(StringField())
-
- # This will match all pages that have the word 'coding' as an item in the
- # 'tags' list
- Page.objects(tags='coding')
-
-Query operators
----------------
-Operators other than equality may also be used in queries; just attach the
-operator name to a key with a double-underscore::
-
- # Only find users whose age is 18 or less
- young_users = Users.objects(age__lte=18)
-
-Available operators are as follows:
-
-* ``neq`` -- not equal to
-* ``lt`` -- less than
-* ``lte`` -- less than or equal to
-* ``gt`` -- greater than
-* ``gte`` -- greater than or equal to
-* ``in`` -- value is in list (a list of values should be provided)
-* ``nin`` -- value is not in list (a list of values should be provided)
-* ``mod`` -- ``value % x == y``, where ``x`` and ``y`` are two provided values
-* ``all`` -- every item in array is in list of values provided
-* ``size`` -- the size of the array is
-* ``exists`` -- value for field exists
-
-Limiting and skipping results
------------------------------
-Just as with traditional ORMs, you may limit the number of results returned, or
-skip a number or results in you query.
-:meth:`~mongoengine.queryset.QuerySet.limit` and
-:meth:`~mongoengine.queryset.QuerySet.skip` and methods are available on
-:class:`~mongoengine.queryset.QuerySet` objects, but the prefered syntax for
-achieving this is using array-slicing syntax::
-
- # Only the first 5 people
- users = User.objects[:5]
-
- # All except for the first 5 people
- users = User.objects[5:]
-
- # 5 users, starting from the 10th user found
- users = User.objects[10:15]
-
-Aggregation
------------
-MongoDB provides some aggregation methods out of the box, but there are not as
-many as you typically get with an RDBMS. MongoEngine provides a wrapper around
-the built-in methods and provides some of its own, which are implemented as
-Javascript code that is executed on the database server.
-
-Counting results
-^^^^^^^^^^^^^^^^
-Just as with limiting and skipping results, there is a method on
-:class:`~mongoengine.queryset.QuerySet` objects --
-:meth:`~mongoengine.queryset.QuerySet.count`, but there is also a more Pythonic
-way of achieving this::
-
- num_users = len(User.objects)
-
-Further aggregation
-^^^^^^^^^^^^^^^^^^^
-You may sum over the values of a specific field on documents using
-:meth:`~mongoengine.queryset.QuerySet.sum`::
-
- yearly_expense = Employee.objects.sum('salary')
-
-.. note::
- If the field isn't present on a document, that document will be ignored from
- the sum.
-
-To get the average (mean) of a field on a collection of documents, use
-:meth:`~mongoengine.queryset.QuerySet.average`::
-
- mean_age = User.objects.average('age')
-
-As MongoDB provides native lists, MongoEngine provides a helper method to get a
-dictionary of the frequencies of items in lists across an entire collection --
-:meth:`~mongoengine.queryset.QuerySet.item_frequencies`. An example of its use
-would be generating "tag-clouds"::
-
- class Article(Document):
- tag = ListField(StringField())
-
- # After adding some tagged articles...
- tag_freqs = Article.objects.item_frequencies('tag', normalize=True)
-
- from operator import itemgetter
- top_tags = sorted(tag_freqs.items(), key=itemgetter(1), reverse=True)[:10]
-
-Advanced queries
-----------------
-Sometimes calling a :class:`~mongoengine.queryset.QuerySet` object with keyword
-arguments can't fully express the query you want to use -- for example if you
-need to combine a number of constraints using *and* and *or*. This is made
-possible in MongoEngine through the :class:`~mongoengine.queryset.Q` class.
-A :class:`~mongoengine.queryset.Q` object represents part of a query, and
-can be initialised using the same keyword-argument syntax you use to query
-documents. To build a complex query, you may combine
-:class:`~mongoengine.queryset.Q` objects using the ``&`` (and) and ``|`` (or)
-operators. To use :class:`~mongoengine.queryset.Q` objects, pass them in
-as positional arguments to :attr:`Document.objects` when you filter it by
-calling it with keyword arguments::
-
- # Get published posts
- Post.objects(Q(published=True) | Q(publish_date__lte=datetime.now()))
-
- # Get top posts
- Post.objects((Q(featured=True) & Q(hits__gte=1000)) | Q(hits__gte=5000))
-
-.. warning::
- Only use these advanced queries if absolutely necessary as they will execute
- significantly slower than regular queries. This is because they are not
- natively supported by MongoDB -- they are compiled to Javascript and sent
- to the server for execution.
-
-Atomic updates
---------------
-Documents may be updated atomically by using the
-:meth:`~mongoengine.queryset.QuerySet.update_one` and
-:meth:`~mongoengine.queryset.QuerySet.update` methods on a
-:meth:`~mongoengine.queryset.QuerySet`. There are several different "modifiers"
-that you may use with these methods:
-
-* ``set`` -- set a particular value
-* ``unset`` -- delete a particular value (since MongoDB v1.3+)
-* ``inc`` -- increment a value by a given amount
-* ``dec`` -- decrement a value by a given amount
-* ``push`` -- append a value to a list
-* ``push_all`` -- append several values to a list
-* ``pull`` -- remove a value from a list
-* ``pull_all`` -- remove several values from a list
-
-The syntax for atomic updates is similar to the querying syntax, but the
-modifier comes before the field, not after it::
-
- >>> post = BlogPost(title='Test', page_views=0, tags=['database'])
- >>> post.save()
- >>> BlogPost.objects(id=post.id).update_one(inc__page_views=1)
- >>> post.reload() # the document has been changed, so we need to reload it
- >>> post.page_views
- 1
- >>> BlogPost.objects(id=post.id).update_one(set__title='Example Post')
- >>> post.reload()
- >>> post.title
- 'Example Post'
- >>> BlogPost.objects(id=post.id).update_one(push__tags='nosql')
- >>> post.reload()
- >>> post.tags
- ['database', 'nosql']
-
diff --git a/mongoengine/__init__.py b/mongoengine/__init__.py
index ebb76b8e..97896878 100644
--- a/mongoengine/__init__.py
+++ b/mongoengine/__init__.py
@@ -12,7 +12,7 @@ __all__ = (document.__all__ + fields.__all__ + connection.__all__ +
__author__ = 'Harry Marr'
-VERSION = (0, 2, 1)
+VERSION = (0, 2, 2)
def get_version():
version = '%s.%s' % (VERSION[0], VERSION[1])
diff --git a/mongoengine/base.py b/mongoengine/base.py
index 3c21f7e6..957a22e1 100644
--- a/mongoengine/base.py
+++ b/mongoengine/base.py
@@ -11,6 +11,9 @@ class BaseField(object):
"""A base class for fields in a MongoDB document. Instances of this class
may be added to subclasses of `Document` to define a document's schema.
"""
+
+ # Fields may have _types inserted into indexes by default
+ _index_with_types = True
def __init__(self, name=None, required=False, default=None, unique=False,
unique_with=None, primary_key=False):
@@ -173,8 +176,6 @@ class TopLevelDocumentMetaclass(DocumentMetaclass):
# Apply document-defined meta options
meta.update(attrs.get('meta', {}))
- meta['indexes'] += base_indexes
-
# Only simple classes - direct subclasses of Document - may set
# allow_inheritance to False
if not simple_class and not meta['allow_inheritance']:
@@ -187,6 +188,10 @@ class TopLevelDocumentMetaclass(DocumentMetaclass):
new_class = super_new(cls, name, bases, attrs)
new_class.objects = QuerySetManager()
+ user_indexes = [QuerySet._build_index_spec(new_class, spec)
+ for spec in meta['indexes']] + base_indexes
+ new_class._meta['indexes'] = user_indexes
+
unique_indexes = []
for field_name, field in new_class._fields.items():
# Generate a list of indexes needed by uniqueness constraints
diff --git a/mongoengine/fields.py b/mongoengine/fields.py
index 75bfd317..54790ced 100644
--- a/mongoengine/fields.py
+++ b/mongoengine/fields.py
@@ -204,6 +204,9 @@ class ListField(BaseField):
of the field to be used as a list in the database.
"""
+ # ListFields cannot be indexed with _types - MongoDB doesn't support this
+ _index_with_types = False
+
def __init__(self, field, **kwargs):
if not isinstance(field, BaseField):
raise ValidationError('Argument to ListField constructor must be '
diff --git a/mongoengine/queryset.py b/mongoengine/queryset.py
index 158b7070..29e90d78 100644
--- a/mongoengine/queryset.py
+++ b/mongoengine/queryset.py
@@ -127,14 +127,19 @@ class QuerySet(object):
construct a multi-field index); keys may be prefixed with a **+**
or a **-** to determine the index ordering
"""
+ index_list = QuerySet._build_index_spec(self._document, key_or_list)
+ self._collection.ensure_index(index_list)
+ return self
+
+ @classmethod
+ def _build_index_spec(cls, doc_cls, key_or_list):
+ """Build a PyMongo index spec from a MongoEngine index spec.
+ """
if isinstance(key_or_list, basestring):
key_or_list = [key_or_list]
index_list = []
- # If _types is being used, prepend it to every specified index
- if self._document._meta.get('allow_inheritance'):
- index_list.append(('_types', 1))
-
+ use_types = doc_cls._meta.get('allow_inheritance', True)
for key in key_or_list:
# Get direction from + or -
direction = pymongo.ASCENDING
@@ -142,11 +147,24 @@ class QuerySet(object):
direction = pymongo.DESCENDING
if key.startswith(("+", "-")):
key = key[1:]
- # Use real field name
- key = QuerySet._translate_field_name(self._document, key)
+
+ # Use real field name, do it manually because we need field
+ # objects for the next part (list field checking)
+ parts = key.split('.')
+ fields = QuerySet._lookup_field(doc_cls, parts)
+ parts = [field.name for field in fields]
+ key = '.'.join(parts)
index_list.append((key, direction))
- self._collection.ensure_index(index_list)
- return self
+
+ # Check if a list field is being used, don't use _types if it is
+ if use_types and not all(f._index_with_types for f in fields):
+ use_types = False
+
+ # If _types is being used, prepend it to every specified index
+ if doc_cls._meta.get('allow_inheritance') and use_types:
+ index_list.insert(0, ('_types', 1))
+
+ return index_list
def __call__(self, *q_objs, **query):
"""Filter the selected documents by calling the
@@ -178,7 +196,8 @@ class QuerySet(object):
# Ensure document-defined indexes are created
if self._document._meta['indexes']:
for key_or_list in self._document._meta['indexes']:
- self.ensure_index(key_or_list)
+ #self.ensure_index(key_or_list)
+ self._collection.ensure_index(key_or_list)
# Ensure indexes created by uniqueness constraints
for index in self._document._meta['unique_indexes']:
diff --git a/tests/document.py b/tests/document.py
index e264f5de..e777d106 100644
--- a/tests/document.py
+++ b/tests/document.py
@@ -227,9 +227,11 @@ class DocumentTest(unittest.TestCase):
class BlogPost(Document):
date = DateTimeField(name='addDate', default=datetime.datetime.now)
category = StringField()
+ tags = ListField(StringField())
meta = {
'indexes': [
'-date',
+ 'tags',
('category', '-date')
],
}
@@ -237,7 +239,8 @@ class DocumentTest(unittest.TestCase):
BlogPost.drop_collection()
info = BlogPost.objects._collection.index_information()
- self.assertEqual(len(info), 4) # _id, types, '-date', ('cat', 'date')
+ # _id, types, '-date', 'tags', ('cat', 'date')
+ self.assertEqual(len(info), 5)
# Indexes are lazy so use list() to perform query
list(BlogPost.objects)
@@ -245,6 +248,8 @@ class DocumentTest(unittest.TestCase):
self.assertTrue([('_types', 1), ('category', 1), ('addDate', -1)]
in info.values())
self.assertTrue([('_types', 1), ('addDate', -1)] in info.values())
+ # tags is a list field so it shouldn't have _types in the index
+ self.assertTrue([('tags', 1)] in info.values())
class ExtendedBlogPost(BlogPost):
title = StringField()