Fixed #29547 -- Added support for partial indexes.

Thanks to Ian Foote, Mariusz Felisiak, Simon Charettes, and Markus Holtermann for comments and feedback.
django · timgraham · Oct 30, 2018 · Sep 13, 2018 · Oct 29, 2018 · a906c9898284a9aecb5f48bdc534e9c1273864a6
commit a906c9898284a9aecb5f48bdc534e9c1273864a6
diff --git a/django/contrib/gis/db/backends/postgis/schema.py b/django/contrib/gis/db/backends/postgis/schema.py
@@ -38,6 +38,7 @@ def _create_index_sql(self, model, fields, **kwargs):
             "using": "USING %s" % self.geom_index_type,
             "columns": field_column,
             "extra": '',
+            "condition": '',
         }
 
     def _alter_column_type_sql(self, table, old_field, new_field, new_type):

diff --git a/django/db/backends/base/features.py b/django/db/backends/base/features.py
@@ -276,6 +276,9 @@ class BaseDatabaseFeatures:
     # in UPDATE statements to ensure the expression has the correct type?
     requires_casted_case_in_updates = False
 
+    # Does the backend support partial indexes (CREATE INDEX ... WHERE ...)?
+    supports_partial_indexes = True
+
     def __init__(self, connection):
         self.connection = connection
 

diff --git a/django/db/backends/base/schema.py b/django/db/backends/base/schema.py
@@ -75,7 +75,7 @@ class BaseDatabaseSchemaEditor:
     sql_create_inline_fk = None
     sql_delete_fk = "ALTER TABLE %(table)s DROP CONSTRAINT %(name)s"
 
-    sql_create_index = "CREATE INDEX %(name)s ON %(table)s (%(columns)s)%(extra)s"
+    sql_create_index = "CREATE INDEX %(name)s ON %(table)s (%(columns)s)%(extra)s%(condition)s"
     sql_delete_index = "DROP INDEX %(name)s"
 
     sql_create_pk = "ALTER TABLE %(table)s ADD CONSTRAINT %(name)s PRIMARY KEY (%(columns)s)"
@@ -326,7 +326,7 @@ def delete_model(self, model):
 
     def add_index(self, model, index):
         """Add an index on a model."""
-        self.execute(index.create_sql(model, self))
+        self.execute(index.create_sql(model, self), params=None)
 
     def remove_index(self, model, index):
         """Remove an index from a model."""
@@ -905,7 +905,8 @@ def _get_index_tablespace_sql(self, model, fields, db_tablespace=None):
         return ''
 
     def _create_index_sql(self, model, fields, *, name=None, suffix='', using='',
-                          db_tablespace=None, col_suffixes=(), sql=None, opclasses=()):
+                          db_tablespace=None, col_suffixes=(), sql=None, opclasses=(),
+                          condition=''):
         """
         Return the SQL statement to create the index for one or several fields.
         `sql` can be specified if the syntax differs from the standard (GIS
@@ -929,6 +930,7 @@ def create_index_name(*args, **kwargs):
             using=using,
             columns=self._index_columns(table, columns, col_suffixes, opclasses),
             extra=tablespace_sql,
+            condition=condition,
         )
 
     def _index_columns(self, table, columns, col_suffixes, opclasses):

diff --git a/django/db/backends/mysql/features.py b/django/db/backends/mysql/features.py
@@ -51,6 +51,8 @@ class DatabaseFeatures(BaseDatabaseFeatures):
     db_functions_convert_bytes_to_str = True
     # Alias MySQL's TRADITIONAL to TEXT for consistency with other backends.
     supported_explain_formats = {'JSON', 'TEXT', 'TRADITIONAL'}
+    # Neither MySQL nor MariaDB support partial indexes.
+    supports_partial_indexes = False
 
     @cached_property
     def _mysql_storage_engine(self):

diff --git a/django/db/backends/mysql/schema.py b/django/db/backends/mysql/schema.py
@@ -24,6 +24,8 @@ class DatabaseSchemaEditor(BaseDatabaseSchemaEditor):
     sql_create_pk = "ALTER TABLE %(table)s ADD CONSTRAINT %(name)s PRIMARY KEY (%(columns)s)"
     sql_delete_pk = "ALTER TABLE %(table)s DROP PRIMARY KEY"
 
+    sql_create_index = 'CREATE INDEX %(name)s ON %(table)s (%(columns)s)%(extra)s'
+
     def quote_value(self, value):
         self.connection.ensure_connection()
         quoted = self.connection.connection.escape(value, self.connection.connection.encoders)

diff --git a/django/db/backends/oracle/features.py b/django/db/backends/oracle/features.py
@@ -55,6 +55,7 @@ class DatabaseFeatures(BaseDatabaseFeatures):
     supports_over_clause = True
     supports_ignore_conflicts = False
     max_query_params = 2**16 - 1
+    supports_partial_indexes = False
 
     @cached_property
     def has_fetch_offset_support(self):

diff --git a/django/db/backends/oracle/schema.py b/django/db/backends/oracle/schema.py
@@ -16,6 +16,7 @@ class DatabaseSchemaEditor(BaseDatabaseSchemaEditor):
     sql_alter_column_no_default = "MODIFY %(column)s DEFAULT NULL"
     sql_delete_column = "ALTER TABLE %(table)s DROP COLUMN %(column)s"
     sql_delete_table = "DROP TABLE %(table)s CASCADE CONSTRAINTS"
+    sql_create_index = "CREATE INDEX %(name)s ON %(table)s (%(columns)s)%(extra)s"
 
     def quote_value(self, value):
         if isinstance(value, (datetime.date, datetime.time, datetime.datetime)):

diff --git a/django/db/backends/postgresql/schema.py b/django/db/backends/postgresql/schema.py
@@ -12,7 +12,7 @@ class DatabaseSchemaEditor(BaseDatabaseSchemaEditor):
     sql_delete_sequence = "DROP SEQUENCE IF EXISTS %(sequence)s CASCADE"
     sql_set_sequence_max = "SELECT setval('%(sequence)s', MAX(%(column)s)) FROM %(table)s"
 
-    sql_create_index = "CREATE INDEX %(name)s ON %(table)s%(using)s (%(columns)s)%(extra)s"
+    sql_create_index = "CREATE INDEX %(name)s ON %(table)s%(using)s (%(columns)s)%(extra)s%(condition)s"
     sql_delete_index = "DROP INDEX IF EXISTS %(name)s"
 
     # Setting the constraint to IMMEDIATE runs any deferred checks to allow

diff --git a/django/db/backends/sqlite3/features.py b/django/db/backends/sqlite3/features.py
@@ -34,6 +34,7 @@ class DatabaseFeatures(BaseDatabaseFeatures):
     supports_cast_with_precision = False
     time_cast_precision = 3
     can_release_savepoints = True
+    supports_partial_indexes = Database.version_info >= (3, 8, 0)
     # Is "ALTER TABLE ... RENAME COLUMN" supported?
     can_alter_table_rename_column = Database.sqlite_version_info >= (3, 25, 0)
 

diff --git a/django/db/models/indexes.py b/django/db/models/indexes.py
@@ -1,4 +1,6 @@
 from django.db.backends.utils import names_digest, split_identifier
+from django.db.models.query_utils import Q
+from django.db.models.sql import Query
 
 __all__ = ['Index']
 
@@ -9,9 +11,13 @@ class Index:
     # cross-database compatibility with Oracle)
     max_name_length = 30
 
-    def __init__(self, *, fields=(), name=None, db_tablespace=None, opclasses=()):
+    def __init__(self, *, fields=(), name=None, db_tablespace=None, opclasses=(), condition=None):
         if opclasses and not name:
             raise ValueError('An index must be named to use opclasses.')
+        if not isinstance(condition, (type(None), Q)):
+            raise ValueError('Index.condition must be a Q instance.')
+        if condition and not name:
+            raise ValueError('An index must be named to use condition.')
         if not isinstance(fields, (list, tuple)):
             raise ValueError('Index.fields must be a list or tuple.')
         if not isinstance(opclasses, (list, tuple)):
@@ -35,6 +41,7 @@ def __init__(self, *, fields=(), name=None, db_tablespace=None, opclasses=()):
                 raise ValueError(errors)
         self.db_tablespace = db_tablespace
         self.opclasses = opclasses
+        self.condition = condition
 
     def check_name(self):
         errors = []
@@ -48,12 +55,25 @@ def check_name(self):
             self.name = 'D%s' % self.name[1:]
         return errors
 
+    def _get_condition_sql(self, model, schema_editor):
+        if self.condition is None:
+            return ''
+        query = Query(model=model)
+        query.add_q(self.condition)
+        compiler = query.get_compiler(connection=schema_editor.connection)
+        # Only the WhereNode is of interest for the partial index.
+        sql, params = query.where.as_sql(compiler=compiler, connection=schema_editor.connection)
+        # BaseDatabaseSchemaEditor does the same map on the params, but since
+        # it's handled outside of that class, the work is done here.
+        return ' WHERE ' + (sql % tuple(map(schema_editor.quote_value, params)))
+
     def create_sql(self, model, schema_editor, using=''):
         fields = [model._meta.get_field(field_name) for field_name, _ in self.fields_orders]
         col_suffixes = [order[1] for order in self.fields_orders]
+        condition = self._get_condition_sql(model, schema_editor)
         return schema_editor._create_index_sql(
             model, fields, name=self.name, using=using, db_tablespace=self.db_tablespace,
-            col_suffixes=col_suffixes, opclasses=self.opclasses,
+            col_suffixes=col_suffixes, opclasses=self.opclasses, condition=condition,
         )
 
     def remove_sql(self, model, schema_editor):
@@ -71,6 +91,8 @@ def deconstruct(self):
             kwargs['db_tablespace'] = self.db_tablespace
         if self.opclasses:
             kwargs['opclasses'] = self.opclasses
+        if self.condition:
+            kwargs['condition'] = self.condition
         return (path, (), kwargs)
 
     def clone(self):
@@ -107,7 +129,10 @@ def set_name_with_model(self, model):
         self.check_name()
 
     def __repr__(self):
-        return "<%s: fields='%s'>" % (self.__class__.__name__, ', '.join(self.fields))
+        return "<%s: fields='%s'%s>" % (
+            self.__class__.__name__, ', '.join(self.fields),
+            '' if self.condition is None else ', condition=%s' % self.condition,
+        )
 
     def __eq__(self, other):
         return (self.__class__ == other.__class__) and (self.deconstruct() == other.deconstruct())
diff --git a/docs/ref/models/indexes.txt b/docs/ref/models/indexes.txt
@@ -21,7 +21,7 @@ options`_.
 ``Index`` options
 =================
 
-.. class:: Index(fields=(), name=None, db_tablespace=None, opclasses=())
+.. class:: Index(fields=(), name=None, db_tablespace=None, opclasses=(), condition=None)
 
     Creates an index (B-Tree) in the database.
 
@@ -92,3 +92,44 @@ opclasses=['jsonb_path_ops'])`` creates a gin index on ``jsonfield`` using
 ``opclasses`` are ignored for databases besides PostgreSQL.
 
 :attr:`Index.name` is required when using ``opclasses``.
+
+``condition``
+-------------
+
+.. attribute:: Index.condition
+
+.. versionadded:: 2.2
+
+If the table is very large and your queries mostly target a subset of rows,
+it may be useful to restrict an index to that subset. Specify a condition as a
+:class:`~django.db.models.Q`. For example, ``condition=Q(pages__gt=400)``
+indexes records with more than 400 pages.
+
+:attr:`Index.name` is required when using ``condition``.
+
+.. admonition:: Restrictions on PostgreSQL
+
+    PostgreSQL requires functions referenced in the condition to be be marked as
+    IMMUTABLE. Django doesn't validate this but PostgreSQL will error. This
+    means that functions such as :ref:`date-functions` and
+    :class:`~django.db.models.functions.Concat` aren't accepted. If you store
+    dates in :class:`~django.db.models.DateTimeField`, comparison to
+    :class:`~datetime.datetime` objects may require the ``tzinfo`` argument
+    to be provided because otherwise the comparison could result in a mutable
+    function due to the casting Django does for :ref:`lookups <field-lookups>`.
+
+.. admonition:: Restrictions on SQLite
+
+    SQLite `imposes restrictions <https://www.sqlite.org/partialindex.html>`_
+    on how a partial index can be constructed.
+
+.. admonition:: Oracle
+
+    Oracle does not support partial indexes. Instead, partial indexes can be
+    emulated using functional indexes. Use a :doc:`migration
+    </topics/migrations>` to add the index using :class:`.RunSQL`.
+
+.. admonition:: MySQL and MariaDB
+
+    The ``condition`` argument is ignored with MySQL and MariaDB as neither
+    supports conditional indexes.
diff --git a/docs/releases/2.2.txt b/docs/releases/2.2.txt
@@ -201,6 +201,8 @@ Models
 
 * Added support for PostgreSQL operator classes (:attr:`.Index.opclasses`).
 
+* Added support for partial indexes (:attr:`.Index.condition`).
+
 * Added many :ref:`math database functions <math-functions>`.
 
 * Setting the new ``ignore_conflicts`` parameter of
@@ -288,6 +290,9 @@ Database backend API
 
 * ``DatabaseFeatures.uses_savepoints`` now defaults to ``True``.
 
+* Third party database backends must implement support for partial indexes or
+  set ``DatabaseFeatures.supports_partial_indexes`` to ``False``.
+
 :mod:`django.contrib.gis`
 -------------------------
 

diff --git a/tests/indexes/models.py b/tests/indexes/models.py
@@ -27,6 +27,7 @@ class ArticleTranslation(models.Model):
 class Article(models.Model):
     headline = models.CharField(max_length=100)
     pub_date = models.DateTimeField()
+    published = models.BooleanField(default=False)
 
     # Add virtual relation to the ArticleTranslation model.
     translation = CurrentTranslation(ArticleTranslation, models.CASCADE, ['id'], ['article'])