diff --git a/.readthedocs.yml b/.readthedocs.yml new file mode 100644 index 000000000000..bde8b64da0f0 --- /dev/null +++ b/.readthedocs.yml @@ -0,0 +1,18 @@ +# Configuration for the Read The Docs (RTD) builds of the documentation. +# Ref: https://docs.readthedocs.io/en/stable/config-file/v2.html +# The python.install.requirements pins the version of Sphinx used. +version: 2 + +build: + os: ubuntu-20.04 + tools: + python: "3.8" + +sphinx: + configuration: docs/conf.py + +python: + install: + - requirements: docs/requirements.txt + +formats: all diff --git a/django/__init__.py b/django/__init__.py index 6db3311eb5cd..28f109f9d8ed 100644 --- a/django/__init__.py +++ b/django/__init__.py @@ -1,6 +1,6 @@ from django.utils.version import get_version -VERSION = (3, 2, 9, 'final', 0) +VERSION = (3, 2, 18, 'final', 0) __version__ = get_version(VERSION) diff --git a/django/conf/global_settings.py b/django/conf/global_settings.py index cf9fae496e3a..4a27887a8f04 100644 --- a/django/conf/global_settings.py +++ b/django/conf/global_settings.py @@ -303,6 +303,10 @@ def gettext_noop(s): # SuspiciousOperation (TooManyFieldsSent) is raised. DATA_UPLOAD_MAX_NUMBER_FIELDS = 1000 +# Maximum number of files encoded in a multipart upload that will be read +# before a SuspiciousOperation (TooManyFilesSent) is raised. +DATA_UPLOAD_MAX_NUMBER_FILES = 100 + # Directory in which upload streamed files will be temporarily saved. A value of # `None` will make Django use the operating system's default temporary directory # (i.e. "/tmp" on *nix systems). diff --git a/django/contrib/auth/password_validation.py b/django/contrib/auth/password_validation.py index 845f4d86d5b2..7beb4bdc0ff2 100644 --- a/django/contrib/auth/password_validation.py +++ b/django/contrib/auth/password_validation.py @@ -115,6 +115,36 @@ def get_help_text(self): ) % {'min_length': self.min_length} +def exceeds_maximum_length_ratio(password, max_similarity, value): + """ + Test that value is within a reasonable range of password. + + The following ratio calculations are based on testing SequenceMatcher like + this: + + for i in range(0,6): + print(10**i, SequenceMatcher(a='A', b='A'*(10**i)).quick_ratio()) + + which yields: + + 1 1.0 + 10 0.18181818181818182 + 100 0.019801980198019802 + 1000 0.001998001998001998 + 10000 0.00019998000199980003 + 100000 1.999980000199998e-05 + + This means a length_ratio of 10 should never yield a similarity higher than + 0.2, for 100 this is down to 0.02 and for 1000 it is 0.002. This can be + calculated via 2 / length_ratio. As a result we avoid the potentially + expensive sequence matching. + """ + pwd_len = len(password) + length_bound_similarity = max_similarity / 2 * pwd_len + value_len = len(value) + return pwd_len >= 10 * value_len and value_len < length_bound_similarity + + class UserAttributeSimilarityValidator: """ Validate whether the password is sufficiently different from the user's @@ -130,19 +160,25 @@ class UserAttributeSimilarityValidator: def __init__(self, user_attributes=DEFAULT_USER_ATTRIBUTES, max_similarity=0.7): self.user_attributes = user_attributes + if max_similarity < 0.1: + raise ValueError('max_similarity must be at least 0.1') self.max_similarity = max_similarity def validate(self, password, user=None): if not user: return + password = password.lower() for attribute_name in self.user_attributes: value = getattr(user, attribute_name, None) if not value or not isinstance(value, str): continue - value_parts = re.split(r'\W+', value) + [value] + value_lower = value.lower() + value_parts = re.split(r'\W+', value_lower) + [value_lower] for value_part in value_parts: - if SequenceMatcher(a=password.lower(), b=value_part.lower()).quick_ratio() >= self.max_similarity: + if exceeds_maximum_length_ratio(password, self.max_similarity, value_part): + continue + if SequenceMatcher(a=password, b=value_part).quick_ratio() >= self.max_similarity: try: verbose_name = str(user._meta.get_field(attribute_name).verbose_name) except FieldDoesNotExist: diff --git a/django/core/exceptions.py b/django/core/exceptions.py index 673d004d5756..83161a58cd66 100644 --- a/django/core/exceptions.py +++ b/django/core/exceptions.py @@ -58,6 +58,15 @@ class TooManyFieldsSent(SuspiciousOperation): pass +class TooManyFilesSent(SuspiciousOperation): + """ + The number of fields in a GET or POST request exceeded + settings.DATA_UPLOAD_MAX_NUMBER_FILES. + """ + + pass + + class RequestDataTooBig(SuspiciousOperation): """ The size of the request (excluding any file uploads) exceeded diff --git a/django/core/files/storage.py b/django/core/files/storage.py index 3e68853b59f8..22984f9498d9 100644 --- a/django/core/files/storage.py +++ b/django/core/files/storage.py @@ -51,7 +51,10 @@ def save(self, name, content, max_length=None): content = File(content, name) name = self.get_available_name(name, max_length=max_length) - return self._save(name, content) + name = self._save(name, content) + # Ensure that the name returned from the storage system is still valid. + validate_file_name(name, allow_relative_path=True) + return name # These methods are part of the public API, with default implementations. @@ -75,6 +78,7 @@ def get_available_name(self, name, max_length=None): Return a filename that's free on the target storage system and available for new content to be written to. """ + name = str(name).replace('\\', '/') dir_name, file_name = os.path.split(name) if '..' in pathlib.PurePath(dir_name).parts: raise SuspiciousFileOperation("Detected path traversal attempt in '%s'" % dir_name) @@ -108,6 +112,7 @@ def generate_filename(self, filename): Validate the filename by calling get_valid_name() and return a filename to be passed to the save() method. """ + filename = str(filename).replace('\\', '/') # `filename` may include a path as returned by FileField.upload_to. dirname, filename = os.path.split(filename) if '..' in pathlib.PurePath(dirname).parts: @@ -297,6 +302,8 @@ def _save(self, name, content): if self.file_permissions_mode is not None: os.chmod(full_path, self.file_permissions_mode) + # Ensure the saved path is always relative to the storage root. + name = os.path.relpath(full_path, self.location) # Store filenames with forward slashes, even on Windows. return str(name).replace('\\', '/') diff --git a/django/core/handlers/exception.py b/django/core/handlers/exception.py index 3005a5eccb11..2ecc2a0fd697 100644 --- a/django/core/handlers/exception.py +++ b/django/core/handlers/exception.py @@ -9,7 +9,7 @@ from django.core import signals from django.core.exceptions import ( BadRequest, PermissionDenied, RequestDataTooBig, SuspiciousOperation, - TooManyFieldsSent, + TooManyFieldsSent, TooManyFilesSent, ) from django.http import Http404 from django.http.multipartparser import MultiPartParserError @@ -88,7 +88,7 @@ def response_for_exception(request, exc): exc_info=sys.exc_info(), ) elif isinstance(exc, SuspiciousOperation): - if isinstance(exc, (RequestDataTooBig, TooManyFieldsSent)): + if isinstance(exc, (RequestDataTooBig, TooManyFieldsSent, TooManyFilesSent)): # POST data can't be accessed again, otherwise the original # exception would be raised. request._mark_post_parse_error() diff --git a/django/db/backends/base/operations.py b/django/db/backends/base/operations.py index 0fcc607bcfb0..cdcd9885ba27 100644 --- a/django/db/backends/base/operations.py +++ b/django/db/backends/base/operations.py @@ -9,6 +9,7 @@ from django.db.backends import utils from django.utils import timezone from django.utils.encoding import force_str +from django.utils.regex_helper import _lazy_re_compile class BaseDatabaseOperations: @@ -53,6 +54,8 @@ class BaseDatabaseOperations: # Prefix for EXPLAIN queries, or None EXPLAIN isn't supported. explain_prefix = None + extract_trunc_lookup_pattern = _lazy_re_compile(r"[\w\-_()]+") + def __init__(self, connection): self.connection = connection self._cache = None diff --git a/django/db/backends/mysql/features.py b/django/db/backends/mysql/features.py index 419b2ba6f059..1a9f5da39ee1 100644 --- a/django/db/backends/mysql/features.py +++ b/django/db/backends/mysql/features.py @@ -47,11 +47,24 @@ class DatabaseFeatures(BaseDatabaseFeatures): supports_order_by_nulls_modifier = False order_by_nulls_first = True - test_collations = { - 'ci': 'utf8_general_ci', - 'non_default': 'utf8_esperanto_ci', - 'swedish_ci': 'utf8_swedish_ci', - } + + @cached_property + def test_collations(self): + charset = 'utf8' + if ( + self.connection.mysql_is_mariadb + and self.connection.mysql_version >= (10, 6) + ) or ( + not self.connection.mysql_is_mariadb + and self.connection.mysql_version >= (8, 0, 30) + ): + # utf8 is an alias for utf8mb3 in MariaDB 10.6+ and MySQL 8.0.30+. + charset = "utf8mb3" + return { + 'ci': f'{charset}_general_ci', + 'non_default': f'{charset}_esperanto_ci', + 'swedish_ci': f'{charset}_swedish_ci', + } @cached_property def django_test_skips(self): diff --git a/django/db/backends/postgresql/features.py b/django/db/backends/postgresql/features.py index 84259c0c19f2..f1456321da39 100644 --- a/django/db/backends/postgresql/features.py +++ b/django/db/backends/postgresql/features.py @@ -54,7 +54,6 @@ class DatabaseFeatures(BaseDatabaseFeatures): only_supports_unbounded_with_preceding_and_following = True supports_aggregate_filter_clause = True supported_explain_formats = {'JSON', 'TEXT', 'XML', 'YAML'} - validates_explain_options = False # A query will error on invalid options. supports_deferrable_unique_constraints = True has_json_operators = True json_key_contains_list_matching_requires_list = True diff --git a/django/db/backends/postgresql/operations.py b/django/db/backends/postgresql/operations.py index 8d19872bea88..0ef622efa869 100644 --- a/django/db/backends/postgresql/operations.py +++ b/django/db/backends/postgresql/operations.py @@ -7,6 +7,18 @@ class DatabaseOperations(BaseDatabaseOperations): cast_char_field_without_max_length = 'varchar' explain_prefix = 'EXPLAIN' + explain_options = frozenset( + [ + "ANALYZE", + "BUFFERS", + "COSTS", + "SETTINGS", + "SUMMARY", + "TIMING", + "VERBOSE", + "WAL", + ] + ) cast_data_types = { 'AutoField': 'integer', 'BigAutoField': 'bigint', @@ -258,15 +270,20 @@ def subtract_temporals(self, internal_type, lhs, rhs): return super().subtract_temporals(internal_type, lhs, rhs) def explain_query_prefix(self, format=None, **options): - prefix = super().explain_query_prefix(format) extra = {} - if format: - extra['FORMAT'] = format + # Normalize options. if options: - extra.update({ - name.upper(): 'true' if value else 'false' + options = { + name.upper(): "true" if value else "false" for name, value in options.items() - }) + } + for valid_option in self.explain_options: + value = options.pop(valid_option, None) + if value is not None: + extra[valid_option.upper()] = value + prefix = super().explain_query_prefix(format, **options) + if format: + extra['FORMAT'] = format if extra: prefix += ' (%s)' % ', '.join('%s %s' % i for i in extra.items()) return prefix diff --git a/django/db/models/base.py b/django/db/models/base.py index f49913eea783..d1638f43c849 100644 --- a/django/db/models/base.py +++ b/django/db/models/base.py @@ -549,6 +549,16 @@ def __getstate__(self): state = self.__dict__.copy() state['_state'] = copy.copy(state['_state']) state['_state'].fields_cache = state['_state'].fields_cache.copy() + # memoryview cannot be pickled, so cast it to bytes and store + # separately. + _memoryview_attrs = [] + for attr, value in state.items(): + if isinstance(value, memoryview): + _memoryview_attrs.append((attr, bytes(value))) + if _memoryview_attrs: + state['_memoryview_attrs'] = _memoryview_attrs + for attr, value in _memoryview_attrs: + state.pop(attr) return state def __setstate__(self, state): @@ -568,6 +578,9 @@ def __setstate__(self, state): RuntimeWarning, stacklevel=2, ) + if '_memoryview_attrs' in state: + for attr, value in state.pop('_memoryview_attrs'): + state[attr] = memoryview(value) self.__dict__.update(state) def _get_pk_val(self, meta=None): diff --git a/django/db/models/functions/datetime.py b/django/db/models/functions/datetime.py index 90e6f41be057..47651d281f19 100644 --- a/django/db/models/functions/datetime.py +++ b/django/db/models/functions/datetime.py @@ -41,6 +41,8 @@ def __init__(self, expression, lookup_name=None, tzinfo=None, **extra): super().__init__(expression, **extra) def as_sql(self, compiler, connection): + if not connection.ops.extract_trunc_lookup_pattern.fullmatch(self.lookup_name): + raise ValueError("Invalid lookup_name: %s" % self.lookup_name) sql, params = compiler.compile(self.lhs) lhs_output_field = self.lhs.output_field if isinstance(lhs_output_field, DateTimeField): @@ -192,6 +194,8 @@ def __init__(self, expression, output_field=None, tzinfo=None, is_dst=None, **ex super().__init__(expression, output_field=output_field, **extra) def as_sql(self, compiler, connection): + if not connection.ops.extract_trunc_lookup_pattern.fullmatch(self.kind): + raise ValueError("Invalid kind: %s" % self.kind) inner_sql, inner_params = compiler.compile(self.lhs) tzname = None if isinstance(self.lhs.output_field, DateTimeField): diff --git a/django/db/models/sql/query.py b/django/db/models/sql/query.py index f5f85a4d348e..230b6fa8610e 100644 --- a/django/db/models/sql/query.py +++ b/django/db/models/sql/query.py @@ -41,10 +41,19 @@ ) from django.utils.deprecation import RemovedInDjango40Warning from django.utils.functional import cached_property +from django.utils.regex_helper import _lazy_re_compile from django.utils.tree import Node __all__ = ['Query', 'RawQuery'] +# Quotation marks ('"`[]), whitespace characters, semicolons, or inline +# SQL comments are forbidden in column aliases. +FORBIDDEN_ALIAS_PATTERN = _lazy_re_compile(r"['`\"\]\[;\s]|--|/\*|\*/") + +# Inspired from +# https://www.postgresql.org/docs/current/sql-syntax-lexical.html#SQL-SYNTAX-IDENTIFIERS +EXPLAIN_OPTIONS_PATTERN = _lazy_re_compile(r"[\w\-]+") + def get_field_names_from_opts(opts): return set(chain.from_iterable( @@ -553,6 +562,12 @@ def has_results(self, using): def explain(self, using, format=None, **options): q = self.clone() + for option_name in options: + if ( + not EXPLAIN_OPTIONS_PATTERN.fullmatch(option_name) or + "--" in option_name + ): + raise ValueError(f"Invalid option name: {option_name!r}.") q.explain_query = True q.explain_format = format q.explain_options = options @@ -1034,8 +1049,16 @@ def join_parent_model(self, opts, model, alias, seen): alias = seen[int_model] = join_info.joins[-1] return alias or seen[None] + def check_alias(self, alias): + if FORBIDDEN_ALIAS_PATTERN.search(alias): + raise ValueError( + "Column aliases cannot contain whitespace characters, quotation marks, " + "semicolons, or SQL comments." + ) + def add_annotation(self, annotation, alias, is_summary=False, select=True): """Add a single annotation expression to the Query.""" + self.check_alias(alias) annotation = annotation.resolve_expression(self, allow_joins=True, reuse=None, summarize=is_summary) if select: @@ -2088,6 +2111,7 @@ def add_extra(self, select, select_params, where, params, tables, order_by): else: param_iter = iter([]) for name, entry in select.items(): + self.check_alias(name) entry = str(entry) entry_params = [] pos = entry.find("%s") diff --git a/django/http/multipartparser.py b/django/http/multipartparser.py index f464caa1b4c5..d8a304d4babe 100644 --- a/django/http/multipartparser.py +++ b/django/http/multipartparser.py @@ -14,6 +14,7 @@ from django.conf import settings from django.core.exceptions import ( RequestDataTooBig, SuspiciousMultipartForm, TooManyFieldsSent, + TooManyFilesSent, ) from django.core.files.uploadhandler import ( SkipFile, StopFutureHandlers, StopUpload, @@ -38,6 +39,7 @@ class InputStreamExhausted(Exception): RAW = "raw" FILE = "file" FIELD = "field" +FIELD_TYPES = frozenset([FIELD, RAW]) class MultiPartParser: @@ -102,6 +104,22 @@ def __init__(self, META, input_data, upload_handlers, encoding=None): self._upload_handlers = upload_handlers def parse(self): + # Call the actual parse routine and close all open files in case of + # errors. This is needed because if exceptions are thrown the + # MultiPartParser will not be garbage collected immediately and + # resources would be kept alive. This is only needed for errors because + # the Request object closes all uploaded files at the end of the + # request. + try: + return self._parse() + except Exception: + if hasattr(self, "_files"): + for _, files in self._files.lists(): + for fileobj in files: + fileobj.close() + raise + + def _parse(self): """ Parse the POST data and break it into a FILES MultiValueDict and a POST MultiValueDict. @@ -147,6 +165,8 @@ def parse(self): num_bytes_read = 0 # To count the number of keys in the request. num_post_keys = 0 + # To count the number of files in the request. + num_files = 0 # To limit the amount of data read from the request. read_size = None # Whether a file upload is finished. @@ -162,6 +182,20 @@ def parse(self): old_field_name = None uploaded_file = True + if ( + item_type in FIELD_TYPES and + settings.DATA_UPLOAD_MAX_NUMBER_FIELDS is not None + ): + # Avoid storing more than DATA_UPLOAD_MAX_NUMBER_FIELDS. + num_post_keys += 1 + # 2 accounts for empty raw fields before and after the + # last boundary. + if settings.DATA_UPLOAD_MAX_NUMBER_FIELDS + 2 < num_post_keys: + raise TooManyFieldsSent( + "The number of GET/POST parameters exceeded " + "settings.DATA_UPLOAD_MAX_NUMBER_FIELDS." + ) + try: disposition = meta_data['content-disposition'][1] field_name = disposition['name'].strip() @@ -174,15 +208,6 @@ def parse(self): field_name = force_str(field_name, encoding, errors='replace') if item_type == FIELD: - # Avoid storing more than DATA_UPLOAD_MAX_NUMBER_FIELDS. - num_post_keys += 1 - if (settings.DATA_UPLOAD_MAX_NUMBER_FIELDS is not None and - settings.DATA_UPLOAD_MAX_NUMBER_FIELDS < num_post_keys): - raise TooManyFieldsSent( - 'The number of GET/POST parameters exceeded ' - 'settings.DATA_UPLOAD_MAX_NUMBER_FIELDS.' - ) - # Avoid reading more than DATA_UPLOAD_MAX_MEMORY_SIZE. if settings.DATA_UPLOAD_MAX_MEMORY_SIZE is not None: read_size = settings.DATA_UPLOAD_MAX_MEMORY_SIZE - num_bytes_read @@ -208,6 +233,16 @@ def parse(self): self._post.appendlist(field_name, force_str(data, encoding, errors='replace')) elif item_type == FILE: + # Avoid storing more than DATA_UPLOAD_MAX_NUMBER_FILES. + num_files += 1 + if ( + settings.DATA_UPLOAD_MAX_NUMBER_FILES is not None and + num_files > settings.DATA_UPLOAD_MAX_NUMBER_FILES + ): + raise TooManyFilesSent( + "The number of files exceeded " + "settings.DATA_UPLOAD_MAX_NUMBER_FILES." + ) # This is a file, use the handler... file_name = disposition.get('filename') if file_name: @@ -248,6 +283,8 @@ def parse(self): remaining = len(stripped_chunk) % 4 while remaining != 0: over_chunk = field_stream.read(4 - remaining) + if not over_chunk: + break stripped_chunk += b"".join(over_chunk.split()) remaining = len(stripped_chunk) % 4 @@ -274,8 +311,13 @@ def parse(self): # Handle file upload completions on next iteration. old_field_name = field_name else: - # If this is neither a FIELD or a FILE, just exhaust the stream. - exhaust(stream) + # If this is neither a FIELD nor a FILE, exhaust the field + # stream. Note: There could be an error here at some point, + # but there will be at least two RAW types (before and + # after the other boundaries). This branch is usually not + # reached at all, because a missing content-disposition + # header will skip the whole boundary. + exhaust(field_stream) except StopUpload as e: self._close_files() if not e.connection_reset: diff --git a/django/http/request.py b/django/http/request.py index 195341ec4b69..b6cd7a372f14 100644 --- a/django/http/request.py +++ b/django/http/request.py @@ -12,7 +12,9 @@ DisallowedHost, ImproperlyConfigured, RequestDataTooBig, TooManyFieldsSent, ) from django.core.files import uploadhandler -from django.http.multipartparser import MultiPartParser, MultiPartParserError +from django.http.multipartparser import ( + MultiPartParser, MultiPartParserError, TooManyFilesSent, +) from django.utils.datastructures import ( CaseInsensitiveMapping, ImmutableList, MultiValueDict, ) @@ -360,7 +362,7 @@ def _load_post_and_files(self): data = self try: self._post, self._files = self.parse_file_upload(self.META, data) - except MultiPartParserError: + except (MultiPartParserError, TooManyFilesSent): # An error occurred while parsing POST data. Since when # formatting the error the request handler might access # self.POST, set self._post and self._file to prevent diff --git a/django/http/response.py b/django/http/response.py index 1c22edaff3d9..73f87d7bda3c 100644 --- a/django/http/response.py +++ b/django/http/response.py @@ -485,7 +485,9 @@ def set_headers(self, filelike): disposition = 'attachment' if self.as_attachment else 'inline' try: filename.encode('ascii') - file_expr = 'filename="{}"'.format(filename) + file_expr = 'filename="{}"'.format( + filename.replace('\\', '\\\\').replace('"', r'\"') + ) except UnicodeEncodeError: file_expr = "filename*=utf-8''{}".format(quote(filename)) self.headers['Content-Disposition'] = '{}; {}'.format(disposition, file_expr) diff --git a/django/template/autoreload.py b/django/template/autoreload.py index 18570b563314..be53693e9d2e 100644 --- a/django/template/autoreload.py +++ b/django/template/autoreload.py @@ -18,7 +18,7 @@ def get_template_directories(): if not isinstance(backend, DjangoTemplates): continue - items.update(Path.cwd() / to_path(dir) for dir in backend.engine.dirs) + items.update(Path.cwd() / to_path(dir) for dir in backend.engine.dirs if dir) for loader in backend.engine.template_loaders: if not hasattr(loader, 'get_dirs'): @@ -26,7 +26,7 @@ def get_template_directories(): items.update( Path.cwd() / to_path(directory) for directory in loader.get_dirs() - if not is_django_path(directory) + if directory and not is_django_path(directory) ) return items diff --git a/django/template/defaultfilters.py b/django/template/defaultfilters.py index 1c844580c651..92050122abdf 100644 --- a/django/template/defaultfilters.py +++ b/django/template/defaultfilters.py @@ -22,7 +22,7 @@ from django.utils.timesince import timesince, timeuntil from django.utils.translation import gettext, ngettext -from .base import Variable, VariableDoesNotExist +from .base import VARIABLE_ATTRIBUTE_SEPARATOR from .library import Library register = Library() @@ -481,7 +481,7 @@ def striptags(value): def _property_resolver(arg): """ When arg is convertible to float, behave like operator.itemgetter(arg) - Otherwise, behave like Variable(arg).resolve + Otherwise, chain __getitem__() and getattr(). >>> _property_resolver(1)('abc') 'b' @@ -499,7 +499,19 @@ def _property_resolver(arg): try: float(arg) except ValueError: - return Variable(arg).resolve + if VARIABLE_ATTRIBUTE_SEPARATOR + '_' in arg or arg[0] == '_': + raise AttributeError('Access to private variables is forbidden.') + parts = arg.split(VARIABLE_ATTRIBUTE_SEPARATOR) + + def resolve(value): + for part in parts: + try: + value = value[part] + except (AttributeError, IndexError, KeyError, TypeError, ValueError): + value = getattr(value, part) + return value + + return resolve else: return itemgetter(arg) @@ -512,7 +524,7 @@ def dictsort(value, arg): """ try: return sorted(value, key=_property_resolver(arg)) - except (TypeError, VariableDoesNotExist): + except (AttributeError, TypeError): return '' @@ -524,7 +536,7 @@ def dictsortreversed(value, arg): """ try: return sorted(value, key=_property_resolver(arg), reverse=True) - except (TypeError, VariableDoesNotExist): + except (AttributeError, TypeError): return '' diff --git a/django/template/defaulttags.py b/django/template/defaulttags.py index 4084189cf0ba..6390d1f8e128 100644 --- a/django/template/defaulttags.py +++ b/django/template/defaulttags.py @@ -9,7 +9,7 @@ from django.conf import settings from django.utils import timezone from django.utils.deprecation import RemovedInDjango40Warning -from django.utils.html import conditional_escape, format_html +from django.utils.html import conditional_escape, escape, format_html from django.utils.lorem_ipsum import paragraphs, words from django.utils.safestring import mark_safe @@ -96,10 +96,13 @@ def reset(self, context): class DebugNode(Node): def render(self, context): + if not settings.DEBUG: + return '' + from pprint import pformat - output = [pformat(val) for val in context] + output = [escape(pformat(val)) for val in context] output.append('\n\n') - output.append(pformat(sys.modules)) + output.append(escape(pformat(sys.modules))) return ''.join(output) diff --git a/django/urls/resolvers.py b/django/urls/resolvers.py index 9b00e24509cf..490e2b2ff230 100644 --- a/django/urls/resolvers.py +++ b/django/urls/resolvers.py @@ -154,7 +154,11 @@ def __init__(self, regex, name=None, is_endpoint=False): self.converters = {} def match(self, path): - match = self.regex.search(path) + match = ( + self.regex.fullmatch(path) + if self._is_endpoint and self.regex.pattern.endswith('$') + else self.regex.search(path) + ) if match: # If there are any named groups, use those as kwargs, ignoring # non-named groups. Otherwise, pass all non-named arguments as @@ -244,7 +248,7 @@ def _route_to_regex(route, is_endpoint=False): converters[parameter] = converter parts.append('(?P<' + parameter + '>' + converter.regex + ')') if is_endpoint: - parts.append('$') + parts.append(r'\Z') return ''.join(parts), converters @@ -299,7 +303,7 @@ def __init__(self, prefix_default_language=True): @property def regex(self): # This is only used by reverse() and cached in _reverse_dict. - return re.compile(self.language_prefix) + return re.compile(re.escape(self.language_prefix)) @property def language_prefix(self): diff --git a/django/utils/translation/trans_real.py b/django/utils/translation/trans_real.py index 8042f6fdc41c..b262a5000a48 100644 --- a/django/utils/translation/trans_real.py +++ b/django/utils/translation/trans_real.py @@ -30,6 +30,11 @@ # magic gettext number to separate context from message CONTEXT_SEPARATOR = "\x04" +# Maximum number of characters that will be parsed from the Accept-Language +# header to prevent possible denial of service or memory exhaustion attacks. +# About 10x longer than the longest value shown on MDN’s Accept-Language page. +ACCEPT_LANGUAGE_HEADER_MAX_LENGTH = 500 + # Format of Accept-Language header values. From RFC 2616, section 14.4 and 3.9 # and RFC 3066, section 2.1 accept_language_re = _lazy_re_compile(r''' @@ -556,7 +561,7 @@ def get_language_from_request(request, check_path=False): @functools.lru_cache(maxsize=1000) -def parse_accept_lang_header(lang_string): +def _parse_accept_lang_header(lang_string): """ Parse the lang_string, which is the body of an HTTP Accept-Language header, and return a tuple of (lang, q-value), ordered by 'q' values. @@ -578,3 +583,28 @@ def parse_accept_lang_header(lang_string): result.append((lang, priority)) result.sort(key=lambda k: k[1], reverse=True) return tuple(result) + + +def parse_accept_lang_header(lang_string): + """ + Parse the value of the Accept-Language header up to a maximum length. + + The value of the header is truncated to a maximum length to avoid potential + denial of service and memory exhaustion attacks. Excessive memory could be + used if the raw value is very large as it would be cached due to the use of + functools.lru_cache() to avoid repetitive parsing of common header values. + """ + # If the header value doesn't exceed the maximum allowed length, parse it. + if len(lang_string) <= ACCEPT_LANGUAGE_HEADER_MAX_LENGTH: + return _parse_accept_lang_header(lang_string) + + # If there is at least one comma in the value, parse up to the last comma + # before the max length, skipping any truncated parts at the end of the + # header value. + index = lang_string.rfind(",", 0, ACCEPT_LANGUAGE_HEADER_MAX_LENGTH) + if index > 0: + return _parse_accept_lang_header(lang_string[:index]) + + # Don't attempt to parse if there is only one language-range value which is + # longer than the maximum allowed length and so truncated. + return () diff --git a/docs/Makefile b/docs/Makefile index 39f84ec0e3ff..2b4483053124 100644 --- a/docs/Makefile +++ b/docs/Makefile @@ -9,6 +9,11 @@ PAPER ?= BUILDDIR ?= _build LANGUAGE ?= +# Set the default language. +ifndef LANGUAGE +override LANGUAGE = en +endif + # Convert something like "en_US" to "en", because Sphinx does not recognize # underscores. Country codes should be passed using a dash, e.g. "pt-BR". LANGUAGEOPT = $(firstword $(subst _, ,$(LANGUAGE))) diff --git a/docs/_ext/djangodocs.py b/docs/_ext/djangodocs.py index b21cfebc9e72..5208a532554b 100644 --- a/docs/_ext/djangodocs.py +++ b/docs/_ext/djangodocs.py @@ -8,7 +8,7 @@ from docutils import nodes from docutils.parsers.rst import Directive from docutils.statemachine import ViewList -from sphinx import addnodes +from sphinx import addnodes, version_info as sphinx_version from sphinx.builders.html import StandaloneHTMLBuilder from sphinx.directives.code import CodeBlock from sphinx.domains.std import Cmdoption @@ -115,11 +115,17 @@ class DjangoHTMLTranslator(HTMLTranslator): def visit_table(self, node): self.context.append(self.compact_p) self.compact_p = True - self._table_row_index = 0 # Needed by Sphinx + # Needed by Sphinx. + if sphinx_version >= (4, 3): + self._table_row_indices.append(0) + else: + self._table_row_index = 0 self.body.append(self.starttag(node, 'table', CLASS='docutils')) def depart_table(self, node): self.compact_p = self.context.pop() + if sphinx_version >= (4, 3): + self._table_row_indices.pop() self.body.append('\n') def visit_desc_parameterlist(self, node): diff --git a/docs/_theme/djangodocs/static/djangodocs.css b/docs/_theme/djangodocs/static/djangodocs.css index bd47749a06aa..0b6a8b9ad3bc 100644 --- a/docs/_theme/djangodocs/static/djangodocs.css +++ b/docs/_theme/djangodocs/static/djangodocs.css @@ -103,6 +103,7 @@ dt .literal, table .literal { background:none; } #bd a.reference { text-decoration: none; } #bd a.reference tt.literal { border-bottom: 1px #234f32 dotted; } div.code-block-caption { color: white; background-color: #234F32; margin: 0; padding: 2px 5px; width: 100%; font-family: monospace; font-size: small; line-height: 1.3em; } +div.code-block-caption .literal {color: white; } div.literal-block-wrapper pre { margin-top: 0; } /* Restore colors of pygments hyperlinked code */ diff --git a/docs/conf.py b/docs/conf.py index f7a690953ff5..42d350052cbf 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -32,7 +32,7 @@ # -- General configuration ----------------------------------------------------- # If your documentation needs a minimal Sphinx version, state it here. -needs_sphinx = '1.6.0' +needs_sphinx = "4.5.0" # Add any Sphinx extension module names here, as strings. They can be extensions # coming with Sphinx (named 'sphinx.ext.*') or your custom ones. @@ -70,8 +70,12 @@ # The encoding of source files. # source_encoding = 'utf-8-sig' -# The master toctree document. -master_doc = 'contents' +# The root toctree document. +root_doc = "contents" + +# Disable auto-created table of contents entries for all domain objects (e.g. +# functions, classes, attributes, etc.) in Sphinx 5.2+. +toc_object_entries = False # General substitutions. project = 'Django' @@ -102,12 +106,12 @@ def django_release(): django_next_version = '4.0' extlinks = { - 'bpo': ('https://bugs.python.org/issue%s', 'bpo-'), - 'commit': ('https://github.com/django/django/commit/%s', ''), - 'cve': ('https://nvd.nist.gov/vuln/detail/CVE-%s', 'CVE-'), + "bpo": ("https://bugs.python.org/issue?@action=redirect&bpo=%s", "bpo-%s"), + "commit": ("https://github.com/django/django/commit/%s", "%s"), + "cve": ("https://nvd.nist.gov/vuln/detail/CVE-%s", "CVE-%s"), # A file or directory. GitHub redirects from blob to tree if needed. - 'source': ('https://github.com/django/django/blob/main/%s', ''), - 'ticket': ('https://code.djangoproject.com/ticket/%s', '#'), + "source": ("https://github.com/django/django/blob/main/%s", "%s"), + "ticket": ("https://code.djangoproject.com/ticket/%s", "#%s"), } # The language for content autogenerated by Sphinx. Refer to documentation @@ -125,7 +129,7 @@ def django_release(): # List of patterns, relative to source directory, that match files and # directories to ignore when looking for source files. -exclude_patterns = ['_build', '_theme'] +exclude_patterns = ['_build', '_theme', 'requirements.txt'] # The reST default role (used for this markup: `text`) to use for all documents. default_role = "default-role-error" @@ -304,7 +308,7 @@ def django_release(): # List of tuples (startdocname, targetname, title, author, dir_entry, # description, category, toctree_only) texinfo_documents = [( - master_doc, "django", "", "", "Django", + root_doc, "django", "", "", "Django", "Documentation of the Django framework", "Web development", False )] diff --git a/docs/howto/overriding-templates.txt b/docs/howto/overriding-templates.txt index 0f880690698c..55d7c66f426b 100644 --- a/docs/howto/overriding-templates.txt +++ b/docs/howto/overriding-templates.txt @@ -112,7 +112,7 @@ For example, you can use this technique to add a custom logo to the ``admin/base_site.html`` template: .. code-block:: html+django - :caption: templates/admin/base_site.html + :caption: ``templates/admin/base_site.html`` {% extends "admin/base_site.html" %} diff --git a/docs/howto/writing-migrations.txt b/docs/howto/writing-migrations.txt index 00dc0dfadfa8..3c571fdc94d8 100644 --- a/docs/howto/writing-migrations.txt +++ b/docs/howto/writing-migrations.txt @@ -40,7 +40,7 @@ You can also provide hints that will be passed to the :meth:`allow_migrate()` method of database routers as ``**hints``: .. code-block:: python - :caption: myapp/dbrouters.py + :caption: ``myapp/dbrouters.py`` class MyRouter: @@ -98,7 +98,7 @@ the respective field according to your needs. ``AlterField``, and add imports of ``uuid`` and ``models``. For example: .. code-block:: python - :caption: 0006_remove_uuid_null.py + :caption: ``0006_remove_uuid_null.py`` # Generated by Django A.B on YYYY-MM-DD HH:MM from django.db import migrations, models @@ -122,7 +122,7 @@ the respective field according to your needs. similar to this: .. code-block:: python - :caption: 0004_add_uuid_field.py + :caption: ``0004_add_uuid_field.py`` class Migration(migrations.Migration): @@ -149,7 +149,7 @@ the respective field according to your needs. of ``uuid``. For example: .. code-block:: python - :caption: 0005_populate_uuid_values.py + :caption: ``0005_populate_uuid_values.py`` # Generated by Django A.B on YYYY-MM-DD HH:MM from django.db import migrations @@ -283,7 +283,7 @@ project anywhere without first installing and then uninstalling the old app. Here's a sample migration: .. code-block:: python - :caption: myapp/migrations/0124_move_old_app_to_new_app.py + :caption: ``myapp/migrations/0124_move_old_app_to_new_app.py`` from django.apps import apps as global_apps from django.db import migrations diff --git a/docs/internals/contributing/writing-code/coding-style.txt b/docs/internals/contributing/writing-code/coding-style.txt index 6d0a9c27b794..61a350821e58 100644 --- a/docs/internals/contributing/writing-code/coding-style.txt +++ b/docs/internals/contributing/writing-code/coding-style.txt @@ -158,8 +158,8 @@ Imports .. console:: - $ python -m pip install isort >= 5.1.0 - $ isort -rc . + $ python -m pip install "isort >= 5.1.0" + $ isort . This runs ``isort`` recursively from your current directory, modifying any files that don't conform to the guidelines. If you need to have imports out @@ -186,7 +186,7 @@ Imports For example (comments are for explanatory purposes only): .. code-block:: python - :caption: django/contrib/admin/example.py + :caption: ``django/contrib/admin/example.py`` # future from __future__ import unicode_literals diff --git a/docs/internals/contributing/writing-code/unit-tests.txt b/docs/internals/contributing/writing-code/unit-tests.txt index 490a4673b692..bd426099d6d6 100644 --- a/docs/internals/contributing/writing-code/unit-tests.txt +++ b/docs/internals/contributing/writing-code/unit-tests.txt @@ -557,7 +557,7 @@ Since this pattern involves a lot of boilerplate, Django provides the installed, you should pass the set of targeted ``app_label`` as arguments: .. code-block:: python - :caption: tests/app_label/tests.py + :caption: ``tests/app_label/tests.py`` from django.db import models from django.test import SimpleTestCase diff --git a/docs/internals/howto-release-django.txt b/docs/internals/howto-release-django.txt index beb3581d54fa..723804a95e3a 100644 --- a/docs/internals/howto-release-django.txt +++ b/docs/internals/howto-release-django.txt @@ -63,7 +63,7 @@ You'll need a few things before getting started: * Access to Django's record on PyPI. Create a file with your credentials: .. code-block:: ini - :caption: ~/.pypirc + :caption: ``~/.pypirc`` [pypi] username:YourUsername diff --git a/docs/intro/contributing.txt b/docs/intro/contributing.txt index 4a1dacf50abe..dd3a63157839 100644 --- a/docs/intro/contributing.txt +++ b/docs/intro/contributing.txt @@ -314,7 +314,7 @@ Writing a test for ticket #99999 -------------------------------- In order to resolve this ticket, we'll add a ``make_toast()`` function to the -top-level ``django`` module. First we are going to write a test that tries to +``django.shortcuts`` module. First we are going to write a test that tries to use the function and check that its output looks correct. Navigate to Django's ``tests/shortcuts/`` folder and create a new file diff --git a/docs/intro/overview.txt b/docs/intro/overview.txt index c0d528527f75..86b172f526f4 100644 --- a/docs/intro/overview.txt +++ b/docs/intro/overview.txt @@ -26,7 +26,7 @@ representing your models -- so far, it's been solving many years' worth of database-schema problems. Here's a quick example: .. code-block:: python - :caption: mysite/news/models.py + :caption: ``mysite/news/models.py`` from django.db import models @@ -146,7 +146,7 @@ a website that lets authenticated users add, change and delete objects. The only step required is to register your model in the admin site: .. code-block:: python - :caption: mysite/news/models.py + :caption: ``mysite/news/models.py`` from django.db import models @@ -157,7 +157,7 @@ only step required is to register your model in the admin site: reporter = models.ForeignKey(Reporter, on_delete=models.CASCADE) .. code-block:: python - :caption: mysite/news/admin.py + :caption: ``mysite/news/admin.py`` from django.contrib import admin @@ -189,7 +189,7 @@ Here's what a URLconf might look like for the ``Reporter``/``Article`` example above: .. code-block:: python - :caption: mysite/news/urls.py + :caption: ``mysite/news/urls.py`` from django.urls import path @@ -229,7 +229,7 @@ and renders the template with the retrieved data. Here's an example view for ``year_archive`` from above: .. code-block:: python - :caption: mysite/news/views.py + :caption: ``mysite/news/views.py`` from django.shortcuts import render @@ -258,7 +258,7 @@ Let's say the ``news/year_archive.html`` template was found. Here's what that might look like: .. code-block:: html+django - :caption: mysite/news/templates/news/year_archive.html + :caption: ``mysite/news/templates/news/year_archive.html`` {% extends "base.html" %} @@ -299,7 +299,7 @@ Here's what the "base.html" template, including the use of :doc:`static files `, might look like: .. code-block:: html+django - :caption: mysite/templates/base.html + :caption: ``mysite/templates/base.html`` {% load static %} diff --git a/docs/intro/reusable-apps.txt b/docs/intro/reusable-apps.txt index d00879ea6037..82bdaf278459 100644 --- a/docs/intro/reusable-apps.txt +++ b/docs/intro/reusable-apps.txt @@ -144,7 +144,7 @@ this. For a small app like polls, this process isn't too difficult. #. Create a file ``django-polls/README.rst`` with the following contents: .. code-block:: rst - :caption: django-polls/README.rst + :caption: ``django-polls/README.rst`` ===== Polls @@ -191,7 +191,7 @@ this. For a small app like polls, this process isn't too difficult. with the following contents: .. code-block:: ini - :caption: django-polls/setup.cfg + :caption: ``django-polls/setup.cfg`` [metadata] name = django-polls @@ -226,7 +226,7 @@ this. For a small app like polls, this process isn't too difficult. Django >= X.Y # Replace "X.Y" as appropriate .. code-block:: python - :caption: django-polls/setup.py + :caption: ``django-polls/setup.py`` from setuptools import setup @@ -240,7 +240,7 @@ this. For a small app like polls, this process isn't too difficult. contents: .. code-block:: text - :caption: django-polls/MANIFEST.in + :caption: ``django-polls/MANIFEST.in`` include LICENSE include README.rst diff --git a/docs/intro/tutorial01.txt b/docs/intro/tutorial01.txt index 05f99b6f7615..f3b080e9bb14 100644 --- a/docs/intro/tutorial01.txt +++ b/docs/intro/tutorial01.txt @@ -245,7 +245,7 @@ Let's write the first view. Open the file ``polls/views.py`` and put the following Python code in it: .. code-block:: python - :caption: polls/views.py + :caption: ``polls/views.py`` from django.http import HttpResponse @@ -273,7 +273,7 @@ Your app directory should now look like:: In the ``polls/urls.py`` file include the following code: .. code-block:: python - :caption: polls/urls.py + :caption: ``polls/urls.py`` from django.urls import path @@ -288,7 +288,7 @@ The next step is to point the root URLconf at the ``polls.urls`` module. In :func:`~django.urls.include` in the ``urlpatterns`` list, so you have: .. code-block:: python - :caption: mysite/urls.py + :caption: ``mysite/urls.py`` from django.contrib import admin from django.urls import include, path diff --git a/docs/intro/tutorial02.txt b/docs/intro/tutorial02.txt index b315027c5de5..2fa780178ae4 100644 --- a/docs/intro/tutorial02.txt +++ b/docs/intro/tutorial02.txt @@ -141,7 +141,7 @@ These concepts are represented by Python classes. Edit the :file:`polls/models.py` file so it looks like this: .. code-block:: python - :caption: polls/models.py + :caption: ``polls/models.py`` from django.db import models @@ -217,7 +217,7 @@ add that dotted path to the :setting:`INSTALLED_APPS` setting. It'll look like this: .. code-block:: python - :caption: mysite/settings.py + :caption: ``mysite/settings.py`` INSTALLED_APPS = [ 'polls.apps.PollsConfig', @@ -424,7 +424,7 @@ representation of this object. Let's fix that by editing the ``Question`` model ``Choice``: .. code-block:: python - :caption: polls/models.py + :caption: ``polls/models.py`` from django.db import models @@ -448,7 +448,7 @@ automatically-generated admin. Let's also add a custom method to this model: .. code-block:: python - :caption: polls/models.py + :caption: ``polls/models.py`` import datetime @@ -646,7 +646,7 @@ have an admin interface. To do this, open the :file:`polls/admin.py` file, and edit it to look like this: .. code-block:: python - :caption: polls/admin.py + :caption: ``polls/admin.py`` from django.contrib import admin diff --git a/docs/intro/tutorial03.txt b/docs/intro/tutorial03.txt index 82fa35e49705..720a83a3e508 100644 --- a/docs/intro/tutorial03.txt +++ b/docs/intro/tutorial03.txt @@ -70,7 +70,7 @@ Now let's add a few more views to ``polls/views.py``. These views are slightly different, because they take an argument: .. code-block:: python - :caption: polls/views.py + :caption: ``polls/views.py`` def detail(request, question_id): return HttpResponse("You're looking at question %s." % question_id) @@ -86,7 +86,7 @@ Wire these new views into the ``polls.urls`` module by adding the following :func:`~django.urls.path` calls: .. code-block:: python - :caption: polls/urls.py + :caption: ``polls/urls.py`` from django.urls import path @@ -147,7 +147,7 @@ view, which displays the latest 5 poll questions in the system, separated by commas, according to publication date: .. code-block:: python - :caption: polls/views.py + :caption: ``polls/views.py`` from django.http import HttpResponse @@ -196,7 +196,7 @@ Django as ``polls/index.html``. Put the following code in that template: .. code-block:: html+django - :caption: polls/templates/polls/index.html + :caption: ``polls/templates/polls/index.html`` {% if latest_question_list %}