diff --git a/README.rst b/README.rst index 2a33fa2e640a..6776b7c39b99 100644 --- a/README.rst +++ b/README.rst @@ -25,7 +25,7 @@ ticket here: https://code.djangoproject.com/newticket To get more help: -* Join the ``#django`` channel on irc.freenode.net. Lots of helpful people hang +* Join the ``#django`` channel on ``irc.libera.chat``. Lots of helpful people out there. See https://en.wikipedia.org/wiki/Wikipedia:IRC/Tutorial if you're new to IRC. diff --git a/django/__init__.py b/django/__init__.py index 4b58367eade6..7963a360df01 100644 --- a/django/__init__.py +++ b/django/__init__.py @@ -1,6 +1,6 @@ from django.utils.version import get_version -VERSION = (2, 2, 20, 'final', 0) +VERSION = (2, 2, 24, 'final', 0) __version__ = get_version(VERSION) diff --git a/django/contrib/admindocs/views.py b/django/contrib/admindocs/views.py index 0474c38fd4d4..5986717d9517 100644 --- a/django/contrib/admindocs/views.py +++ b/django/contrib/admindocs/views.py @@ -15,6 +15,7 @@ from django.http import Http404 from django.template.engine import Engine from django.urls import get_mod_func, get_resolver, get_urlconf +from django.utils._os import safe_join from django.utils.decorators import method_decorator from django.utils.inspect import ( func_accepts_kwargs, func_accepts_var_args, get_func_full_args, @@ -328,7 +329,7 @@ def get_context_data(self, **kwargs): else: # This doesn't account for template loaders (#24128). for index, directory in enumerate(default_engine.dirs): - template_file = Path(directory) / template + template_file = Path(safe_join(directory, template)) if template_file.exists(): with template_file.open() as f: template_contents = f.read() diff --git a/django/core/files/storage.py b/django/core/files/storage.py index 1562614e50d6..89faa626e6ec 100644 --- a/django/core/files/storage.py +++ b/django/core/files/storage.py @@ -1,4 +1,5 @@ import os +import pathlib from datetime import datetime from urllib.parse import urljoin @@ -6,6 +7,7 @@ from django.core.exceptions import SuspiciousFileOperation from django.core.files import File, locks from django.core.files.move import file_move_safe +from django.core.files.utils import validate_file_name from django.core.signals import setting_changed from django.utils import timezone from django.utils._os import safe_join @@ -66,6 +68,9 @@ def get_available_name(self, name, max_length=None): available for new content to be written to. """ dir_name, file_name = os.path.split(name) + if '..' in pathlib.PurePath(dir_name).parts: + raise SuspiciousFileOperation("Detected path traversal attempt in '%s'" % dir_name) + validate_file_name(file_name) file_root, file_ext = os.path.splitext(file_name) # If the filename already exists, add an underscore and a random 7 # character alphanumeric string (before the file extension, if one @@ -98,6 +103,8 @@ def generate_filename(self, filename): """ # `filename` may include a path as returned by FileField.upload_to. dirname, filename = os.path.split(filename) + if '..' in pathlib.PurePath(dirname).parts: + raise SuspiciousFileOperation("Detected path traversal attempt in '%s'" % dirname) return os.path.normpath(os.path.join(dirname, self.get_valid_name(filename))) def path(self, name): diff --git a/django/core/files/uploadedfile.py b/django/core/files/uploadedfile.py index 48007b86823d..f452bcd9a4a1 100644 --- a/django/core/files/uploadedfile.py +++ b/django/core/files/uploadedfile.py @@ -8,6 +8,7 @@ from django.conf import settings from django.core.files import temp as tempfile from django.core.files.base import File +from django.core.files.utils import validate_file_name __all__ = ('UploadedFile', 'TemporaryUploadedFile', 'InMemoryUploadedFile', 'SimpleUploadedFile') @@ -47,6 +48,8 @@ def _set_name(self, name): ext = ext[:255] name = name[:255 - len(ext)] + ext + name = validate_file_name(name) + self._name = name name = property(_get_name, _set_name) diff --git a/django/core/files/utils.py b/django/core/files/utils.py index de896071759b..f28cea107758 100644 --- a/django/core/files/utils.py +++ b/django/core/files/utils.py @@ -1,3 +1,29 @@ +import os +import pathlib + +from django.core.exceptions import SuspiciousFileOperation + + +def validate_file_name(name, allow_relative_path=False): + # Remove potentially dangerous names + if os.path.basename(name) in {'', '.', '..'}: + raise SuspiciousFileOperation("Could not derive file name from '%s'" % name) + + if allow_relative_path: + # Use PurePosixPath() because this branch is checked only in + # FileField.generate_filename() where all file paths are expected to be + # Unix style (with forward slashes). + path = pathlib.PurePosixPath(name) + if path.is_absolute() or '..' in path.parts: + raise SuspiciousFileOperation( + "Detected path traversal attempt in '%s'" % name + ) + elif name != os.path.basename(name): + raise SuspiciousFileOperation("File name '%s' includes path elements" % name) + + return name + + class FileProxyMixin: """ A mixin class used to forward file methods to an underlaying file diff --git a/django/core/validators.py b/django/core/validators.py index 38e4b6aa1d7a..2da0688e28b8 100644 --- a/django/core/validators.py +++ b/django/core/validators.py @@ -75,7 +75,7 @@ class URLValidator(RegexValidator): ul = '\u00a1-\uffff' # unicode letters range (must not be a raw string) # IP patterns - ipv4_re = r'(?:25[0-5]|2[0-4]\d|[0-1]?\d?\d)(?:\.(?:25[0-5]|2[0-4]\d|[0-1]?\d?\d)){3}' + ipv4_re = r'(?:0|25[0-5]|2[0-4]\d|1\d?\d?|[1-9]\d?)(?:\.(?:0|25[0-5]|2[0-4]\d|1\d?\d?|[1-9]\d?)){3}' ipv6_re = r'\[[0-9a-f:\.]+\]' # (simple regex, validated later) # Host patterns @@ -101,6 +101,7 @@ class URLValidator(RegexValidator): r'\Z', re.IGNORECASE) message = _('Enter a valid URL.') schemes = ['http', 'https', 'ftp', 'ftps'] + unsafe_chars = frozenset('\t\r\n') def __init__(self, schemes=None, **kwargs): super().__init__(**kwargs) @@ -108,7 +109,9 @@ def __init__(self, schemes=None, **kwargs): self.schemes = schemes def __call__(self, value): - # Check first if the scheme is valid + if isinstance(value, str) and self.unsafe_chars.intersection(value): + raise ValidationError(self.message, code=self.code) + # Check if the scheme is valid. scheme = value.split('://')[0].lower() if scheme not in self.schemes: raise ValidationError(self.message, code=self.code) @@ -253,6 +256,18 @@ def validate_ipv4_address(value): ipaddress.IPv4Address(value) except ValueError: raise ValidationError(_('Enter a valid IPv4 address.'), code='invalid') + else: + # Leading zeros are forbidden to avoid ambiguity with the octal + # notation. This restriction is included in Python 3.9.5+. + # TODO: Remove when dropping support for PY39. + if any( + octet != '0' and octet[0] == '0' + for octet in value.split('.') + ): + raise ValidationError( + _('Enter a valid IPv4 address.'), + code='invalid', + ) def validate_ipv6_address(value): diff --git a/django/db/models/fields/files.py b/django/db/models/fields/files.py index bd8da95e4649..0f8c3fe48420 100644 --- a/django/db/models/fields/files.py +++ b/django/db/models/fields/files.py @@ -6,6 +6,7 @@ from django.core.files.base import File from django.core.files.images import ImageFile from django.core.files.storage import default_storage +from django.core.files.utils import validate_file_name from django.db.models import signals from django.db.models.fields import Field from django.utils.translation import gettext_lazy as _ @@ -304,6 +305,7 @@ def generate_filename(self, instance, filename): else: dirname = datetime.datetime.now().strftime(self.upload_to) filename = posixpath.join(dirname, filename) + filename = validate_file_name(filename, allow_relative_path=True) return self.storage.generate_filename(filename) def save_form_data(self, instance, data): diff --git a/django/http/multipartparser.py b/django/http/multipartparser.py index 5a9cca89e6bb..4570ebbaeeb8 100644 --- a/django/http/multipartparser.py +++ b/django/http/multipartparser.py @@ -7,7 +7,7 @@ import base64 import binascii import cgi -import os +import html from urllib.parse import unquote from django.conf import settings @@ -19,7 +19,6 @@ ) from django.utils.datastructures import MultiValueDict from django.utils.encoding import force_text -from django.utils.text import unescape_entities __all__ = ('MultiPartParser', 'MultiPartParserError', 'InputStreamExhausted') @@ -295,10 +294,25 @@ def handle_file_complete(self, old_field_name, counters): break def sanitize_file_name(self, file_name): - file_name = unescape_entities(file_name) - # Cleanup Windows-style path separators. - file_name = file_name[file_name.rfind('\\') + 1:].strip() - return os.path.basename(file_name) + """ + Sanitize the filename of an upload. + + Remove all possible path separators, even though that might remove more + than actually required by the target system. Filenames that could + potentially cause problems (current/parent dir) are also discarded. + + It should be noted that this function could still return a "filepath" + like "C:some_file.txt" which is handled later on by the storage layer. + So while this function does sanitize filenames to some extent, the + resulting filename should still be considered as untrusted user input. + """ + file_name = html.unescape(file_name) + file_name = file_name.rsplit('/')[-1] + file_name = file_name.rsplit('\\')[-1] + + if file_name in {'', '.', '..'}: + return None + return file_name IE_sanitize = sanitize_file_name diff --git a/django/utils/text.py b/django/utils/text.py index 853436a38f3f..1fae7b252255 100644 --- a/django/utils/text.py +++ b/django/utils/text.py @@ -4,6 +4,7 @@ from gzip import GzipFile from io import BytesIO +from django.core.exceptions import SuspiciousFileOperation from django.utils.functional import SimpleLazyObject, keep_lazy_text, lazy from django.utils.translation import gettext as _, gettext_lazy, pgettext @@ -216,7 +217,7 @@ def _truncate_html(self, length, truncate, text, truncate_len, words): @keep_lazy_text -def get_valid_filename(s): +def get_valid_filename(name): """ Return the given string converted to a string that can be used for a clean filename. Remove leading and trailing spaces; convert other spaces to @@ -225,8 +226,11 @@ def get_valid_filename(s): >>> get_valid_filename("john's portrait in 2004.jpg") 'johns_portrait_in_2004.jpg' """ - s = str(s).strip().replace(' ', '_') - return re.sub(r'(?u)[^-\w.]', '', s) + s = str(name).strip().replace(' ', '_') + s = re.sub(r'(?u)[^-\w.]', '', s) + if s in {'', '.', '..'}: + raise SuspiciousFileOperation("Could not derive file name from '%s'" % name) + return s @keep_lazy_text diff --git a/docs/faq/help.txt b/docs/faq/help.txt index e2626894eef9..fe76ba6e1e12 100644 --- a/docs/faq/help.txt +++ b/docs/faq/help.txt @@ -9,10 +9,9 @@ If this FAQ doesn't contain an answer to your question, you might want to try the |django-users| mailing list. Feel free to ask any question related to installing, using, or debugging Django. -If you prefer IRC, the `#django IRC channel`_ on the Freenode IRC network is an -active community of helpful individuals who may be able to solve your problem. - -.. _`#django IRC channel`: irc://irc.freenode.net/django +If you prefer IRC, the `#django IRC channel`_ on the Libera.Chat IRC network is +an active community of helpful individuals who may be able to solve your +problem. .. _message-does-not-appear-on-django-users: @@ -40,7 +39,7 @@ As with most open-source mailing lists, the folks on |django-users| are volunteers. If nobody has answered your question, it may be because nobody knows the answer, it may be because nobody can understand the question, or it may be that everybody that can help is busy. One thing you might try is to ask -the question on IRC -- visit the `#django IRC channel`_ on the Freenode IRC +the question on IRC -- visit the `#django IRC channel`_ on the Libera.Chat IRC network. You might notice we have a second mailing list, called |django-developers| -- @@ -69,3 +68,4 @@ while a defect is outstanding, we would like to minimize any damage that could be inflicted through public knowledge of that defect. .. _`policy for handling security issues`: ../contributing/#reporting-security-issues +.. _`#django IRC channel`: irc://irc.libera.chat/django diff --git a/docs/index.txt b/docs/index.txt index 6139c3e9b889..9c00dc438578 100644 --- a/docs/index.txt +++ b/docs/index.txt @@ -23,7 +23,7 @@ Having trouble? We'd like to help! .. _archives: https://groups.google.com/group/django-users/ .. _post a question: https://groups.google.com/d/forum/django-users -.. _#django IRC channel: irc://irc.freenode.net/django +.. _#django IRC channel: irc://irc.libera.chat/django .. _ticket tracker: https://code.djangoproject.com/ How the documentation is organized diff --git a/docs/internals/contributing/bugs-and-features.txt b/docs/internals/contributing/bugs-and-features.txt index 858de4ad0860..dcaa78ee93a8 100644 --- a/docs/internals/contributing/bugs-and-features.txt +++ b/docs/internals/contributing/bugs-and-features.txt @@ -166,4 +166,4 @@ Votes on technical matters should be announced and held in public on the .. _searching: https://code.djangoproject.com/search .. _custom queries: https://code.djangoproject.com/query -.. _#django: irc://irc.freenode.net/django +.. _#django: irc://irc.libera.chat/django diff --git a/docs/internals/contributing/index.txt b/docs/internals/contributing/index.txt index 9a1e5d64d7f5..1915c99dddde 100644 --- a/docs/internals/contributing/index.txt +++ b/docs/internals/contributing/index.txt @@ -16,7 +16,7 @@ contribute in many ways: friendly and helpful atmosphere. If you're new to the Django community, you should read the `posting guidelines`_. -* Join the `#django IRC channel`_ on Freenode and answer questions. By +* Join the `#django IRC channel`_ on Libera.Chat and answer questions. By explaining Django to other users, you're going to learn a lot about the framework yourself. @@ -68,8 +68,8 @@ Browse the following sections to find out how: committing-code .. _posting guidelines: https://code.djangoproject.com/wiki/UsingTheMailingList -.. _#django IRC channel: irc://irc.freenode.net/django -.. _#django-dev IRC channel: irc://irc.freenode.net/django-dev +.. _#django IRC channel: irc://irc.libera.chat/django +.. _#django-dev IRC channel: irc://irc.libera.chat/django-dev .. _community page: https://www.djangoproject.com/community/ .. _Django forum: https://forum.djangoproject.com/ .. _register it here: https://www.djangoproject.com/community/add/blogs/ diff --git a/docs/intro/contributing.txt b/docs/intro/contributing.txt index eb00190ad764..ee889db0993f 100644 --- a/docs/intro/contributing.txt +++ b/docs/intro/contributing.txt @@ -40,11 +40,11 @@ so that it can be of use to the widest audience. .. admonition:: Where to get help: If you're having trouble going through this tutorial, please post a message - to |django-developers| or drop by `#django-dev on irc.freenode.net`__ to + to |django-developers| or drop by `#django-dev on irc.libera.chat`__ to chat with other Django users who might be able to help. __ https://diveinto.org/python3/table-of-contents.html -__ irc://irc.freenode.net/django-dev +__ irc://irc.libera.chat/django-dev What does this tutorial cover? ------------------------------ diff --git a/docs/intro/tutorial01.txt b/docs/intro/tutorial01.txt index 56de527fac3e..e795e2cf44f9 100644 --- a/docs/intro/tutorial01.txt +++ b/docs/intro/tutorial01.txt @@ -36,8 +36,8 @@ older versions of Django and install a newer one. .. admonition:: Where to get help: If you're having trouble going through this tutorial, please post a message - to |django-users| or drop by `#django on irc.freenode.net - `_ to chat with other Django users who might + to |django-users| or drop by `#django on irc.libera.chat + `_ to chat with other Django users who might be able to help. Creating a project diff --git a/docs/intro/whatsnext.txt b/docs/intro/whatsnext.txt index 7d3346a12a70..5b05af4a9f9b 100644 --- a/docs/intro/whatsnext.txt +++ b/docs/intro/whatsnext.txt @@ -127,7 +127,7 @@ particular Django setup, try the |django-users| mailing list or the `#django IRC channel`_ instead. .. _ticket system: https://code.djangoproject.com/ -.. _#django IRC channel: irc://irc.freenode.net/django +.. _#django IRC channel: irc://irc.libera.chat/django In plain text ------------- diff --git a/docs/ref/contrib/gis/install/index.txt b/docs/ref/contrib/gis/install/index.txt index 941afe82d83f..fa1125abbaac 100644 --- a/docs/ref/contrib/gis/install/index.txt +++ b/docs/ref/contrib/gis/install/index.txt @@ -109,9 +109,9 @@ Troubleshooting If you can't find the solution to your problem here then participate in the community! You can: -* Join the ``#geodjango`` IRC channel on Freenode. Please be patient and polite - -- while you may not get an immediate response, someone will attempt to answer - your question as soon as they see it. +* Join the ``#django-geo`` IRC channel on Libera.Chat. Please be patient and + polite -- while you may not get an immediate response, someone will attempt + to answer your question as soon as they see it. * Ask your question on the `GeoDjango`__ mailing list. * File a ticket on the `Django trac`__ if you think there's a bug. Make sure to provide a complete description of the problem, versions used, diff --git a/docs/releases/0.95.txt b/docs/releases/0.95.txt index 21fdd15320aa..4b9b91570856 100644 --- a/docs/releases/0.95.txt +++ b/docs/releases/0.95.txt @@ -109,9 +109,9 @@ many common questions appear with some regularity, and any particular problem may already have been answered. Finally, for those who prefer the more immediate feedback offered by IRC, -there's a `#django` channel on irc.freenode.net that is regularly populated -by Django users and developers from around the world. Friendly people are -usually available at any hour of the day -- to help, or just to chat. +there's a ``#django`` channel on ``irc.libera.chat`` that is regularly +populated by Django users and developers from around the world. Friendly people +are usually available at any hour of the day -- to help, or just to chat. .. _Django website: https://www.djangoproject.com/ .. _django-users: https://groups.google.com/group/django-users diff --git a/docs/releases/1.1.txt b/docs/releases/1.1.txt index 49c375b5ce17..e55ef9c903ef 100644 --- a/docs/releases/1.1.txt +++ b/docs/releases/1.1.txt @@ -441,7 +441,7 @@ What's next? We'll take a short break, and then work on Django 1.2 will begin -- no rest for the weary! If you'd like to help, discussion of Django development, including progress toward the 1.2 release, takes place daily on the |django-developers| -mailing list and in the ``#django-dev`` IRC channel on ``irc.freenode.net``. +mailing list and in the ``#django-dev`` IRC channel on ``irc.libera.chat``. Feel free to join the discussions! Django's online documentation also includes pointers on how to contribute to diff --git a/docs/releases/2.2.21.txt b/docs/releases/2.2.21.txt new file mode 100644 index 000000000000..2302df428520 --- /dev/null +++ b/docs/releases/2.2.21.txt @@ -0,0 +1,16 @@ +=========================== +Django 2.2.21 release notes +=========================== + +*May 4, 2021* + +Django 2.2.21 fixes a security issue in 2.2.20. + +CVE-2021-31542: Potential directory-traversal via uploaded files +================================================================ + +``MultiPartParser``, ``UploadedFile``, and ``FieldFile`` allowed +directory-traversal via uploaded files with suitably crafted file names. + +In order to mitigate this risk, stricter basename and path sanitation is now +applied. diff --git a/docs/releases/2.2.22.txt b/docs/releases/2.2.22.txt new file mode 100644 index 000000000000..6808a267afeb --- /dev/null +++ b/docs/releases/2.2.22.txt @@ -0,0 +1,22 @@ +=========================== +Django 2.2.22 release notes +=========================== + +*May 6, 2021* + +Django 2.2.22 fixes a security issue in 2.2.21. + +CVE-2021-32052: Header injection possibility since ``URLValidator`` accepted newlines in input on Python 3.9.5+ +=============================================================================================================== + +On Python 3.9.5+, :class:`~django.core.validators.URLValidator` didn't prohibit +newlines and tabs. If you used values with newlines in HTTP response, you could +suffer from header injection attacks. Django itself wasn't vulnerable because +:class:`~django.http.HttpResponse` prohibits newlines in HTTP headers. + +Moreover, the ``URLField`` form field which uses ``URLValidator`` silently +removes newlines and tabs on Python 3.9.5+, so the possibility of newlines +entering your data only existed if you are using this validator outside of the +form fields. + +This issue was introduced by the :bpo:`43882` fix. diff --git a/docs/releases/2.2.23.txt b/docs/releases/2.2.23.txt new file mode 100644 index 000000000000..6c39361e5fc7 --- /dev/null +++ b/docs/releases/2.2.23.txt @@ -0,0 +1,15 @@ +=========================== +Django 2.2.23 release notes +=========================== + +*May 13, 2021* + +Django 2.2.23 fixes a regression in 2.2.21. + +Bugfixes +======== + +* Fixed a regression in Django 2.2.21 where saving ``FileField`` would raise a + ``SuspiciousFileOperation`` even when a custom + :attr:`~django.db.models.FileField.upload_to` returns a valid file path + (:ticket:`32718`). diff --git a/docs/releases/2.2.24.txt b/docs/releases/2.2.24.txt new file mode 100644 index 000000000000..1064fc53a004 --- /dev/null +++ b/docs/releases/2.2.24.txt @@ -0,0 +1,32 @@ +=========================== +Django 2.2.24 release notes +=========================== + +*June 2, 2021* + +Django 2.2.24 fixes two security issues in 2.2.23. + +CVE-2021-33203: Potential directory traversal via ``admindocs`` +=============================================================== + +Staff members could use the :mod:`~django.contrib.admindocs` +``TemplateDetailView`` view to check the existence of arbitrary files. +Additionally, if (and only if) the default admindocs templates have been +customized by the developers to also expose the file contents, then not only +the existence but also the file contents would have been exposed. + +As a mitigation, path sanitation is now applied and only files within the +template root directories can be loaded. + +CVE-2021-33571: Possible indeterminate SSRF, RFI, and LFI attacks since validators accepted leading zeros in IPv4 addresses +=========================================================================================================================== + +:class:`~django.core.validators.URLValidator`, +:func:`~django.core.validators.validate_ipv4_address`, and +:func:`~django.core.validators.validate_ipv46_address` didn't prohibit leading +zeros in octal literals. If you used such values you could suffer from +indeterminate SSRF, RFI, and LFI attacks. + +:func:`~django.core.validators.validate_ipv4_address` and +:func:`~django.core.validators.validate_ipv46_address` validators were not +affected on Python 3.9.5+. diff --git a/docs/releases/index.txt b/docs/releases/index.txt index 7ccec86e15f3..38bb561b9c45 100644 --- a/docs/releases/index.txt +++ b/docs/releases/index.txt @@ -25,6 +25,10 @@ versions of the documentation contain the release notes for any later releases. .. toctree:: :maxdepth: 1 + 2.2.24 + 2.2.23 + 2.2.22 + 2.2.21 2.2.20 2.2.19 2.2.18 diff --git a/docs/releases/security.txt b/docs/releases/security.txt index 10f871d563fd..509cc6ce7694 100644 --- a/docs/releases/security.txt +++ b/docs/releases/security.txt @@ -1162,3 +1162,44 @@ Versions affected * Django 3.1 :commit:`(patch) <8f6d431b08cbb418d9144b976e7b972546607851>` * Django 3.0 :commit:`(patch) <326a926beef869d3341bc9ef737887f0449b6b71>` * Django 2.2 :commit:`(patch) ` + +April 6, 2021 - :cve:`2021-28658` +--------------------------------- + +Potential directory-traversal via uploaded files. `Full description +`__ + +Versions affected +~~~~~~~~~~~~~~~~~ + +* Django 3.2 :commit:`(patch) <2820fd1be5dfccbf1216c3845fad8580502473e1>` +* Django 3.1 :commit:`(patch) ` +* Django 3.0 :commit:`(patch) ` +* Django 2.2 :commit:`(patch) <4036d62bda0e9e9f6172943794b744a454ca49c2>` + +May 4, 2021 - :cve:`2021-31542` +------------------------------- + +Potential directory-traversal via uploaded files. `Full description +`__ + +Versions affected +~~~~~~~~~~~~~~~~~ + +* Django 3.2 :commit:`(patch) ` +* Django 3.1 :commit:`(patch) <25d84d64122c15050a0ee739e859f22ddab5ac48>` +* Django 2.2 :commit:`(patch) <04ac1624bdc2fa737188401757cf95ced122d26d>` + +May 6, 2021 - :cve:`2021-32052` +------------------------------- + +Header injection possibility since ``URLValidator`` accepted newlines in input +on Python 3.9.5+. `Full description +`__ + +Versions affected +~~~~~~~~~~~~~~~~~ + +* Django 3.2 :commit:`(patch) <2d2c1d0c97832860fbd6597977e2aae17dd7e5b2>` +* Django 3.1 :commit:`(patch) ` +* Django 2.2 :commit:`(patch) ` diff --git a/docs/spelling_wordlist b/docs/spelling_wordlist index f8a718a3bcf6..c511bdb3c648 100644 --- a/docs/spelling_wordlist +++ b/docs/spelling_wordlist @@ -222,7 +222,6 @@ formfield formset formsets formtools -freenode Frysian functionalities gdal @@ -320,6 +319,7 @@ Kyngesburye latin lawrence lexer +Libera lifecycle lifecycles linearize diff --git a/docs/topics/db/sql.txt b/docs/topics/db/sql.txt index 84420c7e4b30..13d5a83c2c53 100644 --- a/docs/topics/db/sql.txt +++ b/docs/topics/db/sql.txt @@ -23,8 +23,8 @@ __ `executing custom SQL directly`_ :doc:`custom query expressions `. Before using raw SQL, explore :doc:`the ORM `. Ask on - |django-users| or the `#django IRC channel - `_ to see if the ORM supports your use case. + one of :doc:`the support channels ` to see if the ORM supports + your use case. .. warning:: diff --git a/tests/admin_docs/test_views.py b/tests/admin_docs/test_views.py index bcadff7d8a62..dc6d3c127b18 100644 --- a/tests/admin_docs/test_views.py +++ b/tests/admin_docs/test_views.py @@ -134,6 +134,22 @@ def test_no_sites_framework(self): self.assertContains(response, 'View documentation') +@unittest.skipUnless(utils.docutils_is_available, 'no docutils installed.') +class AdminDocViewDefaultEngineOnly(TestDataMixin, AdminDocsTestCase): + + def setUp(self): + self.client.force_login(self.superuser) + + def test_template_detail_path_traversal(self): + cases = ['/etc/passwd', '../passwd'] + for fpath in cases: + with self.subTest(path=fpath): + response = self.client.get( + reverse('django-admindocs-templates', args=[fpath]), + ) + self.assertEqual(response.status_code, 400) + + @override_settings(TEMPLATES=[{ 'NAME': 'ONE', 'BACKEND': 'django.template.backends.django.DjangoTemplates', diff --git a/tests/file_storage/test_generate_filename.py b/tests/file_storage/test_generate_filename.py index b4222f412162..cb6465092047 100644 --- a/tests/file_storage/test_generate_filename.py +++ b/tests/file_storage/test_generate_filename.py @@ -1,7 +1,8 @@ import os +from django.core.exceptions import SuspiciousFileOperation from django.core.files.base import ContentFile -from django.core.files.storage import Storage +from django.core.files.storage import FileSystemStorage, Storage from django.db.models import FileField from django.test import SimpleTestCase @@ -36,6 +37,62 @@ def generate_filename(self, filename): class GenerateFilenameStorageTests(SimpleTestCase): + def test_storage_dangerous_paths(self): + candidates = [ + ('/tmp/..', '..'), + ('/tmp/.', '.'), + ('', ''), + ] + s = FileSystemStorage() + msg = "Could not derive file name from '%s'" + for file_name, base_name in candidates: + with self.subTest(file_name=file_name): + with self.assertRaisesMessage(SuspiciousFileOperation, msg % base_name): + s.get_available_name(file_name) + with self.assertRaisesMessage(SuspiciousFileOperation, msg % base_name): + s.generate_filename(file_name) + + def test_storage_dangerous_paths_dir_name(self): + file_name = '/tmp/../path' + s = FileSystemStorage() + msg = "Detected path traversal attempt in '/tmp/..'" + with self.assertRaisesMessage(SuspiciousFileOperation, msg): + s.get_available_name(file_name) + with self.assertRaisesMessage(SuspiciousFileOperation, msg): + s.generate_filename(file_name) + + def test_filefield_dangerous_filename(self): + candidates = [ + ('..', 'some/folder/..'), + ('.', 'some/folder/.'), + ('', 'some/folder/'), + ('???', '???'), + ('$.$.$', '$.$.$'), + ] + f = FileField(upload_to='some/folder/') + for file_name, msg_file_name in candidates: + msg = "Could not derive file name from '%s'" % msg_file_name + with self.subTest(file_name=file_name): + with self.assertRaisesMessage(SuspiciousFileOperation, msg): + f.generate_filename(None, file_name) + + def test_filefield_dangerous_filename_dot_segments(self): + f = FileField(upload_to='some/folder/') + msg = "Detected path traversal attempt in 'some/folder/../path'" + with self.assertRaisesMessage(SuspiciousFileOperation, msg): + f.generate_filename(None, '../path') + + def test_filefield_generate_filename_absolute_path(self): + f = FileField(upload_to='some/folder/') + candidates = [ + '/tmp/path', + '/tmp/../path', + ] + for file_name in candidates: + msg = "Detected path traversal attempt in '%s'" % file_name + with self.subTest(file_name=file_name): + with self.assertRaisesMessage(SuspiciousFileOperation, msg): + f.generate_filename(None, file_name) def test_filefield_generate_filename(self): f = FileField(upload_to='some/folder/') @@ -54,6 +111,57 @@ def upload_to(instance, filename): os.path.normpath('some/folder/test_with_space.txt') ) + def test_filefield_generate_filename_upload_to_overrides_dangerous_filename(self): + def upload_to(instance, filename): + return 'test.txt' + + f = FileField(upload_to=upload_to) + candidates = [ + '/tmp/.', + '/tmp/..', + '/tmp/../path', + '/tmp/path', + 'some/folder/', + 'some/folder/.', + 'some/folder/..', + 'some/folder/???', + 'some/folder/$.$.$', + 'some/../test.txt', + '', + ] + for file_name in candidates: + with self.subTest(file_name=file_name): + self.assertEqual(f.generate_filename(None, file_name), 'test.txt') + + def test_filefield_generate_filename_upload_to_absolute_path(self): + def upload_to(instance, filename): + return '/tmp/' + filename + + f = FileField(upload_to=upload_to) + candidates = [ + 'path', + '../path', + '???', + '$.$.$', + ] + for file_name in candidates: + msg = "Detected path traversal attempt in '/tmp/%s'" % file_name + with self.subTest(file_name=file_name): + with self.assertRaisesMessage(SuspiciousFileOperation, msg): + f.generate_filename(None, file_name) + + def test_filefield_generate_filename_upload_to_dangerous_filename(self): + def upload_to(instance, filename): + return '/tmp/' + filename + + f = FileField(upload_to=upload_to) + candidates = ['..', '.', ''] + for file_name in candidates: + msg = "Could not derive file name from '/tmp/%s'" % file_name + with self.subTest(file_name=file_name): + with self.assertRaisesMessage(SuspiciousFileOperation, msg): + f.generate_filename(None, file_name) + def test_filefield_awss3_storage(self): """ Simulate a FileField with an S3 storage which uses keys rather than diff --git a/tests/file_uploads/tests.py b/tests/file_uploads/tests.py index 2a08d1ba01bd..3afcbfd4ad60 100644 --- a/tests/file_uploads/tests.py +++ b/tests/file_uploads/tests.py @@ -8,8 +8,9 @@ from io import BytesIO, StringIO from urllib.parse import quote +from django.core.exceptions import SuspiciousFileOperation from django.core.files import temp as tempfile -from django.core.files.uploadedfile import SimpleUploadedFile +from django.core.files.uploadedfile import SimpleUploadedFile, UploadedFile from django.http.multipartparser import ( MultiPartParser, MultiPartParserError, parse_header, ) @@ -37,6 +38,16 @@ '../hax0rd.txt', # HTML entities. ] +CANDIDATE_INVALID_FILE_NAMES = [ + '/tmp/', # Directory, *nix-style. + 'c:\\tmp\\', # Directory, win-style. + '/tmp/.', # Directory dot, *nix-style. + 'c:\\tmp\\.', # Directory dot, *nix-style. + '/tmp/..', # Parent directory, *nix-style. + 'c:\\tmp\\..', # Parent directory, win-style. + '', # Empty filename. +] + @override_settings(MEDIA_ROOT=MEDIA_ROOT, ROOT_URLCONF='file_uploads.urls', MIDDLEWARE=[]) class FileUploadTests(TestCase): @@ -52,6 +63,22 @@ def tearDownClass(cls): shutil.rmtree(MEDIA_ROOT) super().tearDownClass() + def test_upload_name_is_validated(self): + candidates = [ + '/tmp/', + '/tmp/..', + '/tmp/.', + ] + if sys.platform == 'win32': + candidates.extend([ + 'c:\\tmp\\', + 'c:\\tmp\\..', + 'c:\\tmp\\.', + ]) + for file_name in candidates: + with self.subTest(file_name=file_name): + self.assertRaises(SuspiciousFileOperation, UploadedFile, name=file_name) + def test_simple_upload(self): with open(__file__, 'rb') as fp: post_data = { @@ -631,6 +658,15 @@ def test_sanitize_file_name(self): with self.subTest(file_name=file_name): self.assertEqual(parser.sanitize_file_name(file_name), 'hax0rd.txt') + def test_sanitize_invalid_file_name(self): + parser = MultiPartParser({ + 'CONTENT_TYPE': 'multipart/form-data; boundary=_foo', + 'CONTENT_LENGTH': '1', + }, StringIO('x'), [], 'utf-8') + for file_name in CANDIDATE_INVALID_FILE_NAMES: + with self.subTest(file_name=file_name): + self.assertIsNone(parser.sanitize_file_name(file_name)) + def test_rfc2231_parsing(self): test_data = ( (b"Content-Type: application/x-stuff; title*=us-ascii'en-us'This%20is%20%2A%2A%2Afun%2A%2A%2A", diff --git a/tests/forms_tests/field_tests/test_filefield.py b/tests/forms_tests/field_tests/test_filefield.py index fc5c4b5c1e1d..33574446f4cb 100644 --- a/tests/forms_tests/field_tests/test_filefield.py +++ b/tests/forms_tests/field_tests/test_filefield.py @@ -20,10 +20,12 @@ def test_filefield_1(self): f.clean(None, '') self.assertEqual('files/test2.pdf', f.clean(None, 'files/test2.pdf')) no_file_msg = "'No file was submitted. Check the encoding type on the form.'" + file = SimpleUploadedFile(None, b'') + file._name = '' with self.assertRaisesMessage(ValidationError, no_file_msg): - f.clean(SimpleUploadedFile('', b'')) + f.clean(file) with self.assertRaisesMessage(ValidationError, no_file_msg): - f.clean(SimpleUploadedFile('', b''), '') + f.clean(file, '') self.assertEqual('files/test3.pdf', f.clean(None, 'files/test3.pdf')) with self.assertRaisesMessage(ValidationError, no_file_msg): f.clean('some content that is not a file') diff --git a/tests/model_fields/test_filefield.py b/tests/model_fields/test_filefield.py index 9330a2eba25c..0afef7284ee5 100644 --- a/tests/model_fields/test_filefield.py +++ b/tests/model_fields/test_filefield.py @@ -1,8 +1,10 @@ import os import sys +import tempfile import unittest -from django.core.files import temp +from django.core.exceptions import SuspiciousFileOperation +from django.core.files import File, temp from django.core.files.base import ContentFile from django.core.files.uploadedfile import TemporaryUploadedFile from django.db.utils import IntegrityError @@ -59,6 +61,15 @@ def test_refresh_from_db(self): d.refresh_from_db() self.assertIs(d.myfile.instance, d) + @unittest.skipIf(sys.platform == 'win32', "Crashes with OSError on Windows.") + def test_save_without_name(self): + with tempfile.NamedTemporaryFile(suffix='.txt') as tmp: + document = Document.objects.create(myfile='something.txt') + document.myfile = File(tmp) + msg = "Detected path traversal attempt in '%s'" % tmp.name + with self.assertRaisesMessage(SuspiciousFileOperation, msg): + document.save() + def test_defer(self): Document.objects.create(myfile='something.txt') self.assertEqual(Document.objects.defer('myfile')[0].myfile, 'something.txt') diff --git a/tests/utils_tests/test_text.py b/tests/utils_tests/test_text.py index cab324d64edb..27e440b8566d 100644 --- a/tests/utils_tests/test_text.py +++ b/tests/utils_tests/test_text.py @@ -1,6 +1,7 @@ import json import sys +from django.core.exceptions import SuspiciousFileOperation from django.test import SimpleTestCase from django.utils import text from django.utils.functional import lazystr @@ -229,6 +230,13 @@ def test_get_valid_filename(self): filename = "^&'@{}[],$=!-#()%+~_123.txt" self.assertEqual(text.get_valid_filename(filename), "-_123.txt") self.assertEqual(text.get_valid_filename(lazystr(filename)), "-_123.txt") + msg = "Could not derive file name from '???'" + with self.assertRaisesMessage(SuspiciousFileOperation, msg): + text.get_valid_filename('???') + # After sanitizing this would yield '..'. + msg = "Could not derive file name from '$.$.$'" + with self.assertRaisesMessage(SuspiciousFileOperation, msg): + text.get_valid_filename('$.$.$') def test_compress_sequence(self): data = [{'key': i} for i in range(10)] diff --git a/tests/validators/invalid_urls.txt b/tests/validators/invalid_urls.txt index 4a092034ff66..a5a41ba8453e 100644 --- a/tests/validators/invalid_urls.txt +++ b/tests/validators/invalid_urls.txt @@ -46,6 +46,14 @@ http://1.1.1.1.1 http://123.123.123 http://3628126748 http://123 +http://000.000.000.000 +http://016.016.016.016 +http://192.168.000.001 +http://01.2.3.4 +http://01.2.3.4 +http://1.02.3.4 +http://1.2.03.4 +http://1.2.3.04 http://.www.foo.bar/ http://.www.foo.bar./ http://[::1:2::3]:8080/ diff --git a/tests/validators/tests.py b/tests/validators/tests.py index 36d0b2a520b3..1f09fb53fc5f 100644 --- a/tests/validators/tests.py +++ b/tests/validators/tests.py @@ -135,6 +135,16 @@ (validate_ipv4_address, '1.1.1.1\n', ValidationError), (validate_ipv4_address, '٧.2٥.3٣.243', ValidationError), + # Leading zeros are forbidden to avoid ambiguity with the octal notation. + (validate_ipv4_address, '000.000.000.000', ValidationError), + (validate_ipv4_address, '016.016.016.016', ValidationError), + (validate_ipv4_address, '192.168.000.001', ValidationError), + (validate_ipv4_address, '01.2.3.4', ValidationError), + (validate_ipv4_address, '01.2.3.4', ValidationError), + (validate_ipv4_address, '1.02.3.4', ValidationError), + (validate_ipv4_address, '1.2.03.4', ValidationError), + (validate_ipv4_address, '1.2.3.04', ValidationError), + # validate_ipv6_address uses django.utils.ipv6, which # is tested in much greater detail in its own testcase (validate_ipv6_address, 'fe80::1', None), @@ -160,6 +170,16 @@ (validate_ipv46_address, '::zzz', ValidationError), (validate_ipv46_address, '12345::', ValidationError), + # Leading zeros are forbidden to avoid ambiguity with the octal notation. + (validate_ipv46_address, '000.000.000.000', ValidationError), + (validate_ipv46_address, '016.016.016.016', ValidationError), + (validate_ipv46_address, '192.168.000.001', ValidationError), + (validate_ipv46_address, '01.2.3.4', ValidationError), + (validate_ipv46_address, '01.2.3.4', ValidationError), + (validate_ipv46_address, '1.02.3.4', ValidationError), + (validate_ipv46_address, '1.2.03.4', ValidationError), + (validate_ipv46_address, '1.2.3.04', ValidationError), + (validate_comma_separated_integer_list, '1', None), (validate_comma_separated_integer_list, '12', None), (validate_comma_separated_integer_list, '1,2', None), @@ -222,9 +242,15 @@ (URLValidator(EXTENDED_SCHEMES), 'git+ssh://git@github.com/example/hg-git.git', None), (URLValidator(EXTENDED_SCHEMES), 'git://-invalid.com', ValidationError), - # Trailing newlines not accepted + # Newlines and tabs are not accepted. (URLValidator(), 'http://www.djangoproject.com/\n', ValidationError), (URLValidator(), 'http://[::ffff:192.9.5.5]\n', ValidationError), + (URLValidator(), 'http://www.djangoproject.com/\r', ValidationError), + (URLValidator(), 'http://[::ffff:192.9.5.5]\r', ValidationError), + (URLValidator(), 'http://www.django\rproject.com/', ValidationError), + (URLValidator(), 'http://[::\rffff:192.9.5.5]', ValidationError), + (URLValidator(), 'http://\twww.djangoproject.com/', ValidationError), + (URLValidator(), 'http://\t[::ffff:192.9.5.5]', ValidationError), # Trailing junk does not take forever to reject (URLValidator(), 'http://www.asdasdasdasdsadfm.com.br ', ValidationError), (URLValidator(), 'http://www.asdasdasdasdsadfm.com.br z', ValidationError), diff --git a/tests/validators/valid_urls.txt b/tests/validators/valid_urls.txt index f79f94814291..ef9e563f8e6d 100644 --- a/tests/validators/valid_urls.txt +++ b/tests/validators/valid_urls.txt @@ -63,6 +63,12 @@ http://0.0.0.0/ http://255.255.255.255 http://224.0.0.0 http://224.1.1.1 +http://111.112.113.114/ +http://88.88.88.88/ +http://11.12.13.14/ +http://10.20.30.40/ +http://1.2.3.4/ +http://127.0.01.09.home.lan http://aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa.example.com http://example.aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa.com http://example.aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa