From b7679a50f31ef63614da21a6ecda9e4ff43a5754 Mon Sep 17 00:00:00 2001 From: Markus Unterwaditzer Date: Mon, 23 Mar 2020 17:24:22 +0100 Subject: [PATCH 01/17] fix: Test transport rate limits parsing and enforcement (#652) Also fix a bug where missing categories ("123::project") would not enforce a rate limit for all categories, as they were parsed as category "" instead of category None. --- sentry_sdk/transport.py | 39 +++++++++----- tests/test_transport.py | 115 +++++++++++++++++++++++++++++++++++++++- 2 files changed, 139 insertions(+), 15 deletions(-) diff --git a/sentry_sdk/transport.py b/sentry_sdk/transport.py index 60ab611c54..6d6a1c1f91 100644 --- a/sentry_sdk/transport.py +++ b/sentry_sdk/transport.py @@ -15,17 +15,22 @@ from sentry_sdk._types import MYPY if MYPY: - from typing import Type from typing import Any - from typing import Optional + from typing import Callable from typing import Dict + from typing import Iterable + from typing import Optional + from typing import Tuple + from typing import Type from typing import Union - from typing import Callable + from urllib3.poolmanager import PoolManager # type: ignore from urllib3.poolmanager import ProxyManager from sentry_sdk._types import Event + DataCategory = Optional[str] + try: from urllib.request import getproxies except ImportError: @@ -94,6 +99,21 @@ def __del__(self): pass +def _parse_rate_limits(header, now=None): + # type: (Any, Optional[datetime]) -> Iterable[Tuple[DataCategory, datetime]] + if now is None: + now = datetime.utcnow() + + for limit in header.split(","): + try: + retry_after, categories, _ = limit.strip().split(":", 2) + retry_after = now + timedelta(seconds=int(retry_after)) + for category in categories and categories.split(";") or (None,): + yield category, retry_after + except (LookupError, ValueError): + continue + + class HttpTransport(Transport): """The default HTTP transport.""" @@ -107,7 +127,7 @@ def __init__( assert self.parsed_dsn is not None self._worker = BackgroundWorker() self._auth = self.parsed_dsn.to_auth("sentry.python/%s" % VERSION) - self._disabled_until = {} # type: Dict[Any, datetime] + self._disabled_until = {} # type: Dict[DataCategory, datetime] self._retry = urllib3.util.Retry() self.options = options @@ -129,16 +149,7 @@ def _update_rate_limits(self, response): # no matter of the status code to update our internal rate limits. header = response.headers.get("x-sentry-rate-limit") if header: - for limit in header.split(","): - try: - retry_after, categories, _ = limit.strip().split(":", 2) - retry_after = datetime.utcnow() + timedelta( - seconds=int(retry_after) - ) - for category in categories.split(";") or (None,): - self._disabled_until[category] = retry_after - except (LookupError, ValueError): - continue + self._disabled_until.update(_parse_rate_limits(header)) # old sentries only communicate global rate limit hits via the # retry-after header on 429. This header can also be emitted on new diff --git a/tests/test_transport.py b/tests/test_transport.py index 00cdc6c42e..398ff0a6da 100644 --- a/tests/test_transport.py +++ b/tests/test_transport.py @@ -2,11 +2,12 @@ import logging import pickle -from datetime import datetime +from datetime import datetime, timedelta import pytest from sentry_sdk import Hub, Client, add_breadcrumb, capture_message +from sentry_sdk.transport import _parse_rate_limits @pytest.fixture(params=[True, False]) @@ -54,3 +55,115 @@ def test_transport_works( assert httpserver.requests assert any("Sending event" in record.msg for record in caplog.records) == debug + + +NOW = datetime(2014, 6, 2) + + +@pytest.mark.parametrize( + "input,expected", + [ + # Invalid rate limits + ("", {}), + ("invalid", {}), + (",,,", {}), + ( + "42::organization, invalid, 4711:foobar;transaction;security:project", + { + None: NOW + timedelta(seconds=42), + "transaction": NOW + timedelta(seconds=4711), + "security": NOW + timedelta(seconds=4711), + # Unknown data categories + "foobar": NOW + timedelta(seconds=4711), + }, + ), + ( + "4711:foobar;;transaction:organization", + { + "transaction": NOW + timedelta(seconds=4711), + # Unknown data categories + "foobar": NOW + timedelta(seconds=4711), + "": NOW + timedelta(seconds=4711), + }, + ), + ], +) +def test_parse_rate_limits(input, expected): + assert dict(_parse_rate_limits(input, now=NOW)) == expected + + +def test_simple_rate_limits(httpserver, capsys, caplog): + client = Client(dsn="http://foobar@{}/123".format(httpserver.url[len("http://") :])) + httpserver.serve_content("no", 429, headers={"Retry-After": "4"}) + + client.capture_event({"type": "transaction"}) + client.flush() + + assert len(httpserver.requests) == 1 + del httpserver.requests[:] + + assert set(client.transport._disabled_until) == set([None]) + + client.capture_event({"type": "transaction"}) + client.capture_event({"type": "event"}) + client.flush() + + assert not httpserver.requests + + +@pytest.mark.parametrize("response_code", [200, 429]) +def test_data_category_limits(httpserver, capsys, caplog, response_code): + client = Client( + dict(dsn="http://foobar@{}/123".format(httpserver.url[len("http://") :])) + ) + httpserver.serve_content( + "hm", + response_code, + headers={"X-Sentry-Rate-Limit": "4711:transaction:organization"}, + ) + + client.capture_event({"type": "transaction"}) + client.flush() + + assert len(httpserver.requests) == 1 + del httpserver.requests[:] + + assert set(client.transport._disabled_until) == set(["transaction"]) + + client.transport.capture_event({"type": "transaction"}) + client.transport.capture_event({"type": "transaction"}) + client.flush() + + assert not httpserver.requests + + client.capture_event({"type": "event"}) + client.flush() + + assert len(httpserver.requests) == 1 + + +@pytest.mark.parametrize("response_code", [200, 429]) +def test_complex_limits_without_data_category( + httpserver, capsys, caplog, response_code +): + client = Client( + dict(dsn="http://foobar@{}/123".format(httpserver.url[len("http://") :])) + ) + httpserver.serve_content( + "hm", response_code, headers={"X-Sentry-Rate-Limit": "4711::organization"}, + ) + + client.capture_event({"type": "transaction"}) + client.flush() + + assert len(httpserver.requests) == 1 + del httpserver.requests[:] + + assert set(client.transport._disabled_until) == set([None]) + + client.transport.capture_event({"type": "transaction"}) + client.transport.capture_event({"type": "transaction"}) + client.capture_event({"type": "event"}) + client.flush() + + assert len(httpserver.requests) == 0 From 44346360312fb3419bfd07927794e12102d45317 Mon Sep 17 00:00:00 2001 From: Markus Unterwaditzer Date: Wed, 25 Mar 2020 13:39:16 +0100 Subject: [PATCH 02/17] fix: Fix infinite loop in transport (#656) Fix #655 --- sentry_sdk/integrations/logging.py | 8 +++++++- tests/test_transport.py | 18 ++++++++++++++++++ 2 files changed, 25 insertions(+), 1 deletion(-) diff --git a/sentry_sdk/integrations/logging.py b/sentry_sdk/integrations/logging.py index 6edd785e91..c25aef4c09 100644 --- a/sentry_sdk/integrations/logging.py +++ b/sentry_sdk/integrations/logging.py @@ -24,7 +24,13 @@ DEFAULT_LEVEL = logging.INFO DEFAULT_EVENT_LEVEL = logging.ERROR -_IGNORED_LOGGERS = set(["sentry_sdk.errors"]) +# Capturing events from those loggers causes recursion errors. We cannot allow +# the user to unconditionally create events from those loggers under any +# circumstances. +# +# Note: Ignoring by logger name here is better than mucking with thread-locals. +# We do not necessarily know whether thread-locals work 100% correctly in the user's environment. +_IGNORED_LOGGERS = set(["sentry_sdk.errors", "urllib3.connectionpool"]) def ignore_logger( diff --git a/tests/test_transport.py b/tests/test_transport.py index 398ff0a6da..6f8e7fa9d9 100644 --- a/tests/test_transport.py +++ b/tests/test_transport.py @@ -8,6 +8,7 @@ from sentry_sdk import Hub, Client, add_breadcrumb, capture_message from sentry_sdk.transport import _parse_rate_limits +from sentry_sdk.integrations.logging import LoggingIntegration @pytest.fixture(params=[True, False]) @@ -57,6 +58,23 @@ def test_transport_works( assert any("Sending event" in record.msg for record in caplog.records) == debug +def test_transport_infinite_loop(httpserver, request): + httpserver.serve_content("ok", 200) + + client = Client( + "http://foobar@{}/123".format(httpserver.url[len("http://") :]), + debug=True, + # Make sure we cannot create events from our own logging + integrations=[LoggingIntegration(event_level=logging.DEBUG)], + ) + + with Hub(client): + capture_message("hi") + client.flush() + + assert len(httpserver.requests) == 1 + + NOW = datetime(2014, 6, 2) From 301141d87dfa690fe34ab1e11a34c54325cfe13c Mon Sep 17 00:00:00 2001 From: Markus Unterwaditzer Date: Wed, 25 Mar 2020 13:39:26 +0100 Subject: [PATCH 03/17] fix: Fix typo in header name (#657) --- sentry_sdk/transport.py | 2 +- tests/test_transport.py | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/sentry_sdk/transport.py b/sentry_sdk/transport.py index 6d6a1c1f91..c6f926a353 100644 --- a/sentry_sdk/transport.py +++ b/sentry_sdk/transport.py @@ -147,7 +147,7 @@ def _update_rate_limits(self, response): # new sentries with more rate limit insights. We honor this header # no matter of the status code to update our internal rate limits. - header = response.headers.get("x-sentry-rate-limit") + header = response.headers.get("x-sentry-rate-limits") if header: self._disabled_until.update(_parse_rate_limits(header)) diff --git a/tests/test_transport.py b/tests/test_transport.py index 6f8e7fa9d9..05dd47f612 100644 --- a/tests/test_transport.py +++ b/tests/test_transport.py @@ -137,7 +137,7 @@ def test_data_category_limits(httpserver, capsys, caplog, response_code): httpserver.serve_content( "hm", response_code, - headers={"X-Sentry-Rate-Limit": "4711:transaction:organization"}, + headers={"X-Sentry-Rate-Limits": "4711:transaction:organization"}, ) client.capture_event({"type": "transaction"}) @@ -168,7 +168,7 @@ def test_complex_limits_without_data_category( dict(dsn="http://foobar@{}/123".format(httpserver.url[len("http://") :])) ) httpserver.serve_content( - "hm", response_code, headers={"X-Sentry-Rate-Limit": "4711::organization"}, + "hm", response_code, headers={"X-Sentry-Rate-Limits": "4711::organization"}, ) client.capture_event({"type": "transaction"}) From f49d62009dff47bc98fb01da78dcc127ff34235b Mon Sep 17 00:00:00 2001 From: Tatiana Vasilevskaya Date: Tue, 31 Mar 2020 14:35:15 +0200 Subject: [PATCH 04/17] Fix bug in _update_scope() (#662) Introduced in e680a75 --- sentry_sdk/hub.py | 2 +- tests/test_basics.py | 10 ++++++++++ 2 files changed, 11 insertions(+), 1 deletion(-) diff --git a/sentry_sdk/hub.py b/sentry_sdk/hub.py index f0060b9d79..18558761cf 100644 --- a/sentry_sdk/hub.py +++ b/sentry_sdk/hub.py @@ -67,7 +67,7 @@ def _update_scope(base, scope_change, scope_kwargs): final_scope.update_from_scope(scope_change) elif scope_kwargs: final_scope = copy.copy(base) - final_scope.update_from_kwargs(scope_kwargs) + final_scope.update_from_kwargs(**scope_kwargs) else: final_scope = base return final_scope diff --git a/tests/test_basics.py b/tests/test_basics.py index 8953dc8803..3e5bbf0fc6 100644 --- a/tests/test_basics.py +++ b/tests/test_basics.py @@ -6,6 +6,7 @@ Client, push_scope, configure_scope, + capture_event, capture_exception, capture_message, add_breadcrumb, @@ -312,3 +313,12 @@ def bar(event, hint): (event,) = events assert event["message"] == "hifoobarbaz" + + +def test_capture_event_with_scope_kwargs(sentry_init, capture_events): + sentry_init(debug=True) + events = capture_events() + capture_event({}, level="info", extras={"foo": "bar"}) + (event,) = events + assert event["level"] == "info" + assert event["extra"]["foo"] == "bar" From d9ffe894a778e4db04bdfd3339d61977e55f48a2 Mon Sep 17 00:00:00 2001 From: Markus Unterwaditzer Date: Tue, 31 Mar 2020 21:54:37 +0200 Subject: [PATCH 05/17] fix: Fix typo in extras_require, fix #663 --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 045532e7df..bb5314a26f 100644 --- a/setup.py +++ b/setup.py @@ -32,7 +32,7 @@ "sanic": ["sanic>=0.8"], "celery": ["celery>=3"], "beam": ["beam>=2.12"], - "rq": ["0.6"], + "rq": ["rq>=0.6"], "aiohttp": ["aiohttp>=3.5"], "tornado": ["tornado>=5"], "sqlalchemy": ["sqlalchemy>=1.2"], From cd646579d04e2fad6a8994304314ac52fec2f83c Mon Sep 17 00:00:00 2001 From: Markus Unterwaditzer Date: Fri, 3 Apr 2020 09:01:53 +0200 Subject: [PATCH 06/17] fix: Prevent sending infinity in envelopes (#664) --- sentry_sdk/envelope.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/sentry_sdk/envelope.py b/sentry_sdk/envelope.py index fd08553249..701b84a649 100644 --- a/sentry_sdk/envelope.py +++ b/sentry_sdk/envelope.py @@ -86,7 +86,7 @@ def serialize_into( self, f # type: Any ): # type: (...) -> None - f.write(json.dumps(self.headers).encode("utf-8")) + f.write(json.dumps(self.headers, allow_nan=False).encode("utf-8")) f.write(b"\n") for item in self.items: item.serialize_into(f) @@ -142,7 +142,7 @@ def get_bytes(self): with open(self.path, "rb") as f: self.bytes = f.read() elif self.json is not None: - self.bytes = json.dumps(self.json).encode("utf-8") + self.bytes = json.dumps(self.json, allow_nan=False).encode("utf-8") else: self.bytes = b"" return self.bytes @@ -256,7 +256,7 @@ def serialize_into( headers = dict(self.headers) length, writer = self.payload._prepare_serialize() headers["length"] = length - f.write(json.dumps(headers).encode("utf-8")) + f.write(json.dumps(headers, allow_nan=False).encode("utf-8")) f.write(b"\n") writer(f) f.write(b"\n") From 8bd8044de7107c20b5318462142becb5b75c6315 Mon Sep 17 00:00:00 2001 From: Armin Ronacher Date: Fri, 17 Apr 2020 13:50:53 +0200 Subject: [PATCH 07/17] ref: Only send 100 sessions in one envelope (#669) --- sentry_sdk/client.py | 10 ++++++++-- sentry_sdk/sessions.py | 3 +++ 2 files changed, 11 insertions(+), 2 deletions(-) diff --git a/sentry_sdk/client.py b/sentry_sdk/client.py index c0fb8422d8..036fc48340 100644 --- a/sentry_sdk/client.py +++ b/sentry_sdk/client.py @@ -2,6 +2,7 @@ import uuid import random from datetime import datetime +from itertools import islice import socket from sentry_sdk._compat import string_types, text_type, iteritems @@ -99,10 +100,15 @@ def _init_impl(self): def _send_sessions(sessions): # type: (List[Any]) -> None transport = self.transport - if sessions and transport: + if not transport or not sessions: + return + sessions_iter = iter(sessions) + while True: envelope = Envelope() - for session in sessions: + for session in islice(sessions_iter, 100): envelope.add_session(session) + if not envelope.items: + break transport.capture_envelope(envelope) try: diff --git a/sentry_sdk/sessions.py b/sentry_sdk/sessions.py index f4f7137cc0..b8ef201e2a 100644 --- a/sentry_sdk/sessions.py +++ b/sentry_sdk/sessions.py @@ -170,6 +170,7 @@ def update( sid=None, # type: Optional[Union[str, uuid.UUID]] did=None, # type: Optional[str] timestamp=None, # type: Optional[datetime] + started=None, # type: Optional[datetime] duration=None, # type: Optional[float] status=None, # type: Optional[SessionStatus] release=None, # type: Optional[str] @@ -194,6 +195,8 @@ def update( if timestamp is None: timestamp = datetime.utcnow() self.timestamp = timestamp + if started is not None: + self.started = started if duration is not None: self.duration = duration if release is not None: From b866e9b649723a551f19a7177aefe5ce7c190940 Mon Sep 17 00:00:00 2001 From: Markus Unterwaditzer Date: Mon, 20 Apr 2020 10:07:32 +0200 Subject: [PATCH 08/17] fix: Flask-dev dropped Python 2 (#671) --- tox.ini | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tox.ini b/tox.ini index 1dbe7025a4..a11e506585 100644 --- a/tox.ini +++ b/tox.ini @@ -20,7 +20,8 @@ envlist = {pypy,py2.7}-django-1.7 {pypy,py2.7}-django-1.6 - {pypy,py2.7,py3.5,py3.6,py3.7,py3.8}-flask-{1.1,1.0,0.11,0.12,dev} + {pypy,py2.7,py3.5,py3.6,py3.7,py3.8}-flask-{1.1,1.0,0.11,0.12} + {py3.6,py3.7,py3.8}-flask-{1.1,1.0,0.11,0.12,dev} {pypy,py2.7,py3.5,py3.6,py3.7,py3.8}-bottle-0.12 From f90cb062bfc3c675f25b68f71f2375bbe48bfe06 Mon Sep 17 00:00:00 2001 From: Markus Unterwaditzer Date: Mon, 20 Apr 2020 13:07:51 +0200 Subject: [PATCH 09/17] ref: reformat tox.ini --- tox.ini | 34 +++++++++++++++++++--------------- 1 file changed, 19 insertions(+), 15 deletions(-) diff --git a/tox.ini b/tox.ini index a11e506585..14f2a08d8d 100644 --- a/tox.ini +++ b/tox.ini @@ -11,14 +11,19 @@ envlist = # === Integrations === - # Formatting: 1 blank line between different integrations. - - py{3.7,3.8}-django-{2.2,3.0,dev} + # General format is {pythonversion}-{integrationname}-{frameworkversion} + # 1 blank line between different integrations + # Each framework version should only be mentioned once. I.e: + # {py2.7,py3.7}-django-{1.11} + # {py3.7}-django-{2.2} + # instead of: + # {py2.7}-django-{1.11} + # {py2.7,py3.7}-django-{1.11,2.2} + + {pypy,py2.7}-django-{1.6,1.7} + {pypy,py2.7,py3.5}-django-{1.8,1.9,1.10,1.11} {py3.5,py3.6,py3.7}-django-{2.0,2.1} - {pypy,py2.7,py3.5}-django-1.11 - {pypy,py2.7,py3.5}-django-{1.8,1.9,1.10} - {pypy,py2.7}-django-1.7 - {pypy,py2.7}-django-1.6 + {py3.7,py3.8}-django-{2.2,3.0,dev} {pypy,py2.7,py3.5,py3.6,py3.7,py3.8}-flask-{1.1,1.0,0.11,0.12} {py3.6,py3.7,py3.8}-flask-{1.1,1.0,0.11,0.12,dev} @@ -28,14 +33,13 @@ envlist = {pypy,py2.7,py3.5,py3.6,py3.7}-falcon-1.4 {pypy,py2.7,py3.5,py3.6,py3.7,py3.8}-falcon-2.0 - py3.5-sanic-{0.8,18} - {py3.6,py3.7}-sanic-{0.8,18,19} + {py3.5,py3.6,py3.7}-sanic-{0.8,18} + {py3.6,py3.7}-sanic-19 {pypy,py2.7,py3.5,py3.6,py3.7,py3.8}-celery-{4.1,4.2,4.3,4.4} {pypy,py2.7}-celery-3 - py2.7-beam-{2.12,2.13} - py3.7-beam-{2.12,2.13} + {py2.7,py3.7}-beam-{2.12,2.13} # The aws_lambda tests deploy to the real AWS and have their own matrix of Python versions. py3.7-aws_lambda @@ -46,13 +50,13 @@ envlist = {pypy,py2.7,py3.5,py3.6,py3.7,py3.8}-rq-{0.12,0.13,1.0,1.1,1.2,1.3} py3.7-aiohttp-3.5 - py{3.7,3.8}-aiohttp-3.6 + {py3.7,py3.8}-aiohttp-3.6 {py3.7,py3.8}-tornado-{5,6} - {py3.4}-trytond-{4.6,4.8,5.0} - {py3.5}-trytond-{4.6,4.8,5.0,5.2} - {py3.6,py3.7,py3.8}-trytond-{4.6,4.8,5.0,5.2,5.4} + {py3.4,py3.5,py3.6,py3.7,py3.8}-trytond-{4.6,4.8,5.0} + {py3.5,py3.6,py3.7,py3.8}-trytond-{5.2} + {py3.6,py3.7,py3.8}-trytond-{5.4} {py2.7,py3.8}-requests From d617e54688790bfad99deabf7be0f3e9b247d93f Mon Sep 17 00:00:00 2001 From: Hoel IRIS Date: Mon, 20 Apr 2020 21:37:08 +0200 Subject: [PATCH 10/17] fix: Preserve contextvars in aiohttp integration (#674) aiohttp integration currently re-create a task to encapsulate the request handler. But: - aiohttp already does it. - contextvars created in it can't be read by aiohttp. It's an issue for users custom logger. Fix #670 --- sentry_sdk/integrations/aiohttp.py | 75 ++++++++++++++---------------- 1 file changed, 35 insertions(+), 40 deletions(-) diff --git a/sentry_sdk/integrations/aiohttp.py b/sentry_sdk/integrations/aiohttp.py index 02c76df7ef..c00a07d2b2 100644 --- a/sentry_sdk/integrations/aiohttp.py +++ b/sentry_sdk/integrations/aiohttp.py @@ -71,46 +71,41 @@ def setup_once(): async def sentry_app_handle(self, request, *args, **kwargs): # type: (Any, Request, *Any, **Any) -> Any - async def inner(): - # type: () -> Any - hub = Hub.current - if hub.get_integration(AioHttpIntegration) is None: - return await old_handle(self, request, *args, **kwargs) - - weak_request = weakref.ref(request) - - with Hub(Hub.current) as hub: - with hub.configure_scope() as scope: - scope.clear_breadcrumbs() - scope.add_event_processor(_make_request_processor(weak_request)) - - span = Span.continue_from_headers(request.headers) - span.op = "http.server" - # If this transaction name makes it to the UI, AIOHTTP's - # URL resolver did not find a route or died trying. - span.transaction = "generic AIOHTTP request" - - with hub.start_span(span): - try: - response = await old_handle(self, request) - except HTTPException as e: - span.set_http_status(e.status_code) - raise - except asyncio.CancelledError: - span.set_status("cancelled") - raise - except Exception: - # This will probably map to a 500 but seems like we - # have no way to tell. Do not set span status. - reraise(*_capture_exception(hub)) - - span.set_http_status(response.status) - return response - - # Explicitly wrap in task such that current contextvar context is - # copied. Just doing `return await inner()` will leak scope data - # between requests. - return await asyncio.get_event_loop().create_task(inner()) + hub = Hub.current + if hub.get_integration(AioHttpIntegration) is None: + return await old_handle(self, request, *args, **kwargs) + + weak_request = weakref.ref(request) + + with Hub(Hub.current) as hub: + # Scope data will not leak between requests because aiohttp + # create a task to wrap each request. + with hub.configure_scope() as scope: + scope.clear_breadcrumbs() + scope.add_event_processor(_make_request_processor(weak_request)) + + span = Span.continue_from_headers(request.headers) + span.op = "http.server" + # If this transaction name makes it to the UI, AIOHTTP's + # URL resolver did not find a route or died trying. + span.transaction = "generic AIOHTTP request" + + with hub.start_span(span): + try: + response = await old_handle(self, request) + except HTTPException as e: + span.set_http_status(e.status_code) + raise + except asyncio.CancelledError: + span.set_status("cancelled") + raise + except Exception: + # This will probably map to a 500 but seems like we + # have no way to tell. Do not set span status. + reraise(*_capture_exception(hub)) + + span.set_http_status(response.status) + return response Application._handle = sentry_app_handle From 0da369f839ee2c383659c91ea8858abcac04b869 Mon Sep 17 00:00:00 2001 From: "dependabot-preview[bot]" <27856297+dependabot-preview[bot]@users.noreply.github.com> Date: Mon, 20 Apr 2020 21:38:00 +0200 Subject: [PATCH 11/17] build(deps): bump sphinx from 2.3.1 to 3.0.2 (#672) Bumps [sphinx](https://github.com/sphinx-doc/sphinx) from 2.3.1 to 3.0.2. - [Release notes](https://github.com/sphinx-doc/sphinx/releases) - [Changelog](https://github.com/sphinx-doc/sphinx/blob/3.x/CHANGES) - [Commits](https://github.com/sphinx-doc/sphinx/compare/v2.3.1...v3.0.2) Signed-off-by: dependabot-preview[bot] Co-authored-by: dependabot-preview[bot] <27856297+dependabot-preview[bot]@users.noreply.github.com> --- docs-requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs-requirements.txt b/docs-requirements.txt index 78b98c5047..c6cd071555 100644 --- a/docs-requirements.txt +++ b/docs-requirements.txt @@ -1,4 +1,4 @@ -sphinx==2.3.1 +sphinx==3.0.2 sphinx-rtd-theme sphinx-autodoc-typehints[type_comments]>=1.8.0 typing-extensions From 55b1df77a39c9eb844d888e1ada95356fc0c2b81 Mon Sep 17 00:00:00 2001 From: Markus Unterwaditzer Date: Tue, 28 Apr 2020 11:19:18 +0200 Subject: [PATCH 12/17] fix: Pin pytest-asyncio (#681) --- tox.ini | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tox.ini b/tox.ini index 14f2a08d8d..67e957d2ae 100644 --- a/tox.ini +++ b/tox.ini @@ -74,7 +74,7 @@ deps = django-{1.11,2.0,2.1,2.2,3.0}: djangorestframework>=3.0.0,<4.0.0 py3.7-django-{1.11,2.0,2.1,2.2,3.0}: channels>2 - py3.7-django-{1.11,2.0,2.1,2.2,3.0}: pytest-asyncio + py3.7-django-{1.11,2.0,2.1,2.2,3.0}: pytest-asyncio==0.10.0 {py2.7,py3.7}-django-{1.11,2.2,3.0}: psycopg2-binary django-{1.6,1.7,1.8}: pytest-django<3.0 From b8f7953d097d89b97fd341e3676f2283aa2e9728 Mon Sep 17 00:00:00 2001 From: "dependabot-preview[bot]" <27856297+dependabot-preview[bot]@users.noreply.github.com> Date: Tue, 28 Apr 2020 09:39:55 +0000 Subject: [PATCH 13/17] build(deps): bump sphinx from 3.0.2 to 3.0.3 (#680) --- docs-requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs-requirements.txt b/docs-requirements.txt index c6cd071555..d9bb629201 100644 --- a/docs-requirements.txt +++ b/docs-requirements.txt @@ -1,4 +1,4 @@ -sphinx==3.0.2 +sphinx==3.0.3 sphinx-rtd-theme sphinx-autodoc-typehints[type_comments]>=1.8.0 typing-extensions From f46373c220eb7af816c946dcd8decd0cb79276b1 Mon Sep 17 00:00:00 2001 From: Reece Dunham Date: Mon, 11 May 2020 02:54:28 -0400 Subject: [PATCH 14/17] Clarify console warning (#684) --- sentry_sdk/_compat.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sentry_sdk/_compat.py b/sentry_sdk/_compat.py index 4db5f44c33..e7933e53da 100644 --- a/sentry_sdk/_compat.py +++ b/sentry_sdk/_compat.py @@ -87,6 +87,6 @@ def check_thread_support(): "We detected the use of uwsgi with disabled threads. " "This will cause issues with the transport you are " "trying to use. Please enable threading for uwsgi. " - '(Enable the "enable-threads" flag).' + '(Add the "enable-threads" flag).' ) ) From 26ecc05688fb52876978db9973f40d68ad0f09b8 Mon Sep 17 00:00:00 2001 From: Markus Unterwaditzer Date: Wed, 13 May 2020 11:41:32 +0200 Subject: [PATCH 15/17] fix(celery): Vendor parts of functools to avoid conflict with newrelic (#685) --- sentry_sdk/_functools.py | 66 ++++++++++++++++++++ sentry_sdk/integrations/asgi.py | 6 +- sentry_sdk/integrations/beam.py | 2 +- sentry_sdk/integrations/celery.py | 8 +-- sentry_sdk/integrations/django/middleware.py | 6 +- sentry_sdk/integrations/serverless.py | 4 +- sentry_sdk/integrations/wsgi.py | 6 +- sentry_sdk/scope.py | 4 +- test-requirements.txt | 1 + tests/integrations/celery/test_celery.py | 27 ++++++++ 10 files changed, 109 insertions(+), 21 deletions(-) create mode 100644 sentry_sdk/_functools.py diff --git a/sentry_sdk/_functools.py b/sentry_sdk/_functools.py new file mode 100644 index 0000000000..a5abeebf52 --- /dev/null +++ b/sentry_sdk/_functools.py @@ -0,0 +1,66 @@ +""" +A backport of Python 3 functools to Python 2/3. The only important change +we rely upon is that `update_wrapper` handles AttributeError gracefully. +""" + +from functools import partial + +from sentry_sdk._types import MYPY + +if MYPY: + from typing import Any + from typing import Callable + + +WRAPPER_ASSIGNMENTS = ( + "__module__", + "__name__", + "__qualname__", + "__doc__", + "__annotations__", +) +WRAPPER_UPDATES = ("__dict__",) + + +def update_wrapper( + wrapper, wrapped, assigned=WRAPPER_ASSIGNMENTS, updated=WRAPPER_UPDATES +): + # type: (Any, Any, Any, Any) -> Any + """Update a wrapper function to look like the wrapped function + + wrapper is the function to be updated + wrapped is the original function + assigned is a tuple naming the attributes assigned directly + from the wrapped function to the wrapper function (defaults to + functools.WRAPPER_ASSIGNMENTS) + updated is a tuple naming the attributes of the wrapper that + are updated with the corresponding attribute from the wrapped + function (defaults to functools.WRAPPER_UPDATES) + """ + for attr in assigned: + try: + value = getattr(wrapped, attr) + except AttributeError: + pass + else: + setattr(wrapper, attr, value) + for attr in updated: + getattr(wrapper, attr).update(getattr(wrapped, attr, {})) + # Issue #17482: set __wrapped__ last so we don't inadvertently copy it + # from the wrapped function when updating __dict__ + wrapper.__wrapped__ = wrapped + # Return the wrapper so this can be used as a decorator via partial() + return wrapper + + +def wraps(wrapped, assigned=WRAPPER_ASSIGNMENTS, updated=WRAPPER_UPDATES): + # type: (Callable[..., Any], Any, Any) -> Callable[[Callable[..., Any]], Callable[..., Any]] + """Decorator factory to apply update_wrapper() to a wrapper function + + Returns a decorator that invokes update_wrapper() with the decorated + function as the wrapper argument and the arguments to wraps() as the + remaining arguments. Default arguments are as for update_wrapper(). + This is a convenience function to simplify applying partial() to + update_wrapper(). + """ + return partial(update_wrapper, wrapped=wrapped, assigned=assigned, updated=updated) diff --git a/sentry_sdk/integrations/asgi.py b/sentry_sdk/integrations/asgi.py index 762634f82f..25201ccf31 100644 --- a/sentry_sdk/integrations/asgi.py +++ b/sentry_sdk/integrations/asgi.py @@ -5,10 +5,10 @@ """ import asyncio -import functools import inspect import urllib +from sentry_sdk._functools import partial from sentry_sdk._types import MYPY from sentry_sdk.hub import Hub, _should_send_default_pii from sentry_sdk.integrations._wsgi_common import _filter_headers @@ -92,9 +92,7 @@ async def _run_app(self, scope, callback): with hub.configure_scope() as sentry_scope: sentry_scope.clear_breadcrumbs() sentry_scope._name = "asgi" - processor = functools.partial( - self.event_processor, asgi_scope=scope - ) + processor = partial(self.event_processor, asgi_scope=scope) sentry_scope.add_event_processor(processor) if scope["type"] in ("http", "websocket"): diff --git a/sentry_sdk/integrations/beam.py b/sentry_sdk/integrations/beam.py index 7252746a7f..be1615dc4b 100644 --- a/sentry_sdk/integrations/beam.py +++ b/sentry_sdk/integrations/beam.py @@ -2,7 +2,7 @@ import sys import types -from functools import wraps +from sentry_sdk._functools import wraps from sentry_sdk.hub import Hub from sentry_sdk._compat import reraise diff --git a/sentry_sdk/integrations/celery.py b/sentry_sdk/integrations/celery.py index 9b58796173..5ac0d32f40 100644 --- a/sentry_sdk/integrations/celery.py +++ b/sentry_sdk/integrations/celery.py @@ -1,6 +1,5 @@ from __future__ import absolute_import -import functools import sys from sentry_sdk.hub import Hub @@ -10,6 +9,7 @@ from sentry_sdk.integrations import Integration, DidNotEnable from sentry_sdk.integrations.logging import ignore_logger from sentry_sdk._types import MYPY +from sentry_sdk._functools import wraps if MYPY: from typing import Any @@ -87,7 +87,7 @@ def sentry_build_tracer(name, task, *args, **kwargs): def _wrap_apply_async(task, f): # type: (Any, F) -> F - @functools.wraps(f) + @wraps(f) def apply_async(*args, **kwargs): # type: (*Any, **Any) -> Any hub = Hub.current @@ -118,7 +118,7 @@ def _wrap_tracer(task, f): # This is the reason we don't use signals for hooking in the first place. # Also because in Celery 3, signal dispatch returns early if one handler # crashes. - @functools.wraps(f) + @wraps(f) def _inner(*args, **kwargs): # type: (*Any, **Any) -> Any hub = Hub.current @@ -157,7 +157,7 @@ def _wrap_task_call(task, f): # functools.wraps is important here because celery-once looks at this # method's name. # https://github.com/getsentry/sentry-python/issues/421 - @functools.wraps(f) + @wraps(f) def _inner(*args, **kwargs): # type: (*Any, **Any) -> Any try: diff --git a/sentry_sdk/integrations/django/middleware.py b/sentry_sdk/integrations/django/middleware.py index edbeccb093..501f2f4c7c 100644 --- a/sentry_sdk/integrations/django/middleware.py +++ b/sentry_sdk/integrations/django/middleware.py @@ -2,19 +2,17 @@ Create spans from Django middleware invocations """ -from functools import wraps - from django import VERSION as DJANGO_VERSION from sentry_sdk import Hub +from sentry_sdk._functools import wraps +from sentry_sdk._types import MYPY from sentry_sdk.utils import ( ContextVar, transaction_from_function, capture_internal_exceptions, ) -from sentry_sdk._types import MYPY - if MYPY: from typing import Any from typing import Callable diff --git a/sentry_sdk/integrations/serverless.py b/sentry_sdk/integrations/serverless.py index 6dd90b43d0..c6ad3a2f68 100644 --- a/sentry_sdk/integrations/serverless.py +++ b/sentry_sdk/integrations/serverless.py @@ -1,9 +1,9 @@ -import functools import sys from sentry_sdk.hub import Hub from sentry_sdk.utils import event_from_exception from sentry_sdk._compat import reraise +from sentry_sdk._functools import wraps from sentry_sdk._types import MYPY @@ -42,7 +42,7 @@ def serverless_function(f=None, flush=True): # noqa # type: (Optional[F], bool) -> Union[F, Callable[[F], F]] def wrapper(f): # type: (F) -> F - @functools.wraps(f) + @wraps(f) def inner(*args, **kwargs): # type: (*Any, **Any) -> Any with Hub(Hub.current) as hub: diff --git a/sentry_sdk/integrations/wsgi.py b/sentry_sdk/integrations/wsgi.py index 22982d8bb1..bd87663896 100644 --- a/sentry_sdk/integrations/wsgi.py +++ b/sentry_sdk/integrations/wsgi.py @@ -1,6 +1,6 @@ -import functools import sys +from sentry_sdk._functools import partial from sentry_sdk.hub import Hub, _should_send_default_pii from sentry_sdk.utils import ( ContextVar, @@ -121,9 +121,7 @@ def __call__(self, environ, start_response): try: rv = self.app( environ, - functools.partial( - _sentry_start_response, start_response, span - ), + partial(_sentry_start_response, start_response, span), ) except BaseException: reraise(*_capture_exception(hub)) diff --git a/sentry_sdk/scope.py b/sentry_sdk/scope.py index 407af3a2cb..c721b56505 100644 --- a/sentry_sdk/scope.py +++ b/sentry_sdk/scope.py @@ -1,10 +1,10 @@ from copy import copy from collections import deque -from functools import wraps from itertools import chain -from sentry_sdk.utils import logger, capture_internal_exceptions +from sentry_sdk._functools import wraps from sentry_sdk._types import MYPY +from sentry_sdk.utils import logger, capture_internal_exceptions if MYPY: from typing import Any diff --git a/test-requirements.txt b/test-requirements.txt index 5c719bec9e..be051169ad 100644 --- a/test-requirements.txt +++ b/test-requirements.txt @@ -6,3 +6,4 @@ pytest-localserver==0.5.0 pytest-cov==2.8.1 gevent eventlet +newrelic diff --git a/tests/integrations/celery/test_celery.py b/tests/integrations/celery/test_celery.py index 2f76c0957a..ea475f309a 100644 --- a/tests/integrations/celery/test_celery.py +++ b/tests/integrations/celery/test_celery.py @@ -309,3 +309,30 @@ def dummy_task(self): # if this is nonempty, the worker never really forked assert not runs + + +@pytest.mark.forked +@pytest.mark.parametrize("newrelic_order", ["sentry_first", "sentry_last"]) +def test_newrelic_interference(init_celery, newrelic_order, celery_invocation): + def instrument_newrelic(): + import celery.app.trace as celery_mod + from newrelic.hooks.application_celery import instrument_celery_execute_trace + + assert hasattr(celery_mod, "build_tracer") + instrument_celery_execute_trace(celery_mod) + + if newrelic_order == "sentry_first": + celery = init_celery() + instrument_newrelic() + elif newrelic_order == "sentry_last": + instrument_newrelic() + celery = init_celery() + else: + raise ValueError(newrelic_order) + + @celery.task(name="dummy_task", bind=True) + def dummy_task(self, x, y): + return x / y + + assert dummy_task.apply(kwargs={"x": 1, "y": 1}).wait() == 1 + assert celery_invocation(dummy_task, 1, 1)[0].wait() == 1 From 5f9a3508b38b7cacb99a8e3276e2ffcdc6aaba8d Mon Sep 17 00:00:00 2001 From: Markus Unterwaditzer Date: Wed, 13 May 2020 13:03:42 +0200 Subject: [PATCH 16/17] doc: Changelog for 0.14.4 --- CHANGES.md | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/CHANGES.md b/CHANGES.md index 61a1771b5e..fe1d6b6386 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -27,6 +27,18 @@ sentry-sdk==0.10.1 A major release `N` implies the previous release `N-1` will no longer receive updates. We generally do not backport bugfixes to older versions unless they are security relevant. However, feel free to ask for backports of specific commits on the bugtracker. +## 0.14.4 + +* Fix bugs in transport rate limit enforcement for specific data categories. + The bug should not have affected anybody because we do not yet emit rate + limits for specific event types/data categories. +* Fix a bug in `capture_event` where it would crash if given additional kwargs. + Thanks to Tatiana Vasilevskaya! +* Fix a bug where contextvars from the request handler were inaccessible in + AIOHTTP error handlers. +* Fix a bug where the Celery integration would crash if newrelic instrumented Celery as well. + + ## 0.14.3 * Attempt to use a monotonic clock to measure span durations in Performance/APM. From a45ae81a0d284c7a09ea5c5d7b549876e634dee7 Mon Sep 17 00:00:00 2001 From: Markus Unterwaditzer Date: Wed, 13 May 2020 13:03:55 +0200 Subject: [PATCH 17/17] release: 0.14.4 --- docs/conf.py | 2 +- sentry_sdk/consts.py | 2 +- setup.py | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/docs/conf.py b/docs/conf.py index c7925a9c86..0b12b616b8 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -22,7 +22,7 @@ copyright = u"2019, Sentry Team and Contributors" author = u"Sentry Team and Contributors" -release = "0.14.3" +release = "0.14.4" version = ".".join(release.split(".")[:2]) # The short X.Y version. diff --git a/sentry_sdk/consts.py b/sentry_sdk/consts.py index 2fe012e66d..27a078aae5 100644 --- a/sentry_sdk/consts.py +++ b/sentry_sdk/consts.py @@ -89,7 +89,7 @@ def _get_default_options(): del _get_default_options -VERSION = "0.14.3" +VERSION = "0.14.4" SDK_INFO = { "name": "sentry.python", "version": VERSION, diff --git a/setup.py b/setup.py index bb5314a26f..456239d09b 100644 --- a/setup.py +++ b/setup.py @@ -12,7 +12,7 @@ setup( name="sentry-sdk", - version="0.14.3", + version="0.14.4", author="Sentry Team and Contributors", author_email="hello@getsentry.com", url="https://github.com/getsentry/sentry-python",