From 2e467a111c87c55479e34d2d8bc5471f4fc0c261 Mon Sep 17 00:00:00 2001 From: Daniel Sanche Date: Thu, 22 Apr 2021 14:17:23 -0700 Subject: [PATCH 01/12] added code for capturing span data --- google/cloud/logging_v2/handlers/_helpers.py | 19 ++++++++++++++----- .../cloud/logging_v2/handlers/app_engine.py | 2 +- google/cloud/logging_v2/handlers/handlers.py | 3 ++- .../logging_v2/handlers/structured_log.py | 1 + 4 files changed, 18 insertions(+), 7 deletions(-) diff --git a/google/cloud/logging_v2/handlers/_helpers.py b/google/cloud/logging_v2/handlers/_helpers.py index 88eba07a6..c388bdc1e 100644 --- a/google/cloud/logging_v2/handlers/_helpers.py +++ b/google/cloud/logging_v2/handlers/_helpers.py @@ -32,6 +32,8 @@ _FLASK_TRACE_HEADER = "X_CLOUD_TRACE_CONTEXT" _PROTOCOL_HEADER = "SERVER_PROTOCOL" +_ALHPANUM_REGEX = r'"[\dA-Za-z]*",' + def format_stackdriver_json(record, message): """Helper to format a LogRecord in in Stackdriver fluentd format. @@ -77,7 +79,10 @@ def get_request_data_from_flask(): trace_id = None header = flask.request.headers.get(_FLASK_TRACE_HEADER) if header: - trace_id = header.split("/", 1)[0] + trace_id, *extras = header.split("/", 1)[0] + if extras: + # the span is the set of alphanumeric characters after the / + span_id = re.findall(_ALHPANUM_REGEX, extras[0])[0] return http_request, trace_id @@ -114,11 +119,15 @@ def get_request_data_from_django(): # find trace id trace_id = None + span_id = None header = request.META.get(_DJANGO_TRACE_HEADER) if header: - trace_id = header.split("/", 1)[0] + trace_id, *extras = header.split("/", 1)[0] + if extras: + # the span is the set of alphanumeric characters after the / + span_id = re.findall(_ALHPANUM_REGEX, extras[0])[0] - return http_request, trace_id + return http_request, trace_id, span_id def get_request_data(): @@ -136,8 +145,8 @@ def get_request_data(): ) for checker in checkers: - http_request, trace_id = checker() + http_request, trace_id, span_id = checker() if http_request is not None: - return http_request, trace_id + return http_request, trace_id, span_id return None, None diff --git a/google/cloud/logging_v2/handlers/app_engine.py b/google/cloud/logging_v2/handlers/app_engine.py index bc7daa9d0..02584ae35 100644 --- a/google/cloud/logging_v2/handlers/app_engine.py +++ b/google/cloud/logging_v2/handlers/app_engine.py @@ -90,7 +90,7 @@ def get_gae_labels(self): """ gae_labels = {} - _, trace_id = get_request_data() + _, trace_id, _ = get_request_data() if trace_id is not None: gae_labels[_TRACE_ID_LABEL] = trace_id diff --git a/google/cloud/logging_v2/handlers/handlers.py b/google/cloud/logging_v2/handlers/handlers.py index b9cc53a94..38e91524d 100644 --- a/google/cloud/logging_v2/handlers/handlers.py +++ b/google/cloud/logging_v2/handlers/handlers.py @@ -59,7 +59,7 @@ def filter(self, record): } record.msg = "" if record.msg is None else record.msg # find http request data - inferred_http, inferred_trace = get_request_data() + inferred_http, inferred_trace, inferred_span = get_request_data() if inferred_trace is not None and self.project is not None: inferred_trace = f"projects/{self.project}/traces/{inferred_trace}" # set labels @@ -70,6 +70,7 @@ def filter(self, record): ) record.trace = getattr(record, "trace", inferred_trace) or "" + record.span_id = getattr(record, "spanId", inferred_span) or "" record.http_request = getattr(record, "http_request", inferred_http) or {} record.request_method = record.http_request.get("requestMethod", "") record.request_url = record.http_request.get("requestUrl", "") diff --git a/google/cloud/logging_v2/handlers/structured_log.py b/google/cloud/logging_v2/handlers/structured_log.py index e9d036423..76a560538 100644 --- a/google/cloud/logging_v2/handlers/structured_log.py +++ b/google/cloud/logging_v2/handlers/structured_log.py @@ -24,6 +24,7 @@ '"severity": "%(levelname)s", ' '"logging.googleapis.com/labels": { %(total_labels_str)s }, ' '"logging.googleapis.com/trace": "%(trace)s", ' + '"logging.googleapis.com/spanId": "%(span_id)s", ' '"logging.googleapis.com/sourceLocation": { "file": "%(file)s", "line": "%(line)d", "function": "%(function)s"}, ' '"httpRequest": {"requestMethod": "%(request_method)s", "requestUrl": "%(request_url)s", "userAgent": "%(user_agent)s", "protocol": "%(protocol)s"} }' ) From 255d10c074818ffc60e2186a0ae1f402bc87bbed Mon Sep 17 00:00:00 2001 From: Daniel Sanche Date: Thu, 22 Apr 2021 14:58:24 -0700 Subject: [PATCH 02/12] improved helper tests --- google/cloud/logging_v2/handlers/_helpers.py | 36 +++++++----- tests/unit/handlers/test__helpers.py | 61 +++++++++++--------- 2 files changed, 57 insertions(+), 40 deletions(-) diff --git a/google/cloud/logging_v2/handlers/_helpers.py b/google/cloud/logging_v2/handlers/_helpers.py index c388bdc1e..ad0187207 100644 --- a/google/cloud/logging_v2/handlers/_helpers.py +++ b/google/cloud/logging_v2/handlers/_helpers.py @@ -16,6 +16,7 @@ import math import json +import re try: import flask @@ -32,8 +33,6 @@ _FLASK_TRACE_HEADER = "X_CLOUD_TRACE_CONTEXT" _PROTOCOL_HEADER = "SERVER_PROTOCOL" -_ALHPANUM_REGEX = r'"[\dA-Za-z]*",' - def format_stackdriver_json(record, message): """Helper to format a LogRecord in in Stackdriver fluentd format. @@ -62,7 +61,7 @@ def get_request_data_from_flask(): request. Both fields will be None if a flask request isn't found. """ if flask is None or not flask.request: - return None, None + return None, None, None # build http_request http_request = { @@ -75,16 +74,21 @@ def get_request_data_from_flask(): "protocol": flask.request.environ.get(_PROTOCOL_HEADER), } - # find trace id + # find trace id and span id trace_id = None + span_id = None header = flask.request.headers.get(_FLASK_TRACE_HEADER) if header: - trace_id, *extras = header.split("/", 1)[0] - if extras: + split_header = header.split("/", 1) + trace_id = split_header[0] + try: + header_suffix = split_header[1] # the span is the set of alphanumeric characters after the / - span_id = re.findall(_ALHPANUM_REGEX, extras[0])[0] + span_id = re.findall(r"\w+", header_suffix)[0] + except IndexError: + pass - return http_request, trace_id + return http_request, trace_id, span_id def get_request_data_from_django(): @@ -98,7 +102,7 @@ def get_request_data_from_django(): request = _get_django_request() if request is None: - return None, None + return None, None, None # convert content_length to int if it exists content_length = None @@ -117,15 +121,19 @@ def get_request_data_from_django(): "protocol": request.META.get(_PROTOCOL_HEADER), } - # find trace id + # find trace id and span id trace_id = None span_id = None header = request.META.get(_DJANGO_TRACE_HEADER) if header: - trace_id, *extras = header.split("/", 1)[0] - if extras: + split_header = header.split("/", 1) + trace_id = split_header[0] + try: + header_suffix = split_header[1] # the span is the set of alphanumeric characters after the / - span_id = re.findall(_ALHPANUM_REGEX, extras[0])[0] + span_id = re.findall(r"\w+", header_suffix)[0] + except IndexError: + pass return http_request, trace_id, span_id @@ -149,4 +157,4 @@ def get_request_data(): if http_request is not None: return http_request, trace_id, span_id - return None, None + return None, None, None diff --git a/tests/unit/handlers/test__helpers.py b/tests/unit/handlers/test__helpers.py index fd17f6ffd..70146ad76 100644 --- a/tests/unit/handlers/test__helpers.py +++ b/tests/unit/handlers/test__helpers.py @@ -17,8 +17,10 @@ import mock _FLASK_TRACE_ID = "flask-id" +_FLASK_SPAN_ID = "span0flask" _FLASK_HTTP_REQUEST = {"requestUrl": "https://flask.palletsprojects.com/en/1.1.x/"} _DJANGO_TRACE_ID = "django-id" +_DJANGO_SPAN_ID = "span0django" _DJANGO_HTTP_REQUEST = {"requestUrl": "https://www.djangoproject.com/"} @@ -44,15 +46,17 @@ def index(): def test_no_context_header(self): app = self.create_app() with app.test_request_context(path="/", headers={}): - http_request, trace_id = self._call_fut() + http_request, trace_id, span_id = self._call_fut() self.assertIsNone(trace_id) + self.assertIsNone(span_id) self.assertEqual(http_request["requestMethod"], "GET") def test_valid_context_header(self): flask_trace_header = "X_CLOUD_TRACE_CONTEXT" expected_trace_id = _FLASK_TRACE_ID - flask_trace_id = expected_trace_id + "/testspanid" + expected_span_id = _FLASK_SPAN_ID + flask_trace_id = f"{expected_trace_id}/{expected_span_id}" app = self.create_app() context = app.test_request_context( @@ -60,9 +64,10 @@ def test_valid_context_header(self): ) with context: - http_request, trace_id = self._call_fut() + http_request, trace_id, span_id = self._call_fut() self.assertEqual(trace_id, expected_trace_id) + self.assertEqual(span_id, expected_span_id) self.assertEqual(http_request["requestMethod"], "GET") def test_http_request_populated(self): @@ -84,7 +89,7 @@ def test_http_request_populated(self): environ_base={"REMOTE_ADDR": expected_ip}, headers=headers, ) - http_request, trace_id = self._call_fut() + http_request, *_ = self._call_fut() self.assertEqual(http_request["requestMethod"], "PUT") self.assertEqual(http_request["requestUrl"], expected_path) @@ -99,7 +104,7 @@ def test_http_request_sparse(self): app = self.create_app() with app.test_client() as c: c.put(path=expected_path) - http_request, trace_id = self._call_fut() + http_request, *_ = self._call_fut() self.assertEqual(http_request["requestMethod"], "PUT") self.assertEqual(http_request["requestUrl"], expected_path) self.assertEqual(http_request["protocol"], "HTTP/1.1") @@ -135,17 +140,20 @@ def test_no_context_header(self): middleware = request.RequestMiddleware(None) middleware.process_request(django_request) - http_request, trace_id = self._call_fut() + http_request, trace_id, span_id = self._call_fut() + self.assertEqual(http_request["requestMethod"], "GET") self.assertIsNone(trace_id) + self.assertIsNone(span_id) def test_valid_context_header(self): from django.test import RequestFactory from google.cloud.logging_v2.handlers.middleware import request django_trace_header = "HTTP_X_CLOUD_TRACE_CONTEXT" - expected_trace_id = "testtraceiddjango" - django_trace_id = expected_trace_id + "/testspanid" + expected_span_id = _DJANGO_SPAN_ID + expected_trace_id = _DJANGO_TRACE_ID + django_trace_id = f"{expected_trace_id}/{expected_span_id}" django_request = RequestFactory().get( "/", **{django_trace_header: django_trace_id} @@ -153,9 +161,10 @@ def test_valid_context_header(self): middleware = request.RequestMiddleware(None) middleware.process_request(django_request) - http_request, trace_id = self._call_fut() + http_request, trace_id, span_id = self._call_fut() self.assertEqual(trace_id, expected_trace_id) + self.assertEqual(span_id, expected_span_id) self.assertEqual(http_request["requestMethod"], "GET") def test_http_request_populated(self): @@ -178,7 +187,7 @@ def test_http_request_populated(self): middleware = request.RequestMiddleware(None) middleware.process_request(django_request) - http_request, trace_id = self._call_fut() + http_request, *_ = self._call_fut() self.assertEqual(http_request["requestMethod"], "PUT") self.assertEqual(http_request["requestUrl"], expected_path) self.assertEqual(http_request["userAgent"], expected_agent) @@ -195,7 +204,7 @@ def test_http_request_sparse(self): django_request = RequestFactory().put(expected_path) middleware = request.RequestMiddleware(None) middleware.process_request(django_request) - http_request, trace_id = self._call_fut() + http_request, *_ = self._call_fut() self.assertEqual(http_request["requestMethod"], "PUT") self.assertEqual(http_request["requestUrl"], expected_path) self.assertEqual(http_request["remoteIp"], "127.0.0.1") @@ -226,8 +235,8 @@ def _helper(self, django_return, flask_return): return django_mock, flask_mock, result def test_from_django(self): - django_expected = (_DJANGO_HTTP_REQUEST, _DJANGO_TRACE_ID) - flask_expected = (None, None) + django_expected = (_DJANGO_HTTP_REQUEST, _DJANGO_TRACE_ID, _DJANGO_SPAN_ID) + flask_expected = (None, None, None) django_mock, flask_mock, output = self._helper(django_expected, flask_expected) self.assertEqual(output, django_expected) @@ -235,8 +244,8 @@ def test_from_django(self): flask_mock.assert_not_called() def test_from_flask(self): - django_expected = (None, None) - flask_expected = (_FLASK_HTTP_REQUEST, _FLASK_TRACE_ID) + django_expected = (None, None, None) + flask_expected = (_FLASK_HTTP_REQUEST, _FLASK_TRACE_ID, _FLASK_SPAN_ID) django_mock, flask_mock, output = self._helper(django_expected, flask_expected) self.assertEqual(output, flask_expected) @@ -245,8 +254,8 @@ def test_from_flask(self): flask_mock.assert_called_once_with() def test_from_django_and_flask(self): - django_expected = (_DJANGO_HTTP_REQUEST, _DJANGO_TRACE_ID) - flask_expected = (_FLASK_HTTP_REQUEST, _FLASK_TRACE_ID) + django_expected = (_DJANGO_HTTP_REQUEST, _DJANGO_TRACE_ID, _DJANGO_SPAN_ID) + flask_expected = (_FLASK_HTTP_REQUEST, _FLASK_TRACE_ID, _FLASK_SPAN_ID) django_mock, flask_mock, output = self._helper(django_expected, flask_expected) @@ -257,19 +266,19 @@ def test_from_django_and_flask(self): flask_mock.assert_not_called() def test_missing_http_request(self): - flask_expected = (None, _FLASK_TRACE_ID) - django_expected = (None, _DJANGO_TRACE_ID) + flask_expected = (None, _FLASK_TRACE_ID, _FLASK_SPAN_ID) + django_expected = (None, _DJANGO_TRACE_ID, _DJANGO_TRACE_ID) django_mock, flask_mock, output = self._helper(django_expected, flask_expected) # function only returns trace if http_request data is present - self.assertEqual(output, (None, None)) + self.assertEqual(output, (None, None, None)) django_mock.assert_called_once_with() flask_mock.assert_called_once_with() def test_missing_trace_id(self): - flask_expected = (_FLASK_HTTP_REQUEST, None) - django_expected = (None, _DJANGO_TRACE_ID) + flask_expected = (_FLASK_HTTP_REQUEST, None, None) + django_expected = (None, _DJANGO_TRACE_ID, _DJANGO_SPAN_ID) django_mock, flask_mock, output = self._helper(django_expected, flask_expected) # trace_id is optional @@ -279,14 +288,14 @@ def test_missing_trace_id(self): flask_mock.assert_called_once_with() def test_missing_both(self): - flask_expected = (None, None) - django_expected = (None, None) + flask_expected = (None, None, None) + django_expected = (None, None, None) django_mock, flask_mock, output = self._helper(django_expected, flask_expected) - self.assertEqual(output, (None, None)) + self.assertEqual(output, (None, None, None)) django_mock.assert_called_once_with() flask_mock.assert_called_once_with() def test_wo_libraries(self): output = self._call_fut() - self.assertEqual(output, (None, None)) + self.assertEqual(output, (None, None, None)) From 06133739f5f394501ed3df55d94dbe7bc3343fa4 Mon Sep 17 00:00:00 2001 From: Daniel Sanche Date: Thu, 22 Apr 2021 15:05:15 -0700 Subject: [PATCH 03/12] refactored --- google/cloud/logging_v2/handlers/_helpers.py | 52 +++++++++++--------- 1 file changed, 29 insertions(+), 23 deletions(-) diff --git a/google/cloud/logging_v2/handlers/_helpers.py b/google/cloud/logging_v2/handlers/_helpers.py index ad0187207..368809ab8 100644 --- a/google/cloud/logging_v2/handlers/_helpers.py +++ b/google/cloud/logging_v2/handlers/_helpers.py @@ -56,9 +56,10 @@ def get_request_data_from_flask(): """Get http_request and trace data from flask request headers. Returns: - Tuple[Optional[dict], Optional[str]]: - Data related to the current http request and the trace_id for the - request. Both fields will be None if a flask request isn't found. + Tuple[Optional[dict], Optional[str], Optional[str]]: + Data related to the current http request, trace_id, and span_id for + the request. All fields will be None if a django request isn't + found. """ if flask is None or not flask.request: return None, None, None @@ -75,18 +76,8 @@ def get_request_data_from_flask(): } # find trace id and span id - trace_id = None - span_id = None header = flask.request.headers.get(_FLASK_TRACE_HEADER) - if header: - split_header = header.split("/", 1) - trace_id = split_header[0] - try: - header_suffix = split_header[1] - # the span is the set of alphanumeric characters after the / - span_id = re.findall(r"\w+", header_suffix)[0] - except IndexError: - pass + trace_id, span_id = _parse_trace_span(header) return http_request, trace_id, span_id @@ -95,9 +86,10 @@ def get_request_data_from_django(): """Get http_request and trace data from django request headers. Returns: - Tuple[Optional[dict], Optional[str]]: - Data related to the current http request and the trace_id for the - request. Both fields will be None if a django request isn't found. + Tuple[Optional[dict], Optional[str], Optional[str]]: + Data related to the current http request, trace_id, and span_id for + the request. All fields will be None if a django request isn't + found. """ request = _get_django_request() @@ -122,9 +114,23 @@ def get_request_data_from_django(): } # find trace id and span id + header = request.META.get(_DJANGO_TRACE_HEADER) + trace_id, span_id = _parse_trace_span(header) + + return http_request, trace_id, span_id + +def _parse_trace_span(header): + """Given an X_CLOUD_TRACE header, extrace the trace and span ids. + + Args: + header (str): the string extracted from the X_CLOUD_TRACE header + Returns: + Tuple[Optional[dict], Optional[str]]: + The trace_id and span_id extracted from the header + Each field will be None if not found. + """ trace_id = None span_id = None - header = request.META.get(_DJANGO_TRACE_HEADER) if header: split_header = header.split("/", 1) trace_id = split_header[0] @@ -134,8 +140,7 @@ def get_request_data_from_django(): span_id = re.findall(r"\w+", header_suffix)[0] except IndexError: pass - - return http_request, trace_id, span_id + return trace_id, span_id def get_request_data(): @@ -143,9 +148,10 @@ def get_request_data(): frameworks (currently supported: Flask and Django). Returns: - Tuple[Optional[dict], Optional[str]]: - Data related to the current http request and the trace_id for the - request. Both fields will be None if a supported web request isn't found. + Tuple[Optional[dict], Optional[str], Optional[str]]: + Data related to the current http request, trace_id, and span_id for + the request. All fields will be None if a django request isn't + found. """ checkers = ( get_request_data_from_django, From 18e8b954181154a5d685f500c028a635fb576d33 Mon Sep 17 00:00:00 2001 From: Daniel Sanche Date: Thu, 22 Apr 2021 15:18:17 -0700 Subject: [PATCH 04/12] added tests around span parsing --- google/cloud/logging_v2/handlers/_helpers.py | 2 +- tests/unit/handlers/test__helpers.py | 62 ++++++++++++++++++++ 2 files changed, 63 insertions(+), 1 deletion(-) diff --git a/google/cloud/logging_v2/handlers/_helpers.py b/google/cloud/logging_v2/handlers/_helpers.py index 368809ab8..e374254a7 100644 --- a/google/cloud/logging_v2/handlers/_helpers.py +++ b/google/cloud/logging_v2/handlers/_helpers.py @@ -137,7 +137,7 @@ def _parse_trace_span(header): try: header_suffix = split_header[1] # the span is the set of alphanumeric characters after the / - span_id = re.findall(r"\w+", header_suffix)[0] + span_id = re.findall(r"^\w+", header_suffix)[0] except IndexError: pass return trace_id, span_id diff --git a/tests/unit/handlers/test__helpers.py b/tests/unit/handlers/test__helpers.py index 70146ad76..73efa1700 100644 --- a/tests/unit/handlers/test__helpers.py +++ b/tests/unit/handlers/test__helpers.py @@ -299,3 +299,65 @@ def test_missing_both(self): def test_wo_libraries(self): output = self._call_fut() self.assertEqual(output, (None, None, None)) + +class Test__parse_trace_span(unittest.TestCase): + @staticmethod + def _call_fut(header): + from google.cloud.logging_v2.handlers import _helpers + + return _helpers._parse_trace_span(header) + + def test_empty_header(self): + header = "" + trace_id, span_id = self._call_fut(header) + self.assertEqual(trace_id, None) + self.assertEqual(span_id, None) + + def test_no_span(self): + header = "12345" + trace_id, span_id = self._call_fut(header) + self.assertEqual(trace_id, header) + self.assertEqual(span_id, None) + + def test_no_trace(self): + header = "/12345" + trace_id, span_id = self._call_fut(header) + self.assertEqual(trace_id, "") + self.assertEqual(span_id, "12345") + + def test_with_span(self): + expected_trace = "12345" + expected_span = "67890" + header = f"{expected_trace}/{expected_span}" + trace_id, span_id = self._call_fut(header) + self.assertEqual(trace_id, expected_trace) + self.assertEqual(span_id, expected_span) + + def test_with_extra_characters(self): + expected_trace = "12345" + expected_span = "67890" + header = f"{expected_trace}/{expected_span};o=0" + trace_id, span_id = self._call_fut(header) + self.assertEqual(trace_id, expected_trace) + self.assertEqual(span_id, expected_span) + + def test_with_unicode_span(self): + """ + Spans are expected to be alphanumeric + """ + expected_trace = "12345" + header = f"{expected_trace}/😀123" + trace_id, span_id = self._call_fut(header) + self.assertEqual(trace_id, expected_trace) + self.assertEqual(span_id, None) + + def test_with_unicode_trace(self): + """ + Spans are expected to be alphanumeric + """ + expected_trace = "12😀345" + expected_span = "67890" + header = f"{expected_trace}/{expected_span}" + trace_id, span_id = self._call_fut(header) + self.assertEqual(trace_id, expected_trace) + self.assertEqual(span_id, expected_span) From 76445658b5f76bbcd1053c7d38fe80a72a7257ab Mon Sep 17 00:00:00 2001 From: Daniel Sanche Date: Thu, 22 Apr 2021 15:36:10 -0700 Subject: [PATCH 05/12] added handler unit tests --- google/cloud/logging_v2/handlers/app_engine.py | 2 +- tests/unit/handlers/test_app_engine.py | 6 +++--- tests/unit/handlers/test_handlers.py | 2 +- tests/unit/handlers/test_structured_log.py | 15 +++++++++++---- 4 files changed, 16 insertions(+), 9 deletions(-) diff --git a/google/cloud/logging_v2/handlers/app_engine.py b/google/cloud/logging_v2/handlers/app_engine.py index 02584ae35..874a9d608 100644 --- a/google/cloud/logging_v2/handlers/app_engine.py +++ b/google/cloud/logging_v2/handlers/app_engine.py @@ -107,7 +107,7 @@ def emit(self, record): record (logging.LogRecord): The record to be logged. """ message = super(AppEngineHandler, self).format(record) - inferred_http, inferred_trace = get_request_data() + inferred_http, inferred_trace, _ = get_request_data() if inferred_trace is not None: inferred_trace = f"projects/{self.project_id}/traces/{inferred_trace}" # allow user overrides diff --git a/tests/unit/handlers/test_app_engine.py b/tests/unit/handlers/test_app_engine.py index 65e804573..c726c8496 100644 --- a/tests/unit/handlers/test_app_engine.py +++ b/tests/unit/handlers/test_app_engine.py @@ -97,7 +97,7 @@ def test_emit(self): expected_trace_id = f"projects/{self.PROJECT}/traces/{trace_id}" get_request_patch = mock.patch( "google.cloud.logging_v2.handlers.app_engine.get_request_data", - return_value=(expected_http_request, trace_id), + return_value=(expected_http_request, trace_id, None), ) with get_request_patch: # library integrations mocked to return test data @@ -135,7 +135,7 @@ def test_emit_manual_field_override(self): inferred_trace_id = "trace-test" get_request_patch = mock.patch( "google.cloud.logging_v2.handlers.app_engine.get_request_data", - return_value=(inferred_http_request, inferred_trace_id), + return_value=(inferred_http_request, inferred_trace_id, None), ) with get_request_patch: # library integrations mocked to return test data @@ -180,7 +180,7 @@ def test_emit_manual_field_override(self): def _get_gae_labels_helper(self, trace_id): get_request_patch = mock.patch( "google.cloud.logging_v2.handlers.app_engine.get_request_data", - return_value=(None, trace_id), + return_value=(None, trace_id, None), ) client = mock.Mock(project=self.PROJECT, spec=["project"]) diff --git a/tests/unit/handlers/test_handlers.py b/tests/unit/handlers/test_handlers.py index 7fb7033b0..197727106 100644 --- a/tests/unit/handlers/test_handlers.py +++ b/tests/unit/handlers/test_handlers.py @@ -297,7 +297,7 @@ def test_emit_manual_field_override(self): expected_trace = "123" setattr(record, "trace", expected_trace) expected_span = "456" - setattr(record, "span_id", expected_span) + setattr(record, "spanId", expected_span) expected_http = {"reuqest_url": "manual"} setattr(record, "http_request", expected_http) expected_source = {"file": "test-file"} diff --git a/tests/unit/handlers/test_structured_log.py b/tests/unit/handlers/test_structured_log.py index 13719bf53..937bad909 100644 --- a/tests/unit/handlers/test_structured_log.py +++ b/tests/unit/handlers/test_structured_log.py @@ -64,6 +64,7 @@ def test_format(self): "message": message, "severity": record.levelname, "logging.googleapis.com/trace": "", + "logging.googleapis.com/spanId": "", "logging.googleapis.com/sourceLocation": { "file": pathname, "line": str(lineno), @@ -128,8 +129,11 @@ def test_format_with_request(self): expected_path = "http://testserver/123" expected_agent = "Mozilla/5.0" expected_trace = "123" + expected_span = "456" + trace_header = f"{expected_trace}/{expected_span};o=0" expected_payload = { "logging.googleapis.com/trace": expected_trace, + "logging.googleapis.com/spanId": expected_span, "httpRequest": { "requestMethod": "PUT", "requestUrl": expected_path, @@ -145,7 +149,7 @@ def test_format_with_request(self): data="body", headers={ "User-Agent": expected_agent, - "X_CLOUD_TRACE_CONTEXT": expected_trace, + "X_CLOUD_TRACE_CONTEXT": trace_header, }, ) handler.filter(record) @@ -173,16 +177,19 @@ def test_format_overrides(self): record = logging.LogRecord(logname, logging.INFO, "", 0, message, None, None) overwrite_path = "http://overwrite" inferred_path = "http://testserver/123" - overwrite_trace = "456" - inferred_trace = "123" + overwrite_trace = "abc" + overwrite_span = "def" + inferred_trace_span = "123/456;" overwrite_file = "test-file" record.http_request = {"requestUrl": overwrite_path} record.source_location = {"file": overwrite_file} record.trace = overwrite_trace + record.spanId = overwrite_span added_labels = {"added_key": "added_value", "overwritten_key": "new_value"} record.labels = added_labels expected_payload = { "logging.googleapis.com/trace": overwrite_trace, + "logging.googleapis.com/spanId": overwrite_span, "logging.googleapis.com/sourceLocation": { "file": overwrite_file, "function": "", @@ -206,7 +213,7 @@ def test_format_overrides(self): c.put( path=inferred_path, data="body", - headers={"X_CLOUD_TRACE_CONTEXT": inferred_trace}, + headers={"X_CLOUD_TRACE_CONTEXT": inferred_trace_span}, ) handler.filter(record) result = json.loads(handler.format(record)) From 35d5d581a5a037326e10daa34b8d308018464944 Mon Sep 17 00:00:00 2001 From: Daniel Sanche Date: Thu, 22 Apr 2021 15:52:56 -0700 Subject: [PATCH 06/12] updated environment tests --- tests/environment | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/environment b/tests/environment index 94ff68580..f297d64fa 160000 --- a/tests/environment +++ b/tests/environment @@ -1 +1 @@ -Subproject commit 94ff685805510ad8d78c170603798cbe44050bce +Subproject commit f297d64fa305aca3b79501ae4d3edc4d982c7401 From daad4c38080cfaaa4f39d2fe07e42f9f1f2c671e Mon Sep 17 00:00:00 2001 From: Daniel Sanche Date: Thu, 22 Apr 2021 15:57:08 -0700 Subject: [PATCH 07/12] moved some lines into try block, just to be safe --- google/cloud/logging_v2/handlers/_helpers.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/google/cloud/logging_v2/handlers/_helpers.py b/google/cloud/logging_v2/handlers/_helpers.py index e374254a7..ada6b66be 100644 --- a/google/cloud/logging_v2/handlers/_helpers.py +++ b/google/cloud/logging_v2/handlers/_helpers.py @@ -132,9 +132,9 @@ def _parse_trace_span(header): trace_id = None span_id = None if header: - split_header = header.split("/", 1) - trace_id = split_header[0] try: + split_header = header.split("/", 1) + trace_id = split_header[0] header_suffix = split_header[1] # the span is the set of alphanumeric characters after the / span_id = re.findall(r"^\w+", header_suffix)[0] From 3d07db09e40d8fb92387065b296cf04a9f04787d Mon Sep 17 00:00:00 2001 From: Daniel Sanche Date: Thu, 22 Apr 2021 16:01:17 -0700 Subject: [PATCH 08/12] fixed lint issue --- google/cloud/logging_v2/handlers/_helpers.py | 1 + tests/unit/handlers/test__helpers.py | 1 + 2 files changed, 2 insertions(+) diff --git a/google/cloud/logging_v2/handlers/_helpers.py b/google/cloud/logging_v2/handlers/_helpers.py index ada6b66be..560a51a5d 100644 --- a/google/cloud/logging_v2/handlers/_helpers.py +++ b/google/cloud/logging_v2/handlers/_helpers.py @@ -119,6 +119,7 @@ def get_request_data_from_django(): return http_request, trace_id, span_id + def _parse_trace_span(header): """Given an X_CLOUD_TRACE header, extrace the trace and span ids. diff --git a/tests/unit/handlers/test__helpers.py b/tests/unit/handlers/test__helpers.py index 73efa1700..b2c822e7c 100644 --- a/tests/unit/handlers/test__helpers.py +++ b/tests/unit/handlers/test__helpers.py @@ -300,6 +300,7 @@ def test_wo_libraries(self): output = self._call_fut() self.assertEqual(output, (None, None, None)) + class Test__parse_trace_span(unittest.TestCase): @staticmethod def _call_fut(header): From 8312f4ae9f18f6c1a0958515251d0a28082848aa Mon Sep 17 00:00:00 2001 From: Daniel Sanche Date: Fri, 23 Apr 2021 12:20:50 -0700 Subject: [PATCH 09/12] changed spanId to span_id to replicate what the API returns on read logs --- google/cloud/logging_v2/handlers/handlers.py | 2 +- tests/environment | 2 +- tests/unit/handlers/test_handlers.py | 2 +- tests/unit/handlers/test_structured_log.py | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/google/cloud/logging_v2/handlers/handlers.py b/google/cloud/logging_v2/handlers/handlers.py index 38e91524d..3a5599aae 100644 --- a/google/cloud/logging_v2/handlers/handlers.py +++ b/google/cloud/logging_v2/handlers/handlers.py @@ -70,7 +70,7 @@ def filter(self, record): ) record.trace = getattr(record, "trace", inferred_trace) or "" - record.span_id = getattr(record, "spanId", inferred_span) or "" + record.span_id = getattr(record, "span_id", inferred_span) or "" record.http_request = getattr(record, "http_request", inferred_http) or {} record.request_method = record.http_request.get("requestMethod", "") record.request_url = record.http_request.get("requestUrl", "") diff --git a/tests/environment b/tests/environment index f297d64fa..969535b9a 160000 --- a/tests/environment +++ b/tests/environment @@ -1 +1 @@ -Subproject commit f297d64fa305aca3b79501ae4d3edc4d982c7401 +Subproject commit 969535b9a474f407cfdd928ff556c5e122876b7d diff --git a/tests/unit/handlers/test_handlers.py b/tests/unit/handlers/test_handlers.py index 197727106..7fb7033b0 100644 --- a/tests/unit/handlers/test_handlers.py +++ b/tests/unit/handlers/test_handlers.py @@ -297,7 +297,7 @@ def test_emit_manual_field_override(self): expected_trace = "123" setattr(record, "trace", expected_trace) expected_span = "456" - setattr(record, "spanId", expected_span) + setattr(record, "span_id", expected_span) expected_http = {"reuqest_url": "manual"} setattr(record, "http_request", expected_http) expected_source = {"file": "test-file"} diff --git a/tests/unit/handlers/test_structured_log.py b/tests/unit/handlers/test_structured_log.py index 937bad909..66822d74c 100644 --- a/tests/unit/handlers/test_structured_log.py +++ b/tests/unit/handlers/test_structured_log.py @@ -184,7 +184,7 @@ def test_format_overrides(self): record.http_request = {"requestUrl": overwrite_path} record.source_location = {"file": overwrite_file} record.trace = overwrite_trace - record.spanId = overwrite_span + record.span_id = overwrite_span added_labels = {"added_key": "added_value", "overwritten_key": "new_value"} record.labels = added_labels expected_payload = { From 2b28f86dbd2f700171c6d281b5af73af17adb4e0 Mon Sep 17 00:00:00 2001 From: Daniel Sanche Date: Mon, 26 Apr 2021 15:11:04 -0700 Subject: [PATCH 10/12] pulled in new environment tests --- tests/environment | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/environment b/tests/environment index 969535b9a..6e893faf4 160000 --- a/tests/environment +++ b/tests/environment @@ -1 +1 @@ -Subproject commit 969535b9a474f407cfdd928ff556c5e122876b7d +Subproject commit 6e893faf4c19afe3a7444346741d4f4ba2775706 From 4ed0d2a06537316867c871e5c1f250e980caa8ac Mon Sep 17 00:00:00 2001 From: Daniel Sanche Date: Mon, 26 Apr 2021 16:32:12 -0700 Subject: [PATCH 11/12] added environment test --- tests/environment | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/environment b/tests/environment index 6e893faf4..df1b7c131 160000 --- a/tests/environment +++ b/tests/environment @@ -1 +1 @@ -Subproject commit 6e893faf4c19afe3a7444346741d4f4ba2775706 +Subproject commit df1b7c131575f8eb59120cef75709496602b7665 From 32cadbacdedbaba8b1e92d73b7cf88bc15ee6e53 Mon Sep 17 00:00:00 2001 From: Daniel Sanche Date: Mon, 3 May 2021 14:53:33 -0700 Subject: [PATCH 12/12] fixed typo --- google/cloud/logging_v2/handlers/_helpers.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/google/cloud/logging_v2/handlers/_helpers.py b/google/cloud/logging_v2/handlers/_helpers.py index 560a51a5d..ba853edff 100644 --- a/google/cloud/logging_v2/handlers/_helpers.py +++ b/google/cloud/logging_v2/handlers/_helpers.py @@ -121,7 +121,7 @@ def get_request_data_from_django(): def _parse_trace_span(header): - """Given an X_CLOUD_TRACE header, extrace the trace and span ids. + """Given an X_CLOUD_TRACE header, extract the trace and span ids. Args: header (str): the string extracted from the X_CLOUD_TRACE header