From 30189eaa1fc4689b5bfefb682add0faa56555672 Mon Sep 17 00:00:00 2001 From: Daniel Sanche Date: Wed, 6 Sep 2023 22:42:10 +0000 Subject: [PATCH 01/12] feat: add caching to routing header --- google/api_core/gapic_v1/routing_header.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/google/api_core/gapic_v1/routing_header.py b/google/api_core/gapic_v1/routing_header.py index 28b13abf..25559cfc 100644 --- a/google/api_core/gapic_v1/routing_header.py +++ b/google/api_core/gapic_v1/routing_header.py @@ -20,6 +20,7 @@ Generally, these headers are specified as gRPC metadata. """ +import functools from enum import Enum from urllib.parse import urlencode @@ -53,6 +54,7 @@ def to_routing_header(params, qualified_enums=True): ) +@functools.lru_cache(32) def to_grpc_metadata(params, qualified_enums=True): """Returns the gRPC metadata containing the routing headers for the given request parameters. From 9ae434cfa1469f5c1aba4872e6861d3993b1ecf3 Mon Sep 17 00:00:00 2001 From: Daniel Sanche Date: Wed, 6 Sep 2023 22:49:03 +0000 Subject: [PATCH 02/12] added constant for cache size --- google/api_core/gapic_v1/routing_header.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/google/api_core/gapic_v1/routing_header.py b/google/api_core/gapic_v1/routing_header.py index 25559cfc..a3a2205b 100644 --- a/google/api_core/gapic_v1/routing_header.py +++ b/google/api_core/gapic_v1/routing_header.py @@ -25,6 +25,8 @@ from urllib.parse import urlencode ROUTING_METADATA_KEY = "x-goog-request-params" +# use caching to avoid repeated computation +ROUTING_PARAM_CACHE_SIZE = 32 def to_routing_header(params, qualified_enums=True): @@ -54,7 +56,7 @@ def to_routing_header(params, qualified_enums=True): ) -@functools.lru_cache(32) +@functools.lru_cache(ROUTING_PARAM_CACHE_SIZE) def to_grpc_metadata(params, qualified_enums=True): """Returns the gRPC metadata containing the routing headers for the given request parameters. From deb13c3608c8831dfbc947986724f0d8bcf00782 Mon Sep 17 00:00:00 2001 From: Daniel Sanche Date: Fri, 6 Oct 2023 15:20:55 -0700 Subject: [PATCH 03/12] moved cacheing into helper --- google/api_core/gapic_v1/routing_header.py | 37 ++++++++++++++-------- 1 file changed, 23 insertions(+), 14 deletions(-) diff --git a/google/api_core/gapic_v1/routing_header.py b/google/api_core/gapic_v1/routing_header.py index a3a2205b..89595e98 100644 --- a/google/api_core/gapic_v1/routing_header.py +++ b/google/api_core/gapic_v1/routing_header.py @@ -33,7 +33,7 @@ def to_routing_header(params, qualified_enums=True): """Returns a routing header string for the given request parameters. Args: - params (Mapping[str, Any]): A dictionary containing the request + params (Mapping[str, str | bytes | Enum]): A dictionary containing the request parameters used for routing. qualified_enums (bool): Whether to represent enum values as their type-qualified symbol names instead of as their @@ -41,28 +41,19 @@ def to_routing_header(params, qualified_enums=True): Returns: str: The routing header string. - """ + tuples = params.items() if isinstance(params, dict) else params if not qualified_enums: - if isinstance(params, dict): - tuples = params.items() - else: - tuples = params - params = [(x[0], x[1].name) if isinstance(x[1], Enum) else x for x in tuples] - return urlencode( - params, - # Per Google API policy (go/api-url-encoding), / is not encoded. - safe="/", - ) + tuples = [(x[0], x[1].name) if isinstance(x[1], Enum) else x for x in tuples] + return _urlencode_params(*tuples) -@functools.lru_cache(ROUTING_PARAM_CACHE_SIZE) def to_grpc_metadata(params, qualified_enums=True): """Returns the gRPC metadata containing the routing headers for the given request parameters. Args: - params (Mapping[str, Any]): A dictionary containing the request + params (Mapping[str, str | bytes | Enum]): A dictionary containing the request parameters used for routing. qualified_enums (bool): Whether to represent enum values as their type-qualified symbol names instead of as their @@ -73,3 +64,21 @@ def to_grpc_metadata(params, qualified_enums=True): and value. """ return (ROUTING_METADATA_KEY, to_routing_header(params, qualified_enums)) + + +@functools.lru_cache(ROUTING_PARAM_CACHE_SIZE) +def _urlencode_params(*params): + """Cacheable wrapper over urlencode + + Args: + *params ([Tuple[str, str | bytes | Enum]): the reqyest parameters + used for routing. + + Returns: + str: The routing header string. + """ + return urlencode( + params, + # Per Google API policy (go/api-url-encoding), / is not encoded. + safe="/", + ) From 6d7c8079218bda43594a59bce7298bea41c97780 Mon Sep 17 00:00:00 2001 From: Daniel Sanche Date: Fri, 6 Oct 2023 15:39:53 -0700 Subject: [PATCH 04/12] cache individual components --- google/api_core/gapic_v1/routing_header.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/google/api_core/gapic_v1/routing_header.py b/google/api_core/gapic_v1/routing_header.py index 89595e98..ff447cea 100644 --- a/google/api_core/gapic_v1/routing_header.py +++ b/google/api_core/gapic_v1/routing_header.py @@ -45,7 +45,7 @@ def to_routing_header(params, qualified_enums=True): tuples = params.items() if isinstance(params, dict) else params if not qualified_enums: tuples = [(x[0], x[1].name) if isinstance(x[1], Enum) else x for x in tuples] - return _urlencode_params(*tuples) + return "&".join([_urlencode_param(*t) for t in tuples]) def to_grpc_metadata(params, qualified_enums=True): @@ -67,18 +67,18 @@ def to_grpc_metadata(params, qualified_enums=True): @functools.lru_cache(ROUTING_PARAM_CACHE_SIZE) -def _urlencode_params(*params): +def _urlencode_param(key, value): """Cacheable wrapper over urlencode Args: - *params ([Tuple[str, str | bytes | Enum]): the reqyest parameters - used for routing. + key (str): The key of the parameter to encode. + value (str | bytes | Enum): The value of the parameter to encode. Returns: - str: The routing header string. + str: The encoded parameter. """ return urlencode( - params, + {key: value}, # Per Google API policy (go/api-url-encoding), / is not encoded. safe="/", ) From a2e7575efeea25520d2b0c32761f07bddc317588 Mon Sep 17 00:00:00 2001 From: Daniel Sanche Date: Fri, 6 Oct 2023 15:57:23 -0700 Subject: [PATCH 05/12] added tests --- tests/unit/gapic/test_routing_header.py | 25 +++++++++++++++++++++++++ 1 file changed, 25 insertions(+) diff --git a/tests/unit/gapic/test_routing_header.py b/tests/unit/gapic/test_routing_header.py index 9d31eb39..c2950643 100644 --- a/tests/unit/gapic/test_routing_header.py +++ b/tests/unit/gapic/test_routing_header.py @@ -70,3 +70,28 @@ def test_to_grpc_metadata(): params = [("name", "meep"), ("book.read", "1")] metadata = routing_header.to_grpc_metadata(params) assert metadata == (routing_header.ROUTING_METADATA_KEY, "name=meep&book.read=1") + +@pytest.mark.parametrize("key,value,expected", [ + ("book.read", "1", "book.read=1"), + ("name", "me/ep", "name=me/ep"), + ("\\", "=", "%5C=%3D"), + (b"hello", "world", "hello=world"), + ("✔️", "✌️", "%E2%9C%94%EF%B8%8F=%E2%9C%8C%EF%B8%8F"), +]) +def test__urlencode_param(key, value, expected): + result = routing_header._urlencode_param(key, value) + assert result == expected + +def test__urlencode_param_caching(): + import time + key = "key" * 100 + value = "value" * 100 + # time with empty cache + start_time = time.perf_counter() + routing_header._urlencode_param(key, value) + duration = time.perf_counter() - start_time + # second call should be much faster + second_start_time = time.perf_counter() + routing_header._urlencode_param(key, value) + second_duration = time.perf_counter() - second_start_time + assert second_duration < duration / 10 From 651adb9d821081ed731cfe6d81f1964e07d7deba Mon Sep 17 00:00:00 2001 From: Daniel Sanche Date: Fri, 6 Oct 2023 16:00:11 -0700 Subject: [PATCH 06/12] ran blacken --- tests/unit/gapic/test_routing_header.py | 20 +++++++++++++------- 1 file changed, 13 insertions(+), 7 deletions(-) diff --git a/tests/unit/gapic/test_routing_header.py b/tests/unit/gapic/test_routing_header.py index c2950643..7e74762a 100644 --- a/tests/unit/gapic/test_routing_header.py +++ b/tests/unit/gapic/test_routing_header.py @@ -71,19 +71,25 @@ def test_to_grpc_metadata(): metadata = routing_header.to_grpc_metadata(params) assert metadata == (routing_header.ROUTING_METADATA_KEY, "name=meep&book.read=1") -@pytest.mark.parametrize("key,value,expected", [ - ("book.read", "1", "book.read=1"), - ("name", "me/ep", "name=me/ep"), - ("\\", "=", "%5C=%3D"), - (b"hello", "world", "hello=world"), - ("✔️", "✌️", "%E2%9C%94%EF%B8%8F=%E2%9C%8C%EF%B8%8F"), -]) + +@pytest.mark.parametrize( + "key,value,expected", + [ + ("book.read", "1", "book.read=1"), + ("name", "me/ep", "name=me/ep"), + ("\\", "=", "%5C=%3D"), + (b"hello", "world", "hello=world"), + ("✔️", "✌️", "%E2%9C%94%EF%B8%8F=%E2%9C%8C%EF%B8%8F"), + ], +) def test__urlencode_param(key, value, expected): result = routing_header._urlencode_param(key, value) assert result == expected + def test__urlencode_param_caching(): import time + key = "key" * 100 value = "value" * 100 # time with empty cache From 58d911d5a1fb745ee7f0e67f0825ab167ba1e407 Mon Sep 17 00:00:00 2001 From: Daniel Sanche Date: Mon, 9 Oct 2023 12:19:09 -0700 Subject: [PATCH 07/12] Update test name Co-authored-by: Anthonios Partheniou --- tests/unit/gapic/test_routing_header.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/unit/gapic/test_routing_header.py b/tests/unit/gapic/test_routing_header.py index 7e74762a..e9279f76 100644 --- a/tests/unit/gapic/test_routing_header.py +++ b/tests/unit/gapic/test_routing_header.py @@ -87,7 +87,7 @@ def test__urlencode_param(key, value, expected): assert result == expected -def test__urlencode_param_caching(): +def test__urlencode_param_caching_performance(): import time key = "key" * 100 From 916164f5acd7a32f27a0cce204457367b501bef9 Mon Sep 17 00:00:00 2001 From: Daniel Sanche Date: Mon, 9 Oct 2023 12:19:30 -0700 Subject: [PATCH 08/12] add comment Co-authored-by: Anthonios Partheniou --- google/api_core/gapic_v1/routing_header.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/google/api_core/gapic_v1/routing_header.py b/google/api_core/gapic_v1/routing_header.py index ff447cea..c5726e63 100644 --- a/google/api_core/gapic_v1/routing_header.py +++ b/google/api_core/gapic_v1/routing_header.py @@ -66,7 +66,8 @@ def to_grpc_metadata(params, qualified_enums=True): return (ROUTING_METADATA_KEY, to_routing_header(params, qualified_enums)) -@functools.lru_cache(ROUTING_PARAM_CACHE_SIZE) +# use caching to avoid repeated computation +@functools.lru_cache(maxsize=ROUTING_PARAM_CACHE_SIZE) def _urlencode_param(key, value): """Cacheable wrapper over urlencode From 232148a8c0130b48fc92e29867aa9ca921901331 Mon Sep 17 00:00:00 2001 From: Daniel Sanche Date: Mon, 9 Oct 2023 12:19:50 -0700 Subject: [PATCH 09/12] improve comments Co-authored-by: Anthonios Partheniou --- google/api_core/gapic_v1/routing_header.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/google/api_core/gapic_v1/routing_header.py b/google/api_core/gapic_v1/routing_header.py index c5726e63..c0c6f648 100644 --- a/google/api_core/gapic_v1/routing_header.py +++ b/google/api_core/gapic_v1/routing_header.py @@ -25,7 +25,9 @@ from urllib.parse import urlencode ROUTING_METADATA_KEY = "x-goog-request-params" -# use caching to avoid repeated computation +# This is the value for the `maxsize` argument of @functools.lru_cache +# https://docs.python.org/3/library/functools.html#functools.lru_cache +# This represents the number of recent function calls to store. ROUTING_PARAM_CACHE_SIZE = 32 From 3873e2cd35da03dd8d379de671c6b924cb319e01 Mon Sep 17 00:00:00 2001 From: Daniel Sanche Date: Mon, 9 Oct 2023 12:20:11 -0700 Subject: [PATCH 10/12] moved comment Co-authored-by: Anthonios Partheniou --- tests/unit/gapic/test_routing_header.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/unit/gapic/test_routing_header.py b/tests/unit/gapic/test_routing_header.py index e9279f76..2c8c7546 100644 --- a/tests/unit/gapic/test_routing_header.py +++ b/tests/unit/gapic/test_routing_header.py @@ -96,8 +96,8 @@ def test__urlencode_param_caching_performance(): start_time = time.perf_counter() routing_header._urlencode_param(key, value) duration = time.perf_counter() - start_time - # second call should be much faster second_start_time = time.perf_counter() routing_header._urlencode_param(key, value) second_duration = time.perf_counter() - second_start_time + # second call should be approximately 10 times faster assert second_duration < duration / 10 From 2f65050583d8a6d5971147dbaad60a6e936694e6 Mon Sep 17 00:00:00 2001 From: Owl Bot Date: Mon, 9 Oct 2023 19:22:04 +0000 Subject: [PATCH 11/12] =?UTF-8?q?=F0=9F=A6=89=20Updates=20from=20OwlBot=20?= =?UTF-8?q?post-processor?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit See https://github.com/googleapis/repo-automation-bots/blob/main/packages/owl-bot/README.md --- .kokoro/requirements.txt | 6 +++--- .pre-commit-config.yaml | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/.kokoro/requirements.txt b/.kokoro/requirements.txt index 0332d326..96d593c8 100644 --- a/.kokoro/requirements.txt +++ b/.kokoro/requirements.txt @@ -467,9 +467,9 @@ typing-extensions==4.4.0 \ --hash=sha256:1511434bb92bf8dd198c12b1cc812e800d4181cfcb867674e0f8279cc93087aa \ --hash=sha256:16fa4864408f655d35ec496218b85f79b3437c829e93320c7c9215ccfd92489e # via -r requirements.in -urllib3==1.26.17 \ - --hash=sha256:24d6a242c28d29af46c3fae832c36db3bbebcc533dd1bb549172cd739c82df21 \ - --hash=sha256:94a757d178c9be92ef5539b8840d48dc9cf1b2709c9d6b588232a055c524458b +urllib3==1.26.12 \ + --hash=sha256:3fa96cf423e6987997fc326ae8df396db2a8b7c667747d47ddd8ecba91f4a74e \ + --hash=sha256:b930dd878d5a8afb066a637fbb35144fe7901e3b209d1cd4f524bd0e9deee997 # via # requests # twine diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 6a8e1695..19409cbd 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -22,7 +22,7 @@ repos: - id: end-of-file-fixer - id: check-yaml - repo: https://github.com/psf/black - rev: 23.7.0 + rev: 22.3.0 hooks: - id: black - repo: https://github.com/pycqa/flake8 From 5a6afd5cd3c047d07b1f89286f5918823469245a Mon Sep 17 00:00:00 2001 From: Owl Bot Date: Mon, 9 Oct 2023 19:23:53 +0000 Subject: [PATCH 12/12] =?UTF-8?q?=F0=9F=A6=89=20Updates=20from=20OwlBot=20?= =?UTF-8?q?post-processor?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit See https://github.com/googleapis/repo-automation-bots/blob/main/packages/owl-bot/README.md --- .kokoro/requirements.txt | 6 +++--- .pre-commit-config.yaml | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/.kokoro/requirements.txt b/.kokoro/requirements.txt index 96d593c8..0332d326 100644 --- a/.kokoro/requirements.txt +++ b/.kokoro/requirements.txt @@ -467,9 +467,9 @@ typing-extensions==4.4.0 \ --hash=sha256:1511434bb92bf8dd198c12b1cc812e800d4181cfcb867674e0f8279cc93087aa \ --hash=sha256:16fa4864408f655d35ec496218b85f79b3437c829e93320c7c9215ccfd92489e # via -r requirements.in -urllib3==1.26.12 \ - --hash=sha256:3fa96cf423e6987997fc326ae8df396db2a8b7c667747d47ddd8ecba91f4a74e \ - --hash=sha256:b930dd878d5a8afb066a637fbb35144fe7901e3b209d1cd4f524bd0e9deee997 +urllib3==1.26.17 \ + --hash=sha256:24d6a242c28d29af46c3fae832c36db3bbebcc533dd1bb549172cd739c82df21 \ + --hash=sha256:94a757d178c9be92ef5539b8840d48dc9cf1b2709c9d6b588232a055c524458b # via # requests # twine diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 19409cbd..6a8e1695 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -22,7 +22,7 @@ repos: - id: end-of-file-fixer - id: check-yaml - repo: https://github.com/psf/black - rev: 22.3.0 + rev: 23.7.0 hooks: - id: black - repo: https://github.com/pycqa/flake8