From 5f5f3387a5b19908ed015d36ccfd16d716db07fd Mon Sep 17 00:00:00 2001 From: masklinn Date: Sat, 1 Feb 2025 14:16:09 +0100 Subject: [PATCH 1/8] formatting fixes I assume ruff's updated a few things since last time it was run, as it now fails. --- src/ua_parser/__main__.py | 6 ++-- tests/test_legacy.py | 72 +++++++++++++++++++-------------------- 2 files changed, 39 insertions(+), 39 deletions(-) diff --git a/src/ua_parser/__main__.py b/src/ua_parser/__main__.py index 0ed140f..047efaa 100644 --- a/src/ua_parser/__main__.py +++ b/src/ua_parser/__main__.py @@ -101,7 +101,7 @@ def run_stdout(args: argparse.Namespace) -> None: lines = list(args.file) count = len(lines) uniques = len(set(lines)) - print(f"{args.file.name}: {count} lines, {uniques} unique ({uniques/count:.0%})") + print(f"{args.file.name}: {count} lines, {uniques} unique ({uniques / count:.0%})") rules = get_rules(args.bases, args.regexes) @@ -320,7 +320,7 @@ def belady(maxsize: int) -> Cache: overhead / cache_size, ) print( - f"{cache.__name__.lower():8}({cache_size:{w}}): {(total - misses.count)/total*100:2.0f}% hit rate {diff}" + f"{cache.__name__.lower():8}({cache_size:{w}}): {(total - misses.count) / total * 100:2.0f}% hit rate {diff}" ) del misses, parser @@ -378,7 +378,7 @@ def run_threaded(args: argparse.Namespace) -> None: totlines = len(lines) * args.threads # runtime in us t = (time.perf_counter_ns() - st) / 1000 - print(f"{t/totlines:>4.0f}us/line", flush=True) + print(f"{t / totlines:>4.0f}us/line", flush=True) EPILOG = """For good results the sample `file` should be an actual diff --git a/tests/test_legacy.py b/tests/test_legacy.py index 7ada17c..8fafbee 100644 --- a/tests/test_legacy.py +++ b/tests/test_legacy.py @@ -107,18 +107,18 @@ def runUserAgentTestsFromYAML(self, file_name): result = {} result = user_agent_parser.ParseUserAgent(user_agent_string) - assert ( - result == expected - ), "UA: {0}\n expected<{1}, {2}, {3}, {4}> != actual<{5}, {6}, {7}, {8}>".format( - user_agent_string, - expected["family"], - expected["major"], - expected["minor"], - expected["patch"], - result["family"], - result["major"], - result["minor"], - result["patch"], + assert result == expected, ( + "UA: {0}\n expected<{1}, {2}, {3}, {4}> != actual<{5}, {6}, {7}, {8}>".format( + user_agent_string, + expected["family"], + expected["major"], + expected["minor"], + expected["patch"], + result["family"], + result["major"], + result["minor"], + result["patch"], + ) ) assert ( len(user_agent_parser._PARSE_CACHE) <= user_agent_parser.MAX_CACHE_SIZE @@ -143,20 +143,20 @@ def runOSTestsFromYAML(self, file_name): } result = user_agent_parser.ParseOS(user_agent_string) - assert ( - result == expected - ), "UA: {0}\n expected<{1} {2} {3} {4} {5}> != actual<{6} {7} {8} {9} {10}>".format( - user_agent_string, - expected["family"], - expected["major"], - expected["minor"], - expected["patch"], - expected["patch_minor"], - result["family"], - result["major"], - result["minor"], - result["patch"], - result["patch_minor"], + assert result == expected, ( + "UA: {0}\n expected<{1} {2} {3} {4} {5}> != actual<{6} {7} {8} {9} {10}>".format( + user_agent_string, + expected["family"], + expected["major"], + expected["minor"], + expected["patch"], + expected["patch_minor"], + result["family"], + result["major"], + result["minor"], + result["patch"], + result["patch_minor"], + ) ) def runDeviceTestsFromYAML(self, file_name): @@ -176,16 +176,16 @@ def runDeviceTestsFromYAML(self, file_name): } result = user_agent_parser.ParseDevice(user_agent_string) - assert ( - result == expected - ), "UA: {0}\n expected<{1} {2} {3}> != actual<{4} {5} {6}>".format( - user_agent_string, - expected["family"], - expected["brand"], - expected["model"], - result["family"], - result["brand"], - result["model"], + assert result == expected, ( + "UA: {0}\n expected<{1} {2} {3}> != actual<{4} {5} {6}>".format( + user_agent_string, + expected["family"], + expected["brand"], + expected["model"], + result["family"], + result["brand"], + result["model"], + ) ) From ce129055a661eb3be0ab0edaa7669efec0cf8d7d Mon Sep 17 00:00:00 2001 From: masklinn Date: Sat, 1 Feb 2025 14:17:40 +0100 Subject: [PATCH 2/8] Fix memoisation of lazy parser & bump version Reported by @Rafiot: the lazy parser is not memoised, this has limited effect on the basic / pure Python parser as its initialisation is trivial, but it *significantly* impact the re2 and regex parsers as they need to process regexes into a filter tree. The memoization was mistakenly removed in #230: while refactoring initialisation I removed the setting of the `parser` global. - add a test to ensure the parser is correctly memoized, not re-instantiated every time - reinstate setting the global - add a mutex on `__getattr__`, it should only be used on first access and avoids two threads creating an expensive parser at the same time (which is a waste of CPU) Fixes #253 --- pyproject.toml | 2 +- src/ua_parser/__init__.py | 29 +++++++++++++++++++++-------- tests/test_convenience_parser.py | 17 +++++++++++++++++ 3 files changed, 39 insertions(+), 9 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 161c98b..11425fc 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -5,7 +5,7 @@ build-backend = "setuptools.build_meta" [project] name = "ua-parser" description = "Python port of Browserscope's user agent parser" -version = "1.0.0" +version = "1.0.1" readme = "README.rst" requires-python = ">=3.9" dependencies = ["ua-parser-builtins"] diff --git a/src/ua_parser/__init__.py b/src/ua_parser/__init__.py index 040dda3..5b5ba71 100644 --- a/src/ua_parser/__init__.py +++ b/src/ua_parser/__init__.py @@ -41,7 +41,8 @@ ] import importlib.util -from typing import Callable, Optional +import threading +from typing import Callable, Optional, cast from .basic import Resolver as BasicResolver from .caching import CachingResolver, S3Fifo as Cache @@ -78,7 +79,7 @@ ) -VERSION = (1, 0, 0) +VERSION = (1, 0, 1) class Parser: @@ -135,15 +136,27 @@ def parse_device(self: Resolver, ua: str) -> Optional[Device]: initialisation, rather than pay for it at first call. """ +_lazy_globals_lock = threading.Lock() + def __getattr__(name: str) -> Parser: global parser - if name == "parser": - if RegexResolver or Re2Resolver or IS_GRAAL: - matchers = load_lazy_builtins() - else: - matchers = load_builtins() - return Parser.from_matchers(matchers) + with _lazy_globals_lock: + if name == "parser": + # if two threads access `ua_parser.parser` before it's + # initialised, the second one will wait until the first + # one's finished by which time the parser global should be + # set and can be returned with no extra work + if p := globals().get("parser"): + return cast(Parser, p) + + if RegexResolver or Re2Resolver or IS_GRAAL: + matchers = load_lazy_builtins() + else: + matchers = load_builtins() + parser = Parser.from_matchers(matchers) + return parser + raise AttributeError(f"module {__name__!r} has no attribute {name!r}") diff --git a/tests/test_convenience_parser.py b/tests/test_convenience_parser.py index cf1d360..8624061 100644 --- a/tests/test_convenience_parser.py +++ b/tests/test_convenience_parser.py @@ -1,6 +1,23 @@ +import ua_parser from ua_parser import Domain, Parser, PartialResult, Result +def test_parser_memoized() -> None: + """The global parser should be lazily instantiated but memoized""" + # ensure there is no global parser + vars(ua_parser).pop("parser", None) + + p1 = ua_parser.parser + p2 = ua_parser.parser + + assert p1 is p2 + + # force the creation of a clean parser + del ua_parser.parser + p3 = ua_parser.parser + assert p3 is not p1 + + def resolver(s: str, d: Domain) -> PartialResult: return PartialResult(d, None, None, None, s) From 1b64406fce241dec909b03a05383f3b31a073e7e Mon Sep 17 00:00:00 2001 From: William Douglas Date: Sat, 15 Feb 2025 04:24:29 -0800 Subject: [PATCH 3/8] builtins: fallback to package.json for uap-core version In case where uap-core isn't a git repo (e.g. git archive), use uap-core's `package.json` as a fallback for getting a version. --- ua-parser-builtins/hatch_build.py | 24 +++++++++++++++--------- 1 file changed, 15 insertions(+), 9 deletions(-) diff --git a/ua-parser-builtins/hatch_build.py b/ua-parser-builtins/hatch_build.py index 1eac776..9bbe23f 100644 --- a/ua-parser-builtins/hatch_build.py +++ b/ua-parser-builtins/hatch_build.py @@ -1,6 +1,7 @@ from __future__ import annotations import io +import json import os import os.path import tempfile @@ -10,19 +11,24 @@ import yaml from hatchling.builders.hooks.plugin.interface import BuildHookInterface from hatchling.metadata.plugin.interface import MetadataHookInterface -from versioningit import get_version +from versioningit import errors, get_version class MetadataHook(MetadataHookInterface): def update(self, metadata: dict[str, Any]) -> None: - v = get_version( - os.path.join(self.root, "uap-core"), - config={ - "format": { - "distance": "{next_version}.dev{distance}", - } - }, - ) + try: + v = get_version( + os.path.join(self.root, "uap-core"), + config={ + "format": { + "distance": "{next_version}.dev{distance}", + } + }, + ) + except errors.NotSdistError: + with open(os.path.join(self.root, "uap-core", "package.json")) as ufile: + ujson = json.load(ufile) + v = ujson["version"] if v in ("0.15.0", "0.16.0", "0.18.0"): v = f"{v}.post1" From 2ca789ea504afd885a9ee3d33ef64db6d34e8908 Mon Sep 17 00:00:00 2001 From: masklinn Date: Sat, 15 Feb 2025 13:58:37 +0100 Subject: [PATCH 4/8] Fix fallback input for release action Apparently the way submodules repos are configured leads to the branches not being mirrored locally (?) As such, the release job's fallback of checking out `'master'` fails whether triggered[^1] or scheduled[^2]. [^1]: https://github.com/ua-parser/uap-python/actions/runs/13090871627 [^2]: https://github.com/ua-parser/uap-python/actions/runs/13092233962 --- .github/workflows/release-builtins.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/release-builtins.yml b/.github/workflows/release-builtins.yml index 6f41709..f2ad7b8 100644 --- a/.github/workflows/release-builtins.yml +++ b/.github/workflows/release-builtins.yml @@ -28,7 +28,7 @@ jobs: persist-credentials: false - name: update core env: - TAG: ${{ inputs.tag || 'master '}} + TAG: ${{ inputs.tag || 'origin/master '}} # needs to detach because we can update to a tag run: git -C uap-core switch --detach "$TAG" - name: Set up Python From ea7a5ae639150589729946746009d86aa4f8c0c5 Mon Sep 17 00:00:00 2001 From: masklinn Date: Sat, 15 Feb 2025 14:01:06 +0100 Subject: [PATCH 5/8] builtins release: make inputs required for manual triggers --- .github/workflows/release-builtins.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/release-builtins.yml b/.github/workflows/release-builtins.yml index f2ad7b8..fad58c0 100644 --- a/.github/workflows/release-builtins.yml +++ b/.github/workflows/release-builtins.yml @@ -11,9 +11,11 @@ on: tag: description: "uap-core ref to release" type: string + required: true environment: description: "environment to release for (testpypy or pypy)" type: environment + required: true jobs: build: From 60b35ec1fb358005f9e732831600b10e8533c080 Mon Sep 17 00:00:00 2001 From: masklinn Date: Sat, 15 Feb 2025 14:02:10 +0100 Subject: [PATCH 6/8] Clarify environment fallback Since the environment is required via `workflow_dispatch`, the only fallback is scheduled release in which case we're publishing to pypy. --- .github/workflows/release-builtins.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/release-builtins.yml b/.github/workflows/release-builtins.yml index fad58c0..917dd3d 100644 --- a/.github/workflows/release-builtins.yml +++ b/.github/workflows/release-builtins.yml @@ -1,6 +1,6 @@ name: Publish ua-parser builtins -run-name: Publish ${{ inputs.tag || 'master' }} to ${{ inputs.environment || 'dummy' }} +run-name: Publish ${{ inputs.tag || 'master' }} to ${{ inputs.environment || 'pypy (scheduled)' }} on: schedule: From 997990f47f97c64db8d69d650c3b7f7c87e402a7 Mon Sep 17 00:00:00 2001 From: masklinn Date: Mon, 3 Mar 2025 18:53:09 +0100 Subject: [PATCH 7/8] Fix fallback input for release action for real Turns out `'master'` probably worked all along as a fallback, the problem is that I was using `'master '`, with a trailing space, which was not a branch git managed to find for obvious reason, and since I carried the error into the fully qualified reference... is still didn't work. And manual triggers didn't have the issue because the tag was `required`, so I'd have to input the tag by hand every time, and the fallback value would be bypassed. - fix the fallback value - remove the requirement on `tag`, such that it's possible to manually trigger the action in a default state --- .github/workflows/release-builtins.yml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/.github/workflows/release-builtins.yml b/.github/workflows/release-builtins.yml index 917dd3d..408f9c7 100644 --- a/.github/workflows/release-builtins.yml +++ b/.github/workflows/release-builtins.yml @@ -11,7 +11,6 @@ on: tag: description: "uap-core ref to release" type: string - required: true environment: description: "environment to release for (testpypy or pypy)" type: environment @@ -30,7 +29,7 @@ jobs: persist-credentials: false - name: update core env: - TAG: ${{ inputs.tag || 'origin/master '}} + TAG: ${{ inputs.tag || 'origin/master' }} # needs to detach because we can update to a tag run: git -C uap-core switch --detach "$TAG" - name: Set up Python From 911b7a313e3b0ee2c419d1ac66f2b5331a6d2143 Mon Sep 17 00:00:00 2001 From: masklinn Date: Mon, 9 Jun 2025 16:23:47 +0200 Subject: [PATCH 8/8] Update classifiers and version bounds - add classifier for cpython 3.13 - add classifier for graal (now that it's been merged) - add pypy 3.11 to tox - re2 still hasn't published for CPython 3.13 so exclude from tox Fixes #257, fixes #265 --- .github/workflows/ci.yml | 2 +- pyproject.toml | 4 ++-- tox.ini | 15 +++++++-------- ua-parser-builtins/pyproject.toml | 3 ++- 4 files changed, 12 insertions(+), 12 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 4cc5f9f..633c55e 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -90,7 +90,7 @@ jobs: - "3.12" - "3.13" - "pypy-3.10" - # - "pypy-3.11" + - "pypy-3.11" - "graalpy-24" include: - source: sdist diff --git a/pyproject.toml b/pyproject.toml index 11425fc..33fc825 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -37,10 +37,10 @@ classifiers = [ "Programming Language :: Python :: 3.10", "Programming Language :: Python :: 3.11", "Programming Language :: Python :: 3.12", + "Programming Language :: Python :: 3.13", "Programming Language :: Python :: Implementation :: CPython", "Programming Language :: Python :: Implementation :: PyPy", - # no graalpy classifier yet (pypa/trove-classifiers#188) - # "Programming Language :: Python :: Implementation :: GraalPy", + "Programming Language :: Python :: Implementation :: GraalPy", ] [project.urls] diff --git a/tox.ini b/tox.ini index 5bd97e9..9b3f154 100644 --- a/tox.ini +++ b/tox.ini @@ -1,14 +1,13 @@ [tox] min_version = 4.0 -env_list = py3{9,10,11,12} - pypy310 +env_list = py3{9,10,11,12,13} + pypy{310,311} graalpy flake8, black, typecheck labels = - test = py3{9,10,11,12},pypy310,graalpy - cpy = py3{9,10,11,12} - pypy = pypy3.10 - graal = graalpy-24 + test = py3{9,10,11,12,13},pypy{310,311},graalpy + cpy = py3{9,10,11,12,13} + pypy = pypy{310,311} check = flake8, black, typecheck [testenv] @@ -21,16 +20,16 @@ wheel_build_env = .pkg deps = pytest pyyaml - google-re2 ua-parser-rs ./ua-parser-builtins commands = pytest -Werror --doctest-glob="*.rst" {posargs} -[testenv:{pypy310,graalpy}] +[testenv:py3{9,10,11,12}] deps = pytest pyyaml + google-re2 ua-parser-rs ./ua-parser-builtins diff --git a/ua-parser-builtins/pyproject.toml b/ua-parser-builtins/pyproject.toml index a9c6d3e..0086e85 100644 --- a/ua-parser-builtins/pyproject.toml +++ b/ua-parser-builtins/pyproject.toml @@ -29,9 +29,10 @@ classifiers = [ "Programming Language :: Python :: 3.10", "Programming Language :: Python :: 3.11", "Programming Language :: Python :: 3.12", + "Programming Language :: Python :: 3.13", "Programming Language :: Python :: Implementation :: CPython", "Programming Language :: Python :: Implementation :: PyPy", - # "Programming Language :: Python :: Implementation :: GraalPy", + "Programming Language :: Python :: Implementation :: GraalPy", ] [tool.hatch.build.hooks.custom]