From 6741f56094677aea50dfa2b0c4e305650289d36b Mon Sep 17 00:00:00 2001 From: masklinn Date: Thu, 24 Nov 2022 19:27:24 +0100 Subject: [PATCH 1/3] Drop 2.7, 3.6, and 3.7 - 2.7 was EOL'd in 2020 - 3.6 was EOL'd in 2021 - 3.7 will be EOL'd in June 2023 Even if a 1.0 is cut before June it doesn't seem likely that it'll matter a lot. 3.8 doesn't bring a lot that's likely to be useful as far as I know, candidates I can think of are: - positional-only parameters - typed dicts and literals, to add typing to the legacy API While at it, remove the now unnecessary future imports, as well as the module-level docstrings (they're not very helpful) and the per-file license headers (they don't seem useful). --- .github/workflows/ci.yml | 28 ++-------------------------- setup.py | 9 --------- src/ua_parser/user_agent_parser.py | 18 ------------------ tests/test_legacy.py | 21 --------------------- tox.ini | 6 +----- 5 files changed, 3 insertions(+), 79 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 0722a9a..d02b41e 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -33,9 +33,6 @@ jobs: fail-fast: false matrix: python-version: - - "2.7" - - "3.6" - - "3.7" - "3.8" - "3.9" - "3.10" @@ -44,15 +41,7 @@ jobs: - "pypy-3.8" - "pypy-3.9" pyyaml-version: ["5.1.*", "5.4.*", "6.0.*", "6.*"] - include: - - python-version: 3.6 - os: ubuntu-20.04 - exclude: - - python-version: 2.7 - pyyaml-version: 6.0.* - - python-version: 2.7 - pyyaml-version: 6.* - runs-on: ${{ matrix.os || 'ubuntu-latest' }} + runs-on: ubuntu-latest steps: - name: Checkout working copy @@ -77,9 +66,6 @@ jobs: fail-fast: false matrix: python-version: - - "2.7" - - "3.6" - - "3.7" - "3.8" - "3.9" - "3.10" @@ -87,11 +73,7 @@ jobs: - "3.12.0-alpha - 3.12" - "pypy-3.8" - "pypy-3.9" - include: - - python-version: 3.6 - os: ubuntu-20.04 - runs-on: ${{ matrix.os || 'ubuntu-latest' }} - + runs-on: ubuntu-latest steps: - name: Checkout working copy uses: actions/checkout@v3 @@ -115,10 +97,4 @@ jobs: - name: install package in environment run: pip install . - name: run tests - if: ${{ matrix.python-version != '2.7' }} run: pytest -v -Werror -Wignore::ImportWarning --doctest-glob="*.rst" - - name: run tests - # pprint formatting was widely altered in 3.5, so can't run - # doctests before then - if: ${{ matrix.python-version == '2.7' }} - run: pytest -v -Werror -Wignore::ImportWarning diff --git a/setup.py b/setup.py index 26aabf3..a69228b 100644 --- a/setup.py +++ b/setup.py @@ -110,7 +110,6 @@ def write_params(fields): fp.write(b"# instead, re-run `setup.py build_regexes` #\n") fp.write(b"############################################\n") fp.write(b"\n") - fp.write(b"from __future__ import absolute_import, unicode_literals\n") fp.write(b"from .user_agent_parser import (\n") fp.write(b" UserAgentParser, DeviceParser, OSParser,\n") fp.write(b")\n") @@ -223,14 +222,6 @@ class sdist(_sdist): "Topic :: Internet :: WWW/HTTP", "Topic :: Software Development :: Libraries :: Python Modules", "Programming Language :: Python", - "Programming Language :: Python :: 2", - "Programming Language :: Python :: 2.7", - "Programming Language :: Python :: 3", - "Programming Language :: Python :: 3.3", - "Programming Language :: Python :: 3.4", - "Programming Language :: Python :: 3.5", - "Programming Language :: Python :: 3.6", - "Programming Language :: Python :: 3.7", "Programming Language :: Python :: 3.8", "Programming Language :: Python :: 3.9", "Programming Language :: Python :: 3.10", diff --git a/src/ua_parser/user_agent_parser.py b/src/ua_parser/user_agent_parser.py index 4989336..63e59b8 100644 --- a/src/ua_parser/user_agent_parser.py +++ b/src/ua_parser/user_agent_parser.py @@ -1,21 +1,3 @@ -# Copyright 2009 Google Inc. -# -# Licensed under the Apache License, Version 2.0 (the 'License') -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an 'AS IS' BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -"""Python implementation of the UA parser.""" - -from __future__ import absolute_import - import os import re import sys diff --git a/tests/test_legacy.py b/tests/test_legacy.py index 9742a66..ed2b6ab 100644 --- a/tests/test_legacy.py +++ b/tests/test_legacy.py @@ -1,24 +1,3 @@ -# Copyright 2008 Google Inc. -# -# Licensed under the Apache License, Version 2.0 (the 'License') -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an 'AS IS' BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - - -"""User Agent Parser Unit Tests. -""" - - -from __future__ import unicode_literals, absolute_import - __author__ = "slamm@google.com (Stephen Lamm)" import logging diff --git a/tox.ini b/tox.ini index 6100de9..9f04d90 100644 --- a/tox.ini +++ b/tox.ini @@ -1,5 +1,5 @@ [tox] -envlist = py27, py36, py37, py38, py39, py310, py311, +envlist = py38, py39, py310, py311, pypy3.8, pypy3.9, docs, flake8, black skipsdist = True @@ -10,10 +10,6 @@ deps = -rrequirements_dev.txt commands = pytest -Werror --doctest-glob="*.rst" {posargs} -[testenv:py27] -# no doctesting in 2.7 because of formatting divergences -commands = pytest -Werror {posargs} - [testenv:flake8] skip_install = True deps = flake8 From 39c601c7fd57a97a8f78db740e0650ee440057b6 Mon Sep 17 00:00:00 2001 From: masklinn Date: Thu, 24 Nov 2022 13:02:37 +0100 Subject: [PATCH 2/3] Modernize packaging PEP 517 ======= Don't migrate away from setuptools, but move "static" packaging content from `setup.py` to `pyproject.toml` and `setup.cfg`. Building should now be performed using [`build`](https://pypa-build.readthedocs.io/en/stable/), as direct invocations of `setup.py` have been deprecated for a while. Also remove the legacy `__author__` strictures from the source files, as `pyproject.toml` supports an authors section which is more suitable. Codegen Modernization ===================== The modernization via command overrides and self-invocation still generated `setup.py` warnings. Migrate the "invoke commands in commands" method to codegen sub-commands in `build`, it looks like this suffices for what we need. This reuses the existing code, though modernizes it significantly thanks to the previous commit. This is strongly inspired by @abravalheri's comments on codegen in setuptools discussions e.g. pypa/setuptools#3180 and pypa/setuptools#3762, those comments were very helpful in getting a better understanding of the sub_commands system (also @jaraco's comments but those were a touch terse so can't say I really got it until sumbling on @abravalheri's). Leverages the setuptools subcommands protocol to support editable wheels, as that might be useful. Don't use editable wheels for testing though, as I don't see the point. Note: not sure the git submodule stuff is useful during the codegen, so currently commented it out. sdists ====== I'm not entirely clear what sdists should really be about, adding codegen to sdist using subcommands seems not necessarily trivial, so for now make them basically an export: - remove `_regexes.py` from sdist - add uap-core to the manifest, and thus the sdist, at least `regexes.yaml` Currently the test files (both python and yaml) are also included, but it's not clear that they *should* be. That seems like [a whole debate](https://discuss.python.org/t/should-sdists-include-docs-and-tests/14578/117), One strong argument (to me) in favor of a sdist maximalism is that it matches the wheels and more official releases, but in that case the sdist should probably be better crafted than what I threw together. Tox === Switch tox to an explicit `pip install` as its builtin handling for installing packages (whether via sdist or develop) does not seem compatible with PEP-517. Need to think about moving to tox 4, which apparently has native support for PEP 517 packages. Cf #157. Also remove the `docs` tox env, it hasn't been a thing since 827347722bfb2fd8088783fd9705308dd8b0d4b6 but I forgot to remove it from the envlist. --- MANIFEST.in | 8 +- Makefile | 2 +- pyproject.toml | 44 ++++++ setup.cfg | 10 +- setup.py | 211 ++++++++--------------------- src/ua_parser/user_agent_parser.py | 2 - tests/test_legacy.py | 2 - tox.ini | 4 +- 8 files changed, 118 insertions(+), 165 deletions(-) create mode 100644 pyproject.toml diff --git a/MANIFEST.in b/MANIFEST.in index a0600da..9c004de 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -1,3 +1,9 @@ +exclude .* +prune .github +global-exclude *~ + include README.rst include LICENSE -global-exclude *~ +graft uap-core +exclude uap-core/.* +recursive-exclude uap-core *.js diff --git a/Makefile b/Makefile index ff691c4..0604a25 100644 --- a/Makefile +++ b/Makefile @@ -14,7 +14,7 @@ format: @black . release: clean - python setup.py sdist bdist_wheel + pyproject-build twine upload -s dist/* .PHONY: all test clean format release diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 0000000..48b6aa9 --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,44 @@ +[build-system] +requires = ["setuptools", "setuptools-scm", "PyYaml"] +build-backend = "setuptools.build_meta" + +[project] +name = "ua-parser" +description = "Python port of Browserscope's user agent parser" +version = "1.0.0a" +readme = "README.rst" +requires-python = ">=3.8" +dependencies = [] +optional-dependencies = { yaml = ["PyYaml"] } + +license = {text = "Apache 2.0"} +urls = {repository = "https://github.com/ua-parser/uap-python"} + +authors = [ + { name = "Stephen Lamm", email = "slamm@google.com"}, + { name = "PBS", email = "no-reply@pbs.org" }, + { name = "Selwin Ong", email = "selwin.ong@gmail.com" }, + { name = "Matt Robenolt", email = "matt@ydekproductions.com" }, + { name = "Lindsey Simon", email = "lsimon@commoner.com" }, +] +maintainers = [ + { name = "masklinn", email = "uap@masklinn.net" } +] + +classifiers = [ + "Development Status :: 4 - Beta", + "Environment :: Web Environment", + "Intended Audience :: Developers", + "Operating System :: OS Independent", + "License :: OSI Approved :: Apache Software License", + "Programming Language :: Python", + "Topic :: Internet :: WWW/HTTP", + "Topic :: Software Development :: Libraries :: Python Modules", + "Programming Language :: Python", + "Programming Language :: Python :: 3.8", + "Programming Language :: Python :: 3.9", + "Programming Language :: Python :: 3.10", + "Programming Language :: Python :: 3.11", + "Programming Language :: Python :: Implementation :: CPython", + "Programming Language :: Python :: Implementation :: PyPy" +] diff --git a/setup.cfg b/setup.cfg index 2a9acf1..9b07aee 100644 --- a/setup.cfg +++ b/setup.cfg @@ -1,2 +1,8 @@ -[bdist_wheel] -universal = 1 +[options] +packages = find: +package_dir = + =src +setup_requires = pyyaml + +[options.packages.find] +where = src diff --git a/setup.py b/setup.py index a69228b..0e14118 100644 --- a/setup.py +++ b/setup.py @@ -1,68 +1,39 @@ #!/usr/bin/env python # flake8: noqa -import os -from distutils import log -from distutils.core import Command -from distutils.command.build import build as _build -from setuptools import setup, find_packages -from setuptools.command.develop import develop as _develop -from setuptools.command.sdist import sdist as _sdist -from setuptools.command.install import install as _install - - -def check_output(*args, **kwargs): - from subprocess import Popen - - proc = Popen(*args, **kwargs) - output, _ = proc.communicate() - rv = proc.poll() - assert rv == 0, output - - -class build_regexes(Command): - description = "build supporting regular expressions from uap-core" - user_options = [ - ("work-path=", "w", "The working directory for source files. Defaults to ."), - ("build-lib=", "b", "directory for script runtime modules"), - ( - "inplace", - "i", - "ignore build-lib and put compiled javascript files into the source " - + "directory alongside your pure Python modules", - ), - ( - "force", - "f", - "Force rebuilding of static content. Defaults to rebuilding on version " - "change detection.", - ), - ] - boolean_options = ["force"] - - def initialize_options(self): - self.build_lib = None - self.force = None - self.work_path = None - self.inplace = None - - def finalize_options(self): - install = self.distribution.get_command_obj("install") - sdist = self.distribution.get_command_obj("sdist") - build_ext = self.distribution.get_command_obj("build_ext") - - if self.inplace is None: - self.inplace = ( - (build_ext.inplace or install.finalized or sdist.finalized) and 1 or 0 - ) +from contextlib import suppress +from os import fspath +from pathlib import Path +from typing import Optional, List, Dict - if self.inplace: - self.build_lib = "./src" - else: - self.set_undefined_options("build", ("build_lib", "build_lib")) - if self.work_path is None: - self.work_path = os.path.realpath(os.path.join(os.path.dirname(__file__))) +from setuptools import setup, Command, find_namespace_packages +from setuptools.command.build import build, SubCommand +from setuptools.command.editable_wheel import editable_wheel + +import yaml + + +build.sub_commands.insert(0, ("compile-regexes", None)) + + +class CompileRegexes(Command, SubCommand): + def initialize_options(self) -> None: + self.pkg_name: Optional[str] = None - def run(self): + def finalize_options(self) -> None: + self.pkg_name = self.distribution.get_name().replace("-", "_") + + def get_source_files(self) -> List[str]: + return ["uap-core/regexes.yaml"] + + def get_outputs(self) -> List[str]: + return [f"{self.pkg_name}/_regexes.py"] + + def get_output_mapping(self) -> Dict[str, str]: + return dict(zip(self.get_source_files(), self.get_outputs())) + + def run(self) -> None: + # FIXME: check git / submodules? + """ work_path = self.work_path if not os.path.exists(os.path.join(work_path, ".git")): return @@ -70,45 +41,43 @@ def run(self): log.info("initializing git submodules") check_output(["git", "submodule", "init"], cwd=work_path) check_output(["git", "submodule", "update"], cwd=work_path) + """ + if not self.pkg_name: + return # or error? - yaml_src = os.path.join(work_path, "uap-core", "regexes.yaml") - if not os.path.exists(yaml_src): + yaml_src = Path("uap-core", "regexes.yaml") + if not yaml_src.is_file(): raise RuntimeError( - "Unable to find regexes.yaml, should be at %r" % yaml_src + f"Unable to find regexes.yaml, should be at {yaml_src!r}" ) - def force_bytes(text): - if text is None: - return text - return text.encode("utf8") - def write_params(fields): # strip trailing None values while len(fields) > 1 and fields[-1] is None: fields.pop() for field in fields: - fp.write((" %r,\n" % field).encode("utf-8")) + fp.write((f" {field!r},\n").encode()) + + with yaml_src.open("rb") as f: + regexes = yaml.safe_load(f) - import yaml + if self.editable_mode: + dist_dir = Path("src") + else: + dist_dir = Path(self.get_finalized_command("bdist_wheel").bdist_dir) - log.info("compiling regexes.yaml -> _regexes.py") - with open(yaml_src, "rb") as fp: - regexes = yaml.safe_load(fp) + outdir = dist_dir / self.pkg_name + outdir.mkdir(parents=True, exist_ok=True) - lib_dest = os.path.join(self.build_lib, "ua_parser") - if not os.path.exists(lib_dest): - os.makedirs(lib_dest) + dest = outdir / "_regexes.py" - py_dest = os.path.join(lib_dest, "_regexes.py") - with open(py_dest, "wb") as fp: + with dest.open("wb") as fp: # fmt: off fp.write(b"# -*- coding: utf-8 -*-\n") - fp.write(b"############################################\n") - fp.write(b"# NOTICE: This file is autogenerated from #\n") - fp.write(b"# regexes.yaml. Do not edit by hand, #\n") - fp.write(b"# instead, re-run `setup.py build_regexes` #\n") - fp.write(b"############################################\n") + fp.write(b"########################################################\n") + fp.write(b"# NOTICE: This file is autogenerated from regexes.yaml #\n") + fp.write(b"########################################################\n") fp.write(b"\n") fp.write(b"from .user_agent_parser import (\n") fp.write(b" UserAgentParser, DeviceParser, OSParser,\n") @@ -156,77 +125,9 @@ def write_params(fields): fp.write(b"]\n") # fmt: on - self.update_manifest() - - def update_manifest(self): - sdist = self.distribution.get_command_obj("sdist") - if not sdist.finalized: - return - - sdist.filelist.files.append("src/ua_parser/_regexes.py") - - -class develop(_develop): - def run(self): - self.run_command("build_regexes") - _develop.run(self) - - -class install(_install): - def run(self): - self.run_command("build_regexes") - _install.run(self) - - -class build(_build): - def run(self): - self.run_command("build_regexes") - _build.run(self) - - -class sdist(_sdist): - sub_commands = _sdist.sub_commands + [("build_regexes", None)] - - -cmdclass = { - "sdist": sdist, - "develop": develop, - "build": build, - "install": install, - "build_regexes": build_regexes, -} - setup( - name="ua-parser", - version="0.16.1", - description="Python port of Browserscope's user agent parser", - author="PBS", - author_email="no-reply@pbs.org", - packages=find_packages(where="src"), - package_dir={"": "src"}, - license="Apache 2.0", - zip_safe=False, - url="https://github.com/ua-parser/uap-python", - include_package_data=True, - setup_requires=["pyyaml"], - install_requires=[], - cmdclass=cmdclass, - classifiers=[ - "Development Status :: 4 - Beta", - "Environment :: Web Environment", - "Intended Audience :: Developers", - "Operating System :: OS Independent", - "License :: OSI Approved :: Apache Software License", - "Programming Language :: Python", - "Topic :: Internet :: WWW/HTTP", - "Topic :: Software Development :: Libraries :: Python Modules", - "Programming Language :: Python", - "Programming Language :: Python :: 3.8", - "Programming Language :: Python :: 3.9", - "Programming Language :: Python :: 3.10", - "Programming Language :: Python :: 3.11", - "Programming Language :: Python :: Implementation :: CPython", - "Programming Language :: Python :: Implementation :: PyPy", - ], + cmdclass={ + "compile-regexes": CompileRegexes, + } ) diff --git a/src/ua_parser/user_agent_parser.py b/src/ua_parser/user_agent_parser.py index 63e59b8..275c12b 100644 --- a/src/ua_parser/user_agent_parser.py +++ b/src/ua_parser/user_agent_parser.py @@ -3,8 +3,6 @@ import sys import warnings -__author__ = "Lindsey Simon " - class UserAgentParser(object): def __init__( diff --git a/tests/test_legacy.py b/tests/test_legacy.py index ed2b6ab..f4170ce 100644 --- a/tests/test_legacy.py +++ b/tests/test_legacy.py @@ -1,5 +1,3 @@ -__author__ = "slamm@google.com (Stephen Lamm)" - import logging import os import platform diff --git a/tox.ini b/tox.ini index 9f04d90..78420b7 100644 --- a/tox.ini +++ b/tox.ini @@ -1,13 +1,13 @@ [tox] envlist = py38, py39, py310, py311, pypy3.8, pypy3.9, - docs, flake8, black + flake8, black skipsdist = True [testenv] -usedevelop = True deps = -rrequirements_dev.txt commands = + pip install . pytest -Werror --doctest-glob="*.rst" {posargs} [testenv:flake8] From 453f1bbba33b3d395904a80421064b21bd2db633 Mon Sep 17 00:00:00 2001 From: masklinn Date: Wed, 26 Apr 2023 20:50:14 +0200 Subject: [PATCH 3/3] Update CI actions for new packaging - remove old compile step - update python-action to v4 and "primary" python to 3.11 - produce sdist and wheel just once - ensure we're testing after installing from wheel, sdist, and source --- .github/workflows/ci.yml | 71 +++++++++++++++++++++++++--------------- 1 file changed, 45 insertions(+), 26 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index d02b41e..5ac5eb1 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -16,9 +16,9 @@ jobs: - name: Checkout working copy uses: actions/checkout@v3 - name: Set up Python - uses: actions/setup-python@v3 + uses: actions/setup-python@v4 with: - python-version: "3.10" + python-version: "3.11" - name: Install checkers run: | python -mpip install --upgrade pip @@ -28,43 +28,51 @@ jobs: - name: black run: black --check --diff --color --quiet . + # REPLACE BY: job which python -mbuild, and uploads the sdist and wheel to artifacts + # build is not binary so can just build the one using whatever python version compile: - strategy: - fail-fast: false - matrix: - python-version: - - "3.8" - - "3.9" - - "3.10" - - "3.11" - - "3.12.0-alpha - 3.12" - - "pypy-3.8" - - "pypy-3.9" - pyyaml-version: ["5.1.*", "5.4.*", "6.0.*", "6.*"] runs-on: ubuntu-latest steps: - name: Checkout working copy uses: actions/checkout@v3 - - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@v3 with: - python-version: ${{ matrix.python-version }} + submodules: true + - name: Set up Python + uses: actions/setup-python@v4 + with: + python-version: "3.11" - name: Install dependency run: | python -mpip install --upgrade pip - python -mpip install pyyaml==${{ matrix.pyyaml-version }} - - name: Build regexes.py - run: python setup.py build_regexes -i - - name: Check results + python -mpip install build + - name: Build sdist and wheel run: | - # check that _regexes exists, and .eggs does not (== setuptools used our dependency) - test -e src/ua_parser/_regexes.py -a ! -e .eggs + python -mbuild + - name: Upload sdist + uses: actions/upload-artifact@v3 + with: + name: sdist + path: dist/*.tar.gz + retention-days: 1 + + - name: Upload wheel + uses: actions/upload-artifact@v3 + with: + name: wheel + path: dist/*.whl + retention-days: 1 test: + runs-on: ubuntu-latest + needs: compile strategy: fail-fast: false matrix: + source: + - wheel + - sdist + - source python-version: - "3.8" - "3.9" @@ -73,14 +81,18 @@ jobs: - "3.12.0-alpha - 3.12" - "pypy-3.8" - "pypy-3.9" - runs-on: ubuntu-latest + include: + - source: sdist + artifact: dist/*.tar.gz + - source: wheel + artifact: dist/*.whl steps: - name: Checkout working copy uses: actions/checkout@v3 with: submodules: true - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@v3 + uses: actions/setup-python@v4 with: python-version: ${{ matrix.python-version }} - name: Install test dependencies @@ -94,7 +106,14 @@ jobs: fi fi python -mpip install -r requirements_dev.txt + - name: download ${{ matrix.source }} artifact + if: matrix.artifact + uses: actions/download-artifact@v3 + with: + name: ${{ matrix.source }} + path: dist/ - name: install package in environment - run: pip install . + run: | + pip install ${{ matrix.artifact || '.' }} - name: run tests run: pytest -v -Werror -Wignore::ImportWarning --doctest-glob="*.rst"