Thanks to visit codestin.com
Credit goes to github.com

Skip to content

Commit bff5547

Browse files
AvasamAvasam
authored
Use lib/ts_utils for scripts/sync_protobuf (#12913)
- Simplified `download_file` error handling (400+ return codes already raised errors!) - Moved `update_metadata` from `scripts/sync_protobuf/_utils.py` to `lib/ts_utils/metadata.py` - Improved `update_metadata` to support any key (values unvalidated atm) and return the modified dictionary - Updated `scripts/stubsabot.py` to use `update_metadata` - Updated `scripts/sync_protobuf/*` to use `lib/ts_utils` - Updated `scripts/sync_protobuf/tensorflow.py` and `scripts/sync_protobuf/google_protobuf.py` to use the version directly from the `METADATA.toml` file
1 parent c225ac7 commit bff5547

9 files changed

Lines changed: 54 additions & 60 deletions

File tree

‎lib/ts_utils/metadata.py‎

Lines changed: 17 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -14,6 +14,7 @@
1414
from typing_extensions import Annotated, TypeGuard
1515

1616
import tomli
17+
import tomlkit
1718
from packaging.requirements import Requirement
1819
from packaging.specifiers import Specifier
1920

@@ -300,6 +301,22 @@ def read_metadata(distribution: str) -> StubMetadata:
300301
)
301302

302303

304+
def update_metadata(distribution: str, **new_values: object) -> tomlkit.TOMLDocument:
305+
"""Updates a distribution's METADATA.toml.
306+
307+
Return the updated TOML dictionary for use without having to open the file separately."""
308+
path = metadata_path(distribution)
309+
try:
310+
with path.open("rb") as file:
311+
data = tomlkit.load(file)
312+
except FileNotFoundError:
313+
raise NoSuchStubError(f"Typeshed has no stubs for {distribution!r}!") from None
314+
data.update(new_values) # pyright: ignore[reportUnknownMemberType] # tomlkit.TOMLDocument.update is partially typed
315+
with path.open("w", encoding="UTF-8") as file:
316+
tomlkit.dump(data, file) # pyright: ignore[reportUnknownMemberType] # tomlkit.dump has partially unknown Mapping type
317+
return data
318+
319+
303320
def parse_requires(distribution: str, req: object) -> Requirement:
304321
assert isinstance(req, str), f"Invalid requirement {req!r} for {distribution!r}"
305322
return Requirement(req)

‎scripts/stubsabot.py‎

Lines changed: 4 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -29,7 +29,7 @@
2929
from packaging.specifiers import Specifier
3030
from termcolor import colored
3131

32-
from ts_utils.metadata import StubMetadata, metadata_path, read_metadata
32+
from ts_utils.metadata import StubMetadata, read_metadata, update_metadata
3333
from ts_utils.paths import STUBS_PATH, distribution_path
3434

3535
TYPESHED_OWNER = "python"
@@ -653,7 +653,7 @@ def normalize(name: str) -> str:
653653
BRANCH_PREFIX = "stubsabot"
654654

655655

656-
def get_update_pr_body(update: Update, metadata: dict[str, Any]) -> str:
656+
def get_update_pr_body(update: Update, metadata: Mapping[str, Any]) -> str:
657657
body = "\n".join(f"{k}: {v}" for k, v in update.links.items())
658658

659659
if update.diff_analysis is not None:
@@ -689,12 +689,7 @@ async def suggest_typeshed_update(update: Update, session: aiohttp.ClientSession
689689
async with _repo_lock:
690690
branch_name = f"{BRANCH_PREFIX}/{normalize(update.distribution)}"
691691
subprocess.check_call(["git", "checkout", "-B", branch_name, "origin/main"])
692-
with metadata_path(update.distribution).open("rb") as f:
693-
meta = tomlkit.load(f)
694-
meta["version"] = update.new_version
695-
with metadata_path(update.distribution).open("w", encoding="UTF-8") as f:
696-
# tomlkit.dump has partially unknown IO type
697-
tomlkit.dump(meta, f) # pyright: ignore[reportUnknownMemberType]
692+
meta = update_metadata(update.distribution, version=update.new_version)
698693
body = get_update_pr_body(update, meta)
699694
subprocess.check_call(["git", "commit", "--all", "-m", f"{title}\n\n{body}"])
700695
if action_level <= ActionLevel.local:
@@ -716,14 +711,9 @@ async def suggest_typeshed_obsolete(obsolete: Obsolete, session: aiohttp.ClientS
716711
async with _repo_lock:
717712
branch_name = f"{BRANCH_PREFIX}/{normalize(obsolete.distribution)}"
718713
subprocess.check_call(["git", "checkout", "-B", branch_name, "origin/main"])
719-
with metadata_path(obsolete.distribution).open("rb") as f:
720-
meta = tomlkit.load(f)
721714
obs_string = tomlkit.string(obsolete.obsolete_since_version)
722715
obs_string.comment(f"Released on {obsolete.obsolete_since_date.date().isoformat()}")
723-
meta["obsolete_since"] = obs_string
724-
with metadata_path(obsolete.distribution).open("w", encoding="UTF-8") as f:
725-
# tomlkit.dump has partially unknown Mapping type
726-
tomlkit.dump(meta, f) # pyright: ignore[reportUnknownMemberType]
716+
update_metadata(obsolete.distribution, obsolete_since=obs_string)
727717
body = "\n".join(f"{k}: {v}" for k, v in obsolete.links.items())
728718
subprocess.check_call(["git", "commit", "--all", "-m", f"{title}\n\n{body}"])
729719
if action_level <= ActionLevel.local:

‎scripts/sync_protobuf/_utils.py‎

Lines changed: 2 additions & 19 deletions
Original file line numberDiff line numberDiff line change
@@ -3,31 +3,25 @@
33
import subprocess
44
import sys
55
from http.client import HTTPResponse
6-
from pathlib import Path
76
from typing import TYPE_CHECKING, Iterable
87
from urllib.request import urlopen
98
from zipfile import ZipFile
109

11-
import tomlkit
1210
from mypy_protobuf.main import ( # type: ignore[import-untyped] # pyright: ignore[reportMissingTypeStubs]
1311
__version__ as mypy_protobuf__version__,
1412
)
1513

1614
if TYPE_CHECKING:
1715
from _typeshed import StrOrBytesPath, StrPath
1816

19-
REPO_ROOT = Path(__file__).absolute().parent.parent.parent
2017
MYPY_PROTOBUF_VERSION = mypy_protobuf__version__
2118

2219

2320
def download_file(url: str, destination: StrPath) -> None:
2421
print(f"Downloading '{url}' to '{destination}'")
2522
resp: HTTPResponse
26-
with urlopen(url) as resp:
27-
if resp.getcode() != 200:
28-
raise RuntimeError(f"Error downloading {url}")
29-
with open(destination, "wb") as file:
30-
file.write(resp.read())
23+
with urlopen(url) as resp, open(destination, "wb") as file:
24+
file.write(resp.read())
3125

3226

3327
def extract_archive(archive_path: StrPath, destination: StrPath) -> None:
@@ -36,17 +30,6 @@ def extract_archive(archive_path: StrPath, destination: StrPath) -> None:
3630
file_in.extractall(destination)
3731

3832

39-
def update_metadata(metadata_folder: StrPath, new_extra_description: str) -> None:
40-
metadata_path = Path(metadata_folder) / "METADATA.toml"
41-
with open(metadata_path) as file:
42-
metadata = tomlkit.load(file)
43-
metadata["extra_description"] = new_extra_description
44-
with open(metadata_path, "w") as file:
45-
# tomlkit.dump has partially unknown IO type
46-
tomlkit.dump(metadata, file) # pyright: ignore[reportUnknownMemberType]
47-
print(f"Updated {metadata_path}")
48-
49-
5033
def run_protoc(
5134
proto_paths: Iterable[StrPath], mypy_out: StrPath, proto_globs: Iterable[str], cwd: StrOrBytesPath | None = None
5235
) -> str:

‎scripts/sync_protobuf/google_protobuf.py‎

Lines changed: 9 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -14,13 +14,14 @@
1414
from pathlib import Path
1515
from typing import Any
1616

17-
from _utils import MYPY_PROTOBUF_VERSION, REPO_ROOT, download_file, extract_archive, run_protoc, update_metadata
17+
from _utils import MYPY_PROTOBUF_VERSION, download_file, extract_archive, run_protoc
18+
from ts_utils.metadata import read_metadata, update_metadata
19+
from ts_utils.paths import distribution_path
1820

19-
# Whenever you update PACKAGE_VERSION here, version should be updated
20-
# in stubs/protobuf/METADATA.toml and vice-versa.
21-
PACKAGE_VERSION = "28.2"
21+
# PyPi version has an extra "major" number that the Git version doesn't have
22+
PACKAGE_VERSION = read_metadata("protobuf").version_spec.version[2:]
2223

23-
STUBS_FOLDER = REPO_ROOT / "stubs" / "protobuf"
24+
STUBS_FOLDER = distribution_path("protobuf").absolute()
2425
ARCHIVE_FILENAME = f"protobuf-{PACKAGE_VERSION}.zip"
2526
ARCHIVE_URL = f"https://github.com/protocolbuffers/protobuf/releases/download/v{PACKAGE_VERSION}/{ARCHIVE_FILENAME}"
2627
EXTRACTED_PACKAGE_DIR = f"protobuf-{PACKAGE_VERSION}"
@@ -78,13 +79,14 @@ def main() -> None:
7879
shutil.rmtree(temp_dir)
7980

8081
update_metadata(
81-
STUBS_FOLDER,
82-
f"""Partially generated using \
82+
"protobuf",
83+
extra_description=f"""Partially generated using \
8384
[mypy-protobuf=={MYPY_PROTOBUF_VERSION}](https://github.com/nipunn1313/mypy-protobuf/tree/v{MYPY_PROTOBUF_VERSION}) \
8485
and {PROTOC_VERSION} on \
8586
[protobuf v{PACKAGE_VERSION}](https://github.com/protocolbuffers/protobuf/releases/tag/v{PACKAGE_VERSION}) \
8687
(python `protobuf=={PYTHON_PROTOBUF_VERSION}`).""",
8788
)
89+
print("Updated protobuf/METADATA.toml")
8890

8991
# Run pre-commit to cleanup the stubs
9092
subprocess.run((sys.executable, "-m", "pre_commit", "run", "--files", *STUBS_FOLDER.rglob("*_pb2.pyi")))

‎scripts/sync_protobuf/s2clientprotocol.py‎

Lines changed: 7 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -12,13 +12,15 @@
1212
import tempfile
1313
from pathlib import Path
1414

15-
from _utils import MYPY_PROTOBUF_VERSION, REPO_ROOT, download_file, extract_archive, run_protoc, update_metadata
15+
from _utils import MYPY_PROTOBUF_VERSION, download_file, extract_archive, run_protoc
16+
from ts_utils.metadata import update_metadata
17+
from ts_utils.paths import distribution_path
1618

1719
# Whenever you update PACKAGE_VERSION here, version should be updated
1820
# in stubs/s2clientprotocol/METADATA.toml and vice-versa.
1921
PACKAGE_VERSION = "c04df4adbe274858a4eb8417175ee32ad02fd609"
2022

21-
STUBS_FOLDER = REPO_ROOT / "stubs" / "s2clientprotocol"
23+
STUBS_FOLDER = distribution_path("s2clientprotocol").absolute()
2224
ARCHIVE_FILENAME = f"{PACKAGE_VERSION}.zip"
2325
ARCHIVE_URL = f"https://github.com/Blizzard/s2client-proto/archive/{ARCHIVE_FILENAME}"
2426
EXTRACTED_PACKAGE_DIR = f"s2client-proto-{PACKAGE_VERSION}"
@@ -57,12 +59,13 @@ def main() -> None:
5759
shutil.rmtree(temp_dir)
5860

5961
update_metadata(
60-
STUBS_FOLDER,
61-
f"""Partially generated using \
62+
"s2clientprotocol",
63+
extra_description=f"""Partially generated using \
6264
[mypy-protobuf=={MYPY_PROTOBUF_VERSION}](https://github.com/nipunn1313/mypy-protobuf/tree/v{MYPY_PROTOBUF_VERSION}) \
6365
and {PROTOC_VERSION} on \
6466
[s2client-proto {PYTHON_S2_CLIENT_PROTO_VERSION}](https://github.com/Blizzard/s2client-proto/tree/{PACKAGE_VERSION}).""",
6567
)
68+
print("Updated s2clientprotocol/METADATA.toml")
6669

6770
# Run pre-commit to cleanup the stubs
6871
subprocess.run((sys.executable, "-m", "pre_commit", "run", "--files", *STUBS_FOLDER.rglob("*_pb2.pyi")))

‎scripts/sync_protobuf/tensorflow.py‎

Lines changed: 8 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -13,13 +13,13 @@
1313
import tempfile
1414
from pathlib import Path
1515

16-
from _utils import MYPY_PROTOBUF_VERSION, REPO_ROOT, download_file, extract_archive, run_protoc, update_metadata
16+
from _utils import MYPY_PROTOBUF_VERSION, download_file, extract_archive, run_protoc
17+
from ts_utils.metadata import read_metadata, update_metadata
18+
from ts_utils.paths import distribution_path
1719

18-
# Whenever you update PACKAGE_VERSION here, version should be updated
19-
# in stubs/tensorflow/METADATA.toml and vice-versa.
20-
PACKAGE_VERSION = "2.17.0"
20+
PACKAGE_VERSION = read_metadata("tensorflow").version_spec.version
2121

22-
STUBS_FOLDER = REPO_ROOT / "stubs" / "tensorflow"
22+
STUBS_FOLDER = distribution_path("tensorflow").absolute()
2323
ARCHIVE_FILENAME = f"v{PACKAGE_VERSION}.zip"
2424
ARCHIVE_URL = f"https://github.com/tensorflow/tensorflow/archive/refs/tags/{ARCHIVE_FILENAME}"
2525
EXTRACTED_PACKAGE_DIR = f"tensorflow-{PACKAGE_VERSION}"
@@ -123,11 +123,12 @@ def main() -> None:
123123
post_creation()
124124

125125
update_metadata(
126-
STUBS_FOLDER,
127-
f"""Partially generated using \
126+
"tensorflow",
127+
extra_description=f"""Partially generated using \
128128
[mypy-protobuf=={MYPY_PROTOBUF_VERSION}](https://github.com/nipunn1313/mypy-protobuf/tree/v{MYPY_PROTOBUF_VERSION}) \
129129
and {PROTOC_VERSION} on `tensorflow=={PACKAGE_VERSION}`.""",
130130
)
131+
print("Updated tensorflow/METADATA.toml")
131132

132133
# Run pre-commit to cleanup the stubs
133134
subprocess.run((sys.executable, "-m", "pre_commit", "run", "--files", *STUBS_FOLDER.rglob("*_pb2.pyi")))

‎stubs/protobuf/METADATA.toml‎

Lines changed: 3 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -1,8 +1,7 @@
1-
# Whenever you update version here, PACKAGE_VERSION should be updated
2-
# in scripts/sync_proto/google_protobuf.py and vice-versa.
3-
version = "5.28.*"
1+
# Using an exact number in the specifier for scripts/sync_proto/google_protobuf.py
2+
version = "~=5.28.3"
43
upstream_repository = "https://github.com/protocolbuffers/protobuf"
5-
extra_description = "Partially generated using [mypy-protobuf==3.6.0](https://github.com/nipunn1313/mypy-protobuf/tree/v3.6.0) and libprotoc 26.1 on [protobuf v28.2](https://github.com/protocolbuffers/protobuf/releases/tag/v28.2) (python `protobuf==5.28.2`)."
4+
extra_description = "Partially generated using [mypy-protobuf==3.6.0](https://github.com/nipunn1313/mypy-protobuf/tree/v3.6.0) and libprotoc 27.2 on [protobuf v28.3](https://github.com/protocolbuffers/protobuf/releases/tag/v28.3) (python `protobuf==5.28.3`)."
65
partial_stub = true
76

87
[tool.stubtest]

‎stubs/s2clientprotocol/METADATA.toml‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -3,4 +3,4 @@
33
version = "5.*"
44
upstream_repository = "https://github.com/Blizzard/s2client-proto"
55
requires = ["types-protobuf"]
6-
extra_description = "Partially generated using [mypy-protobuf==3.6.0](https://github.com/nipunn1313/mypy-protobuf/tree/v3.6.0) and libprotoc 26.1 on [s2client-proto 5.0.12.91115.0](https://github.com/Blizzard/s2client-proto/tree/c04df4adbe274858a4eb8417175ee32ad02fd609)."
6+
extra_description = "Partially generated using [mypy-protobuf==3.6.0](https://github.com/nipunn1313/mypy-protobuf/tree/v3.6.0) and libprotoc 27.2 on [s2client-proto 5.0.12.91115.0](https://github.com/Blizzard/s2client-proto/tree/c04df4adbe274858a4eb8417175ee32ad02fd609)."

‎stubs/tensorflow/METADATA.toml‎

Lines changed: 3 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -1,12 +1,11 @@
1-
# Whenever you update version here, PACKAGE_VERSION should be updated
2-
# in scripts/sync_proto/tensorflow.py and vice-versa.
3-
version = "2.17.*"
1+
# Using an exact number in the specifier for scripts/sync_proto/tensorflow.py
2+
version = "~=2.17.1"
43
upstream_repository = "https://github.com/tensorflow/tensorflow"
54
# requires a version of numpy with a `py.typed` file
65
# see https://github.com/python/typeshed/issues/12551
76
# on why we need the upper bound for numpy
87
requires = ["numpy>=1.20,<2.1.0", "types-protobuf", "types-requests"]
9-
extra_description = "Partially generated using [mypy-protobuf==3.6.0](https://github.com/nipunn1313/mypy-protobuf/tree/v3.6.0) and libprotoc 26.1 on `tensorflow==2.17.0`."
8+
extra_description = "Partially generated using [mypy-protobuf==3.6.0](https://github.com/nipunn1313/mypy-protobuf/tree/v3.6.0) and libprotoc 27.2 on `tensorflow==2.17.1`."
109
partial_stub = true
1110

1211
[tool.stubtest]

0 commit comments

Comments
 (0)