Thanks to visit codestin.com
Credit goes to github.com

Skip to content

Commit 1017c52

Browse files
AvasamAvasam
authored
Simplify protoc install in protobuf generation scripts (#11785)
1 parent 4872c30 commit 1017c52

4 files changed

Lines changed: 26 additions & 43 deletions

File tree

‎scripts/generate_proto_stubs.sh‎

Lines changed: 12 additions & 20 deletions
Original file line numberDiff line numberDiff line change
@@ -15,38 +15,25 @@ set -ex -o pipefail
1515
PROTOBUF_VERSION=26.1
1616
MYPY_PROTOBUF_VERSION=3.6.0
1717

18-
if uname -a | grep Darwin; then
19-
# brew install coreutils wget
20-
PLAT=osx
21-
else
22-
# sudo apt install -y unzip
23-
PLAT=linux
24-
fi
18+
# brew install coreutils wget
19+
# sudo apt install -y unzip
2520
REPO_ROOT="$(realpath "$(dirname "${BASH_SOURCE[0]}")"/..)"
2621
TMP_DIR="$(mktemp -d)"
2722
PYTHON_PROTOBUF_FILENAME="protobuf-$PROTOBUF_VERSION.zip"
28-
PROTOC_FILENAME="protoc-$PROTOBUF_VERSION-$PLAT-x86_64.zip"
29-
PROTOC_URL="https://github.com/protocolbuffers/protobuf/releases/download/v$PROTOBUF_VERSION/$PROTOC_FILENAME"
3023
PYTHON_PROTOBUF_URL="https://github.com/protocolbuffers/protobuf/releases/download/v$PROTOBUF_VERSION/$PYTHON_PROTOBUF_FILENAME"
24+
PYTHON_PROTOBUF_DIR="protobuf-$PROTOBUF_VERSION"
3125

3226
cd "$TMP_DIR"
3327
echo "Working in $TMP_DIR"
3428

35-
# Install protoc
36-
wget "$PROTOC_URL"
37-
mkdir protoc_install
38-
unzip "$PROTOC_FILENAME" -d protoc_install
39-
protoc_install/bin/protoc --version
40-
4129
# Fetch protoc-python (which contains all the .proto files)
4230
wget "$PYTHON_PROTOBUF_URL"
4331
unzip "$PYTHON_PROTOBUF_FILENAME"
44-
PYTHON_PROTOBUF_DIR="protobuf-$PROTOBUF_VERSION"
4532

4633
# Prepare virtualenv
4734
python3 -m venv .venv
4835
source .venv/bin/activate
49-
python3 -m pip install pre-commit mypy-protobuf=="$MYPY_PROTOBUF_VERSION"
36+
python3 -m pip install grpcio-tools pre-commit mypy-protobuf=="$MYPY_PROTOBUF_VERSION"
5037

5138
# Remove existing pyi
5239
find "$REPO_ROOT/stubs/protobuf/" -name '*_pb2.pyi' -delete
@@ -62,8 +49,9 @@ PROTO_FILES=$(grep '"//:.*_proto"' $PYTHON_PROTOBUF_DIR/python/dist/BUILD.bazel
6249
)
6350

6451
# And regenerate!
65-
# shellcheck disable=SC2086
66-
protoc_install/bin/protoc \
52+
PROTOC_VERSION=$(python3 -m grpc_tools.protoc --version)
53+
echo $PROTOC_VERSION
54+
python3 -m grpc_tools.protoc \
6755
--proto_path="$PYTHON_PROTOBUF_DIR/src" \
6856
--mypy_out="relax_strict_optional_primitives:$REPO_ROOT/stubs/protobuf" \
6957
$PROTO_FILES
@@ -76,7 +64,11 @@ rm -rf "$TMP_DIR"
7664
cd "$REPO_ROOT"
7765

7866
sed --in-place="" \
79-
"s/extra_description = .*$/extra_description = \"Generated using [mypy-protobuf==$MYPY_PROTOBUF_VERSION](https:\/\/github.com\/nipunn1313\/mypy-protobuf\/tree\/v$MYPY_PROTOBUF_VERSION) on [protobuf v$PROTOBUF_VERSION](https:\/\/github.com\/protocolbuffers\/protobuf\/releases\/tag\/v$PROTOBUF_VERSION) (python protobuf==$PYTHON_PROTOBUF_VERSION)\"/" \
67+
"s/extra_description = .*$/extra_description = \"\
68+
Partially generated using [mypy-protobuf==$MYPY_PROTOBUF_VERSION](https:\/\/github.com\/nipunn1313\/mypy-protobuf\/tree\/v$MYPY_PROTOBUF_VERSION) \
69+
and $PROTOC_VERSION \
70+
on [protobuf v$PROTOBUF_VERSION](https:\/\/github.com\/protocolbuffers\/protobuf\/releases\/tag\/v$PROTOBUF_VERSION) \
71+
(python protobuf==$PYTHON_PROTOBUF_VERSION).\"/" \
8072
stubs/protobuf/METADATA.toml
8173

8274
# use `|| true` so the script still continues even if a pre-commit hook

‎scripts/sync_tensorflow_protobuf_stubs.sh‎

Lines changed: 12 additions & 21 deletions
Original file line numberDiff line numberDiff line change
@@ -5,51 +5,39 @@
55

66
set -euxo pipefail
77

8-
# Need protoc >= 3.15 for explicit optional
9-
PROTOBUF_VERSION=25.3 # 4.25.3
108
# Whenever you update TENSORFLOW_VERSION here, version should be updated
119
# in stubs/tensorflow/METADATA.toml and vice-versa.
1210
TENSORFLOW_VERSION=2.12.1
1311
MYPY_PROTOBUF_VERSION=3.6.0
1412

15-
if uname -a | grep Darwin; then
16-
# brew install coreutils wget
17-
PLAT=osx
18-
else
19-
PLAT=linux
20-
fi
13+
# brew install coreutils wget
14+
# sudo apt install -y unzip
2115
REPO_ROOT="$(realpath "$(dirname "${BASH_SOURCE[0]}")"/..)"
2216
TMP_DIR="$(mktemp -d)"
2317
TENSORFLOW_FILENAME="v$TENSORFLOW_VERSION.zip"
24-
PROTOC_FILENAME="protoc-$PROTOBUF_VERSION-$PLAT-x86_64.zip"
25-
PROTOC_URL="https://github.com/protocolbuffers/protobuf/releases/download/v$PROTOBUF_VERSION/$PROTOC_FILENAME"
2618
TENSORFLOW_URL="https://github.com/tensorflow/tensorflow/archive/refs/tags/$TENSORFLOW_FILENAME"
19+
TENSORFLOW_DIR="tensorflow-$TENSORFLOW_VERSION"
2720

2821
cd "$TMP_DIR"
2922
echo "Working in $TMP_DIR"
3023

31-
# Install protoc
32-
wget "$PROTOC_URL"
33-
mkdir protoc_install
34-
unzip "$PROTOC_FILENAME" -d protoc_install
35-
protoc_install/bin/protoc --version
36-
3724
# Fetch tensorflow (which contains all the .proto files)
3825
wget "$TENSORFLOW_URL"
3926
unzip "$TENSORFLOW_FILENAME"
40-
TENSORFLOW_DIR="tensorflow-$TENSORFLOW_VERSION"
4127

4228
# Prepare virtualenv
4329
python3 -m venv .venv
4430
source .venv/bin/activate
45-
python3 -m pip install pre-commit mypy-protobuf=="$MYPY_PROTOBUF_VERSION"
31+
python3 -m pip install grpcio-tools pre-commit mypy-protobuf=="$MYPY_PROTOBUF_VERSION"
4632

4733
# Remove existing pyi
4834
find "$REPO_ROOT/stubs/tensorflow/" -name "*_pb2.pyi" -delete
4935

5036
# Folders here cover the more commonly used protobufs externally and
5137
# their dependencies. Tensorflow has more protobufs and can be added if requested.
52-
protoc_install/bin/protoc \
38+
PROTOC_VERSION=$(python3 -m grpc_tools.protoc --version)
39+
echo $PROTOC_VERSION
40+
python3 -m grpc_tools.protoc \
5341
--proto_path="$TENSORFLOW_DIR" \
5442
--mypy_out "relax_strict_optional_primitives:$REPO_ROOT/stubs/tensorflow" \
5543
$TENSORFLOW_DIR/tensorflow/compiler/xla/*.proto \
@@ -89,8 +77,11 @@ rm \
8977
stubs/tensorflow/tensorflow/core/util/example_proto_fast_parsing_test_pb2.pyi \
9078

9179
sed --in-place="" \
92-
"s/extra_description = .*$/extra_description = \"Partially generated using [mypy-protobuf==$MYPY_PROTOBUF_VERSION](https:\/\/github.com\/nipunn1313\/mypy-protobuf\/tree\/v$MYPY_PROTOBUF_VERSION) on tensorflow==$TENSORFLOW_VERSION\"/" \
93-
stubs/tensorflow/METADATA.toml
80+
"s/extra_description = .*$/extra_description = \"\
81+
Partially generated using [mypy-protobuf==$MYPY_PROTOBUF_VERSION](https:\/\/github.com\/nipunn1313\/mypy-protobuf\/tree\/v$MYPY_PROTOBUF_VERSION) \
82+
and $PROTOC_VERSION \
83+
on tensorflow==$TENSORFLOW_VERSION.\"/" \
84+
stubs/tensorflow/METADATA.toml
9485

9586
# use `|| true` so the script still continues even if a pre-commit hook
9687
# applies autofixes (which will result in a nonzero exit code)

‎stubs/protobuf/METADATA.toml‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -2,7 +2,7 @@
22
# in scripts/generate_proto_stubs.sh and vice-versa.
33
version = "5.26.*"
44
upstream_repository = "https://github.com/protocolbuffers/protobuf"
5-
extra_description = "Generated using [mypy-protobuf==3.6.0](https://github.com/nipunn1313/mypy-protobuf/tree/v3.6.0) on [protobuf v26.1](https://github.com/protocolbuffers/protobuf/releases/tag/v26.1) (python protobuf==5.26.1)"
5+
extra_description = "Generated using [mypy-protobuf==3.6.0](https://github.com/nipunn1313/mypy-protobuf/tree/v3.6.0) and libprotoc 25.1 on [protobuf v26.1](https://github.com/protocolbuffers/protobuf/releases/tag/v26.1) (python protobuf==5.26.1)"
66
partial_stub = true
77

88
[tool.stubtest]

‎stubs/tensorflow/METADATA.toml‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -2,7 +2,7 @@ version = "2.15.*"
22
upstream_repository = "https://github.com/tensorflow/tensorflow"
33
# requires a version of numpy with a `py.typed` file
44
requires = ["numpy>=1.20", "types-protobuf", "types-requests"]
5-
extra_description = "Partially generated using [mypy-protobuf==3.6.0](https://github.com/nipunn1313/mypy-protobuf/tree/v3.6.0) on tensorflow==2.12.1"
5+
extra_description = "Partially generated using [mypy-protobuf==3.6.0](https://github.com/nipunn1313/mypy-protobuf/tree/v3.6.0) and libprotoc 25.1 on tensorflow==2.12.1."
66
partial_stub = true
77

88
[tool.stubtest]

0 commit comments

Comments
 (0)