Thanks to visit codestin.com
Credit goes to github.com

Skip to content

Commit ecf241a

Browse files
vi3k6i5vi3k6i5
authored
docs: add docs build kokoro config for django spanner (#644)
* docs: added trampoline v2 file and docs config for kokoro * fix: change library_type to INTEGRATION * docs: add kokoro docker build for docs * fix:repo metadata * fix: updated PROJECT_ROOT for django spanner docs build using kokoro * fix: remove sudo apt update because sudo is not allowed * fix: add build_nox.sh as original build.sh file is used by kokoro workers to run parallelized tests * refactor: renamed build_nox.sh to build-nox.sh * refactor: change build-nox to nox.sh * refactor: change build-nox to nox.sh * refactor: change kokoro build to run different test for docs and different test for workers * refactor: change kokoro build to run different test for docs and different test for workers * refactor: add log msg for debugging * refactor: remove log msg added for debugging * fix: correct docs build command * refactor: remove unused nox file * feat: add .trampolinerc file * fix: correct repo_metadata release_level to beta
1 parent fe4a062 commit ecf241a

File tree

10 files changed

+890
-21
lines changed

10 files changed

+890
-21
lines changed

‎.kokoro/build.sh

Lines changed: 37 additions & 21 deletions
Original file line numberDiff line numberDiff line change
@@ -15,7 +15,11 @@
1515

1616
set -eo pipefail
1717

18-
cd github/python-spanner-django
18+
if [[ -z "${PROJECT_ROOT:-}" ]]; then
19+
PROJECT_ROOT="github/python-spanner-django"
20+
fi
21+
22+
cd "${PROJECT_ROOT}"
1923

2024
# Disable buffering, so that the logs stream through.
2125
export PYTHONUNBUFFERED=1
@@ -27,26 +31,38 @@ export GOOGLE_CLOUD_PROJECT=$(cat "${KOKORO_GFILE_DIR}/project-id.json")
2731

2832
export RUNNING_SPANNER_BACKEND_TESTS=1
2933

30-
pip3 install .
31-
export DJANGO_TESTS_DIR="django_tests_dir"
32-
mkdir -p $DJANGO_TESTS_DIR && git clone --depth 1 --single-branch --branch "spanner/stable/2.2.x" https://github.com/c24t/django.git $DJANGO_TESTS_DIR/django
33-
34-
# Install dependencies for Django tests.
35-
sudo apt-get update
36-
apt-get install -y libffi-dev libjpeg-dev zlib1g-dev libmemcached-dev
37-
cd $DJANGO_TESTS_DIR/django && pip3 install -e . && pip3 install -r tests/requirements/py3.txt; cd ../../
38-
39-
# Hardcode the max number of workers since Spanner has a very low
40-
# QPS for administrative RPCs of 5QPS (averaged every 100 seconds)
41-
if [[ $KOKORO_JOB_NAME == *"continuous"* ]]
34+
if [[ $KOKORO_JOB_NAME == *"docs"* ]]
4235
then
43-
# Disable continuous build as it creates too many Spanner instances
44-
# ("Quota exceeded for quota metric 'Instance create requests' and
45-
# limit 'Instance create requests per minute' of service
46-
# 'spanner.googleapis.com').
47-
export DJANGO_WORKER_COUNT=0
36+
echo "Running docs generation."
37+
# Remove old nox
38+
python3 -m pip uninstall --yes --quiet nox-automation
39+
40+
# Install nox
41+
python3 -m pip install --upgrade --quiet nox
42+
# Generate docs.
43+
python3 -m nox -s docs docfx
4844
else
49-
export DJANGO_WORKER_COUNT=5
50-
fi
45+
pip3 install .
46+
export DJANGO_TESTS_DIR="django_tests_dir"
47+
mkdir -p $DJANGO_TESTS_DIR && git clone --depth 1 --single-branch --branch "spanner/stable/2.2.x" https://github.com/c24t/django.git $DJANGO_TESTS_DIR/django
48+
49+
# Install dependencies for Django tests.
50+
sudo apt-get update
51+
apt-get install -y libffi-dev libjpeg-dev zlib1g-dev libmemcached-dev
52+
cd $DJANGO_TESTS_DIR/django && pip3 install -e . && pip3 install -r tests/requirements/py3.txt; cd ../../
5153

52-
python3 ./run_testing_worker.py
54+
# Hardcode the max number of workers since Spanner has a very low
55+
# QPS for administrative RPCs of 5QPS (averaged every 100 seconds)
56+
if [[ $KOKORO_JOB_NAME == *"continuous"* ]]
57+
then
58+
# Disable continuous build as it creates too many Spanner instances
59+
# ("Quota exceeded for quota metric 'Instance create requests' and
60+
# limit 'Instance create requests per minute' of service
61+
# 'spanner.googleapis.com').
62+
export DJANGO_WORKER_COUNT=0
63+
else
64+
export DJANGO_WORKER_COUNT=5
65+
fi
66+
67+
python3 ./run_testing_worker.py
68+
fi

‎.kokoro/docker/docs/Dockerfile

Lines changed: 98 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,98 @@
1+
# Copyright 2020 Google LLC
2+
#
3+
# Licensed under the Apache License, Version 2.0 (the "License");
4+
# you may not use this file except in compliance with the License.
5+
# You may obtain a copy of the License at
6+
#
7+
# http://www.apache.org/licenses/LICENSE-2.0
8+
#
9+
# Unless required by applicable law or agreed to in writing, software
10+
# distributed under the License is distributed on an "AS IS" BASIS,
11+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12+
# See the License for the specific language governing permissions and
13+
# limitations under the License.
14+
15+
from ubuntu:20.04
16+
17+
ENV DEBIAN_FRONTEND noninteractive
18+
19+
# Ensure local Python is preferred over distribution Python.
20+
ENV PATH /usr/local/bin:$PATH
21+
22+
# Install dependencies.
23+
RUN apt-get update \
24+
&& apt-get install -y --no-install-recommends \
25+
apt-transport-https \
26+
build-essential \
27+
ca-certificates \
28+
curl \
29+
dirmngr \
30+
git \
31+
gpg-agent \
32+
graphviz \
33+
libbz2-dev \
34+
libdb5.3-dev \
35+
libexpat1-dev \
36+
libffi-dev \
37+
liblzma-dev \
38+
libreadline-dev \
39+
libsnappy-dev \
40+
libssl-dev \
41+
libsqlite3-dev \
42+
portaudio19-dev \
43+
redis-server \
44+
software-properties-common \
45+
ssh \
46+
sudo \
47+
tcl \
48+
tcl-dev \
49+
tk \
50+
tk-dev \
51+
uuid-dev \
52+
wget \
53+
zlib1g-dev \
54+
&& add-apt-repository universe \
55+
&& apt-get update \
56+
&& apt-get -y install jq \
57+
&& apt-get clean autoclean \
58+
&& apt-get autoremove -y \
59+
&& rm -rf /var/lib/apt/lists/* \
60+
&& rm -f /var/cache/apt/archives/*.deb
61+
62+
63+
COPY fetch_gpg_keys.sh /tmp
64+
# Install the desired versions of Python.
65+
RUN set -ex \
66+
&& export GNUPGHOME="$(mktemp -d)" \
67+
&& echo "disable-ipv6" >> "${GNUPGHOME}/dirmngr.conf" \
68+
&& /tmp/fetch_gpg_keys.sh \
69+
&& for PYTHON_VERSION in 3.7.8 3.8.5; do \
70+
wget --no-check-certificate -O python-${PYTHON_VERSION}.tar.xz "https://www.python.org/ftp/python/${PYTHON_VERSION%%[a-z]*}/Python-$PYTHON_VERSION.tar.xz" \
71+
&& wget --no-check-certificate -O python-${PYTHON_VERSION}.tar.xz.asc "https://www.python.org/ftp/python/${PYTHON_VERSION%%[a-z]*}/Python-$PYTHON_VERSION.tar.xz.asc" \
72+
&& gpg --batch --verify python-${PYTHON_VERSION}.tar.xz.asc python-${PYTHON_VERSION}.tar.xz \
73+
&& rm -r python-${PYTHON_VERSION}.tar.xz.asc \
74+
&& mkdir -p /usr/src/python-${PYTHON_VERSION} \
75+
&& tar -xJC /usr/src/python-${PYTHON_VERSION} --strip-components=1 -f python-${PYTHON_VERSION}.tar.xz \
76+
&& rm python-${PYTHON_VERSION}.tar.xz \
77+
&& cd /usr/src/python-${PYTHON_VERSION} \
78+
&& ./configure \
79+
--enable-shared \
80+
# This works only on Python 2.7 and throws a warning on every other
81+
# version, but seems otherwise harmless.
82+
--enable-unicode=ucs4 \
83+
--with-system-ffi \
84+
--without-ensurepip \
85+
&& make -j$(nproc) \
86+
&& make install \
87+
&& ldconfig \
88+
; done \
89+
&& rm -rf "${GNUPGHOME}" \
90+
&& rm -rf /usr/src/python* \
91+
&& rm -rf ~/.cache/
92+
93+
RUN wget -O /tmp/get-pip.py 'https://bootstrap.pypa.io/get-pip.py' \
94+
&& python3.7 /tmp/get-pip.py \
95+
&& python3.8 /tmp/get-pip.py \
96+
&& rm /tmp/get-pip.py
97+
98+
CMD ["python3.7"]

‎.kokoro/docker/docs/fetch_gpg_keys.sh

Lines changed: 45 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,45 @@
1+
#!/bin/bash
2+
# Copyright 2020 Google LLC
3+
#
4+
# Licensed under the Apache License, Version 2.0 (the "License");
5+
# you may not use this file except in compliance with the License.
6+
# You may obtain a copy of the License at
7+
#
8+
# http://www.apache.org/licenses/LICENSE-2.0
9+
#
10+
# Unless required by applicable law or agreed to in writing, software
11+
# distributed under the License is distributed on an "AS IS" BASIS,
12+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13+
# See the License for the specific language governing permissions and
14+
# limitations under the License.
15+
16+
# A script to fetch gpg keys with retry.
17+
# Avoid jinja parsing the file.
18+
#
19+
20+
function retry {
21+
if [[ "${#}" -le 1 ]]; then
22+
echo "Usage: ${0} retry_count commands.."
23+
exit 1
24+
fi
25+
local retries=${1}
26+
local command="${@:2}"
27+
until [[ "${retries}" -le 0 ]]; do
28+
$command && return 0
29+
if [[ $? -ne 0 ]]; then
30+
echo "command failed, retrying"
31+
((retries--))
32+
fi
33+
done
34+
return 1
35+
}
36+
37+
# 3.6.9, 3.7.5 (Ned Deily)
38+
retry 3 gpg --keyserver ha.pool.sks-keyservers.net --recv-keys \
39+
0D96DF4D4110E5C43FBFB17F2D347EA6AA65421D
40+
41+
# 3.8.0 (Łukasz Langa)
42+
retry 3 gpg --keyserver ha.pool.sks-keyservers.net --recv-keys \
43+
E3FF2839C048B25C084DEBE9B26995E310250568
44+
45+
#

‎.kokoro/docs/common.cfg

Lines changed: 65 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,65 @@
1+
# Format: //devtools/kokoro/config/proto/build.proto
2+
3+
# Build logs will be here
4+
action {
5+
define_artifacts {
6+
regex: "**/*sponge_log.xml"
7+
}
8+
}
9+
10+
# Download trampoline resources.
11+
gfile_resources: "/bigstore/cloud-devrel-kokoro-resources/trampoline"
12+
13+
# Use the trampoline script to run in docker.
14+
build_file: "python-spanner-django/.kokoro/trampoline_v2.sh"
15+
16+
# Configure the docker image for kokoro-trampoline.
17+
env_vars: {
18+
key: "TRAMPOLINE_IMAGE"
19+
value: "gcr.io/cloud-devrel-kokoro-resources/python-lib-docs"
20+
}
21+
env_vars: {
22+
key: "TRAMPOLINE_BUILD_FILE"
23+
value: "github/python-spanner-django/.kokoro/publish-docs.sh"
24+
}
25+
26+
env_vars: {
27+
key: "STAGING_BUCKET"
28+
value: "docs-staging"
29+
}
30+
31+
env_vars: {
32+
key: "V2_STAGING_BUCKET"
33+
value: "docs-staging-v2"
34+
}
35+
36+
# It will upload the docker image after successful builds.
37+
env_vars: {
38+
key: "TRAMPOLINE_IMAGE_UPLOAD"
39+
value: "true"
40+
}
41+
42+
# It will always build the docker image.
43+
env_vars: {
44+
key: "TRAMPOLINE_DOCKERFILE"
45+
value: ".kokoro/docker/docs/Dockerfile"
46+
}
47+
48+
# Fetch the token needed for reporting release status to GitHub
49+
before_action {
50+
fetch_keystore {
51+
keystore_resource {
52+
keystore_config_id: 73713
53+
keyname: "yoshi-automation-github-key"
54+
}
55+
}
56+
}
57+
58+
before_action {
59+
fetch_keystore {
60+
keystore_resource {
61+
keystore_config_id: 73713
62+
keyname: "docuploader_service_account"
63+
}
64+
}
65+
}

‎.kokoro/docs/docs-presubmit.cfg

Lines changed: 28 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,28 @@
1+
# Format: //devtools/kokoro/config/proto/build.proto
2+
3+
env_vars: {
4+
key: "STAGING_BUCKET"
5+
value: "gcloud-python-test"
6+
}
7+
8+
env_vars: {
9+
key: "V2_STAGING_BUCKET"
10+
value: "gcloud-python-test"
11+
}
12+
13+
# We only upload the image in the main `docs` build.
14+
env_vars: {
15+
key: "TRAMPOLINE_IMAGE_UPLOAD"
16+
value: "false"
17+
}
18+
19+
env_vars: {
20+
key: "TRAMPOLINE_BUILD_FILE"
21+
value: "github/python-spanner-django/.kokoro/build.sh"
22+
}
23+
24+
# Only run this nox session.
25+
env_vars: {
26+
key: "NOX_SESSION"
27+
value: "docs docfx"
28+
}

‎.kokoro/docs/docs.cfg

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
1+
# Format: //devtools/kokoro/config/proto/build.proto

‎.kokoro/publish-docs.sh

Lines changed: 64 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,64 @@
1+
#!/bin/bash
2+
# Copyright 2020 Google LLC
3+
#
4+
# Licensed under the Apache License, Version 2.0 (the "License");
5+
# you may not use this file except in compliance with the License.
6+
# You may obtain a copy of the License at
7+
#
8+
# https://www.apache.org/licenses/LICENSE-2.0
9+
#
10+
# Unless required by applicable law or agreed to in writing, software
11+
# distributed under the License is distributed on an "AS IS" BASIS,
12+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13+
# See the License for the specific language governing permissions and
14+
# limitations under the License.
15+
16+
set -eo pipefail
17+
18+
# Disable buffering, so that the logs stream through.
19+
export PYTHONUNBUFFERED=1
20+
21+
export PATH="${HOME}/.local/bin:${PATH}"
22+
23+
# Install nox
24+
python3 -m pip install --user --upgrade --quiet nox
25+
python3 -m nox --version
26+
27+
# build docs
28+
nox -s docs
29+
30+
python3 -m pip install --user gcp-docuploader
31+
32+
# create metadata
33+
python3 -m docuploader create-metadata \
34+
--name=$(jq --raw-output '.name // empty' .repo-metadata.json) \
35+
--version=$(python3 setup.py --version) \
36+
--language=$(jq --raw-output '.language // empty' .repo-metadata.json) \
37+
--distribution-name=$(python3 setup.py --name) \
38+
--product-page=$(jq --raw-output '.product_documentation // empty' .repo-metadata.json) \
39+
--github-repository=$(jq --raw-output '.repo // empty' .repo-metadata.json) \
40+
--issue-tracker=$(jq --raw-output '.issue_tracker // empty' .repo-metadata.json)
41+
42+
cat docs.metadata
43+
44+
# upload docs
45+
python3 -m docuploader upload docs/_build/html --metadata-file docs.metadata --staging-bucket "${STAGING_BUCKET}"
46+
47+
48+
# docfx yaml files
49+
nox -s docfx
50+
51+
# create metadata.
52+
python3 -m docuploader create-metadata \
53+
--name=$(jq --raw-output '.name // empty' .repo-metadata.json) \
54+
--version=$(python3 setup.py --version) \
55+
--language=$(jq --raw-output '.language // empty' .repo-metadata.json) \
56+
--distribution-name=$(python3 setup.py --name) \
57+
--product-page=$(jq --raw-output '.product_documentation // empty' .repo-metadata.json) \
58+
--github-repository=$(jq --raw-output '.repo // empty' .repo-metadata.json) \
59+
--issue-tracker=$(jq --raw-output '.issue_tracker // empty' .repo-metadata.json)
60+
61+
cat docs.metadata
62+
63+
# upload docs
64+
python3 -m docuploader upload docs/_build/html/docfx_yaml --metadata-file docs.metadata --destination-prefix docfx --staging-bucket "${V2_STAGING_BUCKET}"

0 commit comments

Comments
 (0)