Thanks to visit codestin.com
Credit goes to github.com

Skip to content

Commit 88f44de

Browse files
author
Takashi Matsuo
committed
testing: prototype trampoline_v2
1 parent ce56213 commit 88f44de

File tree

3 files changed

+339
-6
lines changed

3 files changed

+339
-6
lines changed

‎.kokoro/tests/run_tests.sh

Lines changed: 15 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,4 @@
11
#!/bin/bash
2-
32
# Copyright 2019 Google LLC
43
#
54
# Licensed under the Apache License, Version 2.0 (the "License");
@@ -32,10 +31,17 @@ if [[ $* == *--only-diff-head* ]]; then
3231
DIFF_FROM="HEAD~.."
3332
fi
3433

35-
cd github/python-docs-samples
34+
if [[ -z "${PROJECT_ROOT:-}" ]]; then
35+
PROJECT_ROOT="github/python-docs-samples"
36+
fi
37+
38+
cd "${PROJECT_ROOT}"
39+
40+
# add user's pip binary path to PATH
41+
export PATH="${HOME}/.local/bin:${PATH}"
3642

3743
# install nox for testing
38-
pip install -q nox
44+
pip install --user -q nox
3945

4046
# Use secrets acessor service account to get secrets
4147
if [[ -f "${KOKORO_GFILE_DIR}/secrets_viewer_service_account.json" ]]; then
@@ -63,9 +69,12 @@ export GOOGLE_CLIENT_SECRETS=$(pwd)/testing/client-secrets.json
6369
export DATALABELING_ENDPOINT="test-datalabeling.sandbox.googleapis.com:443"
6470

6571
# Run Cloud SQL proxy (background process exit when script does)
66-
wget --quiet https://dl.google.com/cloudsql/cloud_sql_proxy.linux.amd64 -O cloud_sql_proxy && chmod +x cloud_sql_proxy
67-
./cloud_sql_proxy -instances="${MYSQL_INSTANCE}"=tcp:3306 &>> cloud_sql_proxy.log &
68-
./cloud_sql_proxy -instances="${POSTGRES_INSTANCE}"=tcp:5432 &>> cloud_sql_proxy.log &
72+
wget --quiet https://dl.google.com/cloudsql/cloud_sql_proxy.linux.amd64 \
73+
-O ${HOME}/cloud_sql_proxy && chmod +x ${HOME}/cloud_sql_proxy
74+
${HOME}/cloud_sql_proxy -instances="${MYSQL_INSTANCE}"=tcp:3306 &>> \
75+
${HOME}/cloud_sql_proxy.log &
76+
${HOME}/cloud_sql_proxy -instances="${POSTGRES_INSTANCE}"=tcp:5432 &>> \
77+
${HOME}/cloud_sql_proxy.log &
6978
echo -e "\nCloud SQL proxy started."
7079

7180
echo -e "\n******************** TESTING PROJECTS ********************"

‎.kokoro/trampoline_v2.sh

Lines changed: 284 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,284 @@
1+
#!/usr/bin/env bash
2+
# Copyright 2020 Google LLC
3+
#
4+
# Licensed under the Apache License, Version 2.0 (the "License");
5+
# you may not use this file except in compliance with the License.
6+
# You may obtain a copy of the License at
7+
#
8+
# http://www.apache.org/licenses/LICENSE-2.0
9+
#
10+
# Unless required by applicable law or agreed to in writing, software
11+
# distributed under the License is distributed on an "AS IS" BASIS,
12+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13+
# See the License for the specific language governing permissions and
14+
# limitations under the License.
15+
16+
# trampoline_v2.sh
17+
#
18+
# This script does 3 things.
19+
#
20+
# 1. Prepare the Docker image for the test
21+
# 2. Run the Docker with appropriate flags to run the test
22+
# 3. Upload the newly built Docker image
23+
#
24+
# in a way that is somewhat compatible with trampoline_v1.
25+
#
26+
# To run this script, first download few files from gcs to /dev/shm.
27+
# (/dev/shm is passed into the container as KOKORO_GFILE_DIR).
28+
#
29+
# gsutil cp gs://cloud-devrel-kokoro-resources/python-docs-samples/secrets_viewer_service_account.json /dev/shm
30+
# gsutil cp gs://cloud-devrel-kokoro-resources/python-docs-samples/automl_secrets.txt /dev/shm
31+
#
32+
# Then run the script.
33+
# .kokoro/trampoline_v2.sh
34+
#
35+
# You can optionally change these environment variables:
36+
#
37+
# TRAMPOLINE_IMAGE: The docker image to use.
38+
# TRAMPOLINE_IMAGE_SOURCE: The location of the Dockerfile.
39+
# RUN_TESTS_SESSION: The nox session to run.
40+
# TRAMPOLINE_BUILD_FILE: The script to run in the docker container.
41+
42+
set -euo pipefail
43+
44+
if command -v tput >/dev/null && [[ -n "${TERM:-}" ]]; then
45+
readonly IO_COLOR_RED="$(tput setaf 1)"
46+
readonly IO_COLOR_GREEN="$(tput setaf 2)"
47+
readonly IO_COLOR_YELLOW="$(tput setaf 3)"
48+
readonly IO_COLOR_RESET="$(tput sgr0)"
49+
else
50+
readonly IO_COLOR_RED=""
51+
readonly IO_COLOR_GREEN=""
52+
readonly IO_COLOR_YELLOW=""
53+
readonly IO_COLOR_RESET=""
54+
fi
55+
56+
# Logs a message using the given color. The first argument must be one of the
57+
# IO_COLOR_* variables defined above, such as "${IO_COLOR_YELLOW}". The
58+
# remaining arguments will be logged in the given color. The log message will
59+
# also have an RFC-3339 timestamp prepended (in UTC).
60+
function log_impl() {
61+
local color="$1"
62+
shift
63+
local timestamp="$(date -u "+%Y-%m-%dT%H:%M:%SZ")"
64+
echo "================================================================"
65+
echo "${color}${timestamp}:" "$@" "${IO_COLOR_RESET}"
66+
echo "================================================================"
67+
}
68+
69+
# Logs the given message with normal coloring and a timestamp.
70+
function log() {
71+
log_impl "${IO_COLOR_RESET}" "$@"
72+
}
73+
74+
# Logs the given message in green with a timestamp.
75+
function log_green() {
76+
log_impl "${IO_COLOR_GREEN}" "$@"
77+
}
78+
79+
# Logs the given message in yellow with a timestamp.
80+
function log_yellow() {
81+
log_impl "${IO_COLOR_YELLOW}" "$@"
82+
}
83+
84+
# Logs the given message in red with a timestamp.
85+
function log_red() {
86+
log_impl "${IO_COLOR_RED}" "$@"
87+
}
88+
89+
function repo_root() {
90+
local dir="$1"
91+
while [[ ! -d "${dir}/.git" ]]; do
92+
dir="$(dirname "$dir")"
93+
done
94+
echo "${dir}"
95+
}
96+
97+
readonly tmpdir=$(mktemp -d -t ci-XXXXXXXX)
98+
readonly tmphome="${tmpdir}/h"
99+
mkdir -p "${tmphome}"
100+
101+
PROGRAM_PATH="$(realpath "$0")"
102+
PROGRAM_DIR="$(dirname "${PROGRAM_PATH}")"
103+
PROJECT_ROOT="$(repo_root "${PROGRAM_DIR}")"
104+
105+
RUNNING_IN_CI="false"
106+
107+
if [[ -n "${KOKORO_GFILE_DIR:-}" ]]; then
108+
# descriptive env var for indicating it's on CI.
109+
RUNNING_IN_CI="true"
110+
111+
# Now we're re-using the trampoline service account.
112+
# Potentially we can pass down this key into Docker for
113+
# bootstrapping secret.
114+
SERVICE_ACCOUNT_KEY_FILE="${KOKORO_GFILE_DIR}/kokoro-trampoline.service-account.json"
115+
116+
mkdir -p "${tmpdir}/gcloud"
117+
gcloud_config_dir="${tmpdir}/gcloud"
118+
119+
log "Using isolated gcloud config: ${gcloud_config_dir}."
120+
export CLOUDSDK_CONFIG="${gcloud_config_dir}"
121+
122+
log "Using ${SERVICE_ACCOUNT_KEY_FILE} for authentication."
123+
gcloud auth activate-service-account \
124+
--key-file "${SERVICE_ACCOUNT_KEY_FILE}"
125+
gcloud auth configure-docker --quiet
126+
fi
127+
128+
log_yellow "Changing to the project root: ${PROJECT_ROOT}."
129+
cd "${PROJECT_ROOT}"
130+
131+
required_envvars=(
132+
# The basic trampoline configurations.
133+
"TRAMPOLINE_IMAGE"
134+
"TRAMPOLINE_BUILD_FILE"
135+
)
136+
137+
pass_down_envvars=(
138+
# Default empty list.
139+
)
140+
141+
if [[ -f "${PROJECT_ROOT}/.trampolinerc" ]]; then
142+
source "${PROJECT_ROOT}/.trampolinerc"
143+
fi
144+
145+
log_yellow "Checking environment variables."
146+
for e in "${required_envvars[@]}"
147+
do
148+
if [[ -z "${!e:-}" ]]; then
149+
log "Missing ${e} env var. Aborting."
150+
exit 1
151+
fi
152+
done
153+
154+
log_yellow "Preparing Docker image."
155+
# Download the docker image specified by `TRAMPOLINE_IMAGE`
156+
157+
set +e # ignore error on docker operations
158+
# We may want to add --max-concurrent-downloads flag.
159+
160+
log_yellow "Start pulling the Docker image: ${TRAMPOLINE_IMAGE}."
161+
if docker pull "${TRAMPOLINE_IMAGE}"; then
162+
log_green "Finished pulling the Docker image: ${TRAMPOLINE_IMAGE}."
163+
has_cache="true"
164+
else
165+
log_red "Failed pulling the Docker image: ${TRAMPOLINE_IMAGE}."
166+
has_cache="false"
167+
fi
168+
169+
170+
update_cache="false"
171+
if [[ -n "${TRAMPOLINE_IMAGE_SOURCE:-}" ]]; then
172+
# Build the Docker image from the source.
173+
context_dir=$(dirname "${TRAMPOLINE_IMAGE_SOURCE}")
174+
docker_build_flags=(
175+
"-f" "${TRAMPOLINE_IMAGE_SOURCE}"
176+
"-t" "${TRAMPOLINE_IMAGE}"
177+
)
178+
if [[ "${has_cache}" == "true" ]]; then
179+
docker_build_flags+=("--cache-from" "${TRAMPOLINE_IMAGE}")
180+
fi
181+
182+
log_yellow "Start building the docker image."
183+
if docker build "${docker_build_flags[@]}" "${context_dir}"; then
184+
log_green "Finished building the docker image."
185+
update_cache="true"
186+
else
187+
log_red "Failed to build the Docker image. Aborting."
188+
exit 1
189+
fi
190+
else
191+
if [[ "${has_cache}" != "true" ]]; then
192+
log_red "failed to download the image ${TRAMPOLINE_IMAGE}, aborting."
193+
exit 1
194+
fi
195+
fi
196+
197+
# The default user for a Docker container has uid 0 (root). To avoid
198+
# creating root-owned files in the build directory we tell docker to
199+
# use the current user ID.
200+
docker_uid="$(id -u)"
201+
docker_gid="$(id -g)"
202+
docker_user="$(id -un)"
203+
204+
# We use an array for the flags so they are easier to document.
205+
docker_flags=(
206+
# Remove the container after it exists.
207+
"--rm"
208+
209+
# Use the host network.
210+
"--network=host"
211+
212+
# Run in priviledged mode. We are not using docker for sandboxing or
213+
# isolation, just for packaging our dev tools.
214+
"--privileged"
215+
216+
# Pass down the KOKORO_GFILE_DIR
217+
"--volume" "${KOKORO_GFILE_DIR:-/dev/shm}:/gfile"
218+
"--env" "KOKORO_GFILE_DIR=/gfile"
219+
220+
# Tells scripts whether they are running as part of CI or not.
221+
"--env" "RUNNING_IN_CI=${RUNNING_IN_CI:-no}"
222+
223+
# Run the docker script and this user id. Because the docker image gets to
224+
# write in ${PWD} you typically want this to be your user id.
225+
"--user" "${docker_uid}:${docker_gid}"
226+
227+
# Pass down the USER.
228+
"--env" "USER=${docker_user}"
229+
230+
# Mount the project directory inside the Docker container.
231+
"--volume" "${PWD}:/v"
232+
"--workdir" "/v"
233+
"--env" "PROJECT_ROOT=/v"
234+
235+
# Mount the temporary home directory.
236+
"--volume" "${tmphome}:/h"
237+
"--env" "HOME=/h"
238+
)
239+
240+
# Add an option for nicer output if the build gets a tty.
241+
if [[ -t 0 ]]; then
242+
docker_flags+=("-it")
243+
fi
244+
245+
# Passing down env vars
246+
for e in "${pass_down_envvars[@]}"
247+
do
248+
docker_flags+=("--env" "${e}=${!e}")
249+
done
250+
251+
# If arguments are given, all arguments will become the commands run
252+
# in the container, otherwise run TRAMPOLINE_BUILD_FILE.
253+
254+
if [[ $# -ge 1 ]]; then
255+
log_yellow "Running the given commands '" "${@:1}" "' in the container."
256+
readonly commands=("${@:1}")
257+
else
258+
log_yellow "Running the tests in a Docker container."
259+
readonly commands=("/v/${TRAMPOLINE_BUILD_FILE}")
260+
fi
261+
262+
echo docker run "${docker_flags[@]}" "${TRAMPOLINE_IMAGE}" "${commands[@]}"
263+
docker run "${docker_flags[@]}" "${TRAMPOLINE_IMAGE}" "${commands[@]}"
264+
265+
test_retval=$?
266+
267+
if [[ ${test_retval} -eq 0 ]]; then
268+
log_green "Build finished with ${test_retval}"
269+
else
270+
log_red "Build finished with ${test_retval}"
271+
fi
272+
273+
if [[ "${RUNNING_IN_CI}" == "true" ]] && \
274+
[[ "${update_cache}" == "true" ]] && \
275+
[[ -z "${KOKORO_GITHUB_PULL_REQUEST_NUMBER:-}" ]] && \
276+
[[ $test_retval == 0 ]]; then
277+
# Only upload it when the test passes.
278+
log_yellow "Uploading the Docker image."
279+
if docker push "${TRAMPOLINE_IMAGE}"; then
280+
log_green "Finished uploading the Docker image."
281+
else
282+
log_red "Failed uploading the Docker image."
283+
fi
284+
fi

‎.trampolinerc

Lines changed: 40 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,40 @@
1+
# Copyright 2020 Google LLC
2+
#
3+
# Licensed under the Apache License, Version 2.0 (the "License");
4+
# you may not use this file except in compliance with the License.
5+
# You may obtain a copy of the License at
6+
#
7+
# http://www.apache.org/licenses/LICENSE-2.0
8+
#
9+
# Unless required by applicable law or agreed to in writing, software
10+
# distributed under the License is distributed on an "AS IS" BASIS,
11+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12+
# See the License for the specific language governing permissions and
13+
# limitations under the License.
14+
15+
required_envvars+=(
16+
# The nox session to run.
17+
"RUN_TESTS_SESSION"
18+
)
19+
20+
pass_down_envvars+=(
21+
"RUN_TESTS_SESSION"
22+
)
23+
24+
# Set default values. You can override them with env vars.
25+
if [[ -z "${TRAMPOLINE_IMAGE:-}" ]]; then
26+
# Set default image.
27+
TRAMPOLINE_IMAGE="gcr.io/cloud-devrel-kokoro-resources/python-samples-testing-docker"
28+
fi
29+
30+
if [[ -z "${TRAMPOLINE_IMAGE_SOURCE:-}" ]]; then
31+
TRAMPOLINE_IMAGE_SOURCE=".kokoro/docker/Dockerfile"
32+
fi
33+
34+
if [[ -z "${RUN_TESTS_SESSION:-}" ]]; then
35+
RUN_TESTS_SESSION="py-3.7"
36+
fi
37+
38+
if [[ -z "${TRAMPOLINE_BUILD_FILE:-}" ]]; then
39+
TRAMPOLINE_BUILD_FILE=".kokoro/tests/run_tests_diff_head.sh"
40+
fi

0 commit comments

Comments
 (0)