export JENKINS_PUBLISHED_VERSION=${JENKINS_PUBLISHED_VERSION:-'ci/latest'}

echo "--------------------------------------------------------------------------------"

echo "Test Environment:"

printenv | sort

echo "--------------------------------------------------------------------------------"

if [[ "${E2E_UP,,}" == "true" || "${JENKINS_FORCE_GET_TARS:-}" =~ ^[yY]$ ]]; then

if [[ ${KUBE_RUN_FROM_OUTPUT:-} =~ ^[yY]$ ]]; then

echo "Found KUBE_RUN_FROM_OUTPUT=y; will use binaries from _output"

cp _output/release-tars/kubernetes*.tar.gz .

else

echo "Pulling binaries from GCS"

# In a multi-step job, clean up just the kubernetes build files.

# Otherwise, we want a completely empty directory.

if [[ "${JENKINS_FORCE_GET_TARS:-}" =~ ^[yY]$ ]]; then

rm -rf kubernetes*

elif [[ $(find . | wc -l) != 1 ]]; then

echo $PWD not empty, bailing!

exit 1

fi

if [[ ! -z ${JENKINS_EXPLICIT_VERSION:-} ]]; then

# Use an explicit pinned version like "ci/v0.10.0-101-g6c814c4" or

# "release/v0.19.1"

IFS='/' read -a varr <<< "${JENKINS_EXPLICIT_VERSION}"

bucket="${varr[0]}"

githash="${varr[1]}"

echo "Using explicit version $bucket/$githash"

elif [[ ${JENKINS_USE_SERVER_VERSION:-} =~ ^[yY]$ ]]; then

# for GKE we can use server default version.

bucket="release"

msg=$(gcloud ${CMD_GROUP} container get-server-config --project=${PROJECT} --zone=${ZONE} | grep defaultClusterVersion)

# msg will look like "defaultClusterVersion: 1.0.1". Strip

# everything up to, including ": "

githash="v${msg##*: }"

echo "Using server version $bucket/$githash"

else # use JENKINS_PUBLISHED_VERSION

# Use a published version like "ci/latest" (default),

# "release/latest", "release/latest-1", or "release/stable"

IFS='/' read -a varr <<< "${JENKINS_PUBLISHED_VERSION}"

bucket="${varr[0]}"

githash=$(gsutil cat gs://kubernetes-release/${JENKINS_PUBLISHED_VERSION}.txt)

echo "Using published version $bucket/$githash (from ${JENKINS_PUBLISHED_VERSION})"

fi

# At this point, we want to have the following vars set:

# - bucket

# - githash

gsutil -m cp gs://kubernetes-release/${bucket}/${githash}/kubernetes.tar.gz gs://kubernetes-release/${bucket}/${githash}/kubernetes-test.tar.gz .

# Set by GKE-CI to change the CLUSTER_API_VERSION to the git version

if [[ ! -z ${E2E_SET_CLUSTER_API_VERSION:-} ]]; then

export CLUSTER_API_VERSION=$(echo ${githash} | cut -c 2-)

fi

fi

if [[ ! "${CIRCLECI:-}" == "true" ]]; then

# Copy GCE keys so we don't keep cycling them.

# To set this up, you must know the <project>, <zone>, and <instance>

# on which your jenkins jobs are running. Then do:

#

# # SSH from your computer into the instance.

# $ gcloud compute ssh --project="<prj>" ssh --zone="<zone>" <instance>

#

# # Generate a key by ssh'ing from the instance into itself, then exit.

# $ gcloud compute ssh --project="<prj>" ssh --zone="<zone>" <instance>

# $ ^D

#

# # Copy the keys to the desired location (e.g. /var/lib/jenkins/gce_keys/).

# $ sudo mkdir -p /var/lib/jenkins/gce_keys/

# $ sudo cp ~/.ssh/google_compute_engine /var/lib/jenkins/gce_keys/

# $ sudo cp ~/.ssh/google_compute_engine.pub /var/lib/jenkins/gce_keys/

#

# # Move the permissions for the keys to Jenkins.

# $ sudo chown -R jenkins /var/lib/jenkins/gce_keys/

# $ sudo chgrp -R jenkins /var/lib/jenkins/gce_keys/

if [[ "${KUBERNETES_PROVIDER}" == "aws" ]]; then

echo "Skipping SSH key copying for AWS"

else

mkdir -p ${WORKSPACE}/.ssh/

cp /var/lib/jenkins/gce_keys/google_compute_engine ${WORKSPACE}/.ssh/

cp /var/lib/jenkins/gce_keys/google_compute_engine.pub ${WORKSPACE}/.ssh/

fi

fi

md5sum kubernetes*.tar.gz

tar -xzf kubernetes.tar.gz

tar -xzf kubernetes-test.tar.gz

cd kubernetes

ARTIFACTS=${WORKSPACE}/_artifacts

mkdir -p ${ARTIFACTS}

export E2E_REPORT_DIR=${ARTIFACTS}

declare -r gcp_list_resources_script="./cluster/gce/list-resources.sh"

declare -r gcp_resources_before="${ARTIFACTS}/gcp-resources-before.txt"

declare -r gcp_resources_cluster_up="${ARTIFACTS}/gcp-resources-cluster-up.txt"

declare -r gcp_resources_after="${ARTIFACTS}/gcp-resources-after.txt"

if [[ ( ${KUBERNETES_PROVIDER} == "gce" || ${KUBERNETES_PROVIDER} == "gke" ) && -x "${gcp_list_resources_script}" ]]; then

gcp_list_resources="true"

gcp_list_resources="false"

if [[ ! -z "${CLOUDSDK_BUCKET:-}" ]]; then

gsutil -m cp -r "${CLOUDSDK_BUCKET}" ~

rm -rf ~/repo ~/cloudsdk

mv ~/$(basename "${CLOUDSDK_BUCKET}") ~/repo

mkdir ~/cloudsdk

tar zvxf ~/repo/google-cloud-sdk.tar.gz -C ~/cloudsdk

export CLOUDSDK_CORE_DISABLE_PROMPTS=1

export CLOUDSDK_COMPONENT_MANAGER_SNAPSHOT_URL=file://${HOME}/repo/components-2.json

~/cloudsdk/google-cloud-sdk/install.sh --disable-installation-options --bash-completion=false --path-update=false --usage-reporting=false

export PATH=${HOME}/cloudsdk/google-cloud-sdk/bin:${PATH}

export CLOUDSDK_CONFIG=/var/lib/jenkins/.config/gcloud

if [[ "${E2E_UP,,}" == "true" ]]; then

go run ./hack/e2e.go ${E2E_OPT} -v --down

if [[ "${gcp_list_resources}" == "true" ]]; then

${gcp_list_resources_script} > "${gcp_resources_before}"

fi

go run ./hack/e2e.go ${E2E_OPT} -v --up

go run ./hack/e2e.go -v --ctl="version --match-server-version=false"

if [[ "${gcp_list_resources}" == "true" ]]; then

${gcp_list_resources_script} > "${gcp_resources_cluster_up}"

fi

if [[ "${E2E_TEST,,}" == "true" ]]; then

go run ./hack/e2e.go ${E2E_OPT} -v --test --test_args="${GINKGO_TEST_ARGS}" && exitcode=0 || exitcode=$?

if [[ "${E2E_PUBLISH_GREEN_VERSION:-}" == "true" && ${exitcode} == 0 && -n ${githash:-} ]]; then

echo "publish githash to ci/latest-green.txt: ${githash}"

echo "${githash}" > ${WORKSPACE}/githash.txt

gsutil cp ${WORKSPACE}/githash.txt gs://kubernetes-release/ci/latest-green.txt

fi

if [[ "${USE_KUBEMARK:-}" == "true" ]]; then

export RUN_FROM_DISTRO=true

NUM_MINIONS_BKP=${NUM_MINIONS}

MASTER_SIZE_BKP=${MASTER_SIZE}

./test/kubemark/stop-kubemark.sh

NUM_MINIONS=${KUBEMARK_NUM_MINIONS:-$NUM_MINIONS}

MASTER_SIZE=${KUBEMARK_MASTER_SIZE:-$MASTER_SIZE}

./test/kubemark/start-kubemark.sh

./test/kubemark/run-e2e-tests.sh --ginkgo.focus="should\sallow\sstarting\s30\spods\sper\snode" --delete-namespace="false" --gather-resource-usage="false"

./test/kubemark/stop-kubemark.sh

NUM_MINIONS=${NUM_MINIONS_BKP}

MASTER_SIZE=${MASTER_SIZE_BKP}

unset RUN_FROM_DISTRO

unset NUM_MINIONS_BKP

unset MASTER_SIZE_BKP

if [[ "${E2E_DOWN,,}" == "true" ]]; then

# Sleep before deleting the cluster to give the controller manager time to

# delete any cloudprovider resources still around from the last test.

# This is calibrated to allow enough time for 3 attempts to delete the

# resources. Each attempt is allocated 5 seconds for requests to the

# cloudprovider plus the processingRetryInterval from servicecontroller.go

# for the wait between attempts.

sleep 30

go run ./hack/e2e.go ${E2E_OPT} -v --down

if [[ "${gcp_list_resources}" == "true" ]]; then

${gcp_list_resources_script} > "${gcp_resources_after}"

fi

if [[ -f "${gcp_resources_before}" && -f "${gcp_resources_after}" ]]; then

if ! diff -sw -U0 -F'^\[.*\]$' "${gcp_resources_before}" "${gcp_resources_after}" && [[ "${FAIL_ON_GCP_RESOURCE_LEAK:-}" == "true" ]]; then

echo "!!! FAIL: Google Cloud Platform resources leaked while running tests!"

exit 1

fi