Thanks to visit codestin.com
Credit goes to github.com

Skip to content

Commit e479db2

Browse files
david-mcmahondavid-mcmahon
committed
WIP: Continuous Integration API development.
Need to replace their build/common.sh interface. Adding push-ci-build.sh and supporting files.
1 parent 3eb1fc7 commit e479db2

4 files changed

Lines changed: 216 additions & 44 deletions

File tree

‎README-CI.md‎

Lines changed: 16 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,16 @@
1+
# Continuous Integration How-to
2+
3+
## Before
4+
5+
* Currently called from hack/jenkins/build.sh
6+
7+
```
8+
$ build/push-ci-build.sh
9+
```
10+
11+
## After
12+
13+
```
14+
$ git clone https://github.com/kubernetes/release
15+
$ release/push-ci-build.sh
16+
```

‎anago‎

Lines changed: 8 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -921,17 +921,21 @@ for label in ${!RELEASE_VERSION[@]}; do
921921
common::stepheader "PUSH ${RELEASE_VERSION[$label]} IMAGES"
922922
##############################################################################
923923
common::runstep release::gcs::copy_release_artifacts \
924-
${RELEASE_VERSION[$label]} \
925-
$BUILD_OUTPUT-${RELEASE_VERSION[$label]} $RELEASE_BUCKET \
924+
release/${RELEASE_VERSION[$label]} \
925+
$BUILD_OUTPUT-${RELEASE_VERSION[$label]} \
926+
$RELEASE_BUCKET \
926927
|| common::exit 1 "Exiting..."
927928

928929
common::runstep release::docker::release \
929-
$KUBE_DOCKER_REGISTRY ${RELEASE_VERSION[$label]} \
930+
$KUBE_DOCKER_REGISTRY \
931+
${RELEASE_VERSION[$label]} \
930932
|| common::exit 1 "Exiting..."
931933

932934
common::runstep release::gcs::publish_official \
935+
release \
933936
${RELEASE_VERSION[$label]} \
934-
$BUILD_OUTPUT-${RELEASE_VERSION[$label]} $RELEASE_BUCKET \
937+
$BUILD_OUTPUT-${RELEASE_VERSION[$label]} \
938+
$RELEASE_BUCKET \
935939
|| common::exit 1 "Exiting..."
936940
done
937941

‎lib/releaselib.sh‎

Lines changed: 65 additions & 40 deletions
Original file line numberDiff line numberDiff line change
@@ -433,24 +433,25 @@ release::gcs::stage_and_hash() {
433433

434434
###############################################################################
435435
# Copy the release artifacts to staging and push them up to GS
436-
# TODO: There is a version of this in kubernetes/build/common.sh that is also
437-
# used by the build/push-*-build.sh scripts. Refactor.
438-
# @param version - release version
436+
# @param target_version - something like release/<version>
439437
# @param build_output - build output directory
440438
# @param bucket - GS bucket
439+
# @optparam bucket_mirror - (optional) mirror GS bucket
441440
# @return 1 on failure
442441
release::gcs::copy_release_artifacts() {
443-
local version=$1
442+
local target_version=$1
444443
local build_output=$2
445444
local bucket=$3
445+
local bucket_mirror=$4
446446
local platform
447447
local platforms
448448
local release_stage=$build_output/release-stage
449449
local release_tars=$build_output/release-tars
450450
local gcs_stage=$build_output/gcs-stage
451451
local src
452452
local dst
453-
local gcs_destination=gs://$bucket/release/$version/
453+
local gcs_destination=gs://$bucket/$target_version
454+
local gcs_mirror=gs://$bucket_mirror/$target_version
454455
local gce_path=$release_stage/full/kubernetes/cluster/gce
455456
local gci_path
456457

@@ -500,45 +501,57 @@ release::gcs::copy_release_artifacts() {
500501
common::sha1 $path > "$path.sha1" || return 1
501502
done
502503

504+
# Copy the main set from staging to destination
503505
logecho -n "- Copying release artifacts to $gcs_destination: "
504506
logrun -s $GSUTIL -qm cp -r $gcs_stage/* $gcs_destination || return 1
505507

506-
# TODO(jbeda): Generate an HTML page with links for this release so it is easy
507-
# to see it. For extra credit, generate a dynamic page that builds up the
508-
# release list using the GCS JSON API. Use Angular and Bootstrap for extra
509-
# extra credit.
510-
511508
logecho -n "- Marking all uploaded objects public: "
512509
logrun -s $LOGRUN_MOCK $GSUTIL -q -m acl ch -R -g all:R \
513-
"$gcs_destination" || return 1
510+
"$gcs_destination" || return 1
514511

515512
logecho -n "- Listing final contents to log file: "
516513
logrun -s $GSUTIL ls -lhr "$gcs_destination" || return 1
514+
515+
# Push to mirror if set
516+
if [[ -n "$bucket_mirror" ]]; then
517+
logecho -n "- Mirroring build to $gcs_mirror: "
518+
logrun -s $GSUTIL -q -m rsync -d -r "$gcs_destination" "$gcs_mirror" \
519+
|| return 1
520+
logecho -n "- Marking all uploaded mirror objects public: "
521+
logrun -s $LOGRUN_MOCK $GSUTIL -q -m acl ch -R -g all:R \
522+
"$gcs_mirror" || return 1
523+
fi
517524
}
518525

519526

520527
###############################################################################
521528
# Publish a new official version, (latest or stable,) but only if the release
522529
# files actually exist on GCS and the release we're dealing with is newer than
523530
# the contents in GCS.
524-
# @param version - release version
531+
# @param build_type - One of 'release' or 'ci'
532+
# @param version - The version
525533
# @param build_output - build output directory
526534
# @param bucket - GS bucket
535+
# @optparam bucket_mirror - GS mirror bucket
527536
# @return 1 on failure
528537
release::gcs::publish_official () {
529-
local version=$1
530-
local build_output=$2
531-
local bucket=$3
532-
local release_dir="gs://$bucket/release/$version"
538+
local build_type=$1
539+
local version=$2
540+
local build_output=$3
541+
local bucket=$4
542+
local bucket_mirror=$5
543+
local release_dir="gs://$bucket/$build_type/$version"
533544
local version_major
534545
local version_minor
535546
local publish_file
536-
local publish_files
547+
local -a publish_files
548+
local -a publish_buckets=($bucket $bucket_mirror)
537549
local type="latest"
538-
[[ "$version" =~ alpha|beta ]] || type="stable"
539550

540-
logecho
541-
logecho "Publish official pointer text files to $bucket..."
551+
# For release/ targets, type could be 'stable'
552+
if [[ "$build_type" == release ]]; then
553+
[[ "$version" =~ alpha|beta ]] || type="stable"
554+
fi
542555

543556
if ! $GSUTIL ls $release_dir >/dev/null 2>&1 ; then
544557
logecho "Release files don't exist at $release_dir"
@@ -555,13 +568,18 @@ release::gcs::publish_official () {
555568
$type-$version_major.$version_minor
556569
)
557570

558-
for publish_file in ${publish_files[@]}; do
559-
# If there's a version that's above the one we're trying to release, don't
560-
# do anything, and just try the next one.
561-
release::gcs::verify_release_gt release/$publish_file.txt \
562-
$bucket $version || continue
563-
release::gcs::publish release/$publish_file.txt $build_output \
564-
$bucket $version || return 1
571+
for b in ${publish_buckets[*]}; do
572+
logecho
573+
logecho "Publish official pointer text files to $b..."
574+
575+
for publish_file in ${publish_files[@]}; do
576+
# If there's a version that's above the one we're trying to release, don't
577+
# do anything, and just try the next one.
578+
release::gcs::verify_release_gt $build_type/$publish_file.txt \
579+
$b $version || continue
580+
release::gcs::publish $build_type/$publish_file.txt $build_output \
581+
$b $version || return 1
582+
done
565583
done
566584
}
567585

@@ -580,10 +598,10 @@ release::gcs::verify_release_gt() {
580598
local -r version=$3
581599
local -r publish_file_dst="gs://$bucket/$publish_file"
582600
local gcs_version
583-
local greater=true
601+
local greater=1
584602

585603
logecho -n "Test $version > $publish_file (published): "
586-
if ! [[ $version =~ ${VER_REGEX[release]} ]]; then
604+
if ! [[ $version =~ ${VER_REGEX[release]}(.${VER_REGEX[build]})* ]]; then
587605
logecho -r "$FAILED"
588606
logecho "* Invalid version format! $version"
589607
return 1
@@ -594,9 +612,10 @@ release::gcs::verify_release_gt() {
594612
local -r version_patch="${BASH_REMATCH[3]}"
595613
local -r version_prerelease="${BASH_REMATCH[4]}"
596614
local -r version_prerelease_rev="${BASH_REMATCH[5]}"
615+
local -r version_commits="${BASH_REMATCH[7]}"
597616

598617
if gcs_version="$($GSUTIL cat $publish_file_dst 2>/dev/null)"; then
599-
if ! [[ $gcs_version =~ ${VER_REGEX[release]} ]]; then
618+
if ! [[ $gcs_version =~ ${VER_REGEX[release]}(.${VER_REGEX[build]})* ]]; then
600619
logecho -r "$FAILED"
601620
logecho "* file contains invalid release version," \
602621
"can't compare: '$gcs_version'"
@@ -608,17 +627,18 @@ release::gcs::verify_release_gt() {
608627
local -r gcs_version_patch="${BASH_REMATCH[3]}"
609628
local -r gcs_version_prerelease="${BASH_REMATCH[4]}"
610629
local -r gcs_version_prerelease_rev="${BASH_REMATCH[5]}"
630+
local -r gcs_version_commits="${BASH_REMATCH[7]}"
611631

612632
if [[ "$version_major" -lt "$gcs_version_major" ]]; then
613-
greater=false
633+
greater=0
614634
elif [[ "$version_major" -gt "$gcs_version_major" ]]; then
615635
: # fall out
616636
elif [[ "$version_minor" -lt "$gcs_version_minor" ]]; then
617-
greater=false
637+
greater=0
618638
elif [[ "$version_minor" -gt "$gcs_version_minor" ]]; then
619639
: # fall out
620640
elif [[ "$version_patch" -lt "$gcs_version_patch" ]]; then
621-
greater=false
641+
greater=0
622642
elif [[ "$version_patch" -gt "$gcs_version_patch" ]]; then
623643
: # fall out
624644
# Use lexicographic (instead of integer) comparison because
@@ -628,19 +648,25 @@ release::gcs::verify_release_gt() {
628648
# We have to do this because lexicographically "beta" > "alpha" > "", but
629649
# we want official > beta > alpha.
630650
elif [[ -n "$version_prerelease" && -z "$gcs_version_prerelease" ]]; then
631-
greater=false
651+
greater=0
632652
elif [[ -z "$version_prerelease" && -n "$gcs_version_prerelease" ]]; then
633653
: # fall out
634654
elif [[ "$version_prerelease" < "$gcs_version_prerelease" ]]; then
635-
greater=false
655+
greater=0
636656
elif [[ "$version_prerelease" > "$gcs_version_prerelease" ]]; then
637657
: # fall out
638658
# Finally resort to -le here, since we want strictly-greater-than.
639659
elif [[ "$version_prerelease_rev" -le "$gcs_version_prerelease_rev" ]]; then
640-
greater=false
660+
greater=0
661+
elif [[ "$version_prerelease_rev" -gt "$gcs_version_prerelease_rev" ]]; then
662+
: # fall out
663+
# If either version_commits is empty, it will be considered less-than, as
664+
# expected, (e.g. 1.2.3-beta < 1.2.3-beta.1).
665+
elif [[ "$version_commits" -lt "$gcs_version_commits" ]]; then
666+
greater=0
641667
fi
642668

643-
if $greater; then
669+
if ((greater)); then
644670
logecho -r "$OK"
645671
logecho "* $version > $gcs_version (published), updating"
646672
else
@@ -660,14 +686,13 @@ release::gcs::verify_release_gt() {
660686
###############################################################################
661687
# Publish a release to GCS: upload a version file, if KUBE_GCS_MAKE_PUBLIC,
662688
# make it public, and verify the result.
663-
# TODO: There is a version of this in kubernetes/build/common.sh that is also
664-
# used by kube::release::gcs::publish_ci(). Possible refactor.
665689
# @param publish_file - the GCS location to look in
666690
# @param build_output - build output directory
667691
# @param bucket - GS bucket
668692
# @param version - release version
693+
# @optparam bucket_mirror - (optional) mirror GS bucket
669694
# @return 1 on failure
670-
release::gcs::publish() {
695+
release::gcs::publish () {
671696
local publish_file=$1
672697
local build_output=$2
673698
local bucket=$3

‎push-ci-build.sh‎

Lines changed: 127 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,127 @@
1+
#!/bin/bash
2+
#
3+
# Copyright 2016 The Kubernetes Authors All rights reserved.
4+
#
5+
# Licensed under the Apache License, Version 2.0 (the "License");
6+
# you may not use this file except in compliance with the License.
7+
# You may obtain a copy of the License at
8+
#
9+
# http://www.apache.org/licenses/LICENSE-2.0
10+
#
11+
# Unless required by applicable law or agreed to in writing, software
12+
# distributed under the License is distributed on an "AS IS" BASIS,
13+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14+
# See the License for the specific language governing permissions and
15+
# limitations under the License.
16+
#
17+
# Set PROGram name
18+
PROG=${0##*/}
19+
########################################################################
20+
#+
21+
#+ NAME
22+
#+ $PROG - Push Continuous Integration Build
23+
#+
24+
#+ SYNOPSIS
25+
#+ $PROG [--federation]
26+
#+ $PROG [--helpshort|--usage|-?]
27+
#+ $PROG [--help|-man]
28+
#+
29+
#+ DESCRIPTION
30+
#+ Replaces kubernetes/build/push-ci-build.sh.
31+
#+ Used for pushing CI builds during Jenins' runs.
32+
#+
33+
#+ OPTIONS
34+
#+ [--federation] - Enable FEDERATION push
35+
#+ [--help | -man] - display man page for this script
36+
#+ [--usage | -?] - display in-line usage
37+
#+
38+
#+ EXAMPLES
39+
#+ $PROG - Do a CI push
40+
#+ $PROG --federation - Do a CI push with federation
41+
#+
42+
#+ FILES
43+
#+
44+
#+ SEE ALSO
45+
#+ common.sh - base function entry points
46+
#+ releaselib.sh - ::release:: namespace entry points
47+
#+
48+
#+ BUGS/TODO
49+
#+ Known problems with script
50+
#+
51+
########################################################################
52+
# If NO ARGUMENTS should return *usage*, uncomment the following line:
53+
#usage=${1:-yes}
54+
55+
source $(dirname $(readlink -ne $BASH_SOURCE))/lib/common.sh
56+
source $TOOL_LIB_PATH/gitlib.sh
57+
source $TOOL_LIB_PATH/releaselib.sh
58+
59+
###############################################################################
60+
# MAIN
61+
###############################################################################
62+
# TODO: Why can't we just use describe here?
63+
#LATEST=$(kube::release::semantic_version)
64+
LATEST=$(git describe)
65+
RELEASE_BUCKET="kubernetes-release"
66+
RELEASE_BUCKET_MIRROR="{$RELEASE_BUCKET}-dev"
67+
# This will canonicalize the path
68+
KUBE_ROOT=$(cd $(dirname "${BASH_SOURCE}")/.. && pwd -P)
69+
70+
##############################################################################
71+
# Initialize logs
72+
##############################################################################
73+
# Initialize and save up to 10 (rotated logs)
74+
MYLOG=/tmp/$PROG.log
75+
common::logfileinit $MYLOG 10
76+
77+
# BEGIN script
78+
common::timestamp begin
79+
80+
##############################################################################
81+
common::stepheader CHECK PREREQUISITES
82+
##############################################################################
83+
if ! common::set_cloud_binaries; then
84+
logecho "Releasing Kubernetes requires gsutil and gcloud. Please download,"
85+
logecho "install and authorize through the Google Cloud SDK:"
86+
logecho
87+
logecho "https://developers.google.com/cloud/sdk/"
88+
common::exit 1
89+
fi
90+
91+
logecho -n "Check/make release bucket $RELEASE_BUCKET: "
92+
logrun -s release::gcs::ensure_release_bucket $RELEASE_BUCKET \
93+
|| common::exit 1
94+
95+
# TODO: Does this need the retry nechanism from the original?
96+
##############################################################################
97+
common::stepheader RELEASE
98+
##############################################################################
99+
release::gcs::copy_release_artifacts ci/$LATEST \
100+
$KUBE_ROOT/_output \
101+
$RELEASE_BUCKET \
102+
$RELEASE_BUCKET_MIRROR \
103+
|| common::exit 1 "Exiting..."
104+
105+
##############################################################################
106+
common::stepheader PUBLISH
107+
##############################################################################
108+
common::runstep release::gcs::publish_official ci \
109+
$LATEST \
110+
$KUBE_ROOT/_output \
111+
$RELEASE_BUCKET \
112+
$RELEASE_BUCKET_MIRROR \
113+
|| common::exit 1 "Exiting..."
114+
115+
116+
##############################################################################
117+
common::stepheader PUSH FEDERATION
118+
##############################################################################
119+
# Leave push-federation-images.sh for now. Not sure if this makes sense to
120+
# pull into the release repo.
121+
if ((FLAGS_federation)); then
122+
logecho -n "Push federation images: "
123+
logrun -s ${KUBE_ROOT}/build/push-federation-images.sh
124+
fi
125+
126+
# END script
127+
common::timestamp end

0 commit comments

Comments
 (0)