Thanks to visit codestin.com
Credit goes to github.com

Skip to content

Commit 644aa69

Browse files
author
Nick Sardo
committed
Reserve internal address for ILBs
1 parent ad6c85c commit 644aa69

9 files changed

Lines changed: 512 additions & 19 deletions

‎pkg/cloudprovider/providers/gce/BUILD‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -11,6 +11,7 @@ go_library(
1111
srcs = [
1212
"doc.go",
1313
"gce.go",
14+
"gce_address_manager.go",
1415
"gce_addresses.go",
1516
"gce_addresses_fakes.go",
1617
"gce_alpha.go",
@@ -83,6 +84,7 @@ go_library(
8384
go_test(
8485
name = "go_default_test",
8586
srcs = [
87+
"gce_address_manager_test.go",
8688
"gce_annotations_test.go",
8789
"gce_disks_test.go",
8890
"gce_healthchecks_test.go",
@@ -95,6 +97,7 @@ go_test(
9597
"//pkg/cloudprovider:go_default_library",
9698
"//pkg/kubelet/apis:go_default_library",
9799
"//vendor/github.com/stretchr/testify/assert:go_default_library",
100+
"//vendor/github.com/stretchr/testify/require:go_default_library",
98101
"//vendor/golang.org/x/oauth2/google:go_default_library",
99102
"//vendor/google.golang.org/api/compute/v0.alpha:go_default_library",
100103
"//vendor/google.golang.org/api/compute/v0.beta:go_default_library",

‎pkg/cloudprovider/providers/gce/gce_address_manager.go‎

Lines changed: 198 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,198 @@
1+
/*
2+
Copyright 2017 The Kubernetes Authors.
3+
4+
Licensed under the Apache License, Version 2.0 (the "License");
5+
you may not use this file except in compliance with the License.
6+
You may obtain a copy of the License at
7+
8+
http://www.apache.org/licenses/LICENSE-2.0
9+
10+
Unless required by applicable law or agreed to in writing, software
11+
distributed under the License is distributed on an "AS IS" BASIS,
12+
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13+
See the License for the specific language governing permissions and
14+
limitations under the License.
15+
*/
16+
17+
package gce
18+
19+
import (
20+
"fmt"
21+
"net/http"
22+
23+
computebeta "google.golang.org/api/compute/v0.beta"
24+
25+
"github.com/golang/glog"
26+
)
27+
28+
type addressManager struct {
29+
logPrefix string
30+
svc CloudAddressService
31+
name string
32+
serviceName string
33+
targetIP string
34+
addressType lbScheme
35+
region string
36+
subnetURL string
37+
tryRelease bool
38+
}
39+
40+
func newAddressManager(svc CloudAddressService, serviceName, region, subnetURL, name, targetIP string, addressType lbScheme) *addressManager {
41+
return &addressManager{
42+
svc: svc,
43+
logPrefix: fmt.Sprintf("AddressManager(%q)", name),
44+
region: region,
45+
serviceName: serviceName,
46+
name: name,
47+
targetIP: targetIP,
48+
addressType: addressType,
49+
tryRelease: true,
50+
subnetURL: subnetURL,
51+
}
52+
}
53+
54+
// HoldAddress will ensure that the IP is reserved with an address - either owned by the controller
55+
// or by a user. If the address is not the addressManager.name, then it's assumed to be a user's address.
56+
// The string returned is the reserved IP address.
57+
func (am *addressManager) HoldAddress() (string, error) {
58+
// HoldAddress starts with retrieving the address that we use for this load balancer (by name).
59+
// Retrieving an address by IP will indicate if the IP is reserved and if reserved by the user
60+
// or the controller, but won't tell us the current state of the controller's IP. The address
61+
// could be reserving another address; therefore, it would need to be deleted. In the normal
62+
// case of using a controller address, retrieving the address by name results in the fewest API
63+
// calls since it indicates whether a Delete is necessary before Reserve.
64+
glog.V(4).Infof("%v: attempting hold of IP %q Type %q", am.logPrefix, am.targetIP, am.addressType)
65+
// Get the address in case it was orphaned earlier
66+
addr, err := am.svc.GetBetaRegionAddress(am.name, am.region)
67+
if err != nil && !isNotFound(err) {
68+
return "", err
69+
}
70+
71+
if addr != nil {
72+
// If address exists, check if the address had the expected attributes.
73+
validationError := am.validateAddress(addr)
74+
if validationError == nil {
75+
glog.V(4).Infof("%v: address %q already reserves IP %q Type %q. No further action required.", am.logPrefix, addr.Name, addr.Address, addr.AddressType)
76+
return addr.Address, nil
77+
}
78+
79+
glog.V(2).Infof("%v: deleting existing address because %v", am.logPrefix, validationError)
80+
err := am.svc.DeleteRegionAddress(addr.Name, am.region)
81+
if err != nil {
82+
if isNotFound(err) {
83+
glog.V(4).Infof("%v: address %q was not found. Ignoring.", am.logPrefix, addr.Name)
84+
} else {
85+
return "", err
86+
}
87+
} else {
88+
glog.V(4).Infof("%v: successfully deleted previous address %q", am.logPrefix, addr.Name)
89+
}
90+
}
91+
92+
return am.ensureAddressReservation()
93+
}
94+
95+
// ReleaseAddress will release the address if it's owned by the controller.
96+
func (am *addressManager) ReleaseAddress() error {
97+
if !am.tryRelease {
98+
glog.V(4).Infof("%v: not attempting release of address %q.", am.logPrefix, am.targetIP)
99+
return nil
100+
}
101+
102+
glog.V(4).Infof("%v: releasing address %q named %q", am.logPrefix, am.targetIP, am.name)
103+
// Controller only ever tries to unreserve the address named with the load balancer's name.
104+
err := am.svc.DeleteRegionAddress(am.name, am.region)
105+
if err != nil {
106+
if isNotFound(err) {
107+
glog.Warningf("%v: address %q was not found. Ignoring.", am.logPrefix, am.name)
108+
return nil
109+
}
110+
111+
return err
112+
}
113+
114+
glog.V(4).Infof("%v: successfully released IP %q named %q", am.logPrefix, am.targetIP, am.name)
115+
return nil
116+
}
117+
118+
func (am *addressManager) ensureAddressReservation() (string, error) {
119+
// Try reserving the IP with controller-owned address name
120+
// If am.targetIP is an empty string, a new IP will be created.
121+
newAddr := &computebeta.Address{
122+
Name: am.name,
123+
Description: fmt.Sprintf(`{"kubernetes.io/service-name":"%s"}`, am.serviceName),
124+
Address: am.targetIP,
125+
AddressType: string(am.addressType),
126+
Subnetwork: am.subnetURL,
127+
}
128+
129+
err := am.svc.ReserveBetaRegionAddress(newAddr, am.region)
130+
if err == nil {
131+
if newAddr.Address != "" {
132+
glog.V(4).Infof("%v: successfully reserved IP %q with name %q", am.logPrefix, newAddr.Address, newAddr.Name)
133+
return newAddr.Address, nil
134+
}
135+
136+
addr, err := am.svc.GetRegionAddress(newAddr.Name, am.region)
137+
if err != nil {
138+
return "", err
139+
}
140+
141+
glog.V(4).Infof("%v: successfully created address %q which reserved IP %q", am.logPrefix, addr.Name, addr.Address)
142+
return addr.Address, nil
143+
} else if !isHTTPErrorCode(err, http.StatusConflict) && !isHTTPErrorCode(err, http.StatusBadRequest) {
144+
// If the IP is already reserved:
145+
// by an internal address: a StatusConflict is returned
146+
// by an external address: a BadRequest is returned
147+
return "", err
148+
}
149+
150+
// If the target IP was empty, we cannot try to find which IP caused a conflict.
151+
// If the name was already used, then the next sync will attempt deletion of that address.
152+
if am.targetIP == "" {
153+
return "", fmt.Errorf("failed to reserve address %q, err: %v", am.name, err)
154+
}
155+
156+
// Reserving the address failed due to a conflict or bad request. The address manager just checked that no address
157+
// exists with the name, so it may belong to the user.
158+
addr, err := am.svc.GetBetaRegionAddressByIP(am.region, am.targetIP)
159+
if err != nil {
160+
return "", fmt.Errorf("could not find address with IP %q after getting conflict error while creating address: %q", am.targetIP, err)
161+
}
162+
163+
// Check that the address attributes are as required.
164+
if err := am.validateAddress(addr); err != nil {
165+
return "", err
166+
}
167+
168+
if am.isManagedAddress(addr) {
169+
// The address with this name is checked at the beginning of 'HoldAddress()', but for some reason
170+
// it was re-created by this point. May be possible that two controllers are running.
171+
glog.Warning("%v: address %q unexpectedly existed with IP %q.", am.logPrefix, addr.Name, am.targetIP)
172+
} else {
173+
// If the retrieved address is not named with the loadbalancer name, then the controller does not own it.
174+
glog.V(4).Infof("%v: address %q was already reserved with name: %q, description: %q", am.logPrefix, am.targetIP, addr.Name, addr.Description)
175+
am.tryRelease = false
176+
}
177+
178+
return addr.Address, nil
179+
}
180+
181+
func (am *addressManager) validateAddress(addr *computebeta.Address) error {
182+
if am.targetIP != "" && am.targetIP != addr.Address {
183+
return fmt.Errorf("address %q does not have the expected IP %q, actual: %q", addr.Name, am.targetIP, addr.Address)
184+
}
185+
if addr.AddressType != string(am.addressType) {
186+
return fmt.Errorf("address %q does not have the expected address type %q, actual: %q", addr.Name, am.addressType, addr.AddressType)
187+
}
188+
189+
return nil
190+
}
191+
192+
func (am *addressManager) isManagedAddress(addr *computebeta.Address) bool {
193+
return addr.Name == am.name
194+
}
195+
196+
func ensureAddressDeleted(svc CloudAddressService, name, region string) error {
197+
return ignoreNotFound(svc.DeleteRegionAddress(name, region))
198+
}

‎pkg/cloudprovider/providers/gce/gce_address_manager_test.go‎

Lines changed: 137 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,137 @@
1+
/*
2+
Copyright 2017 The Kubernetes Authors.
3+
4+
Licensed under the Apache License, Version 2.0 (the "License");
5+
you may not use this file except in compliance with the License.
6+
You may obtain a copy of the License at
7+
8+
http://www.apache.org/licenses/LICENSE-2.0
9+
10+
Unless required by applicable law or agreed to in writing, software
11+
distributed under the License is distributed on an "AS IS" BASIS,
12+
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13+
See the License for the specific language governing permissions and
14+
limitations under the License.
15+
*/
16+
17+
package gce
18+
19+
import (
20+
"testing"
21+
22+
"github.com/stretchr/testify/assert"
23+
"github.com/stretchr/testify/require"
24+
computebeta "google.golang.org/api/compute/v0.beta"
25+
)
26+
27+
const testSvcName = "my-service"
28+
const testRegion = "us-central1"
29+
const testSubnet = "/projects/x/testRegions/us-central1/testSubnetworks/customsub"
30+
const testLBName = "a111111111111111"
31+
32+
// TestAddressManagerNoRequestedIP tests the typical case of passing in no requested IP
33+
func TestAddressManagerNoRequestedIP(t *testing.T) {
34+
svc := NewFakeCloudAddressService()
35+
targetIP := ""
36+
37+
mgr := newAddressManager(svc, testSvcName, testRegion, testSubnet, testLBName, targetIP, schemeInternal)
38+
testHoldAddress(t, mgr, svc, testLBName, testRegion, targetIP, string(schemeInternal))
39+
testReleaseAddress(t, mgr, svc, testLBName, testRegion)
40+
}
41+
42+
// TestAddressManagerBasic tests the typical case of reserving and unreserving an address.
43+
func TestAddressManagerBasic(t *testing.T) {
44+
svc := NewFakeCloudAddressService()
45+
targetIP := "1.1.1.1"
46+
47+
mgr := newAddressManager(svc, testSvcName, testRegion, testSubnet, testLBName, targetIP, schemeInternal)
48+
testHoldAddress(t, mgr, svc, testLBName, testRegion, targetIP, string(schemeInternal))
49+
testReleaseAddress(t, mgr, svc, testLBName, testRegion)
50+
}
51+
52+
// TestAddressManagerOrphaned tests the case where the address exists with the IP being equal
53+
// to the requested address (forwarding rule or loadbalancer IP).
54+
func TestAddressManagerOrphaned(t *testing.T) {
55+
svc := NewFakeCloudAddressService()
56+
targetIP := "1.1.1.1"
57+
58+
addr := &computebeta.Address{Name: testLBName, Address: targetIP, AddressType: string(schemeInternal)}
59+
err := svc.ReserveBetaRegionAddress(addr, testRegion)
60+
require.NoError(t, err)
61+
62+
mgr := newAddressManager(svc, testSvcName, testRegion, testSubnet, testLBName, targetIP, schemeInternal)
63+
testHoldAddress(t, mgr, svc, testLBName, testRegion, targetIP, string(schemeInternal))
64+
testReleaseAddress(t, mgr, svc, testLBName, testRegion)
65+
}
66+
67+
// TestAddressManagerOutdatedOrphan tests the case where an address exists but points to
68+
// an IP other than the forwarding rule or loadbalancer IP.
69+
func TestAddressManagerOutdatedOrphan(t *testing.T) {
70+
svc := NewFakeCloudAddressService()
71+
previousAddress := "1.1.0.0"
72+
targetIP := "1.1.1.1"
73+
74+
addr := &computebeta.Address{Name: testLBName, Address: previousAddress, AddressType: string(schemeExternal)}
75+
err := svc.ReserveBetaRegionAddress(addr, testRegion)
76+
require.NoError(t, err)
77+
78+
mgr := newAddressManager(svc, testSvcName, testRegion, testSubnet, testLBName, targetIP, schemeInternal)
79+
testHoldAddress(t, mgr, svc, testLBName, testRegion, targetIP, string(schemeInternal))
80+
testReleaseAddress(t, mgr, svc, testLBName, testRegion)
81+
}
82+
83+
// TestAddressManagerExternallyOwned tests the case where the address exists but isn't
84+
// owned by the controller.
85+
func TestAddressManagerExternallyOwned(t *testing.T) {
86+
svc := NewFakeCloudAddressService()
87+
targetIP := "1.1.1.1"
88+
89+
addr := &computebeta.Address{Name: "my-important-address", Address: targetIP, AddressType: string(schemeInternal)}
90+
err := svc.ReserveBetaRegionAddress(addr, testRegion)
91+
require.NoError(t, err)
92+
93+
mgr := newAddressManager(svc, testSvcName, testRegion, testSubnet, testLBName, targetIP, schemeInternal)
94+
ipToUse, err := mgr.HoldAddress()
95+
require.NoError(t, err)
96+
assert.NotEmpty(t, ipToUse)
97+
98+
_, err = svc.GetRegionAddress(testLBName, testRegion)
99+
assert.True(t, isNotFound(err))
100+
101+
testReleaseAddress(t, mgr, svc, testLBName, testRegion)
102+
}
103+
104+
// TestAddressManagerExternallyOwned tests the case where the address exists but isn't
105+
// owned by the controller. However, this address has the wrong type.
106+
func TestAddressManagerBadExternallyOwned(t *testing.T) {
107+
svc := NewFakeCloudAddressService()
108+
targetIP := "1.1.1.1"
109+
110+
addr := &computebeta.Address{Name: "my-important-address", Address: targetIP, AddressType: string(schemeExternal)}
111+
err := svc.ReserveBetaRegionAddress(addr, testRegion)
112+
require.NoError(t, err)
113+
114+
mgr := newAddressManager(svc, testSvcName, testRegion, testSubnet, testLBName, targetIP, schemeInternal)
115+
_, err = mgr.HoldAddress()
116+
assert.NotNil(t, err)
117+
}
118+
119+
func testHoldAddress(t *testing.T, mgr *addressManager, svc CloudAddressService, name, region, targetIP, scheme string) {
120+
ipToUse, err := mgr.HoldAddress()
121+
require.NoError(t, err)
122+
assert.NotEmpty(t, ipToUse)
123+
124+
addr, err := svc.GetBetaRegionAddress(name, region)
125+
require.NoError(t, err)
126+
if targetIP != "" {
127+
assert.EqualValues(t, targetIP, addr.Address)
128+
}
129+
assert.EqualValues(t, scheme, addr.AddressType)
130+
}
131+
132+
func testReleaseAddress(t *testing.T, mgr *addressManager, svc CloudAddressService, name, region string) {
133+
err := mgr.ReleaseAddress()
134+
require.NoError(t, err)
135+
_, err = svc.GetBetaRegionAddress(name, region)
136+
assert.True(t, isNotFound(err))
137+
}

0 commit comments

Comments
 (0)