From 47a4a0410c75f852fa330fd8eafb7dcac83d856f Mon Sep 17 00:00:00 2001 From: DuodenumL Date: Fri, 4 Mar 2022 12:03:01 +0800 Subject: [PATCH 1/7] fix Eru "Pod Resource" function returned duplicated (#556) --- cluster/calcium/resource.go | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/cluster/calcium/resource.go b/cluster/calcium/resource.go index d71552f2b..a10423297 100644 --- a/cluster/calcium/resource.go +++ b/cluster/calcium/resource.go @@ -26,11 +26,12 @@ func (c *Calcium) PodResource(ctx context.Context, podname string) (chan *types. go func() { defer close(ch) for node := range nodeCh { + nodename := node.Name pool.Go(ctx, func() { - nodeResource, err := c.doGetNodeResource(ctx, node.Name, false) + nodeResource, err := c.doGetNodeResource(ctx, nodename, false) if err != nil { nodeResource = &types.NodeResource{ - Name: node.Name, Diffs: []string{logger.Err(ctx, err).Error()}, + Name: nodename, Diffs: []string{logger.Err(ctx, err).Error()}, } } ch <- nodeResource From 8ea5431f5775aebbd92fe2ce2ad146047032edbc Mon Sep 17 00:00:00 2001 From: DuodenumL Date: Tue, 15 Mar 2022 16:25:52 +0800 Subject: [PATCH 2/7] fix issue of building docker image (#563) --- .github/workflows/dockerimage.yml | 2 ++ Dockerfile | 2 +- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/.github/workflows/dockerimage.yml b/.github/workflows/dockerimage.yml index b21bffa2c..f8787d985 100644 --- a/.github/workflows/dockerimage.yml +++ b/.github/workflows/dockerimage.yml @@ -13,6 +13,8 @@ jobs: steps: - name: checkout uses: actions/checkout@v2 + with: + fetch-depth: 0 - name: build and push to github packages uses: docker/build-push-action@v1 diff --git a/Dockerfile b/Dockerfile index 6e881f0f5..390c8b3df 100644 --- a/Dockerfile +++ b/Dockerfile @@ -2,7 +2,7 @@ FROM golang:alpine AS BUILD # make binary RUN apk add --no-cache build-base musl-dev git curl make cmake -RUN git clone https://github.com/projecteru2/core.git /go/src/github.com/projecteru2/core +COPY . /go/src/github.com/projecteru2/core WORKDIR /go/src/github.com/projecteru2/core ARG KEEP_SYMBOL RUN make build && ./eru-core --version From 02e6e8b39abf06538fee33c93e6153bc304200f9 Mon Sep 17 00:00:00 2001 From: DuodenumL Date: Mon, 28 Mar 2022 22:26:34 +0800 Subject: [PATCH 3/7] close engine connection when ping fails (#568) --- engine/docker/docker.go | 5 +++++ engine/engine.go | 1 + engine/factory/factory.go | 5 +++++ engine/fake/fake.go | 5 +++++ engine/mocks/API.go | 14 ++++++++++++++ engine/virt/virt.go | 5 +++++ go.mod | 2 +- go.sum | 2 ++ 8 files changed, 38 insertions(+), 1 deletion(-) diff --git a/engine/docker/docker.go b/engine/docker/docker.go index 7fe5647e6..6cb1066f6 100644 --- a/engine/docker/docker.go +++ b/engine/docker/docker.go @@ -94,3 +94,8 @@ func (e *Engine) Ping(ctx context.Context) error { _, err := e.client.Ping(ctx) return err } + +// CloseConn close connection +func (e *Engine) CloseConn() error { + return e.client.Close() +} diff --git a/engine/engine.go b/engine/engine.go index b2a117378..2d55c4de3 100644 --- a/engine/engine.go +++ b/engine/engine.go @@ -13,6 +13,7 @@ import ( type API interface { Info(ctx context.Context) (*enginetypes.Info, error) Ping(ctx context.Context) error + CloseConn() error Execute(ctx context.Context, ID string, config *enginetypes.ExecConfig) (result string, stdout, stderr io.ReadCloser, stdin io.WriteCloser, err error) ExecResize(ctx context.Context, ID, result string, height, width uint) (err error) diff --git a/engine/factory/factory.go b/engine/factory/factory.go index d4e92fe50..60dcf2ac6 100644 --- a/engine/factory/factory.go +++ b/engine/factory/factory.go @@ -139,6 +139,11 @@ func validateEngine(ctx context.Context, engine engine.API, timeout time.Duratio utils.WithTimeout(ctx, timeout, func(ctx context.Context) { err = engine.Ping(ctx) }) + if err != nil { + if closeErr := engine.CloseConn(); closeErr != nil { + log.Errorf(ctx, "[validateEngine] close conn error: %v", closeErr) + } + } return err } diff --git a/engine/fake/fake.go b/engine/fake/fake.go index 2ec7d3f7e..b26e453e9 100644 --- a/engine/fake/fake.go +++ b/engine/fake/fake.go @@ -24,6 +24,11 @@ func (f *Engine) Ping(ctx context.Context) error { return f.DefaultErr } +// CloseConn . +func (f *Engine) CloseConn() error { + return nil +} + // Execute . func (f *Engine) Execute(ctx context.Context, ID string, config *enginetypes.ExecConfig) (result string, stdout, stderr io.ReadCloser, stdin io.WriteCloser, err error) { return "", nil, nil, nil, f.DefaultErr diff --git a/engine/mocks/API.go b/engine/mocks/API.go index de04eb33f..9ab5f2d6f 100644 --- a/engine/mocks/API.go +++ b/engine/mocks/API.go @@ -67,6 +67,20 @@ func (_m *API) BuildRefs(ctx context.Context, opts *types.BuildRefOptions) []str return r0 } +// CloseConn provides a mock function with given fields: +func (_m *API) CloseConn() error { + ret := _m.Called() + + var r0 error + if rf, ok := ret.Get(0).(func() error); ok { + r0 = rf() + } else { + r0 = ret.Error(0) + } + + return r0 +} + // ExecExitCode provides a mock function with given fields: ctx, ID, result func (_m *API) ExecExitCode(ctx context.Context, ID string, result string) (int, error) { ret := _m.Called(ctx, ID, result) diff --git a/engine/virt/virt.go b/engine/virt/virt.go index 645360749..fa0344b0e 100644 --- a/engine/virt/virt.go +++ b/engine/virt/virt.go @@ -81,6 +81,11 @@ func (v *Virt) Ping(ctx context.Context) error { return err } +// CloseConn closes the connection. +func (v *Virt) CloseConn() error { + return v.client.Close() +} + // Execute executes a command in vm func (v *Virt) Execute(ctx context.Context, ID string, config *enginetypes.ExecConfig) (pid string, stdout, stderr io.ReadCloser, stdin io.WriteCloser, err error) { if config.Tty { diff --git a/go.mod b/go.mod index 10d850684..1de442856 100644 --- a/go.mod +++ b/go.mod @@ -32,7 +32,7 @@ require ( github.com/opencontainers/runc v1.0.0-rc95 // indirect github.com/patrickmn/go-cache v2.1.0+incompatible github.com/pkg/errors v0.9.1 - github.com/projecteru2/libyavirt v0.0.0-20211217082140-493b61aa9b0d + github.com/projecteru2/libyavirt v0.0.0-20220328133352-2a6302e90fc9 github.com/prometheus/client_golang v1.11.0 github.com/sanity-io/litter v1.5.1 github.com/sirupsen/logrus v1.7.0 diff --git a/go.sum b/go.sum index 76e40a808..3aa266499 100644 --- a/go.sum +++ b/go.sum @@ -432,6 +432,8 @@ github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZN github.com/posener/complete v1.1.1/go.mod h1:em0nMJCgc9GFtwrmVmEMR/ZL6WyhyjMBndrE9hABlRI= github.com/projecteru2/libyavirt v0.0.0-20211217082140-493b61aa9b0d h1:BMFqsvIB3nmK5l53nz8r2ndK8//T0njciu5nr/kj9A4= github.com/projecteru2/libyavirt v0.0.0-20211217082140-493b61aa9b0d/go.mod h1:FOc+hWBMLsMrmx5p3/moizKeSomedZPNwB6LhS+kEnE= +github.com/projecteru2/libyavirt v0.0.0-20220328133352-2a6302e90fc9 h1:7atvx2788Nf+HvY5uZFLlogjplvvF7iw5nWI/8UkjoY= +github.com/projecteru2/libyavirt v0.0.0-20220328133352-2a6302e90fc9/go.mod h1:FOc+hWBMLsMrmx5p3/moizKeSomedZPNwB6LhS+kEnE= github.com/prometheus/client_golang v0.9.1/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw= github.com/prometheus/client_golang v0.9.3/go.mod h1:/TN21ttK/J9q6uSwhBd54HahCDft0ttaMvbicHlPoso= github.com/prometheus/client_golang v1.0.0/go.mod h1:db9x61etRT2tGnBNRi70OPL5FsnadC4Ky3P0J6CfImo= From 83b09a154928d1f6149af89068962e3efe295af8 Mon Sep 17 00:00:00 2001 From: DuodenumL Date: Tue, 29 Mar 2022 18:51:40 +0800 Subject: [PATCH 4/7] reuse http client (#570) --- engine/docker/docker.go | 45 +++--------- engine/docker/helper.go | 20 +----- engine/factory/factory.go | 5 -- engine/virt/virt.go | 4 +- go.mod | 3 +- go.sum | 6 ++ utils/http.go | 140 ++++++++++++++++++++++++++++++++++++++ 7 files changed, 160 insertions(+), 63 deletions(-) create mode 100644 utils/http.go diff --git a/engine/docker/docker.go b/engine/docker/docker.go index 6cb1066f6..bd00c45e7 100644 --- a/engine/docker/docker.go +++ b/engine/docker/docker.go @@ -2,18 +2,16 @@ package docker import ( "context" - "fmt" - "io/ioutil" "net/http" - "os" + "strings" "github.com/projecteru2/core/engine" enginetypes "github.com/projecteru2/core/engine/types" "github.com/projecteru2/core/log" coretypes "github.com/projecteru2/core/types" + "github.com/projecteru2/core/utils" dockerapi "github.com/docker/docker/client" - "github.com/docker/go-connections/tlsconfig" ) const ( @@ -32,44 +30,19 @@ type Engine struct { // MakeClient make docker cli func MakeClient(ctx context.Context, config coretypes.Config, nodename, endpoint, ca, cert, key string) (engine.API, error) { var client *http.Client - if config.CertPath != "" && ca != "" && cert != "" && key != "" { // nolint - caFile, err := ioutil.TempFile(config.CertPath, fmt.Sprintf("ca-%s", nodename)) + var err error + if strings.HasPrefix(endpoint, "unix://") { + client = utils.GetUnixSockClient() + } else { + client, err = utils.GetHTTPSClient(ctx, config.CertPath, nodename, ca, cert, key) if err != nil { + log.Errorf(ctx, "[MakeClient] GetHTTPSClient for %s %s error: %v", nodename, endpoint, err) return nil, err } - certFile, err := ioutil.TempFile(config.CertPath, fmt.Sprintf("cert-%s", nodename)) - if err != nil { - return nil, err - } - keyFile, err := ioutil.TempFile(config.CertPath, fmt.Sprintf("key-%s", nodename)) - if err != nil { - return nil, err - } - if err = dumpFromString(ctx, caFile, certFile, keyFile, ca, cert, key); err != nil { - return nil, err - } - options := tlsconfig.Options{ - CAFile: caFile.Name(), - CertFile: certFile.Name(), - KeyFile: keyFile.Name(), - InsecureSkipVerify: true, - } - defer os.Remove(caFile.Name()) - defer os.Remove(certFile.Name()) - defer os.Remove(keyFile.Name()) - tlsc, err := tlsconfig.Client(options) - if err != nil { - return nil, err - } - client = &http.Client{ - Transport: &http.Transport{ - TLSClientConfig: tlsc, - }, - } } log.Debugf(ctx, "[MakeDockerEngine] Create new http.Client for %s, %s", endpoint, config.Docker.APIVersion) - return makeRawClient(ctx, config, client, endpoint) + return makeDockerClient(ctx, config, client, endpoint) } // Info show node info diff --git a/engine/docker/helper.go b/engine/docker/helper.go index 8df5fd70b..eda709c38 100644 --- a/engine/docker/helper.go +++ b/engine/docker/helper.go @@ -257,7 +257,7 @@ func GetIP(ctx context.Context, daemonHost string) string { return u.Hostname() } -func makeRawClient(_ context.Context, config coretypes.Config, client *http.Client, endpoint string) (engine.API, error) { +func makeDockerClient(_ context.Context, config coretypes.Config, client *http.Client, endpoint string) (engine.API, error) { cli, err := dockerapi.NewClientWithOpts( dockerapi.WithHost(endpoint), dockerapi.WithVersion(config.Docker.APIVersion), @@ -268,24 +268,6 @@ func makeRawClient(_ context.Context, config coretypes.Config, client *http.Clie return &Engine{cli, config}, nil } -func dumpFromString(ctx context.Context, ca, cert, key *os.File, caStr, certStr, keyStr string) error { - files := []*os.File{ca, cert, key} - data := []string{caStr, certStr, keyStr} - for i := 0; i < 3; i++ { - if _, err := files[i].WriteString(data[i]); err != nil { - return err - } - if err := files[i].Chmod(0444); err != nil { - return err - } - if err := files[i].Close(); err != nil { - return err - } - } - log.Debug(ctx, "[dumpFromString] Dump ca.pem, cert.pem, key.pem from string") - return nil -} - func useCNI(labels map[string]string) bool { for k, v := range labels { if k == "cni" && v == "1" { diff --git a/engine/factory/factory.go b/engine/factory/factory.go index 60dcf2ac6..d4e92fe50 100644 --- a/engine/factory/factory.go +++ b/engine/factory/factory.go @@ -139,11 +139,6 @@ func validateEngine(ctx context.Context, engine engine.API, timeout time.Duratio utils.WithTimeout(ctx, timeout, func(ctx context.Context) { err = engine.Ping(ctx) }) - if err != nil { - if closeErr := engine.CloseConn(); closeErr != nil { - log.Errorf(ctx, "[validateEngine] close conn error: %v", closeErr) - } - } return err } diff --git a/engine/virt/virt.go b/engine/virt/virt.go index fa0344b0e..482a7744c 100644 --- a/engine/virt/virt.go +++ b/engine/virt/virt.go @@ -90,7 +90,7 @@ func (v *Virt) CloseConn() error { func (v *Virt) Execute(ctx context.Context, ID string, config *enginetypes.ExecConfig) (pid string, stdout, stderr io.ReadCloser, stdin io.WriteCloser, err error) { if config.Tty { flags := virttypes.AttachGuestFlags{Safe: true, Force: true} - stream, err := v.client.AttachGuest(ctx, ID, config.Cmd, flags) + _, stream, err := v.client.AttachGuest(ctx, ID, config.Cmd, flags) if err != nil { return "", nil, nil, nil, err } @@ -274,7 +274,7 @@ func (v *Virt) VirtualizationLogs(ctx context.Context, opts *enginetypes.Virtual // VirtualizationAttach attaches something to a guest. func (v *Virt) VirtualizationAttach(ctx context.Context, ID string, stream, openStdin bool) (stdout, stderr io.ReadCloser, stdin io.WriteCloser, err error) { flags := virttypes.AttachGuestFlags{Safe: true, Force: true} - attachGuest, err := v.client.AttachGuest(ctx, ID, []string{}, flags) + _, attachGuest, err := v.client.AttachGuest(ctx, ID, []string{}, flags) if err != nil { return nil, nil, nil, err } diff --git a/go.mod b/go.mod index 1de442856..e3d969767 100644 --- a/go.mod +++ b/go.mod @@ -10,6 +10,7 @@ require ( github.com/cenkalti/backoff/v4 v4.0.2 github.com/containerd/containerd v1.4.8 // indirect github.com/containerd/continuity v0.0.0-20200710164510-efbc4488d8fe // indirect + github.com/cornelk/hashmap v1.0.1 // indirect github.com/docker/distribution v2.7.1+incompatible github.com/docker/docker v20.10.0+incompatible github.com/docker/go-connections v0.4.0 @@ -32,7 +33,7 @@ require ( github.com/opencontainers/runc v1.0.0-rc95 // indirect github.com/patrickmn/go-cache v2.1.0+incompatible github.com/pkg/errors v0.9.1 - github.com/projecteru2/libyavirt v0.0.0-20220328133352-2a6302e90fc9 + github.com/projecteru2/libyavirt v0.0.0-20220329021434-9b7b732e9f8e github.com/prometheus/client_golang v1.11.0 github.com/sanity-io/litter v1.5.1 github.com/sirupsen/logrus v1.7.0 diff --git a/go.sum b/go.sum index 3aa266499..9942caaa0 100644 --- a/go.sum +++ b/go.sum @@ -107,6 +107,8 @@ github.com/coreos/go-systemd/v22 v22.3.1/go.mod h1:Y58oyj3AT4RCenI/lSvhwexgC+NSV github.com/coreos/go-systemd/v22 v22.3.2 h1:D9/bQk5vlXQFZ6Kwuu6zaiXJ9oTPe68++AzAJc1DzSI= github.com/coreos/go-systemd/v22 v22.3.2/go.mod h1:Y58oyj3AT4RCenI/lSvhwexgC+NSVTIJ3seZv2GcEnc= github.com/coreos/pkg v0.0.0-20180928190104-399ea9e2e55f/go.mod h1:E3G3o1h8I7cfcXa63jLwjI0eiQQMgzzUDFVpN/nH/eA= +github.com/cornelk/hashmap v1.0.1 h1:RXGcy29hEdLLV8T6aK4s+BAd4tq4+3Hq50N2GoG0uIg= +github.com/cornelk/hashmap v1.0.1/go.mod h1:8wbysTUDnwJGrPZ1Iwsou3m+An6sldFrJItjRhfegCw= github.com/cpuguy83/go-md2man v1.0.10 h1:BSKMNlYxDvnunlTymqtgONjNnaRV1sTpcovwwjF22jk= github.com/cpuguy83/go-md2man v1.0.10/go.mod h1:SmD6nW6nTyfqj6ABTjUi3V3JVMnlJmwcJI5acqYI6dE= github.com/cpuguy83/go-md2man/v2 v2.0.0-20190314233015-f79a8a8ca69d/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU= @@ -119,6 +121,8 @@ github.com/davecgh/go-spew v0.0.0-20161028175848-04cdfd42973b/go.mod h1:J7Y8YcW2 github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/dchest/siphash v1.1.0 h1:1Rs9eTUlZLPBEvV+2sTaM8O0NWn0ppbgqS7p11aWawI= +github.com/dchest/siphash v1.1.0/go.mod h1:q+IRvb2gOSrUnYoPqHiyHXS0FOBBOdl6tONBlVnOnt4= github.com/dgraph-io/badger v1.6.0/go.mod h1:zwt7syl517jmP8s94KqSxTlM6IMsdhYy6psNgSztDR4= github.com/dgrijalva/jwt-go v3.2.0+incompatible/go.mod h1:E3ru+11k8xSBh+hMPgOLZmtrrCbhqsmaPHjLKYnJCaQ= github.com/dgryski/go-farm v0.0.0-20190423205320-6a90982ecee2/go.mod h1:SqUrOPUnsFjfmXRMNPybcSiG0BgUW2AuFH8PAnS2iTw= @@ -434,6 +438,8 @@ github.com/projecteru2/libyavirt v0.0.0-20211217082140-493b61aa9b0d h1:BMFqsvIB3 github.com/projecteru2/libyavirt v0.0.0-20211217082140-493b61aa9b0d/go.mod h1:FOc+hWBMLsMrmx5p3/moizKeSomedZPNwB6LhS+kEnE= github.com/projecteru2/libyavirt v0.0.0-20220328133352-2a6302e90fc9 h1:7atvx2788Nf+HvY5uZFLlogjplvvF7iw5nWI/8UkjoY= github.com/projecteru2/libyavirt v0.0.0-20220328133352-2a6302e90fc9/go.mod h1:FOc+hWBMLsMrmx5p3/moizKeSomedZPNwB6LhS+kEnE= +github.com/projecteru2/libyavirt v0.0.0-20220329021434-9b7b732e9f8e h1:IBRdnQybu4DRixX0grv4H4+tN8X+6Tp6LgVxx0ApIfM= +github.com/projecteru2/libyavirt v0.0.0-20220329021434-9b7b732e9f8e/go.mod h1:FOc+hWBMLsMrmx5p3/moizKeSomedZPNwB6LhS+kEnE= github.com/prometheus/client_golang v0.9.1/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw= github.com/prometheus/client_golang v0.9.3/go.mod h1:/TN21ttK/J9q6uSwhBd54HahCDft0ttaMvbicHlPoso= github.com/prometheus/client_golang v1.0.0/go.mod h1:db9x61etRT2tGnBNRi70OPL5FsnadC4Ky3P0J6CfImo= diff --git a/utils/http.go b/utils/http.go new file mode 100644 index 000000000..6ad3cc5ed --- /dev/null +++ b/utils/http.go @@ -0,0 +1,140 @@ +package utils + +import ( + "context" + "fmt" + "io/ioutil" + "net" + "net/http" + "os" + "runtime" + "strings" + "time" + + "github.com/cornelk/hashmap" + "github.com/docker/go-connections/tlsconfig" + + "github.com/projecteru2/core/log" +) + +var defaultHTTPClient = &http.Client{ + CheckRedirect: checkRedirect, + Transport: getDefaultTransport(), +} + +var defaultUnixSockClient = &http.Client{ + Transport: getDefaultUnixSockTransport(), +} + +var httpsClientCache = hashmap.New(32) + +// GetHTTPClient returns a HTTP client +func GetHTTPClient() *http.Client { + return defaultHTTPClient +} + +// GetUnixSockClient . +func GetUnixSockClient() *http.Client { + return defaultUnixSockClient +} + +// GetHTTPSClient returns an HTTPS client +// if cert_path/ca/cert/key is empty, it returns an HTTP client instead +func GetHTTPSClient(ctx context.Context, certPath, name, ca, cert, key string) (client *http.Client, err error) { + if certPath == "" || ca == "" || cert == "" || key == "" { + return GetHTTPClient(), nil + } + + cacheKey := name + SHA256(fmt.Sprintf("%s-%s-%s-%s-%s", certPath, name, ca, cert, key))[:8] + if httpsClient, ok := httpsClientCache.Get(cacheKey); ok { + return httpsClient.(*http.Client), nil + } + + caFile, err := ioutil.TempFile(certPath, fmt.Sprintf("ca-%s", name)) + if err != nil { + return nil, err + } + certFile, err := ioutil.TempFile(certPath, fmt.Sprintf("cert-%s", name)) + if err != nil { + return nil, err + } + keyFile, err := ioutil.TempFile(certPath, fmt.Sprintf("key-%s", name)) + if err != nil { + return nil, err + } + if err = dumpFromString(ctx, caFile, certFile, keyFile, ca, cert, key); err != nil { + return nil, err + } + options := tlsconfig.Options{ + CAFile: caFile.Name(), + CertFile: certFile.Name(), + KeyFile: keyFile.Name(), + InsecureSkipVerify: true, + } + defer os.Remove(caFile.Name()) + defer os.Remove(certFile.Name()) + defer os.Remove(keyFile.Name()) + tlsc, err := tlsconfig.Client(options) + if err != nil { + return nil, err + } + transport := getDefaultTransport() + transport.TLSClientConfig = tlsc + + client = &http.Client{ + CheckRedirect: checkRedirect, + Transport: transport, + } + httpsClientCache.Set(cacheKey, client) + return client, nil +} + +func getDefaultTransport() *http.Transport { + return &http.Transport{ + DialContext: (&net.Dialer{ + KeepAlive: time.Second * 30, + Timeout: time.Second * 30, + }).DialContext, + + IdleConnTimeout: time.Second * 90, + MaxIdleConnsPerHost: runtime.GOMAXPROCS(0) + 1, + Proxy: http.ProxyFromEnvironment, + } +} + +func getDefaultUnixSockTransport() *http.Transport { + return &http.Transport{ + DialContext: func(_ context.Context, _, addr string) (net.Conn, error) { + return net.DialTimeout("unix", strings.Split(addr, ":")[0], time.Second*30) + }, + + IdleConnTimeout: time.Second * 90, + MaxIdleConnsPerHost: runtime.GOMAXPROCS(0) + 1, + DisableCompression: true, + } +} + +func dumpFromString(ctx context.Context, ca, cert, key *os.File, caStr, certStr, keyStr string) error { + files := []*os.File{ca, cert, key} + data := []string{caStr, certStr, keyStr} + for i := 0; i < 3; i++ { + if _, err := files[i].WriteString(data[i]); err != nil { + return err + } + if err := files[i].Chmod(0444); err != nil { + return err + } + if err := files[i].Close(); err != nil { + return err + } + } + log.Debug(ctx, "[dumpFromString] Dump ca.pem, cert.pem, key.pem from string") + return nil +} + +func checkRedirect(req *http.Request, via []*http.Request) error { + if via[0].Method == http.MethodGet { + return http.ErrUseLastResponse + } + return fmt.Errorf("unexpected redirect") +} From 91cd77c806ba87f3ce5856e8dd144c596b2bdec1 Mon Sep 17 00:00:00 2001 From: DuodenumL Date: Mon, 6 Jun 2022 20:44:51 +0800 Subject: [PATCH 5/7] [SPPSP-4850] fix the issue of realloc algorithm (#577) --- scheduler/complex/cpu_test.go | 9 ++++++++- scheduler/complex/potassium.go | 1 + 2 files changed, 9 insertions(+), 1 deletion(-) diff --git a/scheduler/complex/cpu_test.go b/scheduler/complex/cpu_test.go index 7691858de..25e8466e4 100644 --- a/scheduler/complex/cpu_test.go +++ b/scheduler/complex/cpu_test.go @@ -80,6 +80,7 @@ func TestCPUReallocPlan(t *testing.T) { "1": 30, "2": 40, } + scheduleInfo.CPU.Add(CPU) si, remain, aff := cpuReallocPlan(scheduleInfo, 1, CPU, 100) assert.EqualValues(t, 0, remain) assert.True(t, reflect.DeepEqual(aff, types.CPUMap{"0": 100})) @@ -101,6 +102,7 @@ func TestCPUReallocPlan(t *testing.T) { "1": 30, "2": 40, } + scheduleInfo.CPU.Add(CPU) si, remain, aff = cpuReallocPlan(scheduleInfo, 1.2, CPU, 100) assert.EqualValues(t, 0, remain) assert.True(t, reflect.DeepEqual(aff, types.CPUMap{"0": 100, "2": 20})) @@ -124,6 +126,7 @@ func TestCPUReallocPlan(t *testing.T) { "2": 40, "3": 10, } + scheduleInfo.CPU.Add(CPU) si, remain, aff = cpuReallocPlan(scheduleInfo, 2, CPU, 100) assert.EqualValues(t, 0, remain) assert.True(t, reflect.DeepEqual(aff, types.CPUMap{"0": 100, "1": 100})) @@ -145,6 +148,7 @@ func TestCPUReallocPlan(t *testing.T) { "1": 30, "2": 40, } + scheduleInfo.CPU.Add(CPU) si, remain, aff = cpuReallocPlan(scheduleInfo, 2, CPU, 100) assert.EqualValues(t, 1, remain) assert.True(t, reflect.DeepEqual(aff, types.CPUMap{"0": 100})) @@ -174,6 +178,7 @@ func TestCPUReallocWithPriorPlan(t *testing.T) { "1": 30, "2": 40, } + scheduleInfo.CPU.Add(CPU) si, cpuPlans, total, err := po.ReselectCPUNodes(context.TODO(), scheduleInfo, CPU, 2, 0) assert.Nil(t, err) assert.EqualValues(t, 1, total) @@ -198,6 +203,7 @@ func TestCPUReallocWithPriorPlan(t *testing.T) { "1": 30, "2": 40, } + scheduleInfo.CPU.Add(CPU) si, cpuPlans, total, err = po.ReselectCPUNodes(context.TODO(), scheduleInfo, CPU, 2, 0) assert.Nil(t, err) assert.EqualValues(t, 3, total) @@ -233,6 +239,7 @@ func TestCPUReallocWithPriorPlan(t *testing.T) { "1": 30, "2": 40, } + scheduleInfo.CPU.Add(CPU) _, _, _, err = po.ReselectCPUNodes(context.TODO(), scheduleInfo, CPU, 2, 0) assert.EqualError(t, err, "failed to reschedule cpu: no node remains 1.00 pieces of cpu and 0 bytes of memory at the same time: not enough resource") } @@ -262,7 +269,7 @@ func TestGetFullResult(t *testing.T) { {"1": 100, "2": 100}, }) - res = h.getFullResult(2, []resourceInfo{ + res = h.getFullResult(2, []resourceInfo{ { id: "0", pieces: 200, diff --git a/scheduler/complex/potassium.go b/scheduler/complex/potassium.go index 71827ac34..8bf88d182 100644 --- a/scheduler/complex/potassium.go +++ b/scheduler/complex/potassium.go @@ -204,6 +204,7 @@ func cpuReallocPlan(scheduleInfo resourcetypes.ScheduleInfo, quota float64, CPU cpuIDs = append(cpuIDs, cpuID) } sort.Slice(cpuIDs, func(i, j int) bool { return CPU[cpuIDs[i]] < CPU[cpuIDs[j]] }) + scheduleInfo.CPU.Sub(CPU) // shrink, ensure affinity if diff <= 0 { From 2091d84b2bfc6ff0830aa3ae7f1060a0ff796d43 Mon Sep 17 00:00:00 2001 From: DuodenumL Date: Tue, 21 Jun 2022 15:34:55 +0800 Subject: [PATCH 6/7] [SPCLD-6491] fix bugs of cpu realloc algorithm (#579) --- scheduler/complex/potassium.go | 8 ++++---- scheduler/complex/resource.go | 17 ++++++++++++----- types/helper.go | 5 +++++ types/helper_test.go | 9 +++++++++ 4 files changed, 30 insertions(+), 9 deletions(-) diff --git a/scheduler/complex/potassium.go b/scheduler/complex/potassium.go index 8bf88d182..f017ed273 100644 --- a/scheduler/complex/potassium.go +++ b/scheduler/complex/potassium.go @@ -197,7 +197,7 @@ func (m *Potassium) ReselectCPUNodes(ctx context.Context, scheduleInfo resourcet func cpuReallocPlan(scheduleInfo resourcetypes.ScheduleInfo, quota float64, CPU types.CPUMap, sharebase int64) (resourcetypes.ScheduleInfo, float64, types.CPUMap) { affinityPlan := make(types.CPUMap) - diff := int64(quota*float64(sharebase)) - CPU.Total() + diff := types.RoundToInt(quota*float64(sharebase)) - CPU.Total() // sort by pieces cpuIDs := []string{} for cpuID := range CPU { @@ -226,7 +226,7 @@ func cpuReallocPlan(scheduleInfo resourcetypes.ScheduleInfo, quota float64, CPU } // expand, prioritize full cpus - needPieces := int64(quota * float64(sharebase)) + needPieces := types.RoundToInt(quota * float64(sharebase)) for i := len(cpuIDs) - 1; i >= 0; i-- { cpuID := cpuIDs[i] if needPieces == 0 { @@ -243,10 +243,10 @@ func cpuReallocPlan(scheduleInfo resourcetypes.ScheduleInfo, quota float64, CPU // fragments, try to find complement if available := scheduleInfo.CPU[cpuID]; available == sharebase-CPU[cpuID] { - expand := utils.Min64(available, needPieces) + expand := utils.Min64(available, needPieces-CPU[cpuID]) affinityPlan[cpuID] = CPU[cpuID] + expand scheduleInfo.CPU[cpuID] -= expand - needPieces -= sharebase + needPieces -= affinityPlan[cpuID] continue } diff --git a/scheduler/complex/resource.go b/scheduler/complex/resource.go index 7527daa39..3f1112e1e 100644 --- a/scheduler/complex/resource.go +++ b/scheduler/complex/resource.go @@ -5,6 +5,7 @@ import ( "sort" "github.com/projecteru2/core/types" + "github.com/projecteru2/core/utils" ) type resourceInfo struct { @@ -121,7 +122,10 @@ func (h *host) getFragmentResult(fragment int64, resources []resourceInfo) []typ resourceMaps := h.getFragmentsResult(resources, fragment) result := make([]types.ResourceMap, len(resourceMaps)) for i, resourceMap := range resourceMaps { - result[i] = resourceMap[0] + result[i] = types.ResourceMap{} + for id, pieces := range resourceMap[0] { + result[i][id] = pieces + } } // to pass tests due to new algorithm returns unsorted list @@ -259,16 +263,19 @@ func (h *host) getFullResult(full int, resources []resourceInfo) []types.Resourc func (h *host) distributeOneRation(ration float64, maxShare int) []types.ResourceMap { ration *= float64(h.share) - fullRequire := int64(ration) / int64(h.share) - fragmentRequire := int64(ration) % int64(h.share) + fullRequire := types.RoundToInt(ration) / int64(h.share) + fragmentRequire := types.RoundToInt(ration) % int64(h.share) if fullRequire == 0 { - if maxShare == -1 { + if maxShare < 0 { // 这个时候就把所有的资源都当成碎片 maxShare = len(h.full) + len(h.fragment) } + maxShare = utils.Min(maxShare, len(h.full)+len(h.fragment)) diff := maxShare - len(h.fragment) - h.fragment = append(h.fragment, h.full[:diff]...) + if diff > 0 { + h.fragment = append(h.fragment, h.full[:diff]...) + } return h.getFragmentResult(fragmentRequire, h.fragment) } diff --git a/types/helper.go b/types/helper.go index 0da15b75b..e3ee73bc7 100644 --- a/types/helper.go +++ b/types/helper.go @@ -6,3 +6,8 @@ import "math" func Round(f float64) float64 { return math.Round(f*1000000000) / 1000000000 } + +// RoundToInt for float64 to int +func RoundToInt(f float64) int64 { + return int64(math.Round(f)) +} diff --git a/types/helper_test.go b/types/helper_test.go index 87b67548e..9138f3225 100644 --- a/types/helper_test.go +++ b/types/helper_test.go @@ -16,3 +16,12 @@ func TestRound(t *testing.T) { a = 19.99998 assert.InDelta(t, (Round(a)), 19.99998, 1e-6) } + +func TestRoundToInt(t *testing.T) { + a := 0.0199999998 + assert.EqualValues(t, RoundToInt(a), 0) + a = 0.1999998 + assert.EqualValues(t, RoundToInt(a), 0) + a = 1.999998 + assert.EqualValues(t, RoundToInt(a), 2) +} From b951956f1100a9898e8f78f1097dba5fcf72e826 Mon Sep 17 00:00:00 2001 From: DuodenumL Date: Mon, 25 Jul 2022 15:35:55 +0800 Subject: [PATCH 7/7] [SPPSP-5975] optimize "GetOutboundAddress" --- cluster/calcium/calcium_test.go | 1 + cluster/calcium/service.go | 2 +- types/config.go | 21 +++++++++++---------- utils/service.go | 23 +++++++++++++++++++---- utils/service_test.go | 2 +- 5 files changed, 33 insertions(+), 16 deletions(-) diff --git a/cluster/calcium/calcium_test.go b/cluster/calcium/calcium_test.go index e79a1e1ef..8b983cec5 100644 --- a/cluster/calcium/calcium_test.go +++ b/cluster/calcium/calcium_test.go @@ -56,6 +56,7 @@ func NewTestCluster() *Calcium { WALFile: filepath.Join(walDir, "core.wal.log"), MaxConcurrency: 10, HAKeepaliveInterval: 16 * time.Second, + DialTarget: "8.8.8.8:80", } c.store = &storemocks.Store{} c.scheduler = &schedulermocks.Scheduler{} diff --git a/cluster/calcium/service.go b/cluster/calcium/service.go index e7f578a3c..8329cc7ff 100644 --- a/cluster/calcium/service.go +++ b/cluster/calcium/service.go @@ -26,7 +26,7 @@ func (c *Calcium) WatchServiceStatus(ctx context.Context) (<-chan types.ServiceS // RegisterService writes self service address in store func (c *Calcium) RegisterService(ctx context.Context) (unregister func(), err error) { - serviceAddress, err := utils.GetOutboundAddress(c.config.Bind) + serviceAddress, err := utils.GetOutboundAddress(c.config.Bind, c.config.DialTarget) if err != nil { log.Errorf(ctx, "[RegisterService] failed to get outbound address: %v", err) return diff --git a/types/config.go b/types/config.go index 7bce93d1a..6764d9fe6 100644 --- a/types/config.go +++ b/types/config.go @@ -18,16 +18,17 @@ const ( // Config holds eru-core config type Config struct { LogLevel string `yaml:"log_level" required:"true" default:"INFO"` - Bind string `yaml:"bind" required:"true" default:"5001"` // HTTP API address - LockTimeout time.Duration `yaml:"lock_timeout" required:"true" default:"30s"` // timeout for lock (ttl) - GlobalTimeout time.Duration `yaml:"global_timeout" required:"true" default:"300s"` // timeout for remove, run_and_wait and build, in second - ConnectionTimeout time.Duration `yaml:"connection_timeout" default:"10s"` // timeout for connections - HAKeepaliveInterval time.Duration `yaml:"ha_keepalive_interval" default:"16s"` // interval for node status watcher - Statsd string `yaml:"statsd"` // statsd host and port - Profile string `yaml:"profile"` // profile ip:port - CertPath string `yaml:"cert_path"` // docker cert files path - MaxConcurrency int64 `yaml:"max_concurrency" default:"20"` // concurrently call single runtime in the same time - Store string `yaml:"store" default:"etcd"` // store type + Bind string `yaml:"bind" required:"true" default:"5001"` // HTTP API address + DialTarget string `yaml:"dial_target" required:"false" default:"8.8.8.8:80"` // for getting outbound address + LockTimeout time.Duration `yaml:"lock_timeout" required:"true" default:"30s"` // timeout for lock (ttl) + GlobalTimeout time.Duration `yaml:"global_timeout" required:"true" default:"300s"` // timeout for remove, run_and_wait and build, in second + ConnectionTimeout time.Duration `yaml:"connection_timeout" default:"10s"` // timeout for connections + HAKeepaliveInterval time.Duration `yaml:"ha_keepalive_interval" default:"16s"` // interval for node status watcher + Statsd string `yaml:"statsd"` // statsd host and port + Profile string `yaml:"profile"` // profile ip:port + CertPath string `yaml:"cert_path"` // docker cert files path + MaxConcurrency int64 `yaml:"max_concurrency" default:"20"` // concurrently call single runtime in the same time + Store string `yaml:"store" default:"etcd"` // store type Auth AuthConfig `yaml:"auth"` // grpc auth GRPCConfig GRPCConfig `yaml:"grpc"` // grpc config diff --git a/utils/service.go b/utils/service.go index 2feccda10..80d61c91d 100644 --- a/utils/service.go +++ b/utils/service.go @@ -6,15 +6,30 @@ import ( "strings" ) -// GetOutboundAddress finds out self service address -func GetOutboundAddress(bind string) (string, error) { - conn, err := net.Dial("udp", "8.8.8.8:80") +// GetOutboundAddress finds out self-service address +func GetOutboundAddress(bind string, dialTarget string) (string, error) { + parts := strings.Split(bind, ":") + if len(parts) != 2 { + return "", fmt.Errorf("invalid bind address %s", bind) + } + ip := parts[0] + port := parts[1] + + address := net.ParseIP(ip) + if ip == "" || address == nil || address.IsUnspecified() { + return getOutboundAddress(port, dialTarget) + } + + return bind, nil +} + +func getOutboundAddress(port string, dialTarget string) (string, error) { + conn, err := net.Dial("udp", dialTarget) if err != nil { return "", err } defer conn.Close() localAddr := conn.LocalAddr().(*net.UDPAddr) - port := strings.Split(bind, ":")[1] return fmt.Sprintf("%s:%s", localAddr.IP, port), nil } diff --git a/utils/service_test.go b/utils/service_test.go index c7f3cd885..18848e477 100644 --- a/utils/service_test.go +++ b/utils/service_test.go @@ -8,7 +8,7 @@ import ( func TestGetOutboundAddress(t *testing.T) { bind := "1.1.1.1:1234" - addr, err := GetOutboundAddress(bind) + addr, err := GetOutboundAddress(bind, "8.8.8.8:80") assert.NoError(t, err) assert.Contains(t, addr, "1234") }