projecteru2 · DuodenumL · Jul 25, 2022 · Mar 4, 2022 · Mar 15, 2022 · Mar 28, 2022
diff --git a/.github/workflows/dockerimage.yml b/.github/workflows/dockerimage.yml
@@ -13,6 +13,8 @@ jobs:
     steps:
       - name: checkout
         uses: actions/checkout@v2
+        with:
+          fetch-depth: 0
 
       - name: build and push to github packages
         uses: docker/build-push-action@v1

diff --git a/Dockerfile b/Dockerfile
@@ -2,7 +2,7 @@ FROM golang:alpine AS BUILD
 
 # make binary
 RUN apk add --no-cache build-base musl-dev git curl make cmake
-RUN git clone https://github.com/projecteru2/core.git /go/src/github.com/projecteru2/core
+COPY . /go/src/github.com/projecteru2/core
 WORKDIR /go/src/github.com/projecteru2/core
 ARG KEEP_SYMBOL
 RUN make build && ./eru-core --version

diff --git a/cluster/calcium/calcium_test.go b/cluster/calcium/calcium_test.go
@@ -56,6 +56,7 @@ func NewTestCluster() *Calcium {
 		WALFile:             filepath.Join(walDir, "core.wal.log"),
 		MaxConcurrency:      10,
 		HAKeepaliveInterval: 16 * time.Second,
+		DialTarget:          "8.8.8.8:80",
 	}
 	c.store = &storemocks.Store{}
 	c.scheduler = &schedulermocks.Scheduler{}

diff --git a/cluster/calcium/resource.go b/cluster/calcium/resource.go
@@ -26,11 +26,12 @@ func (c *Calcium) PodResource(ctx context.Context, podname string) (chan *types.
 	go func() {
 		defer close(ch)
 		for node := range nodeCh {
+			nodename := node.Name
 			pool.Go(ctx, func() {
-				nodeResource, err := c.doGetNodeResource(ctx, node.Name, false)
+				nodeResource, err := c.doGetNodeResource(ctx, nodename, false)
 				if err != nil {
 					nodeResource = &types.NodeResource{
-						Name: node.Name, Diffs: []string{logger.Err(ctx, err).Error()},
+						Name: nodename, Diffs: []string{logger.Err(ctx, err).Error()},
 					}
 				}
 				ch <- nodeResource

diff --git a/cluster/calcium/service.go b/cluster/calcium/service.go
@@ -26,7 +26,7 @@ func (c *Calcium) WatchServiceStatus(ctx context.Context) (<-chan types.ServiceS
 
 // RegisterService writes self service address in store
 func (c *Calcium) RegisterService(ctx context.Context) (unregister func(), err error) {
-	serviceAddress, err := utils.GetOutboundAddress(c.config.Bind)
+	serviceAddress, err := utils.GetOutboundAddress(c.config.Bind, c.config.DialTarget)
 	if err != nil {
 		log.Errorf(ctx, "[RegisterService] failed to get outbound address: %v", err)
 		return

diff --git a/engine/docker/docker.go b/engine/docker/docker.go
@@ -2,18 +2,16 @@ package docker
 
 import (
 	"context"
-	"fmt"
-	"io/ioutil"
 	"net/http"
-	"os"
+	"strings"
 
 	"github.com/projecteru2/core/engine"
 	enginetypes "github.com/projecteru2/core/engine/types"
 	"github.com/projecteru2/core/log"
 	coretypes "github.com/projecteru2/core/types"
+	"github.com/projecteru2/core/utils"
 
 	dockerapi "github.com/docker/docker/client"
-	"github.com/docker/go-connections/tlsconfig"
 )
 
 const (
@@ -32,44 +30,19 @@ type Engine struct {
 // MakeClient make docker cli
 func MakeClient(ctx context.Context, config coretypes.Config, nodename, endpoint, ca, cert, key string) (engine.API, error) {
 	var client *http.Client
-	if config.CertPath != "" && ca != "" && cert != "" && key != "" { // nolint
-		caFile, err := ioutil.TempFile(config.CertPath, fmt.Sprintf("ca-%s", nodename))
+	var err error
+	if strings.HasPrefix(endpoint, "unix://") {
+		client = utils.GetUnixSockClient()
+	} else {
+		client, err = utils.GetHTTPSClient(ctx, config.CertPath, nodename, ca, cert, key)
 		if err != nil {
+			log.Errorf(ctx, "[MakeClient] GetHTTPSClient for %s %s error: %v", nodename, endpoint, err)
 			return nil, err
 		}
-		certFile, err := ioutil.TempFile(config.CertPath, fmt.Sprintf("cert-%s", nodename))
-		if err != nil {
-			return nil, err
-		}
-		keyFile, err := ioutil.TempFile(config.CertPath, fmt.Sprintf("key-%s", nodename))
-		if err != nil {
-			return nil, err
-		}
-		if err = dumpFromString(ctx, caFile, certFile, keyFile, ca, cert, key); err != nil {
-			return nil, err
-		}
-		options := tlsconfig.Options{
-			CAFile:             caFile.Name(),
-			CertFile:           certFile.Name(),
-			KeyFile:            keyFile.Name(),
-			InsecureSkipVerify: true,
-		}
-		defer os.Remove(caFile.Name())
-		defer os.Remove(certFile.Name())
-		defer os.Remove(keyFile.Name())
-		tlsc, err := tlsconfig.Client(options)
-		if err != nil {
-			return nil, err
-		}
-		client = &http.Client{
-			Transport: &http.Transport{
-				TLSClientConfig: tlsc,
-			},
-		}
 	}
 
 	log.Debugf(ctx, "[MakeDockerEngine] Create new http.Client for %s, %s", endpoint, config.Docker.APIVersion)
-	return makeRawClient(ctx, config, client, endpoint)
+	return makeDockerClient(ctx, config, client, endpoint)
 }
 
 // Info show node info
@@ -94,3 +67,8 @@ func (e *Engine) Ping(ctx context.Context) error {
 	_, err := e.client.Ping(ctx)
 	return err
 }
+
+// CloseConn close connection
+func (e *Engine) CloseConn() error {
+	return e.client.Close()
+}
diff --git a/engine/docker/helper.go b/engine/docker/helper.go
@@ -257,7 +257,7 @@ func GetIP(ctx context.Context, daemonHost string) string {
 	return u.Hostname()
 }
 
-func makeRawClient(_ context.Context, config coretypes.Config, client *http.Client, endpoint string) (engine.API, error) {
+func makeDockerClient(_ context.Context, config coretypes.Config, client *http.Client, endpoint string) (engine.API, error) {
 	cli, err := dockerapi.NewClientWithOpts(
 		dockerapi.WithHost(endpoint),
 		dockerapi.WithVersion(config.Docker.APIVersion),
@@ -268,24 +268,6 @@ func makeRawClient(_ context.Context, config coretypes.Config, client *http.Clie
 	return &Engine{cli, config}, nil
 }
 
-func dumpFromString(ctx context.Context, ca, cert, key *os.File, caStr, certStr, keyStr string) error {
-	files := []*os.File{ca, cert, key}
-	data := []string{caStr, certStr, keyStr}
-	for i := 0; i < 3; i++ {
-		if _, err := files[i].WriteString(data[i]); err != nil {
-			return err
-		}
-		if err := files[i].Chmod(0444); err != nil {
-			return err
-		}
-		if err := files[i].Close(); err != nil {
-			return err
-		}
-	}
-	log.Debug(ctx, "[dumpFromString] Dump ca.pem, cert.pem, key.pem from string")
-	return nil
-}
-
 func useCNI(labels map[string]string) bool {
 	for k, v := range labels {
 		if k == "cni" && v == "1" {

diff --git a/engine/engine.go b/engine/engine.go
@@ -13,6 +13,7 @@ import (
 type API interface {
 	Info(ctx context.Context) (*enginetypes.Info, error)
 	Ping(ctx context.Context) error
+	CloseConn() error
 
 	Execute(ctx context.Context, ID string, config *enginetypes.ExecConfig) (result string, stdout, stderr io.ReadCloser, stdin io.WriteCloser, err error)
 	ExecResize(ctx context.Context, ID, result string, height, width uint) (err error)

diff --git a/engine/fake/fake.go b/engine/fake/fake.go
@@ -24,6 +24,11 @@ func (f *Engine) Ping(ctx context.Context) error {
 	return f.DefaultErr
 }
 
+// CloseConn .
+func (f *Engine) CloseConn() error {
+	return nil
+}
+
 // Execute .
 func (f *Engine) Execute(ctx context.Context, ID string, config *enginetypes.ExecConfig) (result string, stdout, stderr io.ReadCloser, stdin io.WriteCloser, err error) {
 	return "", nil, nil, nil, f.DefaultErr

diff --git a/engine/mocks/API.go b/engine/mocks/API.go
diff --git a/engine/virt/virt.go b/engine/virt/virt.go
@@ -81,11 +81,16 @@ func (v *Virt) Ping(ctx context.Context) error {
 	return err
 }
 
+// CloseConn closes the connection.
+func (v *Virt) CloseConn() error {
+	return v.client.Close()
+}
+
 // Execute executes a command in vm
 func (v *Virt) Execute(ctx context.Context, ID string, config *enginetypes.ExecConfig) (pid string, stdout, stderr io.ReadCloser, stdin io.WriteCloser, err error) {
 	if config.Tty {
 		flags := virttypes.AttachGuestFlags{Safe: true, Force: true}
-		stream, err := v.client.AttachGuest(ctx, ID, config.Cmd, flags)
+		_, stream, err := v.client.AttachGuest(ctx, ID, config.Cmd, flags)
 		if err != nil {
 			return "", nil, nil, nil, err
 		}
@@ -269,7 +274,7 @@ func (v *Virt) VirtualizationLogs(ctx context.Context, opts *enginetypes.Virtual
 // VirtualizationAttach attaches something to a guest.
 func (v *Virt) VirtualizationAttach(ctx context.Context, ID string, stream, openStdin bool) (stdout, stderr io.ReadCloser, stdin io.WriteCloser, err error) {
 	flags := virttypes.AttachGuestFlags{Safe: true, Force: true}
-	attachGuest, err := v.client.AttachGuest(ctx, ID, []string{}, flags)
+	_, attachGuest, err := v.client.AttachGuest(ctx, ID, []string{}, flags)
 	if err != nil {
 		return nil, nil, nil, err
 	}

diff --git a/go.mod b/go.mod
@@ -10,6 +10,7 @@ require (
 	github.com/cenkalti/backoff/v4 v4.0.2
 	github.com/containerd/containerd v1.4.8 // indirect
 	github.com/containerd/continuity v0.0.0-20200710164510-efbc4488d8fe // indirect
+	github.com/cornelk/hashmap v1.0.1 // indirect
 	github.com/docker/distribution v2.7.1+incompatible
 	github.com/docker/docker v20.10.0+incompatible
 	github.com/docker/go-connections v0.4.0
@@ -32,7 +33,7 @@ require (
 	github.com/opencontainers/runc v1.0.0-rc95 // indirect
 	github.com/patrickmn/go-cache v2.1.0+incompatible
 	github.com/pkg/errors v0.9.1
-	github.com/projecteru2/libyavirt v0.0.0-20211217082140-493b61aa9b0d
+	github.com/projecteru2/libyavirt v0.0.0-20220329021434-9b7b732e9f8e
 	github.com/prometheus/client_golang v1.11.0
 	github.com/sanity-io/litter v1.5.1
 	github.com/sirupsen/logrus v1.7.0

diff --git a/go.sum b/go.sum
@@ -107,6 +107,8 @@ github.com/coreos/go-systemd/v22 v22.3.1/go.mod h1:Y58oyj3AT4RCenI/lSvhwexgC+NSV
 github.com/coreos/go-systemd/v22 v22.3.2 h1:D9/bQk5vlXQFZ6Kwuu6zaiXJ9oTPe68++AzAJc1DzSI=
 github.com/coreos/go-systemd/v22 v22.3.2/go.mod h1:Y58oyj3AT4RCenI/lSvhwexgC+NSVTIJ3seZv2GcEnc=
 github.com/coreos/pkg v0.0.0-20180928190104-399ea9e2e55f/go.mod h1:E3G3o1h8I7cfcXa63jLwjI0eiQQMgzzUDFVpN/nH/eA=
+github.com/cornelk/hashmap v1.0.1 h1:RXGcy29hEdLLV8T6aK4s+BAd4tq4+3Hq50N2GoG0uIg=
+github.com/cornelk/hashmap v1.0.1/go.mod h1:8wbysTUDnwJGrPZ1Iwsou3m+An6sldFrJItjRhfegCw=
 github.com/cpuguy83/go-md2man v1.0.10 h1:BSKMNlYxDvnunlTymqtgONjNnaRV1sTpcovwwjF22jk=
 github.com/cpuguy83/go-md2man v1.0.10/go.mod h1:SmD6nW6nTyfqj6ABTjUi3V3JVMnlJmwcJI5acqYI6dE=
 github.com/cpuguy83/go-md2man/v2 v2.0.0-20190314233015-f79a8a8ca69d/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU=
@@ -119,6 +121,8 @@ github.com/davecgh/go-spew v0.0.0-20161028175848-04cdfd42973b/go.mod h1:J7Y8YcW2
 github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
 github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
 github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
+github.com/dchest/siphash v1.1.0 h1:1Rs9eTUlZLPBEvV+2sTaM8O0NWn0ppbgqS7p11aWawI=
+github.com/dchest/siphash v1.1.0/go.mod h1:q+IRvb2gOSrUnYoPqHiyHXS0FOBBOdl6tONBlVnOnt4=
 github.com/dgraph-io/badger v1.6.0/go.mod h1:zwt7syl517jmP8s94KqSxTlM6IMsdhYy6psNgSztDR4=
 github.com/dgrijalva/jwt-go v3.2.0+incompatible/go.mod h1:E3ru+11k8xSBh+hMPgOLZmtrrCbhqsmaPHjLKYnJCaQ=
 github.com/dgryski/go-farm v0.0.0-20190423205320-6a90982ecee2/go.mod h1:SqUrOPUnsFjfmXRMNPybcSiG0BgUW2AuFH8PAnS2iTw=
@@ -432,6 +436,10 @@ github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZN
 github.com/posener/complete v1.1.1/go.mod h1:em0nMJCgc9GFtwrmVmEMR/ZL6WyhyjMBndrE9hABlRI=
 github.com/projecteru2/libyavirt v0.0.0-20211217082140-493b61aa9b0d h1:BMFqsvIB3nmK5l53nz8r2ndK8//T0njciu5nr/kj9A4=
 github.com/projecteru2/libyavirt v0.0.0-20211217082140-493b61aa9b0d/go.mod h1:FOc+hWBMLsMrmx5p3/moizKeSomedZPNwB6LhS+kEnE=
+github.com/projecteru2/libyavirt v0.0.0-20220328133352-2a6302e90fc9 h1:7atvx2788Nf+HvY5uZFLlogjplvvF7iw5nWI/8UkjoY=
+github.com/projecteru2/libyavirt v0.0.0-20220328133352-2a6302e90fc9/go.mod h1:FOc+hWBMLsMrmx5p3/moizKeSomedZPNwB6LhS+kEnE=
+github.com/projecteru2/libyavirt v0.0.0-20220329021434-9b7b732e9f8e h1:IBRdnQybu4DRixX0grv4H4+tN8X+6Tp6LgVxx0ApIfM=
+github.com/projecteru2/libyavirt v0.0.0-20220329021434-9b7b732e9f8e/go.mod h1:FOc+hWBMLsMrmx5p3/moizKeSomedZPNwB6LhS+kEnE=
 github.com/prometheus/client_golang v0.9.1/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw=
 github.com/prometheus/client_golang v0.9.3/go.mod h1:/TN21ttK/J9q6uSwhBd54HahCDft0ttaMvbicHlPoso=
 github.com/prometheus/client_golang v1.0.0/go.mod h1:db9x61etRT2tGnBNRi70OPL5FsnadC4Ky3P0J6CfImo=

diff --git a/scheduler/complex/cpu_test.go b/scheduler/complex/cpu_test.go
@@ -80,6 +80,7 @@ func TestCPUReallocPlan(t *testing.T) {
 		"1": 30,
 		"2": 40,
 	}
+	scheduleInfo.CPU.Add(CPU)
 	si, remain, aff := cpuReallocPlan(scheduleInfo, 1, CPU, 100)
 	assert.EqualValues(t, 0, remain)
 	assert.True(t, reflect.DeepEqual(aff, types.CPUMap{"0": 100}))
@@ -101,6 +102,7 @@ func TestCPUReallocPlan(t *testing.T) {
 		"1": 30,
 		"2": 40,
 	}
+	scheduleInfo.CPU.Add(CPU)
 	si, remain, aff = cpuReallocPlan(scheduleInfo, 1.2, CPU, 100)
 	assert.EqualValues(t, 0, remain)
 	assert.True(t, reflect.DeepEqual(aff, types.CPUMap{"0": 100, "2": 20}))
@@ -124,6 +126,7 @@ func TestCPUReallocPlan(t *testing.T) {
 		"2": 40,
 		"3": 10,
 	}
+	scheduleInfo.CPU.Add(CPU)
 	si, remain, aff = cpuReallocPlan(scheduleInfo, 2, CPU, 100)
 	assert.EqualValues(t, 0, remain)
 	assert.True(t, reflect.DeepEqual(aff, types.CPUMap{"0": 100, "1": 100}))
@@ -145,6 +148,7 @@ func TestCPUReallocPlan(t *testing.T) {
 		"1": 30,
 		"2": 40,
 	}
+	scheduleInfo.CPU.Add(CPU)
 	si, remain, aff = cpuReallocPlan(scheduleInfo, 2, CPU, 100)
 	assert.EqualValues(t, 1, remain)
 	assert.True(t, reflect.DeepEqual(aff, types.CPUMap{"0": 100}))
@@ -174,6 +178,7 @@ func TestCPUReallocWithPriorPlan(t *testing.T) {
 		"1": 30,
 		"2": 40,
 	}
+	scheduleInfo.CPU.Add(CPU)
 	si, cpuPlans, total, err := po.ReselectCPUNodes(context.TODO(), scheduleInfo, CPU, 2, 0)
 	assert.Nil(t, err)
 	assert.EqualValues(t, 1, total)
@@ -198,6 +203,7 @@ func TestCPUReallocWithPriorPlan(t *testing.T) {
 		"1": 30,
 		"2": 40,
 	}
+	scheduleInfo.CPU.Add(CPU)
 	si, cpuPlans, total, err = po.ReselectCPUNodes(context.TODO(), scheduleInfo, CPU, 2, 0)
 	assert.Nil(t, err)
 	assert.EqualValues(t, 3, total)
@@ -233,6 +239,7 @@ func TestCPUReallocWithPriorPlan(t *testing.T) {
 		"1": 30,
 		"2": 40,
 	}
+	scheduleInfo.CPU.Add(CPU)
 	_, _, _, err = po.ReselectCPUNodes(context.TODO(), scheduleInfo, CPU, 2, 0)
 	assert.EqualError(t, err, "failed to reschedule cpu: no node remains 1.00 pieces of cpu and 0 bytes of memory at the same time: not enough resource")
 }
@@ -262,7 +269,7 @@ func TestGetFullResult(t *testing.T) {
 		{"1": 100, "2": 100},
 	})
 
-	res = h.getFullResult(2,  []resourceInfo{
+	res = h.getFullResult(2, []resourceInfo{
 		{
 			id:     "0",
 			pieces: 200,

diff --git a/scheduler/complex/potassium.go b/scheduler/complex/potassium.go
@@ -197,13 +197,14 @@ func (m *Potassium) ReselectCPUNodes(ctx context.Context, scheduleInfo resourcet
 
 func cpuReallocPlan(scheduleInfo resourcetypes.ScheduleInfo, quota float64, CPU types.CPUMap, sharebase int64) (resourcetypes.ScheduleInfo, float64, types.CPUMap) {
 	affinityPlan := make(types.CPUMap)
-	diff := int64(quota*float64(sharebase)) - CPU.Total()
+	diff := types.RoundToInt(quota*float64(sharebase)) - CPU.Total()
 	// sort by pieces
 	cpuIDs := []string{}
 	for cpuID := range CPU {
 		cpuIDs = append(cpuIDs, cpuID)
 	}
 	sort.Slice(cpuIDs, func(i, j int) bool { return CPU[cpuIDs[i]] < CPU[cpuIDs[j]] })
+	scheduleInfo.CPU.Sub(CPU)
 
 	// shrink, ensure affinity
 	if diff <= 0 {
@@ -225,7 +226,7 @@ func cpuReallocPlan(scheduleInfo resourcetypes.ScheduleInfo, quota float64, CPU
 	}
 
 	// expand, prioritize full cpus
-	needPieces := int64(quota * float64(sharebase))
+	needPieces := types.RoundToInt(quota * float64(sharebase))
 	for i := len(cpuIDs) - 1; i >= 0; i-- {
 		cpuID := cpuIDs[i]
 		if needPieces == 0 {
@@ -242,10 +243,10 @@ func cpuReallocPlan(scheduleInfo resourcetypes.ScheduleInfo, quota float64, CPU
 
 		// fragments, try to find complement
 		if available := scheduleInfo.CPU[cpuID]; available == sharebase-CPU[cpuID] {
-			expand := utils.Min64(available, needPieces)
+			expand := utils.Min64(available, needPieces-CPU[cpuID])
 			affinityPlan[cpuID] = CPU[cpuID] + expand
 			scheduleInfo.CPU[cpuID] -= expand
-			needPieces -= sharebase
+			needPieces -= affinityPlan[cpuID]
 			continue
 		}