From 1f6988754439018e46a6f22c3cb6a34a1f9d13f8 Mon Sep 17 00:00:00 2001 From: Yvonnick Esnault Date: Fri, 13 Mar 2020 14:42:05 +0100 Subject: [PATCH 1/2] fix(hatchery): check maxWorker with MaxProv this will avoid to start many workers (in the same time) if maxProv > maxWorker in the configuration Signed-off-by: Yvonnick Esnault --- sdk/hatchery/starter.go | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/sdk/hatchery/starter.go b/sdk/hatchery/starter.go index cd980f4962..930c16380c 100644 --- a/sdk/hatchery/starter.go +++ b/sdk/hatchery/starter.go @@ -123,8 +123,9 @@ func spawnWorkerForJob(ctx context.Context, h Interface, j workerStarterRequest) if maxProv < 1 { maxProv = defaultMaxProvisioning } - if atomic.LoadInt64(&nbWorkerToStart) >= int64(maxProv) { - log.Debug("hatchery> spawnWorkerForJob> max concurrent provisioning reached") + if atomic.LoadInt64(&nbWorkerToStart) >= int64(maxProv) || + atomic.LoadInt64(&nbWorkerToStart) > int64(h.Configuration().Provision.MaxWorker) { + log.Debug("hatchery> spawnWorkerForJob> max concurrent provisioning or max workers reached") return false } From cce4ebd03c854393bfed58d5f4f5275982038e29 Mon Sep 17 00:00:00 2001 From: Yvonnick Esnault Date: Mon, 16 Mar 2020 11:59:06 +0100 Subject: [PATCH 2/2] fix: cr Signed-off-by: Yvonnick Esnault --- engine/hatchery/kubernetes/kubernetes.go | 14 +++--------- engine/hatchery/local/local.go | 14 +++--------- engine/hatchery/marathon/marathon.go | 14 +++--------- engine/hatchery/openstack/openstack.go | 14 +++--------- engine/hatchery/swarm/swarm_conf.go | 14 +++--------- engine/hatchery/vsphere/vsphere.go | 14 +++--------- engine/service/types.go | 27 ++++++++++++++++++++++++ sdk/hatchery/starter.go | 5 ++--- 8 files changed, 47 insertions(+), 69 deletions(-) diff --git a/engine/hatchery/kubernetes/kubernetes.go b/engine/hatchery/kubernetes/kubernetes.go index 6f6cf7eedf..db40b8ba9a 100644 --- a/engine/hatchery/kubernetes/kubernetes.go +++ b/engine/hatchery/kubernetes/kubernetes.go @@ -183,19 +183,11 @@ func (h *HatcheryKubernetes) getStartingConfig() (*clientcmdapi.Config, error) { func (h *HatcheryKubernetes) CheckConfiguration(cfg interface{}) error { hconfig, ok := cfg.(HatcheryConfiguration) if !ok { - return fmt.Errorf("Invalid configuration") - } - - if hconfig.API.HTTP.URL == "" { - return fmt.Errorf("API HTTP(s) URL is mandatory") - } - - if hconfig.API.Token == "" { - return fmt.Errorf("API Token URL is mandatory") + return fmt.Errorf("Invalid hatchery kubernetes configuration") } - if hconfig.Name == "" { - return fmt.Errorf("please enter a name in your kubernetes hatchery configuration") + if err := hconfig.Check(); err != nil { + return fmt.Errorf("Invalid hatchery kubernetes configuration: %v", err) } if hconfig.Namespace == "" { diff --git a/engine/hatchery/local/local.go b/engine/hatchery/local/local.go index a09a731c4c..d4d1db9b4f 100644 --- a/engine/hatchery/local/local.go +++ b/engine/hatchery/local/local.go @@ -93,25 +93,17 @@ func (h *HatcheryLocal) Status(ctx context.Context) sdk.MonitoringStatus { func (h *HatcheryLocal) CheckConfiguration(cfg interface{}) error { hconfig, ok := cfg.(HatcheryConfiguration) if !ok { - return fmt.Errorf("Invalid configuration") - } - - if hconfig.API.HTTP.URL == "" { - return fmt.Errorf("API HTTP(s) URL is mandatory") + return fmt.Errorf("Invalid hatchery local configuration") } - if hconfig.API.Token == "" { - return fmt.Errorf("API Token URL is mandatory") + if err := hconfig.Check(); err != nil { + return fmt.Errorf("Invalid hatchery local configuration: %v", err) } if hconfig.Basedir == "" { return fmt.Errorf("Invalid basedir directory") } - if hconfig.Name == "" { - return fmt.Errorf("please enter a name in your local hatchery configuration") - } - if ok, err := sdk.DirectoryExists(hconfig.Basedir); !ok { return fmt.Errorf("Basedir doesn't exist") } else if err != nil { diff --git a/engine/hatchery/marathon/marathon.go b/engine/hatchery/marathon/marathon.go index 39d087f8ed..3f53f51845 100644 --- a/engine/hatchery/marathon/marathon.go +++ b/engine/hatchery/marathon/marathon.go @@ -88,15 +88,11 @@ func (h *HatcheryMarathon) Status(ctx context.Context) sdk.MonitoringStatus { func (h *HatcheryMarathon) CheckConfiguration(cfg interface{}) error { hconfig, ok := cfg.(HatcheryConfiguration) if !ok { - return fmt.Errorf("Invalid configuration") - } - - if hconfig.API.HTTP.URL == "" { - return fmt.Errorf("API HTTP(s) URL is mandatory") + return fmt.Errorf("Invalid hatchery marathon configuration") } - if hconfig.API.Token == "" { - return fmt.Errorf("API Token URL is mandatory") + if err := hconfig.Check(); err != nil { + return fmt.Errorf("Invalid marathon configuration: %v", err) } if hconfig.MarathonURL == "" { @@ -107,10 +103,6 @@ func (h *HatcheryMarathon) CheckConfiguration(cfg interface{}) error { return fmt.Errorf("Marathon ID Prefix is mandatory") } - if hconfig.Name == "" { - return fmt.Errorf("please enter a name in your marathon hatchery configuration") - } - h.marathonLabels = map[string]string{} if hconfig.MarathonLabels != "" { array := strings.Split(hconfig.MarathonLabels, ",") diff --git a/engine/hatchery/openstack/openstack.go b/engine/hatchery/openstack/openstack.go index 167af86cfe..5eab70c9d7 100644 --- a/engine/hatchery/openstack/openstack.go +++ b/engine/hatchery/openstack/openstack.go @@ -94,15 +94,11 @@ func (h *HatcheryOpenstack) Status(ctx context.Context) sdk.MonitoringStatus { func (h *HatcheryOpenstack) CheckConfiguration(cfg interface{}) error { hconfig, ok := cfg.(HatcheryConfiguration) if !ok { - return fmt.Errorf("Invalid configuration") - } - - if hconfig.API.HTTP.URL == "" { - return fmt.Errorf("API HTTP(s) URL is mandatory") + return fmt.Errorf("Invalid hatchery openstack configuration") } - if hconfig.API.Token == "" { - return fmt.Errorf("API Token URL is mandatory") + if err := hconfig.Check(); err != nil { + return fmt.Errorf("Invalid hatchery openstack configuration: %v", err) } if hconfig.Tenant == "" && hconfig.Domain == "" { @@ -125,10 +121,6 @@ func (h *HatcheryOpenstack) CheckConfiguration(cfg interface{}) error { return fmt.Errorf("Openstack-region is mandatory") } - if hconfig.Name == "" { - return fmt.Errorf("please enter a name in your openstack hatchery configuration") - } - if hconfig.IPRange != "" { ips, err := IPinRanges(context.Background(), hconfig.IPRange) if err != nil { diff --git a/engine/hatchery/swarm/swarm_conf.go b/engine/hatchery/swarm/swarm_conf.go index 647266b36d..05faa49c1c 100644 --- a/engine/hatchery/swarm/swarm_conf.go +++ b/engine/hatchery/swarm/swarm_conf.go @@ -85,15 +85,11 @@ func (h *HatcherySwarm) Status(ctx context.Context) sdk.MonitoringStatus { func (h *HatcherySwarm) CheckConfiguration(cfg interface{}) error { hconfig, ok := cfg.(HatcheryConfiguration) if !ok { - return fmt.Errorf("Invalid configuration") - } - - if hconfig.API.HTTP.URL == "" { - return fmt.Errorf("API HTTP(s) URL is mandatory") + return fmt.Errorf("Invalid hatchery swarm configuration") } - if hconfig.API.Token == "" { - return fmt.Errorf("API Token URL is mandatory") + if err := hconfig.Check(); err != nil { + return fmt.Errorf("Invalid hatchery swarm configuration: %v", err) } if hconfig.WorkerTTL <= 0 { @@ -103,9 +99,5 @@ func (h *HatcherySwarm) CheckConfiguration(cfg interface{}) error { return fmt.Errorf("worker-memory must be > 1") } - if hconfig.Name == "" { - return fmt.Errorf("please enter a name in your swarm hatchery configuration") - } - return nil } diff --git a/engine/hatchery/vsphere/vsphere.go b/engine/hatchery/vsphere/vsphere.go index 474a0e677d..6cae99b53d 100644 --- a/engine/hatchery/vsphere/vsphere.go +++ b/engine/hatchery/vsphere/vsphere.go @@ -80,15 +80,11 @@ func (h *HatcheryVSphere) Status(ctx context.Context) sdk.MonitoringStatus { func (h *HatcheryVSphere) CheckConfiguration(cfg interface{}) error { hconfig, ok := cfg.(HatcheryConfiguration) if !ok { - return fmt.Errorf("Invalid configuration") - } - - if hconfig.API.HTTP.URL == "" { - return fmt.Errorf("API HTTP(s) URL is mandatory") + return fmt.Errorf("Invalid hatchery vsphere configuration") } - if hconfig.API.Token == "" { - return fmt.Errorf("API Token URL is mandatory") + if err := hconfig.Check(); err != nil { + return fmt.Errorf("Invalid hatchery vsphere configuration: %v", err) } if hconfig.VSphereUser == "" { @@ -107,10 +103,6 @@ func (h *HatcheryVSphere) CheckConfiguration(cfg interface{}) error { return fmt.Errorf("vsphere-datacenter is mandatory") } - if hconfig.Name == "" { - return fmt.Errorf("please enter a name in your vsphere hatchery configuration") - } - return nil } diff --git a/engine/service/types.go b/engine/service/types.go index 6362a86642..2c9b16b44a 100644 --- a/engine/service/types.go +++ b/engine/service/types.go @@ -3,6 +3,7 @@ package service import ( "context" "crypto/rsa" + "fmt" "time" "github.com/ovh/cds/sdk" @@ -63,6 +64,32 @@ type HatcheryCommonConfiguration struct { } `toml:"logOptions" comment:"Hatchery Log Configuration" json:"logOptions"` } +func (hcc HatcheryCommonConfiguration) Check() error { + if hcc.Provision.MaxConcurrentProvisioning > hcc.Provision.MaxWorker { + return fmt.Errorf("maxConcurrentProvisioning (value: %d) cannot be less than maxWorker (value: %d) ", + hcc.Provision.MaxConcurrentProvisioning, hcc.Provision.MaxWorker) + } + + if hcc.Provision.MaxConcurrentRegistering > hcc.Provision.MaxWorker { + return fmt.Errorf("maxConcurrentRegistering (value: %d) cannot be less than maxWorker (value: %d) ", + hcc.Provision.MaxConcurrentRegistering, hcc.Provision.MaxWorker) + } + + if hcc.API.HTTP.URL == "" { + return fmt.Errorf("API HTTP(s) URL is mandatory") + } + + if hcc.API.Token == "" { + return fmt.Errorf("API Token URL is mandatory") + } + + if hcc.Name == "" { + return fmt.Errorf("please enter a name in your hatchery configuration") + } + + return nil +} + // Common is the struct representing a CDS µService type Common struct { Client cdsclient.Interface diff --git a/sdk/hatchery/starter.go b/sdk/hatchery/starter.go index 930c16380c..cd980f4962 100644 --- a/sdk/hatchery/starter.go +++ b/sdk/hatchery/starter.go @@ -123,9 +123,8 @@ func spawnWorkerForJob(ctx context.Context, h Interface, j workerStarterRequest) if maxProv < 1 { maxProv = defaultMaxProvisioning } - if atomic.LoadInt64(&nbWorkerToStart) >= int64(maxProv) || - atomic.LoadInt64(&nbWorkerToStart) > int64(h.Configuration().Provision.MaxWorker) { - log.Debug("hatchery> spawnWorkerForJob> max concurrent provisioning or max workers reached") + if atomic.LoadInt64(&nbWorkerToStart) >= int64(maxProv) { + log.Debug("hatchery> spawnWorkerForJob> max concurrent provisioning reached") return false }