Thanks to visit codestin.com
Credit goes to github.com

Skip to content

Commit 1ee1e3c

Browse files
sguiheuxsguiheux
authored
feat: add maintenance mode (#3592)
1 parent 91c841b commit 1ee1e3c

File tree

10 files changed

+143
-20
lines changed

10 files changed

+143
-20
lines changed

‎cli/cdsctl/admin.go‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -22,6 +22,7 @@ func admin() *cobra.Command {
2222
adminPlatformModels,
2323
adminPlugins,
2424
adminBroadcasts,
25+
adminMaintenance,
2526
adminErrors,
2627
usr,
2728
group,
@@ -34,6 +35,7 @@ func admin() *cobra.Command {
3435
adminServices,
3536
adminHooks,
3637
adminPlatformModels,
38+
adminMaintenance,
3739
adminPlugins,
3840
adminBroadcasts,
3941
adminErrors,

‎cli/cdsctl/admin_maintenance.go‎

Lines changed: 38 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,38 @@
1+
package main
2+
3+
import (
4+
"github.com/spf13/cobra"
5+
6+
"github.com/ovh/cds/cli"
7+
)
8+
9+
var (
10+
adminMaintenancesCmd = cli.Command{
11+
Name: "maintenance",
12+
Short: "Manage CDS maintenance",
13+
}
14+
15+
adminMaintenance = cli.NewCommand(adminMaintenancesCmd, nil,
16+
[]*cobra.Command{
17+
cli.NewCommand(adminMaintenanceEnableCmd, adminMaintenanceEnable, nil),
18+
cli.NewCommand(adminMaintenanceDisableCmd, adminMaintenanceDisable, nil),
19+
})
20+
)
21+
22+
var adminMaintenanceEnableCmd = cli.Command{
23+
Name: "enable",
24+
Short: "Enable CDS maintenance",
25+
}
26+
27+
func adminMaintenanceEnable(v cli.Values) error {
28+
return client.Maintenance(true)
29+
}
30+
31+
var adminMaintenanceDisableCmd = cli.Command{
32+
Name: "disable",
33+
Short: "Disable CDS maintenance",
34+
}
35+
36+
func adminMaintenanceDisable(v cli.Values) error {
37+
return client.Maintenance(false)
38+
}

‎engine/api/admin.go‎

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -14,6 +14,13 @@ import (
1414
"github.com/ovh/cds/sdk/log"
1515
)
1616

17+
func (api *API) postMaintenanceHandler() service.Handler {
18+
return func(ctx context.Context, w http.ResponseWriter, r *http.Request) error {
19+
enable := FormString(r, "enable")
20+
api.Cache.Publish(maintenanceQueueName, enable)
21+
return nil
22+
}
23+
}
1724
func (api *API) adminTruncateWarningsHandler() service.Handler {
1825
return func(ctx context.Context, w http.ResponseWriter, r *http.Request) error {
1926
if _, err := api.mustDB().Exec("delete from warning"); err != nil {

‎engine/api/api.go‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -227,6 +227,7 @@ type API struct {
227227
Config Configuration
228228
DBConnectionFactory *database.DBConnectionFactory
229229
StartupTime time.Time
230+
Maintenance bool
230231
eventsBroker *eventsBroker
231232
warnChan chan sdk.Event
232233
Cache cache.Store
@@ -642,6 +643,9 @@ func (a *API) Serve(ctx context.Context) error {
642643
event.Subscribe(a.warnChan)
643644

644645
log.Info("Initializing internal routines...")
646+
sdk.GoRoutine(ctx, "maintenance.Subscribe", func(ctx context.Context) {
647+
a.listenMaintenance(ctx)
648+
})
645649

646650
sdk.GoRoutine(ctx, "worker.Initialize", func(ctx context.Context) {
647651
if err := worker.Initialize(ctx, a.DBConnectionFactory.GetDBMap, a.Cache); err != nil {

‎engine/api/api_routes.go‎

Lines changed: 21 additions & 20 deletions
Original file line numberDiff line numberDiff line change
@@ -12,7 +12,7 @@ import (
1212
func (api *API) InitRouter() {
1313
api.Router.URL = api.Config.URL.API
1414
api.Router.SetHeaderFunc = DefaultHeaders
15-
api.Router.Middlewares = append(api.Router.Middlewares, api.authMiddleware, api.tracingMiddleware)
15+
api.Router.Middlewares = append(api.Router.Middlewares, api.authMiddleware, api.tracingMiddleware, api.maintenanceMiddleware)
1616
api.Router.PostMiddlewares = append(api.Router.PostMiddlewares, api.deletePermissionMiddleware, TracingPostMiddleware)
1717

1818
api.eventsBroker = &eventsBroker{
@@ -38,6 +38,7 @@ func (api *API) InitRouter() {
3838
r.Handle("/action/{actionID}/audit", r.GET(api.getActionAuditHandler, NeedAdmin(true)))
3939

4040
// Admin
41+
r.Handle("/admin/maintenance", r.POST(api.postMaintenanceHandler, NeedAdmin(true)))
4142
r.Handle("/admin/warning", r.DELETE(api.adminTruncateWarningsHandler, NeedAdmin(true)))
4243
r.Handle("/admin/debug", r.GET(api.getProfileIndexHandler, Auth(false)))
4344
r.Handle("/admin/debug/trace", r.POST(api.getTraceHandler, NeedAdmin(true)), r.GET(api.getTraceHandler, NeedAdmin(true)))
@@ -320,25 +321,25 @@ func (api *API) InitRouter() {
320321
r.Handle("/build/{id}/step", r.POST(api.updateStepStatusHandler))
321322

322323
//Workflow queue
323-
r.Handle("/queue/workflows", r.GET(api.getWorkflowJobQueueHandler, EnableTracing()))
324-
r.Handle("/queue/workflows/count", r.GET(api.countWorkflowJobQueueHandler, EnableTracing()))
325-
r.Handle("/queue/workflows/{id}/take", r.POST(api.postTakeWorkflowJobHandler, NeedWorker(), EnableTracing()))
326-
r.Handle("/queue/workflows/{id}/book", r.POST(api.postBookWorkflowJobHandler, NeedHatchery(), EnableTracing()), r.DELETE(api.deleteBookWorkflowJobHandler, NeedHatchery(), EnableTracing()))
327-
r.Handle("/queue/workflows/{id}/attempt", r.POST(api.postIncWorkflowJobAttemptHandler, NeedHatchery(), EnableTracing()))
328-
r.Handle("/queue/workflows/{id}/infos", r.GET(api.getWorkflowJobHandler, NeedWorker(), NeedHatchery(), EnableTracing()))
329-
r.Handle("/queue/workflows/{permID}/vulnerability", r.POSTEXECUTE(api.postVulnerabilityReportHandler, NeedWorker(), EnableTracing()))
330-
r.Handle("/queue/workflows/{id}/spawn/infos", r.POST(r.Asynchronous(api.postSpawnInfosWorkflowJobHandler, 1), NeedHatchery(), EnableTracing()))
331-
r.Handle("/queue/workflows/{permID}/result", r.POSTEXECUTE(api.postWorkflowJobResultHandler, NeedWorker(), EnableTracing()))
332-
r.Handle("/queue/workflows/{permID}/log", r.POSTEXECUTE(r.Asynchronous(api.postWorkflowJobLogsHandler, 1), NeedWorker()))
333-
r.Handle("/queue/workflows/log/service", r.POSTEXECUTE(r.Asynchronous(api.postWorkflowJobServiceLogsHandler, 1), NeedHatchery()))
334-
r.Handle("/queue/workflows/{permID}/coverage", r.POSTEXECUTE(api.postWorkflowJobCoverageResultsHandler, NeedWorker(), EnableTracing()))
335-
r.Handle("/queue/workflows/{permID}/test", r.POSTEXECUTE(api.postWorkflowJobTestsResultsHandler, NeedWorker(), EnableTracing()))
336-
r.Handle("/queue/workflows/{permID}/tag", r.POSTEXECUTE(api.postWorkflowJobTagsHandler, NeedWorker(), EnableTracing()))
337-
r.Handle("/queue/workflows/{permID}/variable", r.POSTEXECUTE(api.postWorkflowJobVariableHandler, NeedWorker(), EnableTracing()))
338-
r.Handle("/queue/workflows/{permID}/step", r.POSTEXECUTE(api.postWorkflowJobStepStatusHandler, NeedWorker(), EnableTracing()))
339-
r.Handle("/queue/workflows/{permID}/artifact/{ref}", r.POSTEXECUTE(api.postWorkflowJobArtifactHandler, NeedWorker(), EnableTracing()))
340-
r.Handle("/queue/workflows/{permID}/artifact/{ref}/url", r.POSTEXECUTE(api.postWorkflowJobArtifacWithTempURLHandler, NeedWorker(), EnableTracing()))
341-
r.Handle("/queue/workflows/{permID}/artifact/{ref}/url/callback", r.POSTEXECUTE(api.postWorkflowJobArtifactWithTempURLCallbackHandler, NeedWorker(), EnableTracing()))
324+
r.Handle("/queue/workflows", r.GET(api.getWorkflowJobQueueHandler, EnableTracing(), MaintenanceAware()))
325+
r.Handle("/queue/workflows/count", r.GET(api.countWorkflowJobQueueHandler, EnableTracing(), MaintenanceAware()))
326+
r.Handle("/queue/workflows/{id}/take", r.POST(api.postTakeWorkflowJobHandler, NeedWorker(), EnableTracing(), MaintenanceAware()))
327+
r.Handle("/queue/workflows/{id}/book", r.POST(api.postBookWorkflowJobHandler, NeedHatchery(), EnableTracing(), MaintenanceAware()), r.DELETE(api.deleteBookWorkflowJobHandler, NeedHatchery(), EnableTracing(), MaintenanceAware()))
328+
r.Handle("/queue/workflows/{id}/attempt", r.POST(api.postIncWorkflowJobAttemptHandler, NeedHatchery(), EnableTracing(), MaintenanceAware()))
329+
r.Handle("/queue/workflows/{id}/infos", r.GET(api.getWorkflowJobHandler, NeedWorker(), NeedHatchery(), EnableTracing(), MaintenanceAware()))
330+
r.Handle("/queue/workflows/{permID}/vulnerability", r.POSTEXECUTE(api.postVulnerabilityReportHandler, NeedWorker(), EnableTracing(), MaintenanceAware()))
331+
r.Handle("/queue/workflows/{id}/spawn/infos", r.POST(r.Asynchronous(api.postSpawnInfosWorkflowJobHandler, 1), NeedHatchery(), EnableTracing(), MaintenanceAware()))
332+
r.Handle("/queue/workflows/{permID}/result", r.POSTEXECUTE(api.postWorkflowJobResultHandler, NeedWorker(), EnableTracing(), MaintenanceAware()))
333+
r.Handle("/queue/workflows/{permID}/log", r.POSTEXECUTE(r.Asynchronous(api.postWorkflowJobLogsHandler, 1), NeedWorker(), MaintenanceAware()))
334+
r.Handle("/queue/workflows/log/service", r.POSTEXECUTE(r.Asynchronous(api.postWorkflowJobServiceLogsHandler, 1), NeedHatchery(), MaintenanceAware()))
335+
r.Handle("/queue/workflows/{permID}/coverage", r.POSTEXECUTE(api.postWorkflowJobCoverageResultsHandler, NeedWorker(), EnableTracing(), MaintenanceAware()))
336+
r.Handle("/queue/workflows/{permID}/test", r.POSTEXECUTE(api.postWorkflowJobTestsResultsHandler, NeedWorker(), EnableTracing(), MaintenanceAware()))
337+
r.Handle("/queue/workflows/{permID}/tag", r.POSTEXECUTE(api.postWorkflowJobTagsHandler, NeedWorker(), EnableTracing(), MaintenanceAware()))
338+
r.Handle("/queue/workflows/{permID}/variable", r.POSTEXECUTE(api.postWorkflowJobVariableHandler, NeedWorker(), EnableTracing(), MaintenanceAware()))
339+
r.Handle("/queue/workflows/{permID}/step", r.POSTEXECUTE(api.postWorkflowJobStepStatusHandler, NeedWorker(), EnableTracing(), MaintenanceAware()))
340+
r.Handle("/queue/workflows/{permID}/artifact/{ref}", r.POSTEXECUTE(api.postWorkflowJobArtifactHandler, NeedWorker(), EnableTracing(), MaintenanceAware()))
341+
r.Handle("/queue/workflows/{permID}/artifact/{ref}/url", r.POSTEXECUTE(api.postWorkflowJobArtifacWithTempURLHandler, NeedWorker(), EnableTracing(), MaintenanceAware()))
342+
r.Handle("/queue/workflows/{permID}/artifact/{ref}/url/callback", r.POSTEXECUTE(api.postWorkflowJobArtifactWithTempURLCallbackHandler, NeedWorker(), EnableTracing(), MaintenanceAware()))
342343

343344
r.Handle("/variable/type", r.GET(api.getVariableTypeHandler))
344345
r.Handle("/parameter/type", r.GET(api.getParameterTypeHandler))

‎engine/api/maintenance.go‎

Lines changed: 39 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,39 @@
1+
package api
2+
3+
import (
4+
"context"
5+
"strconv"
6+
"time"
7+
8+
"github.com/ovh/cds/sdk/log"
9+
)
10+
11+
const (
12+
maintenanceQueueName = "cds_maintenance"
13+
)
14+
15+
func (a *API) listenMaintenance(c context.Context) {
16+
pubSub := a.Cache.Subscribe(maintenanceQueueName)
17+
tick := time.NewTicker(50 * time.Millisecond)
18+
defer tick.Stop()
19+
for {
20+
select {
21+
case <-c.Done():
22+
if c.Err() != nil {
23+
log.Error("listenMaintenance> Exiting: %v", c.Err())
24+
return
25+
}
26+
case <-tick.C:
27+
msg, err := a.Cache.GetMessageFromSubscription(c, pubSub)
28+
if err != nil {
29+
log.Warning("listenMaintenance> Cannot get message %s: %s", msg, err)
30+
continue
31+
}
32+
b, err := strconv.ParseBool(msg)
33+
if err != nil {
34+
log.Warning("listenMaintenance> Cannot parse value %s: %s", msg, err)
35+
}
36+
a.Maintenance = b
37+
}
38+
}
39+
}

‎engine/api/router.go‎

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -495,6 +495,14 @@ func Auth(v bool) HandlerConfigParam {
495495
return f
496496
}
497497

498+
// MaintenanceAware route need CDS maintenance off
499+
func MaintenanceAware() HandlerConfigParam {
500+
f := func(rc *service.HandlerConfig) {
501+
rc.Options["maintenance_aware"] = "true"
502+
}
503+
return f
504+
}
505+
498506
// EnableTracing on a route
499507
func EnableTracing() HandlerConfigParam {
500508
f := func(rc *service.HandlerConfig) {

‎engine/api/router_middleware.go‎

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -216,3 +216,10 @@ func TracingMiddlewareFunc(serviceName string, db gorp.SqlExecutor, store cache.
216216
return ctx, err
217217
}
218218
}
219+
220+
func (api *API) maintenanceMiddleware(ctx context.Context, w http.ResponseWriter, req *http.Request, rc *service.HandlerConfig) (context.Context, error) {
221+
if rc.Options["maintenance_aware"] == "true" && api.Maintenance {
222+
return ctx, sdk.WrapError(sdk.ErrServiceUnavailable, "CDS Maintenance ON")
223+
}
224+
return ctx, nil
225+
}

‎sdk/cdsclient/client_maintenance.go‎

Lines changed: 11 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,11 @@
1+
package cdsclient
2+
3+
import (
4+
"context"
5+
"fmt"
6+
)
7+
8+
func (c *client) Maintenance(enable bool) error {
9+
_, err := c.PostJSON(context.Background(), fmt.Sprintf("/admin/maintenance?enable=%v", enable), nil, nil)
10+
return err
11+
}

‎sdk/cdsclient/interface.go‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -160,6 +160,11 @@ type PipelineClient interface {
160160
PipelineList(projectKey string) ([]sdk.Pipeline, error)
161161
}
162162

163+
// MaintenanceClient manage maintenance mode on CDS
164+
type MaintenanceClient interface {
165+
Maintenance(enable bool) error
166+
}
167+
163168
// ProjectClient exposes project related functions
164169
type ProjectClient interface {
165170
ProjectCreate(proj *sdk.Project, groupName string) error
@@ -303,6 +308,7 @@ type Interface interface {
303308
GRPCPluginsClient
304309
HatcheryClient
305310
BroadcastClient
311+
MaintenanceClient
306312
PipelineClient
307313
PlatformClient
308314
ProjectClient

0 commit comments

Comments
 (0)