From 21e7eef83b6283661a3c33455a1c6d026d910705 Mon Sep 17 00:00:00 2001 From: NorthRealm <155140859+NorthRealm@users.noreply.github.com> Date: Tue, 5 Aug 2025 09:54:07 +0800 Subject: [PATCH 01/29] fix --- services/mailer/mail_workflow_run.go | 16 +++++++++++----- 1 file changed, 11 insertions(+), 5 deletions(-) diff --git a/services/mailer/mail_workflow_run.go b/services/mailer/mail_workflow_run.go index 29b3abda8ee29..4602e2b6abcff 100644 --- a/services/mailer/mail_workflow_run.go +++ b/services/mailer/mail_workflow_run.go @@ -34,6 +34,17 @@ func generateMessageIDForActionsWorkflowRunStatusEmail(repo *repo_model.Reposito } func composeAndSendActionsWorkflowRunStatusEmail(ctx context.Context, repo *repo_model.Repository, run *actions_model.ActionRun, sender *user_model.User, recipients []*user_model.User) { + jobs, err := actions_model.GetRunJobsByRunID(ctx, run.ID) + if err != nil { + log.Error("GetRunJobsByRunID: %v", err) + return + } + for _, job := range jobs { + if !job.Status.IsDone() { + return + } + } + subject := "Run" switch run.Status { case actions_model.StatusFailure: @@ -48,11 +59,6 @@ func composeAndSendActionsWorkflowRunStatusEmail(ctx context.Context, repo *repo messageID := generateMessageIDForActionsWorkflowRunStatusEmail(repo, run) metadataHeaders := generateMetadataHeaders(repo) - jobs, err := actions_model.GetRunJobsByRunID(ctx, run.ID) - if err != nil { - log.Error("GetRunJobsByRunID: %v", err) - return - } sort.SliceStable(jobs, func(i, j int) bool { si, sj := jobs[i].Status, jobs[j].Status /* From 0ec1bb3bbadd0e51db05531948dd6cbbe1ed76ea Mon Sep 17 00:00:00 2001 From: NorthRealm <155140859+NorthRealm@users.noreply.github.com> Date: Tue, 5 Aug 2025 10:24:18 +0800 Subject: [PATCH 02/29] add trace --- services/mailer/mail_workflow_run.go | 2 ++ 1 file changed, 2 insertions(+) diff --git a/services/mailer/mail_workflow_run.go b/services/mailer/mail_workflow_run.go index 4602e2b6abcff..ee41a3f03ad82 100644 --- a/services/mailer/mail_workflow_run.go +++ b/services/mailer/mail_workflow_run.go @@ -122,6 +122,7 @@ func composeAndSendActionsWorkflowRunStatusEmail(ctx context.Context, repo *repo } msgs := make([]*sender_service.Message, 0, len(tos)) for _, rec := range tos { + log.Trace("Composing actions email and send to %s (UID: %d)", rec.Name, rec.ID) msg := sender_service.NewMessageFrom( rec.Email, displayName, @@ -166,6 +167,7 @@ func MailActionsTrigger(ctx context.Context, sender *user_model.User, repo *repo } if len(recipients) > 0 { + log.Trace("MailActionsTrigger: will try to send actions email") composeAndSendActionsWorkflowRunStatusEmail(ctx, repo, run, sender, recipients) } } From 3957352ecdf8020d5809551f7f4577d3cda551be Mon Sep 17 00:00:00 2001 From: NorthRealm <155140859+NorthRealm@users.noreply.github.com> Date: Tue, 5 Aug 2025 10:29:08 +0800 Subject: [PATCH 03/29] add trace --- services/mailer/mail_workflow_run.go | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/services/mailer/mail_workflow_run.go b/services/mailer/mail_workflow_run.go index ee41a3f03ad82..38603be83324f 100644 --- a/services/mailer/mail_workflow_run.go +++ b/services/mailer/mail_workflow_run.go @@ -122,7 +122,7 @@ func composeAndSendActionsWorkflowRunStatusEmail(ctx context.Context, repo *repo } msgs := make([]*sender_service.Message, 0, len(tos)) for _, rec := range tos { - log.Trace("Composing actions email and send to %s (UID: %d)", rec.Name, rec.ID) + log.Trace("Composing actions email and sending to %s (UID: %d)", rec.Name, rec.ID) msg := sender_service.NewMessageFrom( rec.Email, displayName, @@ -167,7 +167,6 @@ func MailActionsTrigger(ctx context.Context, sender *user_model.User, repo *repo } if len(recipients) > 0 { - log.Trace("MailActionsTrigger: will try to send actions email") composeAndSendActionsWorkflowRunStatusEmail(ctx, repo, run, sender, recipients) } } From cdb1e80bf95c5d1f35c2b4839621e32d9704bb25 Mon Sep 17 00:00:00 2001 From: NorthRealm <155140859+NorthRealm@users.noreply.github.com> Date: Tue, 5 Aug 2025 18:02:31 +0800 Subject: [PATCH 04/29] assume --- routers/web/repo/actions/view.go | 16 +++++++--------- 1 file changed, 7 insertions(+), 9 deletions(-) diff --git a/routers/web/repo/actions/view.go b/routers/web/repo/actions/view.go index 52b2e9995e3ba..020d881e49e58 100644 --- a/routers/web/repo/actions/view.go +++ b/routers/web/repo/actions/view.go @@ -560,9 +560,9 @@ func Cancel(ctx *context_module.Context) { if len(updatedjobs) > 0 { job := updatedjobs[0] actions_service.NotifyWorkflowRunStatusUpdateWithReload(ctx, job) - notify_service.WorkflowRunStatusUpdate(ctx, job.Run.Repo, job.Run.TriggerUser, job.Run) } - ctx.JSON(http.StatusOK, struct{}{}) + + ctx.JSONOK() } func Approve(ctx *context_module.Context) { @@ -603,18 +603,16 @@ func Approve(ctx *context_module.Context) { actions_service.CreateCommitStatus(ctx, jobs...) - if len(updatedjobs) > 0 { - job := updatedjobs[0] - actions_service.NotifyWorkflowRunStatusUpdateWithReload(ctx, job) - notify_service.WorkflowRunStatusUpdate(ctx, job.Run.Repo, job.Run.TriggerUser, job.Run) - } - for _, job := range updatedjobs { _ = job.LoadAttributes(ctx) notify_service.WorkflowJobStatusUpdate(ctx, job.Run.Repo, job.Run.TriggerUser, job, nil) } + if len(updatedjobs) > 0 { + job := updatedjobs[0] + actions_service.NotifyWorkflowRunStatusUpdateWithReload(ctx, job) + } - ctx.JSON(http.StatusOK, struct{}{}) + ctx.JSONOK() } func Delete(ctx *context_module.Context) { From 7384a652ebb3ce767df57a8b6f14979b1fcdc6d4 Mon Sep 17 00:00:00 2001 From: NorthRealm <155140859+NorthRealm@users.noreply.github.com> Date: Tue, 5 Aug 2025 18:05:45 +0800 Subject: [PATCH 05/29] add trace --- services/mailer/mail_workflow_run.go | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/services/mailer/mail_workflow_run.go b/services/mailer/mail_workflow_run.go index 38603be83324f..59203c5d947ec 100644 --- a/services/mailer/mail_workflow_run.go +++ b/services/mailer/mail_workflow_run.go @@ -41,6 +41,7 @@ func composeAndSendActionsWorkflowRunStatusEmail(ctx context.Context, repo *repo } for _, job := range jobs { if !job.Status.IsDone() { + log.Trace("composeAndSendActionsWorkflowRunStatusEmail: A job is not done. Will not compose and send actions email.") return } } @@ -122,7 +123,7 @@ func composeAndSendActionsWorkflowRunStatusEmail(ctx context.Context, repo *repo } msgs := make([]*sender_service.Message, 0, len(tos)) for _, rec := range tos { - log.Trace("Composing actions email and sending to %s (UID: %d)", rec.Name, rec.ID) + log.Trace("Sending actions email to %s (UID: %d)", rec.Name, rec.ID) msg := sender_service.NewMessageFrom( rec.Email, displayName, @@ -167,6 +168,7 @@ func MailActionsTrigger(ctx context.Context, sender *user_model.User, repo *repo } if len(recipients) > 0 { + log.Trace("MailActionsTrigger: Initiate email composition") composeAndSendActionsWorkflowRunStatusEmail(ctx, repo, run, sender, recipients) } } From 1d9c378cc84f7c613c2d453b4434f2afd1f52450 Mon Sep 17 00:00:00 2001 From: NorthRealm <155140859+NorthRealm@users.noreply.github.com> Date: Tue, 5 Aug 2025 19:02:03 +0800 Subject: [PATCH 06/29] update --- services/actions/clear_tasks.go | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/services/actions/clear_tasks.go b/services/actions/clear_tasks.go index 274c04aa57f08..139f3e2a53433 100644 --- a/services/actions/clear_tasks.go +++ b/services/actions/clear_tasks.go @@ -42,10 +42,8 @@ func notifyWorkflowJobStatusUpdate(ctx context.Context, jobs []*actions_model.Ac _ = job.LoadAttributes(ctx) notify_service.WorkflowJobStatusUpdate(ctx, job.Run.Repo, job.Run.TriggerUser, job, nil) } - if len(jobs) > 0 { - job := jobs[0] - notify_service.WorkflowRunStatusUpdate(ctx, job.Run.Repo, job.Run.TriggerUser, job.Run) - } + job := jobs[0] + notify_service.WorkflowRunStatusUpdate(ctx, job.Run.Repo, job.Run.TriggerUser, job.Run) } } From c3a7f15bc4c4e83d533f7960f6752adc0cadb1a8 Mon Sep 17 00:00:00 2001 From: NorthRealm <155140859+NorthRealm@users.noreply.github.com> Date: Tue, 5 Aug 2025 19:51:16 +0800 Subject: [PATCH 07/29] Revert "assume" This reverts commit cdb1e80bf95c5d1f35c2b4839621e32d9704bb25. --- routers/web/repo/actions/view.go | 16 +++++++++------- 1 file changed, 9 insertions(+), 7 deletions(-) diff --git a/routers/web/repo/actions/view.go b/routers/web/repo/actions/view.go index 020d881e49e58..52b2e9995e3ba 100644 --- a/routers/web/repo/actions/view.go +++ b/routers/web/repo/actions/view.go @@ -560,9 +560,9 @@ func Cancel(ctx *context_module.Context) { if len(updatedjobs) > 0 { job := updatedjobs[0] actions_service.NotifyWorkflowRunStatusUpdateWithReload(ctx, job) + notify_service.WorkflowRunStatusUpdate(ctx, job.Run.Repo, job.Run.TriggerUser, job.Run) } - - ctx.JSONOK() + ctx.JSON(http.StatusOK, struct{}{}) } func Approve(ctx *context_module.Context) { @@ -603,16 +603,18 @@ func Approve(ctx *context_module.Context) { actions_service.CreateCommitStatus(ctx, jobs...) - for _, job := range updatedjobs { - _ = job.LoadAttributes(ctx) - notify_service.WorkflowJobStatusUpdate(ctx, job.Run.Repo, job.Run.TriggerUser, job, nil) - } if len(updatedjobs) > 0 { job := updatedjobs[0] actions_service.NotifyWorkflowRunStatusUpdateWithReload(ctx, job) + notify_service.WorkflowRunStatusUpdate(ctx, job.Run.Repo, job.Run.TriggerUser, job.Run) } - ctx.JSONOK() + for _, job := range updatedjobs { + _ = job.LoadAttributes(ctx) + notify_service.WorkflowJobStatusUpdate(ctx, job.Run.Repo, job.Run.TriggerUser, job, nil) + } + + ctx.JSON(http.StatusOK, struct{}{}) } func Delete(ctx *context_module.Context) { From bc3a4674ea8a588af962edfb06d3861731d3c36b Mon Sep 17 00:00:00 2001 From: Christopher Homberger Date: Wed, 6 Aug 2025 15:31:57 +0200 Subject: [PATCH 08/29] remove duplicate workflow run trigger + add test for cancel * missing test for approve --- routers/web/repo/actions/view.go | 2 - tests/integration/repo_webhook_test.go | 128 +++++++++++++++++++++++++ 2 files changed, 128 insertions(+), 2 deletions(-) diff --git a/routers/web/repo/actions/view.go b/routers/web/repo/actions/view.go index 52b2e9995e3ba..68f7d246e7ae1 100644 --- a/routers/web/repo/actions/view.go +++ b/routers/web/repo/actions/view.go @@ -560,7 +560,6 @@ func Cancel(ctx *context_module.Context) { if len(updatedjobs) > 0 { job := updatedjobs[0] actions_service.NotifyWorkflowRunStatusUpdateWithReload(ctx, job) - notify_service.WorkflowRunStatusUpdate(ctx, job.Run.Repo, job.Run.TriggerUser, job.Run) } ctx.JSON(http.StatusOK, struct{}{}) } @@ -606,7 +605,6 @@ func Approve(ctx *context_module.Context) { if len(updatedjobs) > 0 { job := updatedjobs[0] actions_service.NotifyWorkflowRunStatusUpdateWithReload(ctx, job) - notify_service.WorkflowRunStatusUpdate(ctx, job.Run.Repo, job.Run.TriggerUser, job.Run) } for _, job := range updatedjobs { diff --git a/tests/integration/repo_webhook_test.go b/tests/integration/repo_webhook_test.go index 1da7bc9d3c80f..52832330a9c3d 100644 --- a/tests/integration/repo_webhook_test.go +++ b/tests/integration/repo_webhook_test.go @@ -12,6 +12,7 @@ import ( "path" "strings" "testing" + "time" auth_model "code.gitea.io/gitea/models/auth" "code.gitea.io/gitea/models/repo" @@ -1058,6 +1059,10 @@ func Test_WebhookWorkflowRun(t *testing.T) { name: "WorkflowRunDepthLimit", callback: testWebhookWorkflowRunDepthLimit, }, + { + name: "WorkflowRunDuplicateEvents", + callback: testWorkflowRunDuplicateEvents, + }, } for _, test := range tests { t.Run(test.name, func(t *testing.T) { @@ -1070,6 +1075,129 @@ func Test_WebhookWorkflowRun(t *testing.T) { } } +func testWorkflowRunDuplicateEvents(t *testing.T, webhookData *workflowRunWebhook) { + // 1. create a new webhook with special webhook for repo1 + user2 := unittest.AssertExistsAndLoadBean(t, &user_model.User{ID: 2}) + session := loginUser(t, "user2") + token := getTokenForLoggedInUser(t, session, auth_model.AccessTokenScopeWriteRepository, auth_model.AccessTokenScopeWriteUser) + + testAPICreateWebhookForRepo(t, session, "user2", "repo1", webhookData.URL, "workflow_run") + + repo1 := unittest.AssertExistsAndLoadBean(t, &repo.Repository{ID: 1}) + + gitRepo1, err := gitrepo.OpenRepository(t.Context(), repo1) + assert.NoError(t, err) + + // 2.2 trigger the webhooks + + // add workflow file to the repo + // init the workflow + wfTreePath := ".gitea/workflows/push.yml" + wfFileContent := `on: + push: + workflow_dispatch: + +jobs: + test: + runs-on: ubuntu-latest + steps: + - run: exit 0 + + test2: + needs: [test] + runs-on: ubuntu-latest + steps: + - run: exit 0 + + test3: + needs: [test, test2] + runs-on: ubuntu-latest + steps: + - run: exit 0 + + test4: + needs: [test, test2, test3] + runs-on: ubuntu-latest + steps: + - run: exit 0 + + test5: + needs: [test, test2, test4] + runs-on: ubuntu-latest + steps: + - run: exit 0 + + test6: + strategy: + matrix: + os: [ubuntu-20.04, ubuntu-22.04, ubuntu-24.04] + needs: [test, test2, test3] + runs-on: ${{ matrix.os }} + steps: + - run: exit 0 + + test7: + needs: test6 + runs-on: ubuntu-latest + steps: + - run: exit 0 + + test8: + runs-on: ubuntu-latest + steps: + - run: exit 0 + + test9: + strategy: + matrix: + os: [ubuntu-20.04, ubuntu-22.04, ubuntu-24.04, ubuntu-25.04, windows-2022, windows-2025, macos-13, macos-14, macos-15] + runs-on: ${{ matrix.os }} + steps: + - run: exit 0 + + test10: + runs-on: ubuntu-latest + steps: + - run: exit 0` + opts := getWorkflowCreateFileOptions(user2, repo1.DefaultBranch, "create "+wfTreePath, wfFileContent) + createWorkflowFile(t, token, "user2", "repo1", wfTreePath, opts) + + commitID, err := gitRepo1.GetBranchCommitID(repo1.DefaultBranch) + assert.NoError(t, err) + + // 3. validate the webhook is triggered + assert.Equal(t, "workflow_run", webhookData.triggeredEvent) + assert.Len(t, webhookData.payloads, 1) + assert.Equal(t, "requested", webhookData.payloads[0].Action) + assert.Equal(t, "queued", webhookData.payloads[0].WorkflowRun.Status) + assert.Equal(t, repo1.DefaultBranch, webhookData.payloads[0].WorkflowRun.HeadBranch) + assert.Equal(t, commitID, webhookData.payloads[0].WorkflowRun.HeadSha) + assert.Equal(t, "repo1", webhookData.payloads[0].Repo.Name) + assert.Equal(t, "user2/repo1", webhookData.payloads[0].Repo.FullName) + + time.Sleep(15 * time.Second) // wait for the workflow to be processed + + // Call cancel ui api + // Only a web UI API exists for cancelling workflow runs, so use the UI endpoint. + cancelURL := fmt.Sprintf("/user2/repo1/actions/runs/%d/cancel", webhookData.payloads[0].WorkflowRun.RunNumber) + req := NewRequestWithValues(t, "POST", cancelURL, map[string]string{ + "_csrf": GetUserCSRFToken(t, session), + }) + session.MakeRequest(t, req, http.StatusOK) + + assert.Len(t, webhookData.payloads, 2) + + // 4. Validate the second webhook payload + assert.Equal(t, "workflow_run", webhookData.triggeredEvent) + assert.Equal(t, "completed", webhookData.payloads[1].Action) + assert.Equal(t, "push", webhookData.payloads[1].WorkflowRun.Event) + assert.Equal(t, "completed", webhookData.payloads[1].WorkflowRun.Status) + assert.Equal(t, repo1.DefaultBranch, webhookData.payloads[1].WorkflowRun.HeadBranch) + assert.Equal(t, commitID, webhookData.payloads[1].WorkflowRun.HeadSha) + assert.Equal(t, "repo1", webhookData.payloads[1].Repo.Name) + assert.Equal(t, "user2/repo1", webhookData.payloads[1].Repo.FullName) +} + func testWebhookWorkflowRun(t *testing.T, webhookData *workflowRunWebhook) { // 1. create a new webhook with special webhook for repo1 user2 := unittest.AssertExistsAndLoadBean(t, &user_model.User{ID: 2}) From cf547bb15d64a2f9fc7d204f1c0622fd61a8f38c Mon Sep 17 00:00:00 2001 From: NorthRealm <155140859+NorthRealm@users.noreply.github.com> Date: Wed, 6 Aug 2025 21:44:25 +0800 Subject: [PATCH 09/29] update --- routers/web/repo/actions/view.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/routers/web/repo/actions/view.go b/routers/web/repo/actions/view.go index 68f7d246e7ae1..e0565c0de5695 100644 --- a/routers/web/repo/actions/view.go +++ b/routers/web/repo/actions/view.go @@ -561,7 +561,7 @@ func Cancel(ctx *context_module.Context) { job := updatedjobs[0] actions_service.NotifyWorkflowRunStatusUpdateWithReload(ctx, job) } - ctx.JSON(http.StatusOK, struct{}{}) + ctx.JSONOK() } func Approve(ctx *context_module.Context) { @@ -612,7 +612,7 @@ func Approve(ctx *context_module.Context) { notify_service.WorkflowJobStatusUpdate(ctx, job.Run.Repo, job.Run.TriggerUser, job, nil) } - ctx.JSON(http.StatusOK, struct{}{}) + ctx.JSONOK() } func Delete(ctx *context_module.Context) { From e212011fcab1377da21a9bfa204ef517c1a47108 Mon Sep 17 00:00:00 2001 From: Christopher Homberger Date: Fri, 8 Aug 2025 22:49:08 +0200 Subject: [PATCH 10/29] remove timeout --- tests/integration/repo_webhook_test.go | 3 --- 1 file changed, 3 deletions(-) diff --git a/tests/integration/repo_webhook_test.go b/tests/integration/repo_webhook_test.go index 2a14119da27aa..4a24f4eb769e4 100644 --- a/tests/integration/repo_webhook_test.go +++ b/tests/integration/repo_webhook_test.go @@ -12,7 +12,6 @@ import ( "path" "strings" "testing" - "time" auth_model "code.gitea.io/gitea/models/auth" "code.gitea.io/gitea/models/repo" @@ -1246,8 +1245,6 @@ jobs: assert.Equal(t, "repo1", webhookData.payloads[0].Repo.Name) assert.Equal(t, "user2/repo1", webhookData.payloads[0].Repo.FullName) - time.Sleep(15 * time.Second) // wait for the workflow to be processed - // Call cancel ui api // Only a web UI API exists for cancelling workflow runs, so use the UI endpoint. cancelURL := fmt.Sprintf("/user2/repo1/actions/runs/%d/cancel", webhookData.payloads[0].WorkflowRun.RunNumber) From 0506162f17c367df2837343289f55595491ae51d Mon Sep 17 00:00:00 2001 From: NorthRealm <155140859+NorthRealm@users.noreply.github.com> Date: Sat, 9 Aug 2025 13:11:43 +0800 Subject: [PATCH 11/29] update --- services/mailer/mail_workflow_run.go | 2 +- services/mailer/notify.go | 3 --- 2 files changed, 1 insertion(+), 4 deletions(-) diff --git a/services/mailer/mail_workflow_run.go b/services/mailer/mail_workflow_run.go index 59203c5d947ec..ec6f123139db9 100644 --- a/services/mailer/mail_workflow_run.go +++ b/services/mailer/mail_workflow_run.go @@ -149,7 +149,7 @@ func MailActionsTrigger(ctx context.Context, sender *user_model.User, repo *repo if setting.MailService == nil { return } - if run.Status.IsSkipped() { + if !run.Status.IsDone() || run.Status.IsSkipped() { return } diff --git a/services/mailer/notify.go b/services/mailer/notify.go index c008685e131c2..ae16b2b429a93 100644 --- a/services/mailer/notify.go +++ b/services/mailer/notify.go @@ -208,8 +208,5 @@ func (m *mailNotifier) RepoPendingTransfer(ctx context.Context, doer, newOwner * } func (m *mailNotifier) WorkflowRunStatusUpdate(ctx context.Context, repo *repo_model.Repository, sender *user_model.User, run *actions_model.ActionRun) { - if !run.Status.IsDone() { - return - } MailActionsTrigger(ctx, sender, repo, run) } From 6bcdd7e532a395dfc01071b33c9a9b5003bffe3f Mon Sep 17 00:00:00 2001 From: Christopher Homberger Date: Sun, 10 Aug 2025 12:46:24 +0200 Subject: [PATCH 12/29] fix more workflow_run completion events * Not every abandoned job per run sends an completion event * Rerun multiple Jobs only send requested event once --- routers/web/repo/actions/view.go | 7 +- services/actions/clear_tasks.go | 20 +- tests/integration/repo_webhook_test.go | 292 +++++++++++++++++++++++++ 3 files changed, 317 insertions(+), 2 deletions(-) diff --git a/routers/web/repo/actions/view.go b/routers/web/repo/actions/view.go index e0565c0de5695..3422128026ffd 100644 --- a/routers/web/repo/actions/view.go +++ b/routers/web/repo/actions/view.go @@ -429,6 +429,12 @@ func Rerun(ctx *context_module.Context) { ctx.ServerError("UpdateRun", err) return } + + if err := run.LoadAttributes(ctx); err != nil { + ctx.ServerError("run.LoadAttributes", err) + return + } + notify_service.WorkflowRunStatusUpdate(ctx, run.Repo, run.TriggerUser, run) } job, jobs := getRunJobs(ctx, runIndex, jobIndex) @@ -485,7 +491,6 @@ func rerunJob(ctx *context_module.Context, job *actions_model.ActionRunJob, shou } actions_service.CreateCommitStatus(ctx, job) - actions_service.NotifyWorkflowRunStatusUpdateWithReload(ctx, job) notify_service.WorkflowJobStatusUpdate(ctx, job.Run.Repo, job.Run.TriggerUser, job, nil) return nil diff --git a/services/actions/clear_tasks.go b/services/actions/clear_tasks.go index 139f3e2a53433..bca38e1af5fa8 100644 --- a/services/actions/clear_tasks.go +++ b/services/actions/clear_tasks.go @@ -111,6 +111,10 @@ func CancelAbandonedJobs(ctx context.Context) error { } now := timeutil.TimeStampNow() + + // Collect one job per run to send workflow run status update + updatedRuns := map[int64]*actions_model.ActionRunJob{} + for _, job := range jobs { job.Status = actions_model.StatusCancelled job.Stopped = now @@ -125,10 +129,24 @@ func CancelAbandonedJobs(ctx context.Context) error { } CreateCommitStatus(ctx, job) if updated { - NotifyWorkflowRunStatusUpdateWithReload(ctx, job) + updatedRuns[job.RunID] = job notify_service.WorkflowJobStatusUpdate(ctx, job.Run.Repo, job.Run.TriggerUser, job, nil) } } + for _, job := range updatedRuns { + c, err := db.Count[actions_model.ActionRunJob](ctx, actions_model.FindRunJobOptions{ + RunID: job.RunID, + Statuses: []actions_model.Status{actions_model.StatusWaiting, actions_model.StatusBlocked, actions_model.StatusRunning}, + }) + if err != nil { + log.Error("Count waiting jobs for run %d: %v", job.RunID, err) + continue + } + if c == 0 { + NotifyWorkflowRunStatusUpdateWithReload(ctx, job) + } + } + return nil } diff --git a/tests/integration/repo_webhook_test.go b/tests/integration/repo_webhook_test.go index 4a24f4eb769e4..f933eaf2c6a0b 100644 --- a/tests/integration/repo_webhook_test.go +++ b/tests/integration/repo_webhook_test.go @@ -12,6 +12,7 @@ import ( "path" "strings" "testing" + "time" auth_model "code.gitea.io/gitea/models/auth" "code.gitea.io/gitea/models/repo" @@ -24,7 +25,9 @@ import ( "code.gitea.io/gitea/modules/json" "code.gitea.io/gitea/modules/setting" api "code.gitea.io/gitea/modules/structs" + "code.gitea.io/gitea/modules/test" webhook_module "code.gitea.io/gitea/modules/webhook" + "code.gitea.io/gitea/services/actions" "code.gitea.io/gitea/tests" runnerv1 "code.gitea.io/actions-proto-go/runner/v1" @@ -1133,6 +1136,22 @@ func Test_WebhookWorkflowRun(t *testing.T) { name: "WorkflowRunDuplicateEvents", callback: testWorkflowRunDuplicateEvents, }, + { + name: "WorkflowRunEventDuplicateEventsRerun", + callback: testWorkflowRunDuplicateEventsRerun, + }, + { + name: "WorkflowRunDuplicateEventsCancelAbandoned", + callback: func(t *testing.T, webhookData *workflowRunWebhook) { + testWorkflowRunDuplicateEventsCancelAbandoned(t, webhookData, true) + }, + }, + { + name: "WorkflowRunDuplicateEventsCancelAbandoned", + callback: func(t *testing.T, webhookData *workflowRunWebhook) { + testWorkflowRunDuplicateEventsCancelAbandoned(t, webhookData, false) + }, + }, } for _, test := range tests { t.Run(test.name, func(t *testing.T) { @@ -1266,6 +1285,279 @@ jobs: assert.Equal(t, "user2/repo1", webhookData.payloads[1].Repo.FullName) } +func testWorkflowRunDuplicateEventsRerun(t *testing.T, webhookData *workflowRunWebhook) { + // 1. create a new webhook with special webhook for repo1 + user2 := unittest.AssertExistsAndLoadBean(t, &user_model.User{ID: 2}) + session := loginUser(t, "user2") + token := getTokenForLoggedInUser(t, session, auth_model.AccessTokenScopeWriteRepository, auth_model.AccessTokenScopeWriteUser) + + runners := make([]*mockRunner, 2) + for i := 0; i < len(runners); i++ { + runners[i] = newMockRunner() + runners[i].registerAsRepoRunner(t, "user2", "repo1", fmt.Sprintf("mock-runner-%d", i), []string{"ubuntu-latest"}, false) + } + + testAPICreateWebhookForRepo(t, session, "user2", "repo1", webhookData.URL, "workflow_run") + + repo1 := unittest.AssertExistsAndLoadBean(t, &repo.Repository{ID: 1}) + + gitRepo1, err := gitrepo.OpenRepository(t.Context(), repo1) + assert.NoError(t, err) + + // 2.2 trigger the webhooks + + // add workflow file to the repo + // init the workflow + wfTreePath := ".gitea/workflows/push.yml" + wfFileContent := `on: + push: + workflow_dispatch: + +jobs: + test: + runs-on: ubuntu-latest + steps: + - run: exit 0 + + test2: + needs: [test] + runs-on: ubuntu-latest + steps: + - run: exit 0 + + test3: + needs: [test, test2] + runs-on: ubuntu-latest + steps: + - run: exit 0 + + test4: + needs: [test, test2, test3] + runs-on: ubuntu-latest + steps: + - run: exit 0 + + test5: + needs: [test, test2, test4] + runs-on: ubuntu-latest + steps: + - run: exit 0 + + test6: + strategy: + matrix: + os: [ubuntu-20.04, ubuntu-22.04, ubuntu-24.04] + needs: [test, test2, test3] + runs-on: ${{ matrix.os }} + steps: + - run: exit 0 + + test7: + needs: test6 + runs-on: ubuntu-latest + steps: + - run: exit 0 + + test8: + runs-on: ubuntu-latest + steps: + - run: exit 0 + + test9: + strategy: + matrix: + os: [ubuntu-20.04, ubuntu-22.04, ubuntu-24.04, ubuntu-25.04, windows-2022, windows-2025, macos-13, macos-14, macos-15] + runs-on: ${{ matrix.os }} + steps: + - run: exit 0 + + test10: + runs-on: ubuntu-latest + steps: + - run: exit 0` + opts := getWorkflowCreateFileOptions(user2, repo1.DefaultBranch, "create "+wfTreePath, wfFileContent) + createWorkflowFile(t, token, "user2", "repo1", wfTreePath, opts) + + commitID, err := gitRepo1.GetBranchCommitID(repo1.DefaultBranch) + assert.NoError(t, err) + + // 3. validate the webhook is triggered + assert.Equal(t, "workflow_run", webhookData.triggeredEvent) + assert.Len(t, webhookData.payloads, 1) + assert.Equal(t, "requested", webhookData.payloads[0].Action) + assert.Equal(t, "queued", webhookData.payloads[0].WorkflowRun.Status) + assert.Equal(t, repo1.DefaultBranch, webhookData.payloads[0].WorkflowRun.HeadBranch) + assert.Equal(t, commitID, webhookData.payloads[0].WorkflowRun.HeadSha) + assert.Equal(t, "repo1", webhookData.payloads[0].Repo.Name) + assert.Equal(t, "user2/repo1", webhookData.payloads[0].Repo.FullName) + + tasks := make([]*runnerv1.Task, len(runners)) + for i := 0; i < len(runners); i++ { + tasks[i] = runners[i].fetchTask(t) + runners[i].execTask(t, tasks[i], &mockTaskOutcome{ + result: runnerv1.Result_RESULT_SUCCESS, + }) + } + + // Call cancel ui api + // Only a web UI API exists for cancelling workflow runs, so use the UI endpoint. + cancelURL := fmt.Sprintf("/user2/repo1/actions/runs/%d/cancel", webhookData.payloads[0].WorkflowRun.RunNumber) + req := NewRequestWithValues(t, "POST", cancelURL, map[string]string{ + "_csrf": GetUserCSRFToken(t, session), + }) + session.MakeRequest(t, req, http.StatusOK) + + assert.Len(t, webhookData.payloads, 2) + + // 4. Validate the second webhook payload + assert.Equal(t, "workflow_run", webhookData.triggeredEvent) + assert.Equal(t, "completed", webhookData.payloads[1].Action) + assert.Equal(t, "push", webhookData.payloads[1].WorkflowRun.Event) + assert.Equal(t, "completed", webhookData.payloads[1].WorkflowRun.Status) + assert.Equal(t, repo1.DefaultBranch, webhookData.payloads[1].WorkflowRun.HeadBranch) + assert.Equal(t, commitID, webhookData.payloads[1].WorkflowRun.HeadSha) + assert.Equal(t, "repo1", webhookData.payloads[1].Repo.Name) + assert.Equal(t, "user2/repo1", webhookData.payloads[1].Repo.FullName) + + // Call rerun ui api + // Only a web UI API exists for cancelling workflow runs, so use the UI endpoint. + rerunURL := fmt.Sprintf("/user2/repo1/actions/runs/%d/rerun", webhookData.payloads[0].WorkflowRun.RunNumber) + req = NewRequestWithValues(t, "POST", rerunURL, map[string]string{ + "_csrf": GetUserCSRFToken(t, session), + }) + session.MakeRequest(t, req, http.StatusOK) + + assert.Len(t, webhookData.payloads, 3) +} + +func testWorkflowRunDuplicateEventsCancelAbandoned(t *testing.T, webhookData *workflowRunWebhook, partiallyAbandoned bool) { + // 1. create a new webhook with special webhook for repo1 + user2 := unittest.AssertExistsAndLoadBean(t, &user_model.User{ID: 2}) + session := loginUser(t, "user2") + token := getTokenForLoggedInUser(t, session, auth_model.AccessTokenScopeWriteRepository, auth_model.AccessTokenScopeWriteUser) + + runners := make([]*mockRunner, 2) + for i := 0; i < len(runners); i++ { + runners[i] = newMockRunner() + runners[i].registerAsRepoRunner(t, "user2", "repo1", fmt.Sprintf("mock-runner-%d", i), []string{"ubuntu-latest"}, false) + } + + testAPICreateWebhookForRepo(t, session, "user2", "repo1", webhookData.URL, "workflow_run") + + repo1 := unittest.AssertExistsAndLoadBean(t, &repo.Repository{ID: 1}) + + gitRepo1, err := gitrepo.OpenRepository(t.Context(), repo1) + assert.NoError(t, err) + + // 2.2 trigger the webhooks + + // add workflow file to the repo + // init the workflow + wfTreePath := ".gitea/workflows/push.yml" + wfFileContent := `on: + push: + workflow_dispatch: + +jobs: + test: + runs-on: ubuntu-latest + steps: + - run: exit 0 + + test2: + needs: [test] + runs-on: ubuntu-latest + steps: + - run: exit 0 + + test3: + needs: [test, test2] + runs-on: ubuntu-latest + steps: + - run: exit 0 + + test4: + needs: [test, test2, test3] + runs-on: ubuntu-latest + steps: + - run: exit 0 + + test5: + needs: [test, test2, test4] + runs-on: ubuntu-latest + steps: + - run: exit 0 + + test6: + strategy: + matrix: + os: [ubuntu-20.04, ubuntu-22.04, ubuntu-24.04] + needs: [test, test2, test3] + runs-on: ${{ matrix.os }} + steps: + - run: exit 0 + + test7: + needs: test6 + runs-on: ubuntu-latest + steps: + - run: exit 0 + + test8: + runs-on: ubuntu-latest + steps: + - run: exit 0 + + test9: + strategy: + matrix: + os: [ubuntu-20.04, ubuntu-22.04, ubuntu-24.04, ubuntu-25.04, windows-2022, windows-2025, macos-13, macos-14, macos-15] + runs-on: ${{ matrix.os }} + steps: + - run: exit 0 + + test10: + runs-on: ubuntu-latest + steps: + - run: exit 0` + opts := getWorkflowCreateFileOptions(user2, repo1.DefaultBranch, "create "+wfTreePath, wfFileContent) + createWorkflowFile(t, token, "user2", "repo1", wfTreePath, opts) + + commitID, err := gitRepo1.GetBranchCommitID(repo1.DefaultBranch) + assert.NoError(t, err) + + // 3. validate the webhook is triggered + assert.Equal(t, "workflow_run", webhookData.triggeredEvent) + assert.Len(t, webhookData.payloads, 1) + assert.Equal(t, "requested", webhookData.payloads[0].Action) + assert.Equal(t, "queued", webhookData.payloads[0].WorkflowRun.Status) + assert.Equal(t, repo1.DefaultBranch, webhookData.payloads[0].WorkflowRun.HeadBranch) + assert.Equal(t, commitID, webhookData.payloads[0].WorkflowRun.HeadSha) + assert.Equal(t, "repo1", webhookData.payloads[0].Repo.Name) + assert.Equal(t, "user2/repo1", webhookData.payloads[0].Repo.FullName) + + tasks := make([]*runnerv1.Task, len(runners)) + for i := 0; i < len(runners); i++ { + tasks[i] = runners[i].fetchTask(t) + if !partiallyAbandoned { + runners[i].execTask(t, tasks[i], &mockTaskOutcome{ + result: runnerv1.Result_RESULT_SUCCESS, + }) + } + } + + defer test.MockVariableValue(&setting.Actions.AbandonedJobTimeout, (time.Duration)(0))() + + err = actions.CancelAbandonedJobs(t.Context()) + assert.NoError(t, err) + + if partiallyAbandoned { + assert.Len(t, webhookData.payloads, 1) + } else { + assert.Len(t, webhookData.payloads, 2) + } +} + func testWebhookWorkflowRun(t *testing.T, webhookData *workflowRunWebhook) { // 1. create a new webhook with special webhook for repo1 user2 := unittest.AssertExistsAndLoadBean(t, &user_model.User{ID: 2}) From 3a442482d53788e727f74be06e3397b93787ffa7 Mon Sep 17 00:00:00 2001 From: Christopher Homberger Date: Sun, 10 Aug 2025 13:45:10 +0200 Subject: [PATCH 13/29] modernize --- tests/integration/repo_webhook_test.go | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/tests/integration/repo_webhook_test.go b/tests/integration/repo_webhook_test.go index f933eaf2c6a0b..8ae9d474248ce 100644 --- a/tests/integration/repo_webhook_test.go +++ b/tests/integration/repo_webhook_test.go @@ -1292,7 +1292,7 @@ func testWorkflowRunDuplicateEventsRerun(t *testing.T, webhookData *workflowRunW token := getTokenForLoggedInUser(t, session, auth_model.AccessTokenScopeWriteRepository, auth_model.AccessTokenScopeWriteUser) runners := make([]*mockRunner, 2) - for i := 0; i < len(runners); i++ { + for i := range runners { runners[i] = newMockRunner() runners[i].registerAsRepoRunner(t, "user2", "repo1", fmt.Sprintf("mock-runner-%d", i), []string{"ubuntu-latest"}, false) } @@ -1392,7 +1392,7 @@ jobs: assert.Equal(t, "user2/repo1", webhookData.payloads[0].Repo.FullName) tasks := make([]*runnerv1.Task, len(runners)) - for i := 0; i < len(runners); i++ { + for i := range runners { tasks[i] = runners[i].fetchTask(t) runners[i].execTask(t, tasks[i], &mockTaskOutcome{ result: runnerv1.Result_RESULT_SUCCESS, @@ -1437,7 +1437,7 @@ func testWorkflowRunDuplicateEventsCancelAbandoned(t *testing.T, webhookData *wo token := getTokenForLoggedInUser(t, session, auth_model.AccessTokenScopeWriteRepository, auth_model.AccessTokenScopeWriteUser) runners := make([]*mockRunner, 2) - for i := 0; i < len(runners); i++ { + for i := range runners { runners[i] = newMockRunner() runners[i].registerAsRepoRunner(t, "user2", "repo1", fmt.Sprintf("mock-runner-%d", i), []string{"ubuntu-latest"}, false) } @@ -1537,7 +1537,7 @@ jobs: assert.Equal(t, "user2/repo1", webhookData.payloads[0].Repo.FullName) tasks := make([]*runnerv1.Task, len(runners)) - for i := 0; i < len(runners); i++ { + for i := range runners { tasks[i] = runners[i].fetchTask(t) if !partiallyAbandoned { runners[i].execTask(t, tasks[i], &mockTaskOutcome{ From 5eaa9c862e3b5f8d5663b8bd083865b4611ce422 Mon Sep 17 00:00:00 2001 From: NorthRealm <155140859+NorthRealm@users.noreply.github.com> Date: Wed, 13 Aug 2025 14:53:56 +0800 Subject: [PATCH 14/29] error logs --- services/mailer/mail_workflow_run.go | 28 ++++++++++++++-------------- services/mailer/notify.go | 4 +++- 2 files changed, 17 insertions(+), 15 deletions(-) diff --git a/services/mailer/mail_workflow_run.go b/services/mailer/mail_workflow_run.go index ec6f123139db9..62e8552159117 100644 --- a/services/mailer/mail_workflow_run.go +++ b/services/mailer/mail_workflow_run.go @@ -33,16 +33,15 @@ func generateMessageIDForActionsWorkflowRunStatusEmail(repo *repo_model.Reposito return fmt.Sprintf("<%s/actions/runs/%d@%s>", repo.FullName(), run.Index, setting.Domain) } -func composeAndSendActionsWorkflowRunStatusEmail(ctx context.Context, repo *repo_model.Repository, run *actions_model.ActionRun, sender *user_model.User, recipients []*user_model.User) { +func composeAndSendActionsWorkflowRunStatusEmail(ctx context.Context, repo *repo_model.Repository, run *actions_model.ActionRun, sender *user_model.User, recipients []*user_model.User) error { jobs, err := actions_model.GetRunJobsByRunID(ctx, run.ID) if err != nil { - log.Error("GetRunJobsByRunID: %v", err) - return + return err } for _, job := range jobs { if !job.Status.IsDone() { - log.Trace("composeAndSendActionsWorkflowRunStatusEmail: A job is not done. Will not compose and send actions email.") - return + log.Debug("composeAndSendActionsWorkflowRunStatusEmail: A job is not done. Will not compose and send actions email.") + return nil } } @@ -118,8 +117,7 @@ func composeAndSendActionsWorkflowRunStatusEmail(ctx context.Context, repo *repo "Jobs": convertedJobs, "locale": locale, }); err != nil { - log.Error("ExecuteTemplate [%s]: %v", tplWorkflowRun, err) - return + return err } msgs := make([]*sender_service.Message, 0, len(tos)) for _, rec := range tos { @@ -143,14 +141,16 @@ func composeAndSendActionsWorkflowRunStatusEmail(ctx context.Context, repo *repo } SendAsync(msgs...) } + + return nil } -func MailActionsTrigger(ctx context.Context, sender *user_model.User, repo *repo_model.Repository, run *actions_model.ActionRun) { +func MailActionsTrigger(ctx context.Context, sender *user_model.User, repo *repo_model.Repository, run *actions_model.ActionRun) error { if setting.MailService == nil { - return + return nil } if !run.Status.IsDone() || run.Status.IsSkipped() { - return + return nil } recipients := make([]*user_model.User, 0) @@ -159,8 +159,7 @@ func MailActionsTrigger(ctx context.Context, sender *user_model.User, repo *repo notifyPref, err := user_model.GetUserSetting(ctx, sender.ID, user_model.SettingsKeyEmailNotificationGiteaActions, user_model.SettingEmailNotificationGiteaActionsFailureOnly) if err != nil { - log.Error("GetUserSetting: %v", err) - return + return err } if notifyPref == user_model.SettingEmailNotificationGiteaActionsAll || !run.Status.IsSuccess() && notifyPref != user_model.SettingEmailNotificationGiteaActionsDisabled { recipients = append(recipients, sender) @@ -168,7 +167,8 @@ func MailActionsTrigger(ctx context.Context, sender *user_model.User, repo *repo } if len(recipients) > 0 { - log.Trace("MailActionsTrigger: Initiate email composition") - composeAndSendActionsWorkflowRunStatusEmail(ctx, repo, run, sender, recipients) + log.Debug("MailActionsTrigger: Initiate email composition") + return composeAndSendActionsWorkflowRunStatusEmail(ctx, repo, run, sender, recipients) } + return nil } diff --git a/services/mailer/notify.go b/services/mailer/notify.go index ae16b2b429a93..a7df0052bce0e 100644 --- a/services/mailer/notify.go +++ b/services/mailer/notify.go @@ -208,5 +208,7 @@ func (m *mailNotifier) RepoPendingTransfer(ctx context.Context, doer, newOwner * } func (m *mailNotifier) WorkflowRunStatusUpdate(ctx context.Context, repo *repo_model.Repository, sender *user_model.User, run *actions_model.ActionRun) { - MailActionsTrigger(ctx, sender, repo, run) + if err := MailActionsTrigger(ctx, sender, repo, run); err != nil { + log.Error("MailActionsTrigger: %v", err) + } } From 1539d9fdd88212048eaf472f52fe63f9b06c78fd Mon Sep 17 00:00:00 2001 From: NorthRealm <155140859+NorthRealm@users.noreply.github.com> Date: Thu, 14 Aug 2025 20:19:38 +0800 Subject: [PATCH 15/29] update --- tests/integration/repo_webhook_test.go | 158 ++++++++++++------------- 1 file changed, 78 insertions(+), 80 deletions(-) diff --git a/tests/integration/repo_webhook_test.go b/tests/integration/repo_webhook_test.go index 8ae9d474248ce..a0f92aa751feb 100644 --- a/tests/integration/repo_webhook_test.go +++ b/tests/integration/repo_webhook_test.go @@ -1120,51 +1120,51 @@ func Test_WebhookWorkflowRun(t *testing.T) { defer provider.Close() webhookData.URL = provider.URL() - tests := []struct { + testCases := []struct { name string - callback func(t *testing.T, webhookData *workflowRunWebhook) + testFunc func(t *testing.T, webhookData *workflowRunWebhook) }{ { name: "WorkflowRun", - callback: testWebhookWorkflowRun, + testFunc: testWebhookWorkflowRun, }, { name: "WorkflowRunDepthLimit", - callback: testWebhookWorkflowRunDepthLimit, + testFunc: testWebhookWorkflowRunDepthLimit, }, { - name: "WorkflowRunDuplicateEvents", - callback: testWorkflowRunDuplicateEvents, + name: "WorkflowRunEvents", + testFunc: testWorkflowRunEvents, }, { - name: "WorkflowRunEventDuplicateEventsRerun", - callback: testWorkflowRunDuplicateEventsRerun, + name: "WorkflowRunEventsOnRerun", + testFunc: testWorkflowRunEventsOnRerun, }, { - name: "WorkflowRunDuplicateEventsCancelAbandoned", - callback: func(t *testing.T, webhookData *workflowRunWebhook) { - testWorkflowRunDuplicateEventsCancelAbandoned(t, webhookData, true) + name: "WorkflowRunEventsOnCancellingAllJobsAbandonedRun", + testFunc: func(t *testing.T, webhookData *workflowRunWebhook) { + testWorkflowRunEventsOnCancellingAbandonedRun(t, webhookData, true) }, }, { - name: "WorkflowRunDuplicateEventsCancelAbandoned", - callback: func(t *testing.T, webhookData *workflowRunWebhook) { - testWorkflowRunDuplicateEventsCancelAbandoned(t, webhookData, false) + name: "WorkflowRunEventsOnCancellingPartiallyAbandonedRun", + testFunc: func(t *testing.T, webhookData *workflowRunWebhook) { + testWorkflowRunEventsOnCancellingAbandonedRun(t, webhookData, false) }, }, } - for _, test := range tests { - t.Run(test.name, func(t *testing.T) { + for _, obj := range testCases { + t.Run(obj.name, func(t *testing.T) { webhookData.payloads = nil webhookData.triggeredEvent = "" onGiteaRun(t, func(t *testing.T, giteaURL *url.URL) { - test.callback(t, webhookData) + obj.testFunc(t, webhookData) }) }) } } -func testWorkflowRunDuplicateEvents(t *testing.T, webhookData *workflowRunWebhook) { +func testWorkflowRunEvents(t *testing.T, webhookData *workflowRunWebhook) { // 1. create a new webhook with special webhook for repo1 user2 := unittest.AssertExistsAndLoadBean(t, &user_model.User{ID: 2}) session := loginUser(t, "user2") @@ -1187,56 +1187,56 @@ func testWorkflowRunDuplicateEvents(t *testing.T, webhookData *workflowRunWebhoo workflow_dispatch: jobs: - test: + test: runs-on: ubuntu-latest steps: - run: exit 0 - test2: + test2: needs: [test] runs-on: ubuntu-latest steps: - run: exit 0 - test3: + test3: needs: [test, test2] runs-on: ubuntu-latest steps: - run: exit 0 - - test4: + + test4: needs: [test, test2, test3] runs-on: ubuntu-latest steps: - run: exit 0 - - test5: + + test5: needs: [test, test2, test4] runs-on: ubuntu-latest steps: - run: exit 0 - + test6: strategy: matrix: - os: [ubuntu-20.04, ubuntu-22.04, ubuntu-24.04] + os: [ubuntu-20.04, ubuntu-22.04, ubuntu-24.04] needs: [test, test2, test3] runs-on: ${{ matrix.os }} steps: - run: exit 0 - - test7: + + test7: needs: test6 runs-on: ubuntu-latest steps: - run: exit 0 - - test8: + + test8: runs-on: ubuntu-latest steps: - run: exit 0 - - test9: + + test9: strategy: matrix: os: [ubuntu-20.04, ubuntu-22.04, ubuntu-24.04, ubuntu-25.04, windows-2022, windows-2025, macos-13, macos-14, macos-15] @@ -1244,7 +1244,7 @@ jobs: steps: - run: exit 0 - test10: + test10: runs-on: ubuntu-latest steps: - run: exit 0` @@ -1285,7 +1285,7 @@ jobs: assert.Equal(t, "user2/repo1", webhookData.payloads[1].Repo.FullName) } -func testWorkflowRunDuplicateEventsRerun(t *testing.T, webhookData *workflowRunWebhook) { +func testWorkflowRunEventsOnRerun(t *testing.T, webhookData *workflowRunWebhook) { // 1. create a new webhook with special webhook for repo1 user2 := unittest.AssertExistsAndLoadBean(t, &user_model.User{ID: 2}) session := loginUser(t, "user2") @@ -1314,56 +1314,56 @@ func testWorkflowRunDuplicateEventsRerun(t *testing.T, webhookData *workflowRunW workflow_dispatch: jobs: - test: + test: runs-on: ubuntu-latest steps: - run: exit 0 - test2: + test2: needs: [test] runs-on: ubuntu-latest steps: - run: exit 0 - test3: + test3: needs: [test, test2] runs-on: ubuntu-latest steps: - run: exit 0 - - test4: + + test4: needs: [test, test2, test3] runs-on: ubuntu-latest steps: - run: exit 0 - - test5: + + test5: needs: [test, test2, test4] runs-on: ubuntu-latest steps: - run: exit 0 - + test6: strategy: matrix: - os: [ubuntu-20.04, ubuntu-22.04, ubuntu-24.04] + os: [ubuntu-20.04, ubuntu-22.04, ubuntu-24.04] needs: [test, test2, test3] runs-on: ${{ matrix.os }} steps: - run: exit 0 - - test7: + + test7: needs: test6 runs-on: ubuntu-latest steps: - run: exit 0 - - test8: + + test8: runs-on: ubuntu-latest steps: - run: exit 0 - - test9: + + test9: strategy: matrix: os: [ubuntu-20.04, ubuntu-22.04, ubuntu-24.04, ubuntu-25.04, windows-2022, windows-2025, macos-13, macos-14, macos-15] @@ -1371,7 +1371,7 @@ jobs: steps: - run: exit 0 - test10: + test10: runs-on: ubuntu-latest steps: - run: exit 0` @@ -1391,10 +1391,9 @@ jobs: assert.Equal(t, "repo1", webhookData.payloads[0].Repo.Name) assert.Equal(t, "user2/repo1", webhookData.payloads[0].Repo.FullName) - tasks := make([]*runnerv1.Task, len(runners)) - for i := range runners { - tasks[i] = runners[i].fetchTask(t) - runners[i].execTask(t, tasks[i], &mockTaskOutcome{ + for _, runner := range runners { + task := runner.fetchTask(t) + runner.execTask(t, task, &mockTaskOutcome{ result: runnerv1.Result_RESULT_SUCCESS, }) } @@ -1430,7 +1429,9 @@ jobs: assert.Len(t, webhookData.payloads, 3) } -func testWorkflowRunDuplicateEventsCancelAbandoned(t *testing.T, webhookData *workflowRunWebhook, partiallyAbandoned bool) { +func testWorkflowRunEventsOnCancellingAbandonedRun(t *testing.T, webhookData *workflowRunWebhook, allJobsAbandoned bool) { + defer test.MockVariableValue(&setting.Actions.AbandonedJobTimeout, (time.Duration)(0))() + // 1. create a new webhook with special webhook for repo1 user2 := unittest.AssertExistsAndLoadBean(t, &user_model.User{ID: 2}) session := loginUser(t, "user2") @@ -1459,56 +1460,56 @@ func testWorkflowRunDuplicateEventsCancelAbandoned(t *testing.T, webhookData *wo workflow_dispatch: jobs: - test: + test: runs-on: ubuntu-latest steps: - run: exit 0 - test2: + test2: needs: [test] runs-on: ubuntu-latest steps: - run: exit 0 - test3: + test3: needs: [test, test2] runs-on: ubuntu-latest steps: - run: exit 0 - - test4: + + test4: needs: [test, test2, test3] runs-on: ubuntu-latest steps: - run: exit 0 - - test5: + + test5: needs: [test, test2, test4] runs-on: ubuntu-latest steps: - run: exit 0 - + test6: strategy: matrix: - os: [ubuntu-20.04, ubuntu-22.04, ubuntu-24.04] + os: [ubuntu-20.04, ubuntu-22.04, ubuntu-24.04] needs: [test, test2, test3] runs-on: ${{ matrix.os }} steps: - run: exit 0 - - test7: + + test7: needs: test6 runs-on: ubuntu-latest steps: - run: exit 0 - - test8: + + test8: runs-on: ubuntu-latest steps: - run: exit 0 - - test9: + + test9: strategy: matrix: os: [ubuntu-20.04, ubuntu-22.04, ubuntu-24.04, ubuntu-25.04, windows-2022, windows-2025, macos-13, macos-14, macos-15] @@ -1516,7 +1517,7 @@ jobs: steps: - run: exit 0 - test10: + test10: runs-on: ubuntu-latest steps: - run: exit 0` @@ -1536,22 +1537,19 @@ jobs: assert.Equal(t, "repo1", webhookData.payloads[0].Repo.Name) assert.Equal(t, "user2/repo1", webhookData.payloads[0].Repo.FullName) - tasks := make([]*runnerv1.Task, len(runners)) - for i := range runners { - tasks[i] = runners[i].fetchTask(t) - if !partiallyAbandoned { - runners[i].execTask(t, tasks[i], &mockTaskOutcome{ + for _, runner := range runners { + task := runner.fetchTask(t) + if !allJobsAbandoned { + runner.execTask(t, task, &mockTaskOutcome{ result: runnerv1.Result_RESULT_SUCCESS, }) } } - defer test.MockVariableValue(&setting.Actions.AbandonedJobTimeout, (time.Duration)(0))() - err = actions.CancelAbandonedJobs(t.Context()) assert.NoError(t, err) - if partiallyAbandoned { + if allJobsAbandoned { assert.Len(t, webhookData.payloads, 1) } else { assert.Len(t, webhookData.payloads, 2) From bab5ff20a5ec72fce01d923f61b6305f63d4ec52 Mon Sep 17 00:00:00 2001 From: NorthRealm <155140859+NorthRealm@users.noreply.github.com> Date: Fri, 15 Aug 2025 18:51:41 +0800 Subject: [PATCH 16/29] update --- services/actions/clear_tasks.go | 26 ++++++++++++++++---------- 1 file changed, 16 insertions(+), 10 deletions(-) diff --git a/services/actions/clear_tasks.go b/services/actions/clear_tasks.go index bca38e1af5fa8..a2fe4ef94a940 100644 --- a/services/actions/clear_tasks.go +++ b/services/actions/clear_tasks.go @@ -106,7 +106,7 @@ func CancelAbandonedJobs(ctx context.Context) error { UpdatedBefore: timeutil.TimeStamp(time.Now().Add(-setting.Actions.AbandonedJobTimeout).Unix()), }) if err != nil { - log.Warn("find abandoned tasks: %v", err) + log.Error("CancelAbandonedJobs: %v", err) return err } @@ -124,7 +124,7 @@ func CancelAbandonedJobs(ctx context.Context) error { updated = err == nil && n > 0 return err }); err != nil { - log.Warn("cancel abandoned job %v: %v", job.ID, err) + log.Warn("CancelAbandonedJobs jobid %v: %v", job.ID, err) // go on } CreateCommitStatus(ctx, job) @@ -134,18 +134,24 @@ func CancelAbandonedJobs(ctx context.Context) error { } } - for _, job := range updatedRuns { - c, err := db.Count[actions_model.ActionRunJob](ctx, actions_model.FindRunJobOptions{ - RunID: job.RunID, - Statuses: []actions_model.Status{actions_model.StatusWaiting, actions_model.StatusBlocked, actions_model.StatusRunning}, - }) + for _, obj := range updatedRuns { + jobs, err := actions_model.GetRunJobsByRunID(ctx, obj.RunID) if err != nil { - log.Error("Count waiting jobs for run %d: %v", job.RunID, err) + log.Error("CancelAbandonedJobs runid %d: %v", obj.RunID, err) continue } - if c == 0 { - NotifyWorkflowRunStatusUpdateWithReload(ctx, job) + + unfinished := false + for _, job := range jobs { + if !job.Status.IsDone() { + unfinished = true + break + } + } + if unfinished { + continue } + NotifyWorkflowRunStatusUpdateWithReload(ctx, obj) } return nil From ec62f814b7982f6f69a2ab8dce082e5b8b96903e Mon Sep 17 00:00:00 2001 From: NorthRealm <155140859+NorthRealm@users.noreply.github.com> Date: Fri, 15 Aug 2025 18:51:46 +0800 Subject: [PATCH 17/29] Revert "update" This reverts commit a592e969fcf7c215e6991d6d4d22ffb7a3167fb2. --- services/actions/clear_tasks.go | 26 ++++++++++---------------- 1 file changed, 10 insertions(+), 16 deletions(-) diff --git a/services/actions/clear_tasks.go b/services/actions/clear_tasks.go index a2fe4ef94a940..bca38e1af5fa8 100644 --- a/services/actions/clear_tasks.go +++ b/services/actions/clear_tasks.go @@ -106,7 +106,7 @@ func CancelAbandonedJobs(ctx context.Context) error { UpdatedBefore: timeutil.TimeStamp(time.Now().Add(-setting.Actions.AbandonedJobTimeout).Unix()), }) if err != nil { - log.Error("CancelAbandonedJobs: %v", err) + log.Warn("find abandoned tasks: %v", err) return err } @@ -124,7 +124,7 @@ func CancelAbandonedJobs(ctx context.Context) error { updated = err == nil && n > 0 return err }); err != nil { - log.Warn("CancelAbandonedJobs jobid %v: %v", job.ID, err) + log.Warn("cancel abandoned job %v: %v", job.ID, err) // go on } CreateCommitStatus(ctx, job) @@ -134,24 +134,18 @@ func CancelAbandonedJobs(ctx context.Context) error { } } - for _, obj := range updatedRuns { - jobs, err := actions_model.GetRunJobsByRunID(ctx, obj.RunID) + for _, job := range updatedRuns { + c, err := db.Count[actions_model.ActionRunJob](ctx, actions_model.FindRunJobOptions{ + RunID: job.RunID, + Statuses: []actions_model.Status{actions_model.StatusWaiting, actions_model.StatusBlocked, actions_model.StatusRunning}, + }) if err != nil { - log.Error("CancelAbandonedJobs runid %d: %v", obj.RunID, err) + log.Error("Count waiting jobs for run %d: %v", job.RunID, err) continue } - - unfinished := false - for _, job := range jobs { - if !job.Status.IsDone() { - unfinished = true - break - } - } - if unfinished { - continue + if c == 0 { + NotifyWorkflowRunStatusUpdateWithReload(ctx, job) } - NotifyWorkflowRunStatusUpdateWithReload(ctx, obj) } return nil From 20b5777e3d460ea74976b7ae08e7ab9b1579cd29 Mon Sep 17 00:00:00 2001 From: NorthRealm <155140859+NorthRealm@users.noreply.github.com> Date: Fri, 15 Aug 2025 19:03:31 +0800 Subject: [PATCH 18/29] update --- services/actions/clear_tasks.go | 21 +++++++++++++-------- 1 file changed, 13 insertions(+), 8 deletions(-) diff --git a/services/actions/clear_tasks.go b/services/actions/clear_tasks.go index bca38e1af5fa8..6a27b5c7e3adb 100644 --- a/services/actions/clear_tasks.go +++ b/services/actions/clear_tasks.go @@ -134,18 +134,23 @@ func CancelAbandonedJobs(ctx context.Context) error { } } - for _, job := range updatedRuns { - c, err := db.Count[actions_model.ActionRunJob](ctx, actions_model.FindRunJobOptions{ - RunID: job.RunID, - Statuses: []actions_model.Status{actions_model.StatusWaiting, actions_model.StatusBlocked, actions_model.StatusRunning}, - }) + for runid, job := range updatedRuns { + jobs, err := actions_model.GetRunJobsByRunID(ctx, runid) if err != nil { - log.Error("Count waiting jobs for run %d: %v", job.RunID, err) + log.Error("Count waiting jobs for run %d: %v", runid, err) continue } - if c == 0 { - NotifyWorkflowRunStatusUpdateWithReload(ctx, job) + unfinished := false + for _, job := range jobs { + if !job.Status.IsDone() { + unfinished = true + break + } + } + if unfinished { + continue } + NotifyWorkflowRunStatusUpdateWithReload(ctx, job) } return nil From a1e64e7dd6876bcb0c9f4b051405415c78ac0f0a Mon Sep 17 00:00:00 2001 From: NorthRealm <155140859+NorthRealm@users.noreply.github.com> Date: Fri, 15 Aug 2025 19:09:38 +0800 Subject: [PATCH 19/29] update --- services/actions/clear_tasks.go | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/services/actions/clear_tasks.go b/services/actions/clear_tasks.go index 6a27b5c7e3adb..ca377aee15a17 100644 --- a/services/actions/clear_tasks.go +++ b/services/actions/clear_tasks.go @@ -106,7 +106,6 @@ func CancelAbandonedJobs(ctx context.Context) error { UpdatedBefore: timeutil.TimeStamp(time.Now().Add(-setting.Actions.AbandonedJobTimeout).Unix()), }) if err != nil { - log.Warn("find abandoned tasks: %v", err) return err } @@ -124,7 +123,7 @@ func CancelAbandonedJobs(ctx context.Context) error { updated = err == nil && n > 0 return err }); err != nil { - log.Warn("cancel abandoned job %v: %v", job.ID, err) + log.Warn("CancelAbandonedJobs jobid %v: %v", job.ID, err) // go on } CreateCommitStatus(ctx, job) @@ -137,7 +136,7 @@ func CancelAbandonedJobs(ctx context.Context) error { for runid, job := range updatedRuns { jobs, err := actions_model.GetRunJobsByRunID(ctx, runid) if err != nil { - log.Error("Count waiting jobs for run %d: %v", runid, err) + log.Error("CancelAbandonedJobs runid %d: %v", runid, err) continue } unfinished := false From 1bc9097a01beb2035959ffc2ffc57d1c2b464187 Mon Sep 17 00:00:00 2001 From: NorthRealm <155140859+NorthRealm@users.noreply.github.com> Date: Fri, 15 Aug 2025 20:31:47 +0800 Subject: [PATCH 20/29] Revert "update" This reverts commit a1e64e7dd6876bcb0c9f4b051405415c78ac0f0a. --- services/actions/clear_tasks.go | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/services/actions/clear_tasks.go b/services/actions/clear_tasks.go index ca377aee15a17..6a27b5c7e3adb 100644 --- a/services/actions/clear_tasks.go +++ b/services/actions/clear_tasks.go @@ -106,6 +106,7 @@ func CancelAbandonedJobs(ctx context.Context) error { UpdatedBefore: timeutil.TimeStamp(time.Now().Add(-setting.Actions.AbandonedJobTimeout).Unix()), }) if err != nil { + log.Warn("find abandoned tasks: %v", err) return err } @@ -123,7 +124,7 @@ func CancelAbandonedJobs(ctx context.Context) error { updated = err == nil && n > 0 return err }); err != nil { - log.Warn("CancelAbandonedJobs jobid %v: %v", job.ID, err) + log.Warn("cancel abandoned job %v: %v", job.ID, err) // go on } CreateCommitStatus(ctx, job) @@ -136,7 +137,7 @@ func CancelAbandonedJobs(ctx context.Context) error { for runid, job := range updatedRuns { jobs, err := actions_model.GetRunJobsByRunID(ctx, runid) if err != nil { - log.Error("CancelAbandonedJobs runid %d: %v", runid, err) + log.Error("Count waiting jobs for run %d: %v", runid, err) continue } unfinished := false From 2330dcfe8ade9c74219dee4474c978b13a46e5c9 Mon Sep 17 00:00:00 2001 From: NorthRealm <155140859+NorthRealm@users.noreply.github.com> Date: Fri, 15 Aug 2025 20:31:47 +0800 Subject: [PATCH 21/29] Revert "update" This reverts commit 20b5777e3d460ea74976b7ae08e7ab9b1579cd29. --- services/actions/clear_tasks.go | 21 ++++++++------------- 1 file changed, 8 insertions(+), 13 deletions(-) diff --git a/services/actions/clear_tasks.go b/services/actions/clear_tasks.go index 6a27b5c7e3adb..bca38e1af5fa8 100644 --- a/services/actions/clear_tasks.go +++ b/services/actions/clear_tasks.go @@ -134,23 +134,18 @@ func CancelAbandonedJobs(ctx context.Context) error { } } - for runid, job := range updatedRuns { - jobs, err := actions_model.GetRunJobsByRunID(ctx, runid) + for _, job := range updatedRuns { + c, err := db.Count[actions_model.ActionRunJob](ctx, actions_model.FindRunJobOptions{ + RunID: job.RunID, + Statuses: []actions_model.Status{actions_model.StatusWaiting, actions_model.StatusBlocked, actions_model.StatusRunning}, + }) if err != nil { - log.Error("Count waiting jobs for run %d: %v", runid, err) + log.Error("Count waiting jobs for run %d: %v", job.RunID, err) continue } - unfinished := false - for _, job := range jobs { - if !job.Status.IsDone() { - unfinished = true - break - } - } - if unfinished { - continue + if c == 0 { + NotifyWorkflowRunStatusUpdateWithReload(ctx, job) } - NotifyWorkflowRunStatusUpdateWithReload(ctx, job) } return nil From 341c5d85e3aa294e81d005fe09a98255ddbebf66 Mon Sep 17 00:00:00 2001 From: NorthRealm <155140859+NorthRealm@users.noreply.github.com> Date: Tue, 19 Aug 2025 15:14:59 +0800 Subject: [PATCH 22/29] fix maybe --- services/actions/clear_tasks.go | 28 +++--- tests/integration/repo_webhook_test.go | 132 +++++++++++++++---------- 2 files changed, 93 insertions(+), 67 deletions(-) diff --git a/services/actions/clear_tasks.go b/services/actions/clear_tasks.go index bca38e1af5fa8..3c7aa0b1a5918 100644 --- a/services/actions/clear_tasks.go +++ b/services/actions/clear_tasks.go @@ -99,7 +99,7 @@ func stopTasks(ctx context.Context, opts actions_model.FindTaskOptions) error { return nil } -// CancelAbandonedJobs cancels the jobs which have waiting status, but haven't been picked by a runner for a long time +// CancelAbandonedJobs cancels jobs that have not been picked by any runner for a long time func CancelAbandonedJobs(ctx context.Context) error { jobs, err := db.Find[actions_model.ActionRunJob](ctx, actions_model.FindRunJobOptions{ Statuses: []actions_model.Status{actions_model.StatusWaiting, actions_model.StatusBlocked}, @@ -121,31 +121,29 @@ func CancelAbandonedJobs(ctx context.Context) error { updated := false if err := db.WithTx(ctx, func(ctx context.Context) error { n, err := actions_model.UpdateRunJob(ctx, job, nil, "status", "stopped") - updated = err == nil && n > 0 - return err + if err != nil { + return err + } + if err := job.LoadAttributes(ctx); err != nil { + return err + } + updated = n > 0 + if updated && job.Run.Status.IsDone() { + updatedRuns[job.RunID] = job + } + return nil }); err != nil { log.Warn("cancel abandoned job %v: %v", job.ID, err) // go on } CreateCommitStatus(ctx, job) if updated { - updatedRuns[job.RunID] = job notify_service.WorkflowJobStatusUpdate(ctx, job.Run.Repo, job.Run.TriggerUser, job, nil) } } for _, job := range updatedRuns { - c, err := db.Count[actions_model.ActionRunJob](ctx, actions_model.FindRunJobOptions{ - RunID: job.RunID, - Statuses: []actions_model.Status{actions_model.StatusWaiting, actions_model.StatusBlocked, actions_model.StatusRunning}, - }) - if err != nil { - log.Error("Count waiting jobs for run %d: %v", job.RunID, err) - continue - } - if c == 0 { - NotifyWorkflowRunStatusUpdateWithReload(ctx, job) - } + notify_service.WorkflowRunStatusUpdate(ctx, job.Run.Repo, job.Run.TriggerUser, job.Run) } return nil diff --git a/tests/integration/repo_webhook_test.go b/tests/integration/repo_webhook_test.go index a0f92aa751feb..1c3c368ffae10 100644 --- a/tests/integration/repo_webhook_test.go +++ b/tests/integration/repo_webhook_test.go @@ -1105,21 +1105,6 @@ type workflowRunWebhook struct { } func Test_WebhookWorkflowRun(t *testing.T) { - webhookData := &workflowRunWebhook{} - provider := newMockWebhookProvider(func(r *http.Request) { - assert.Contains(t, r.Header["X-Github-Event-Type"], "workflow_run", "X-GitHub-Event-Type should contain workflow_run") - assert.Contains(t, r.Header["X-Gitea-Event-Type"], "workflow_run", "X-Gitea-Event-Type should contain workflow_run") - assert.Contains(t, r.Header["X-Gogs-Event-Type"], "workflow_run", "X-Gogs-Event-Type should contain workflow_run") - content, _ := io.ReadAll(r.Body) - var payload api.WorkflowRunPayload - err := json.Unmarshal(content, &payload) - assert.NoError(t, err) - webhookData.payloads = append(webhookData.payloads, payload) - webhookData.triggeredEvent = "workflow_run" - }, http.StatusOK) - defer provider.Close() - webhookData.URL = provider.URL() - testCases := []struct { name string testFunc func(t *testing.T, webhookData *workflowRunWebhook) @@ -1141,23 +1126,29 @@ func Test_WebhookWorkflowRun(t *testing.T) { testFunc: testWorkflowRunEventsOnRerun, }, { - name: "WorkflowRunEventsOnCancellingAllJobsAbandonedRun", - testFunc: func(t *testing.T, webhookData *workflowRunWebhook) { - testWorkflowRunEventsOnCancellingAbandonedRun(t, webhookData, true) - }, - }, - { - name: "WorkflowRunEventsOnCancellingPartiallyAbandonedRun", - testFunc: func(t *testing.T, webhookData *workflowRunWebhook) { - testWorkflowRunEventsOnCancellingAbandonedRun(t, webhookData, false) - }, + name: "WorkflowRunEventsOnCancellingAbandonedRun", + testFunc: testWorkflowRunEventsOnCancellingAbandonedRun, }, } for _, obj := range testCases { t.Run(obj.name, func(t *testing.T) { - webhookData.payloads = nil - webhookData.triggeredEvent = "" onGiteaRun(t, func(t *testing.T, giteaURL *url.URL) { + webhookData := &workflowRunWebhook{} + provider := newMockWebhookProvider(func(r *http.Request) { + assert.Contains(t, r.Header["X-Github-Event-Type"], "workflow_run", "X-GitHub-Event-Type should contain workflow_run") + assert.Contains(t, r.Header["X-Gitea-Event-Type"], "workflow_run", "X-Gitea-Event-Type should contain workflow_run") + assert.Contains(t, r.Header["X-Gogs-Event-Type"], "workflow_run", "X-Gogs-Event-Type should contain workflow_run") + content, _ := io.ReadAll(r.Body) + var payload api.WorkflowRunPayload + err := json.Unmarshal(content, &payload) + assert.NoError(t, err) + webhookData.payloads = append(webhookData.payloads, payload) + webhookData.triggeredEvent = "workflow_run" + }, http.StatusOK) + defer provider.Close() + webhookData.URL = provider.URL() + webhookData.payloads = nil + webhookData.triggeredEvent = "" obj.testFunc(t, webhookData) }) }) @@ -1429,32 +1420,35 @@ jobs: assert.Len(t, webhookData.payloads, 3) } -func testWorkflowRunEventsOnCancellingAbandonedRun(t *testing.T, webhookData *workflowRunWebhook, allJobsAbandoned bool) { - defer test.MockVariableValue(&setting.Actions.AbandonedJobTimeout, (time.Duration)(0))() +func testWorkflowRunEventsOnCancellingAbandonedRun(t *testing.T, webhookData *workflowRunWebhook) { + defer test.MockVariableValue(&setting.Actions.AbandonedJobTimeout, 0*time.Nanosecond)() // 1. create a new webhook with special webhook for repo1 user2 := unittest.AssertExistsAndLoadBean(t, &user_model.User{ID: 2}) session := loginUser(t, "user2") token := getTokenForLoggedInUser(t, session, auth_model.AccessTokenScopeWriteRepository, auth_model.AccessTokenScopeWriteUser) + repoName := "test-workflow-run-cancelling-abandoned-run" + testRepo := unittest.AssertExistsAndLoadBean(t, &repo.Repository{ID: createActionsTestRepo(t, token, repoName, false).ID}) + runners := make([]*mockRunner, 2) for i := range runners { runners[i] = newMockRunner() - runners[i].registerAsRepoRunner(t, "user2", "repo1", fmt.Sprintf("mock-runner-%d", i), []string{"ubuntu-latest"}, false) + runners[i].registerAsRepoRunner(t, "user2", repoName, fmt.Sprintf("mock-runner-%d", i), []string{"ubuntu-latest"}, false) } - testAPICreateWebhookForRepo(t, session, "user2", "repo1", webhookData.URL, "workflow_run") - - repo1 := unittest.AssertExistsAndLoadBean(t, &repo.Repository{ID: 1}) + testAPICreateWebhookForRepo(t, session, "user2", repoName, webhookData.URL, "workflow_run") - gitRepo1, err := gitrepo.OpenRepository(t.Context(), repo1) + ctx := t.Context() + gitRepo, err := gitrepo.OpenRepository(ctx, testRepo) assert.NoError(t, err) // 2.2 trigger the webhooks // add workflow file to the repo // init the workflow - wfTreePath := ".gitea/workflows/push.yml" + wfilename := "push.yml" + wfTreePath := ".gitea/workflows/" + wfilename wfFileContent := `on: push: workflow_dispatch: @@ -1521,10 +1515,11 @@ jobs: runs-on: ubuntu-latest steps: - run: exit 0` - opts := getWorkflowCreateFileOptions(user2, repo1.DefaultBranch, "create "+wfTreePath, wfFileContent) - createWorkflowFile(t, token, "user2", "repo1", wfTreePath, opts) - commitID, err := gitRepo1.GetBranchCommitID(repo1.DefaultBranch) + opts := getWorkflowCreateFileOptions(user2, testRepo.DefaultBranch, "create "+wfTreePath, wfFileContent) + createWorkflowFile(t, token, "user2", repoName, wfTreePath, opts) + + commitID, err := gitRepo.GetBranchCommitID(testRepo.DefaultBranch) assert.NoError(t, err) // 3. validate the webhook is triggered @@ -1532,28 +1527,61 @@ jobs: assert.Len(t, webhookData.payloads, 1) assert.Equal(t, "requested", webhookData.payloads[0].Action) assert.Equal(t, "queued", webhookData.payloads[0].WorkflowRun.Status) - assert.Equal(t, repo1.DefaultBranch, webhookData.payloads[0].WorkflowRun.HeadBranch) + assert.Equal(t, testRepo.DefaultBranch, webhookData.payloads[0].WorkflowRun.HeadBranch) assert.Equal(t, commitID, webhookData.payloads[0].WorkflowRun.HeadSha) - assert.Equal(t, "repo1", webhookData.payloads[0].Repo.Name) - assert.Equal(t, "user2/repo1", webhookData.payloads[0].Repo.FullName) + assert.Equal(t, repoName, webhookData.payloads[0].Repo.Name) + assert.Equal(t, "user2/"+repoName, webhookData.payloads[0].Repo.FullName) for _, runner := range runners { - task := runner.fetchTask(t) - if !allJobsAbandoned { - runner.execTask(t, task, &mockTaskOutcome{ - result: runnerv1.Result_RESULT_SUCCESS, - }) - } + runner.fetchTask(t) } - err = actions.CancelAbandonedJobs(t.Context()) + err = actions.CancelAbandonedJobs(ctx) assert.NoError(t, err) + assert.Len(t, webhookData.payloads, 2) + assert.Equal(t, "completed", webhookData.payloads[1].Action) + assert.Equal(t, "completed", webhookData.payloads[1].WorkflowRun.Status) + assert.Equal(t, testRepo.DefaultBranch, webhookData.payloads[1].WorkflowRun.HeadBranch) + assert.Equal(t, commitID, webhookData.payloads[1].WorkflowRun.HeadSha) + assert.Equal(t, repoName, webhookData.payloads[1].Repo.Name) + assert.Equal(t, "user2/"+repoName, webhookData.payloads[1].Repo.FullName) - if allJobsAbandoned { - assert.Len(t, webhookData.payloads, 1) - } else { - assert.Len(t, webhookData.payloads, 2) + apiReqValues := url.Values{} + apiReqValues.Set("ref", testRepo.DefaultBranch) + req := NewRequestWithURLValues(t, "POST", + fmt.Sprintf("/api/v1/repos/%s/actions/workflows/%s/dispatches", testRepo.FullName(), wfilename), + apiReqValues).AddTokenAuth(token) + MakeRequest(t, req, http.StatusNoContent) + + for i := range runners { + runners[i] = newMockRunner() + runners[i].registerAsRepoRunner(t, "user2", repoName, fmt.Sprintf("mock-runner-2-%d", i), []string{"ubuntu-latest"}, false) } + + assert.Len(t, webhookData.payloads, 3) + assert.Equal(t, "requested", webhookData.payloads[2].Action) + assert.Equal(t, "queued", webhookData.payloads[2].WorkflowRun.Status) + assert.Equal(t, testRepo.DefaultBranch, webhookData.payloads[2].WorkflowRun.HeadBranch) + assert.Equal(t, commitID, webhookData.payloads[2].WorkflowRun.HeadSha) + assert.Equal(t, repoName, webhookData.payloads[2].Repo.Name) + assert.Equal(t, "user2/"+repoName, webhookData.payloads[2].Repo.FullName) + + for _, runner := range runners { + task := runner.fetchTask(t) + runner.execTask(t, task, &mockTaskOutcome{ + result: runnerv1.Result_RESULT_SUCCESS, + }) + } + + err = actions.CancelAbandonedJobs(ctx) + assert.NoError(t, err) + assert.Len(t, webhookData.payloads, 4) + assert.Equal(t, "completed", webhookData.payloads[3].Action) + assert.Equal(t, "completed", webhookData.payloads[3].WorkflowRun.Status) + assert.Equal(t, testRepo.DefaultBranch, webhookData.payloads[3].WorkflowRun.HeadBranch) + assert.Equal(t, commitID, webhookData.payloads[3].WorkflowRun.HeadSha) + assert.Equal(t, repoName, webhookData.payloads[3].Repo.Name) + assert.Equal(t, "user2/"+repoName, webhookData.payloads[3].Repo.FullName) } func testWebhookWorkflowRun(t *testing.T, webhookData *workflowRunWebhook) { From 846731ae5626f1b8bba487874c89836abbad8ed2 Mon Sep 17 00:00:00 2001 From: NorthRealm <155140859+NorthRealm@users.noreply.github.com> Date: Tue, 19 Aug 2025 15:18:16 +0800 Subject: [PATCH 23/29] wtf --- tests/integration/repo_webhook_test.go | 36 -------------------------- 1 file changed, 36 deletions(-) diff --git a/tests/integration/repo_webhook_test.go b/tests/integration/repo_webhook_test.go index 1c3c368ffae10..82b167b18e222 100644 --- a/tests/integration/repo_webhook_test.go +++ b/tests/integration/repo_webhook_test.go @@ -1546,42 +1546,6 @@ jobs: assert.Equal(t, repoName, webhookData.payloads[1].Repo.Name) assert.Equal(t, "user2/"+repoName, webhookData.payloads[1].Repo.FullName) - apiReqValues := url.Values{} - apiReqValues.Set("ref", testRepo.DefaultBranch) - req := NewRequestWithURLValues(t, "POST", - fmt.Sprintf("/api/v1/repos/%s/actions/workflows/%s/dispatches", testRepo.FullName(), wfilename), - apiReqValues).AddTokenAuth(token) - MakeRequest(t, req, http.StatusNoContent) - - for i := range runners { - runners[i] = newMockRunner() - runners[i].registerAsRepoRunner(t, "user2", repoName, fmt.Sprintf("mock-runner-2-%d", i), []string{"ubuntu-latest"}, false) - } - - assert.Len(t, webhookData.payloads, 3) - assert.Equal(t, "requested", webhookData.payloads[2].Action) - assert.Equal(t, "queued", webhookData.payloads[2].WorkflowRun.Status) - assert.Equal(t, testRepo.DefaultBranch, webhookData.payloads[2].WorkflowRun.HeadBranch) - assert.Equal(t, commitID, webhookData.payloads[2].WorkflowRun.HeadSha) - assert.Equal(t, repoName, webhookData.payloads[2].Repo.Name) - assert.Equal(t, "user2/"+repoName, webhookData.payloads[2].Repo.FullName) - - for _, runner := range runners { - task := runner.fetchTask(t) - runner.execTask(t, task, &mockTaskOutcome{ - result: runnerv1.Result_RESULT_SUCCESS, - }) - } - - err = actions.CancelAbandonedJobs(ctx) - assert.NoError(t, err) - assert.Len(t, webhookData.payloads, 4) - assert.Equal(t, "completed", webhookData.payloads[3].Action) - assert.Equal(t, "completed", webhookData.payloads[3].WorkflowRun.Status) - assert.Equal(t, testRepo.DefaultBranch, webhookData.payloads[3].WorkflowRun.HeadBranch) - assert.Equal(t, commitID, webhookData.payloads[3].WorkflowRun.HeadSha) - assert.Equal(t, repoName, webhookData.payloads[3].Repo.Name) - assert.Equal(t, "user2/"+repoName, webhookData.payloads[3].Repo.FullName) } func testWebhookWorkflowRun(t *testing.T, webhookData *workflowRunWebhook) { From 7a4109655999191df15c22a630a9290eac22bc4b Mon Sep 17 00:00:00 2001 From: NorthRealm <155140859+NorthRealm@users.noreply.github.com> Date: Tue, 19 Aug 2025 19:34:51 +0800 Subject: [PATCH 24/29] Revert "wtf" This reverts commit 846731ae5626f1b8bba487874c89836abbad8ed2. --- tests/integration/repo_webhook_test.go | 36 ++++++++++++++++++++++++++ 1 file changed, 36 insertions(+) diff --git a/tests/integration/repo_webhook_test.go b/tests/integration/repo_webhook_test.go index 82b167b18e222..1c3c368ffae10 100644 --- a/tests/integration/repo_webhook_test.go +++ b/tests/integration/repo_webhook_test.go @@ -1546,6 +1546,42 @@ jobs: assert.Equal(t, repoName, webhookData.payloads[1].Repo.Name) assert.Equal(t, "user2/"+repoName, webhookData.payloads[1].Repo.FullName) + apiReqValues := url.Values{} + apiReqValues.Set("ref", testRepo.DefaultBranch) + req := NewRequestWithURLValues(t, "POST", + fmt.Sprintf("/api/v1/repos/%s/actions/workflows/%s/dispatches", testRepo.FullName(), wfilename), + apiReqValues).AddTokenAuth(token) + MakeRequest(t, req, http.StatusNoContent) + + for i := range runners { + runners[i] = newMockRunner() + runners[i].registerAsRepoRunner(t, "user2", repoName, fmt.Sprintf("mock-runner-2-%d", i), []string{"ubuntu-latest"}, false) + } + + assert.Len(t, webhookData.payloads, 3) + assert.Equal(t, "requested", webhookData.payloads[2].Action) + assert.Equal(t, "queued", webhookData.payloads[2].WorkflowRun.Status) + assert.Equal(t, testRepo.DefaultBranch, webhookData.payloads[2].WorkflowRun.HeadBranch) + assert.Equal(t, commitID, webhookData.payloads[2].WorkflowRun.HeadSha) + assert.Equal(t, repoName, webhookData.payloads[2].Repo.Name) + assert.Equal(t, "user2/"+repoName, webhookData.payloads[2].Repo.FullName) + + for _, runner := range runners { + task := runner.fetchTask(t) + runner.execTask(t, task, &mockTaskOutcome{ + result: runnerv1.Result_RESULT_SUCCESS, + }) + } + + err = actions.CancelAbandonedJobs(ctx) + assert.NoError(t, err) + assert.Len(t, webhookData.payloads, 4) + assert.Equal(t, "completed", webhookData.payloads[3].Action) + assert.Equal(t, "completed", webhookData.payloads[3].WorkflowRun.Status) + assert.Equal(t, testRepo.DefaultBranch, webhookData.payloads[3].WorkflowRun.HeadBranch) + assert.Equal(t, commitID, webhookData.payloads[3].WorkflowRun.HeadSha) + assert.Equal(t, repoName, webhookData.payloads[3].Repo.Name) + assert.Equal(t, "user2/"+repoName, webhookData.payloads[3].Repo.FullName) } func testWebhookWorkflowRun(t *testing.T, webhookData *workflowRunWebhook) { From 42b6b4e809571f2cf8b96fba1420fe117b883603 Mon Sep 17 00:00:00 2001 From: NorthRealm <155140859+NorthRealm@users.noreply.github.com> Date: Tue, 19 Aug 2025 19:39:14 +0800 Subject: [PATCH 25/29] f --- tests/integration/repo_webhook_test.go | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/tests/integration/repo_webhook_test.go b/tests/integration/repo_webhook_test.go index 1c3c368ffae10..dc8473b3c3ba8 100644 --- a/tests/integration/repo_webhook_test.go +++ b/tests/integration/repo_webhook_test.go @@ -1536,6 +1536,9 @@ jobs: runner.fetchTask(t) } + // Add this sleep to ensure the func can find the tasks by timestamp. + time.Sleep(time.Second) + err = actions.CancelAbandonedJobs(ctx) assert.NoError(t, err) assert.Len(t, webhookData.payloads, 2) @@ -1573,6 +1576,8 @@ jobs: }) } + time.Sleep(time.Second) + err = actions.CancelAbandonedJobs(ctx) assert.NoError(t, err) assert.Len(t, webhookData.payloads, 4) From cef6ebab773402a5fae8b7527a23b158e3484866 Mon Sep 17 00:00:00 2001 From: NorthRealm <155140859+NorthRealm@users.noreply.github.com> Date: Wed, 20 Aug 2025 23:48:26 +0800 Subject: [PATCH 26/29] update --- tests/integration/repo_webhook_test.go | 18 ++++++++++-------- 1 file changed, 10 insertions(+), 8 deletions(-) diff --git a/tests/integration/repo_webhook_test.go b/tests/integration/repo_webhook_test.go index dc8473b3c3ba8..dfcc844a12738 100644 --- a/tests/integration/repo_webhook_test.go +++ b/tests/integration/repo_webhook_test.go @@ -1434,7 +1434,8 @@ func testWorkflowRunEventsOnCancellingAbandonedRun(t *testing.T, webhookData *wo runners := make([]*mockRunner, 2) for i := range runners { runners[i] = newMockRunner() - runners[i].registerAsRepoRunner(t, "user2", repoName, fmt.Sprintf("mock-runner-%d", i), []string{"ubuntu-latest"}, false) + runners[i].registerAsRepoRunner(t, "user2", repoName, + fmt.Sprintf("mock-runner-1-%d", i), []string{"ubuntu-latest"}, false) } testAPICreateWebhookForRepo(t, session, "user2", repoName, webhookData.URL, "workflow_run") @@ -1549,16 +1550,17 @@ jobs: assert.Equal(t, repoName, webhookData.payloads[1].Repo.Name) assert.Equal(t, "user2/"+repoName, webhookData.payloads[1].Repo.FullName) - apiReqValues := url.Values{} - apiReqValues.Set("ref", testRepo.DefaultBranch) - req := NewRequestWithURLValues(t, "POST", - fmt.Sprintf("/api/v1/repos/%s/actions/workflows/%s/dispatches", testRepo.FullName(), wfilename), - apiReqValues).AddTokenAuth(token) - MakeRequest(t, req, http.StatusNoContent) + session.MakeRequest(t, + NewRequestWithURLValues(t, "POST", + fmt.Sprintf("/api/v1/repos/%s/actions/workflows/%s/dispatches", testRepo.FullName(), wfilename), + url.Values{"ref": {testRepo.DefaultBranch}}). + AddTokenAuth(token), + http.StatusNoContent) for i := range runners { runners[i] = newMockRunner() - runners[i].registerAsRepoRunner(t, "user2", repoName, fmt.Sprintf("mock-runner-2-%d", i), []string{"ubuntu-latest"}, false) + runners[i].registerAsRepoRunner(t, "user2", repoName, + fmt.Sprintf("mock-runner-2-%d", i), []string{"ubuntu-latest"}, false) } assert.Len(t, webhookData.payloads, 3) From d00d5cfe21c02d28bef0cead3f19db038dfa601f Mon Sep 17 00:00:00 2001 From: Christopher Homberger Date: Thu, 21 Aug 2025 20:52:34 +0200 Subject: [PATCH 27/29] do not explicitly expect a workflow run completion event while jobs are running --- tests/integration/repo_webhook_test.go | 45 +++----------------------- 1 file changed, 4 insertions(+), 41 deletions(-) diff --git a/tests/integration/repo_webhook_test.go b/tests/integration/repo_webhook_test.go index dfcc844a12738..955e0494a38b4 100644 --- a/tests/integration/repo_webhook_test.go +++ b/tests/integration/repo_webhook_test.go @@ -1534,7 +1534,10 @@ jobs: assert.Equal(t, "user2/"+repoName, webhookData.payloads[0].Repo.FullName) for _, runner := range runners { - runner.fetchTask(t) + task := runner.fetchTask(t) + runner.execTask(t, task, &mockTaskOutcome{ + result: runnerv1.Result_RESULT_SUCCESS, + }) } // Add this sleep to ensure the func can find the tasks by timestamp. @@ -1549,46 +1552,6 @@ jobs: assert.Equal(t, commitID, webhookData.payloads[1].WorkflowRun.HeadSha) assert.Equal(t, repoName, webhookData.payloads[1].Repo.Name) assert.Equal(t, "user2/"+repoName, webhookData.payloads[1].Repo.FullName) - - session.MakeRequest(t, - NewRequestWithURLValues(t, "POST", - fmt.Sprintf("/api/v1/repos/%s/actions/workflows/%s/dispatches", testRepo.FullName(), wfilename), - url.Values{"ref": {testRepo.DefaultBranch}}). - AddTokenAuth(token), - http.StatusNoContent) - - for i := range runners { - runners[i] = newMockRunner() - runners[i].registerAsRepoRunner(t, "user2", repoName, - fmt.Sprintf("mock-runner-2-%d", i), []string{"ubuntu-latest"}, false) - } - - assert.Len(t, webhookData.payloads, 3) - assert.Equal(t, "requested", webhookData.payloads[2].Action) - assert.Equal(t, "queued", webhookData.payloads[2].WorkflowRun.Status) - assert.Equal(t, testRepo.DefaultBranch, webhookData.payloads[2].WorkflowRun.HeadBranch) - assert.Equal(t, commitID, webhookData.payloads[2].WorkflowRun.HeadSha) - assert.Equal(t, repoName, webhookData.payloads[2].Repo.Name) - assert.Equal(t, "user2/"+repoName, webhookData.payloads[2].Repo.FullName) - - for _, runner := range runners { - task := runner.fetchTask(t) - runner.execTask(t, task, &mockTaskOutcome{ - result: runnerv1.Result_RESULT_SUCCESS, - }) - } - - time.Sleep(time.Second) - - err = actions.CancelAbandonedJobs(ctx) - assert.NoError(t, err) - assert.Len(t, webhookData.payloads, 4) - assert.Equal(t, "completed", webhookData.payloads[3].Action) - assert.Equal(t, "completed", webhookData.payloads[3].WorkflowRun.Status) - assert.Equal(t, testRepo.DefaultBranch, webhookData.payloads[3].WorkflowRun.HeadBranch) - assert.Equal(t, commitID, webhookData.payloads[3].WorkflowRun.HeadSha) - assert.Equal(t, repoName, webhookData.payloads[3].Repo.Name) - assert.Equal(t, "user2/"+repoName, webhookData.payloads[3].Repo.FullName) } func testWebhookWorkflowRun(t *testing.T, webhookData *workflowRunWebhook) { From 24d46df99a8a73d79d4096d7e7df1f9f06dbf56d Mon Sep 17 00:00:00 2001 From: NorthRealm <155140859+NorthRealm@users.noreply.github.com> Date: Fri, 22 Aug 2025 10:47:25 +0800 Subject: [PATCH 28/29] update --- tests/integration/repo_webhook_test.go | 24 +++++++++++++++++------- 1 file changed, 17 insertions(+), 7 deletions(-) diff --git a/tests/integration/repo_webhook_test.go b/tests/integration/repo_webhook_test.go index 955e0494a38b4..3ac017239a0e8 100644 --- a/tests/integration/repo_webhook_test.go +++ b/tests/integration/repo_webhook_test.go @@ -1126,8 +1126,16 @@ func Test_WebhookWorkflowRun(t *testing.T) { testFunc: testWorkflowRunEventsOnRerun, }, { - name: "WorkflowRunEventsOnCancellingAbandonedRun", - testFunc: testWorkflowRunEventsOnCancellingAbandonedRun, + name: "WorkflowRunEventsOnCancellingAbandonedRunAllJobsAbandoned", + testFunc: func(t *testing.T, webhookData *workflowRunWebhook) { + testWorkflowRunEventsOnCancellingAbandonedRun(t, webhookData, true) + }, + }, + { + name: "WorkflowRunEventsOnCancellingAbandonedRunPartiallyAbandoned", + testFunc: func(t *testing.T, webhookData *workflowRunWebhook) { + testWorkflowRunEventsOnCancellingAbandonedRun(t, webhookData, false) + }, }, } for _, obj := range testCases { @@ -1420,7 +1428,7 @@ jobs: assert.Len(t, webhookData.payloads, 3) } -func testWorkflowRunEventsOnCancellingAbandonedRun(t *testing.T, webhookData *workflowRunWebhook) { +func testWorkflowRunEventsOnCancellingAbandonedRun(t *testing.T, webhookData *workflowRunWebhook, allJobsAbandoned bool) { defer test.MockVariableValue(&setting.Actions.AbandonedJobTimeout, 0*time.Nanosecond)() // 1. create a new webhook with special webhook for repo1 @@ -1435,7 +1443,7 @@ func testWorkflowRunEventsOnCancellingAbandonedRun(t *testing.T, webhookData *wo for i := range runners { runners[i] = newMockRunner() runners[i].registerAsRepoRunner(t, "user2", repoName, - fmt.Sprintf("mock-runner-1-%d", i), []string{"ubuntu-latest"}, false) + fmt.Sprintf("mock-runner-%d", i), []string{"ubuntu-latest"}, false) } testAPICreateWebhookForRepo(t, session, "user2", repoName, webhookData.URL, "workflow_run") @@ -1535,9 +1543,11 @@ jobs: for _, runner := range runners { task := runner.fetchTask(t) - runner.execTask(t, task, &mockTaskOutcome{ - result: runnerv1.Result_RESULT_SUCCESS, - }) + if !allJobsAbandoned { + runner.execTask(t, task, &mockTaskOutcome{ + result: runnerv1.Result_RESULT_SUCCESS, + }) + } } // Add this sleep to ensure the func can find the tasks by timestamp. From 5efd4b966ec26c5a1d63282b77a8dd884558536d Mon Sep 17 00:00:00 2001 From: NorthRealm <155140859+NorthRealm@users.noreply.github.com> Date: Fri, 22 Aug 2025 21:42:09 +0800 Subject: [PATCH 29/29] UPDATE Co-authored-by: ChristopherHX Signed-off-by: NorthRealm <155140859+NorthRealm@users.noreply.github.com> --- tests/integration/repo_webhook_test.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tests/integration/repo_webhook_test.go b/tests/integration/repo_webhook_test.go index 3ac017239a0e8..d54a604655561 100644 --- a/tests/integration/repo_webhook_test.go +++ b/tests/integration/repo_webhook_test.go @@ -1541,9 +1541,9 @@ jobs: assert.Equal(t, repoName, webhookData.payloads[0].Repo.Name) assert.Equal(t, "user2/"+repoName, webhookData.payloads[0].Repo.FullName) - for _, runner := range runners { - task := runner.fetchTask(t) - if !allJobsAbandoned { + if !allJobsAbandoned { + for _, runner := range runners { + task := runner.fetchTask(t) runner.execTask(t, task, &mockTaskOutcome{ result: runnerv1.Result_RESULT_SUCCESS, })