From 7d36fa8188f5847be0e41b8d781a99652c4cceb9 Mon Sep 17 00:00:00 2001 From: Samuel Barabas Date: Wed, 26 Jan 2022 14:22:06 +0100 Subject: [PATCH 1/4] Don't attempt to delete busy runners --- .../lambdas/runners/src/gh-auth/gh-auth.ts | 1 + .../src/scale-runners/scale-down.test.ts | 6 ++++++ .../runners/src/scale-runners/scale-down.ts | 20 +++++++++++-------- 3 files changed, 19 insertions(+), 8 deletions(-) diff --git a/modules/runners/lambdas/runners/src/gh-auth/gh-auth.ts b/modules/runners/lambdas/runners/src/gh-auth/gh-auth.ts index 93ff477fe1..1c742ecef1 100644 --- a/modules/runners/lambdas/runners/src/gh-auth/gh-auth.ts +++ b/modules/runners/lambdas/runners/src/gh-auth/gh-auth.ts @@ -19,6 +19,7 @@ const logger = rootLogger.getChildLogger({ name: 'gh-auth' }); export async function createOctoClient(token: string, ghesApiUrl = ''): Promise { const ocktokitOptions: OctokitOptions = { auth: token, + log: console, }; if (ghesApiUrl) { ocktokitOptions.baseUrl = ghesApiUrl; diff --git a/modules/runners/lambdas/runners/src/scale-runners/scale-down.test.ts b/modules/runners/lambdas/runners/src/scale-runners/scale-down.test.ts index e17afd79ed..bff76bea53 100644 --- a/modules/runners/lambdas/runners/src/scale-runners/scale-down.test.ts +++ b/modules/runners/lambdas/runners/src/scale-runners/scale-down.test.ts @@ -149,26 +149,32 @@ const DEFAULT_REGISTERED_RUNNERS = [ { id: 101, name: 'i-idle-101', + busy: false, }, { id: 102, name: 'i-idle-102', + busy: false, }, { id: 103, name: 'i-oldest-idle-103', + busy: false, }, { id: 104, name: 'i-oldest-idle-104', + busy: false, }, { id: 105, name: 'i-running-105', + busy: false, }, { id: 106, name: 'i-running-106', + busy: false, }, ]; diff --git a/modules/runners/lambdas/runners/src/scale-runners/scale-down.ts b/modules/runners/lambdas/runners/src/scale-runners/scale-down.ts index 1c48af50e7..3b9a839d73 100644 --- a/modules/runners/lambdas/runners/src/scale-runners/scale-down.ts +++ b/modules/runners/lambdas/runners/src/scale-runners/scale-down.ts @@ -135,14 +135,18 @@ async function evaluateAndRemoveRunners( const ghRunners = await listGitHubRunners(ec2Runner); const ghRunner = ghRunners.find((runner) => runner.name === ec2Runner.instanceId); if (ghRunner) { - if (runnerMinimumTimeExceeded(ec2Runner)) { - if (idleCounter > 0) { - idleCounter--; - logger.info(`Runner '${ec2Runner.instanceId}' will kept idle.`, LogFields.print()); - } else { - logger.info(`Runner '${ec2Runner.instanceId}' will be terminated.`, LogFields.print()); - await removeRunner(ec2Runner, ghRunner.id); - } + if (!runnerMinimumTimeExceeded(ec2Runner)) { + idleCounter--; + console.debug(`Runner '${ec2Runner.instanceId}' will be kept - too young.`); + } else if (ghRunner.busy) { + idleCounter--; + console.debug(`Runner '${ec2Runner.instanceId}' will be kept - busy.`); + } else if (idleCounter > 0) { + idleCounter--; + console.debug(`Runner '${ec2Runner.instanceId}' will be kept - scale down config.`); + } else { + console.debug(`Runner '${ec2Runner.instanceId}' will be terminated.`); + await removeRunner(ec2Runner, ghRunner.id); } } else { if (bootTimeExceeded(ec2Runner)) { From 8088b0120a45f3c236bbe164d91ebd4e51259c9d Mon Sep 17 00:00:00 2001 From: Timo Ulich Date: Wed, 9 Mar 2022 08:17:40 +0100 Subject: [PATCH 2/4] fix idleCounter logic --- .../runners/src/scale-runners/scale-down.ts | 25 +++++++------------ 1 file changed, 9 insertions(+), 16 deletions(-) diff --git a/modules/runners/lambdas/runners/src/scale-runners/scale-down.ts b/modules/runners/lambdas/runners/src/scale-runners/scale-down.ts index 3b9a839d73..df4ee1fe97 100644 --- a/modules/runners/lambdas/runners/src/scale-runners/scale-down.ts +++ b/modules/runners/lambdas/runners/src/scale-runners/scale-down.ts @@ -111,10 +111,7 @@ async function removeRunner(ec2runner: RunnerInfo, ghRunnerId: number): Promise< logger.error(`Failed to de-register GitHub runner: ${result.status}`, LogFields.print()); } } catch (e) { - logger.info( - `Runner '${ec2runner.instanceId}' cannot be de-registered, most likely the runner is active.`, - LogFields.print(), - ); + logger.error(`Runner '${ec2runner.instanceId}' cannot be de-registered. Error: ${e}`, LogFields.print()); } } @@ -135,18 +132,14 @@ async function evaluateAndRemoveRunners( const ghRunners = await listGitHubRunners(ec2Runner); const ghRunner = ghRunners.find((runner) => runner.name === ec2Runner.instanceId); if (ghRunner) { - if (!runnerMinimumTimeExceeded(ec2Runner)) { - idleCounter--; - console.debug(`Runner '${ec2Runner.instanceId}' will be kept - too young.`); - } else if (ghRunner.busy) { - idleCounter--; - console.debug(`Runner '${ec2Runner.instanceId}' will be kept - busy.`); - } else if (idleCounter > 0) { - idleCounter--; - console.debug(`Runner '${ec2Runner.instanceId}' will be kept - scale down config.`); - } else { - console.debug(`Runner '${ec2Runner.instanceId}' will be terminated.`); - await removeRunner(ec2Runner, ghRunner.id); + if (!ghRunner.busy && runnerMinimumTimeExceeded(ec2Runner)) { + if (idleCounter > 0) { + idleCounter--; + logger.info(`Runner '${ec2Runner.instanceId}' will be kept idle.`, LogFields.print()); + } else { + logger.info(`Runner '${ec2Runner.instanceId}' will be terminated.`, LogFields.print()); + await removeRunner(ec2Runner, ghRunner.id); + } } } else { if (bootTimeExceeded(ec2Runner)) { From 41116af09897b16a2655422e1f51594ddaa5dd62 Mon Sep 17 00:00:00 2001 From: Timo Ulich Date: Wed, 9 Mar 2022 08:18:00 +0100 Subject: [PATCH 3/4] adjust tests to cover the new case --- .../runners/src/scale-runners/scale-down.test.ts | 16 +++++++++++++++- 1 file changed, 15 insertions(+), 1 deletion(-) diff --git a/modules/runners/lambdas/runners/src/scale-runners/scale-down.test.ts b/modules/runners/lambdas/runners/src/scale-runners/scale-down.test.ts index bff76bea53..45f8afd8b9 100644 --- a/modules/runners/lambdas/runners/src/scale-runners/scale-down.test.ts +++ b/modules/runners/lambdas/runners/src/scale-runners/scale-down.test.ts @@ -143,6 +143,14 @@ const DEFAULT_RUNNERS_ORIGINAL = [ launchTime: moment(new Date()).toDate(), repo: `${TEST_DATA.repositoryOwner}/${TEST_DATA.repositoryName}`, }, + { + instanceId: 'i-busy-112', + launchTime: moment(new Date()) + .subtract(minimumRunningTimeInMinutes + 27, 'minutes') + .toDate(), + type: 'Org', + owner: TEST_DATA.repositoryOwner, + }, ]; const DEFAULT_REGISTERED_RUNNERS = [ @@ -176,6 +184,11 @@ const DEFAULT_REGISTERED_RUNNERS = [ name: 'i-running-106', busy: false, }, + { + id: 112, + name: 'i-busy-112', + busy: true, + }, ]; describe('scaleDown', () => { @@ -266,7 +279,8 @@ describe('scaleDown', () => { ); RUNNERS_ALL_REMOVED = DEFAULT_RUNNERS_ORG.filter( - (r) => !r.instanceId.includes('running') && !r.instanceId.includes('registered'), + (r) => + !r.instanceId.includes('running') && !r.instanceId.includes('registered') && !r.instanceId.includes('busy'), ); DEFAULT_RUNNERS_ORPHANED = DEFAULT_RUNNERS_ORIGINAL.filter( (r) => r.instanceId.includes('orphan') && !r.instanceId.includes('not-registered'), From 34d8ef97e276c99eb72072909d64caacfa608dcf Mon Sep 17 00:00:00 2001 From: Timo Ulich Date: Wed, 9 Mar 2022 08:27:01 +0100 Subject: [PATCH 4/4] remove debugging code --- modules/runners/lambdas/runners/src/gh-auth/gh-auth.ts | 1 - 1 file changed, 1 deletion(-) diff --git a/modules/runners/lambdas/runners/src/gh-auth/gh-auth.ts b/modules/runners/lambdas/runners/src/gh-auth/gh-auth.ts index 1c742ecef1..93ff477fe1 100644 --- a/modules/runners/lambdas/runners/src/gh-auth/gh-auth.ts +++ b/modules/runners/lambdas/runners/src/gh-auth/gh-auth.ts @@ -19,7 +19,6 @@ const logger = rootLogger.getChildLogger({ name: 'gh-auth' }); export async function createOctoClient(token: string, ghesApiUrl = ''): Promise { const ocktokitOptions: OctokitOptions = { auth: token, - log: console, }; if (ghesApiUrl) { ocktokitOptions.baseUrl = ghesApiUrl;