From f360cb5b1ebd87a1def58a24695893ad1ffb1bf0 Mon Sep 17 00:00:00 2001 From: Guilherme Caulada Date: Wed, 23 Sep 2026 11:35:29 -0300 Subject: [PATCH] feat(termination-watcher): support additional GitHub Apps --- .../src/deregister.test.ts | 25 +++ .../termination-watcher/src/deregister.ts | 120 +------------ .../src/github-app-client.test.ts | 161 +++++++++++++++++ .../src/github-app-client.ts | 167 ++++++++++++++++++ modules/multi-runner/termination-watcher.tf | 6 +- modules/termination-watcher/README.md | 4 +- .../termination-watcher/deregister-retry.tf | 2 +- modules/termination-watcher/main.tf | 6 +- .../termination-watcher/notification/main.tf | 2 +- .../termination-watcher/termination/main.tf | 2 +- .../tests/multi-app.tftest.hcl | 78 ++++++++ modules/termination-watcher/variables.tf | 8 +- 12 files changed, 455 insertions(+), 126 deletions(-) create mode 100644 lambdas/functions/termination-watcher/src/github-app-client.test.ts create mode 100644 lambdas/functions/termination-watcher/src/github-app-client.ts create mode 100644 modules/termination-watcher/tests/multi-app.tftest.hcl diff --git a/lambdas/functions/termination-watcher/src/deregister.test.ts b/lambdas/functions/termination-watcher/src/deregister.test.ts index 45585d3cb7..05ca8403c7 100644 --- a/lambdas/functions/termination-watcher/src/deregister.test.ts +++ b/lambdas/functions/termination-watcher/src/deregister.test.ts @@ -34,8 +34,11 @@ const mockApps = { getRepoInstallation: vi.fn(), }; +const mockHookAfter = vi.fn(); + function MockOctokit() { return { + hook: { after: mockHookAfter }, actions: mockActions, apps: mockApps, paginate: mockPaginate, @@ -114,6 +117,28 @@ describe('deregisterRunner', () => { setupAuthMocks(); }); + it('deregisters through an additional App when it is selected', async () => { + mockGetCredentials.mockResolvedValue([ + { appId: 1, privateKey: 'one' }, + { appId: 2, privateKey: 'two' }, + ]); + mockApps.getOrgInstallation.mockResolvedValue({ data: { id: 222 } }); + mockPaginate.iterator.mockReturnValue( + (async function* () { + yield { data: [{ id: 42, name: `runner-${orgInstance.InstanceId}` }] }; + })(), + ); + mockActions.deleteSelfHostedRunnerFromOrg.mockResolvedValue({}); + const random = vi.spyOn(Math, 'random').mockReturnValue(0.99); + try { + await deregisterRunner(orgInstance, baseConfig); + expect(mockCreateAppAuth).toHaveBeenCalledWith({ appId: 2, privateKey: 'two', installationId: 222 }); + expect(mockActions.deleteSelfHostedRunnerFromOrg).toHaveBeenCalledWith({ org: 'test-org', runner_id: 42 }); + } finally { + random.mockRestore(); + } + }); + it('should skip deregistration when disabled', async () => { await deregisterRunner(orgInstance, { ...baseConfig, enableRunnerDeregistration: false }); expect(mockGetCredentials).not.toHaveBeenCalled(); diff --git a/lambdas/functions/termination-watcher/src/deregister.ts b/lambdas/functions/termination-watcher/src/deregister.ts index 56cc067828..8b5fa93696 100644 --- a/lambdas/functions/termination-watcher/src/deregister.ts +++ b/lambdas/functions/termination-watcher/src/deregister.ts @@ -1,13 +1,10 @@ -import { createAppAuth } from '@octokit/auth-app'; -import { Octokit } from '@octokit/rest'; -import { throttling } from '@octokit/plugin-throttling'; -import { request } from '@octokit/request'; +import type { Octokit } from '@octokit/rest'; import { Instance } from '@aws-sdk/client-ec2'; import { SQSClient, SendMessageCommand } from '@aws-sdk/client-sqs'; import { createChildLogger } from '@aws-github-runner/aws-powertools-util'; -import { createCommonStorage, type GitHubAppCredential } from '@aws-github-runner/storage-providers'; -import type { EndpointDefaults } from '@octokit/types'; import type { Config } from './ConfigResolver'; +import { createRunnerInstallationClient } from './github-app-client'; +export { createThrottleOptions, resetAppCredentialsCache } from './github-app-client'; export interface DeregisterRetryMessage { instanceId: string; @@ -21,73 +18,6 @@ const sqsClient = new SQSClient({ region: process.env.AWS_REGION }); const logger = createChildLogger('deregister'); -let appCredentialsPromise: Promise | undefined; - -export function createThrottleOptions() { - return { - onRateLimit: (_retryAfter: number, options: Required) => { - logger.warn(`Rate limit hit for ${options.method} ${options.url}`); - return false; - }, - onSecondaryRateLimit: (_retryAfter: number, options: Required) => { - logger.warn(`Secondary rate limit hit for ${options.method} ${options.url}`); - return false; - }, - }; -} - -async function loadAppCredentials(): Promise { - const credentials = await createCommonStorage().githubAppCredentials.get(); - const credential = credentials[0]; - if (!credential) { - throw new Error('No GitHub App credentials found'); - } - return credential; -} - -function getAppCredentials(): Promise { - if (!appCredentialsPromise) { - appCredentialsPromise = loadAppCredentials().catch((error: unknown) => { - appCredentialsPromise = undefined; - throw error; - }); - } - return appCredentialsPromise; -} - -export function resetAppCredentialsCache(): void { - appCredentialsPromise = undefined; -} - -function createOctokitInstance(token: string, ghesApiUrl: string): Octokit { - const CustomOctokit = Octokit.plugin(throttling); - const octokitOptions: ConstructorParameters[0] = { - auth: token, - }; - if (ghesApiUrl) { - octokitOptions.baseUrl = ghesApiUrl; - } - return new CustomOctokit({ - ...octokitOptions, - userAgent: 'github-aws-runners-termination-watcher', - throttle: createThrottleOptions(), - }); -} - -async function createAuthenticatedClient(ghesApiUrl: string): Promise { - const { appId, privateKey } = await getAppCredentials(); - const authOptions: { appId: number; privateKey: string; request?: typeof request } = { - appId, - privateKey, - }; - if (ghesApiUrl) { - authOptions.request = request.defaults({ baseUrl: ghesApiUrl }); - } - const auth = createAppAuth(authOptions); - const appAuth = await auth({ type: 'app' }); - return createOctokitInstance(appAuth.token, ghesApiUrl); -} - function getOwnerFromTags(instance: Instance): string | undefined { return instance.Tags?.find((tag) => tag.Key === 'ghr:Owner')?.Value; } @@ -96,44 +26,6 @@ function getRunnerTypeFromTags(instance: Instance): string | undefined { return instance.Tags?.find((tag) => tag.Key === 'ghr:Type')?.Value; } -async function getInstallationId(octokit: Octokit, owner: string): Promise { - const { data: installation } = await octokit.apps.getOrgInstallation({ org: owner }); - return installation.id; -} - -async function getInstallationIdForRepo(octokit: Octokit, owner: string, repo: string): Promise { - const { data: installation } = await octokit.apps.getRepoInstallation({ owner, repo }); - return installation.id; -} - -async function createInstallationClient( - appOctokit: Octokit, - owner: string, - runnerType: string, - ghesApiUrl: string, -): Promise { - let installationId: number; - if (runnerType === 'Repo') { - const [repoOwner, repo] = owner.split('/'); - installationId = await getInstallationIdForRepo(appOctokit, repoOwner, repo); - } else { - installationId = await getInstallationId(appOctokit, owner); - } - - const { appId, privateKey } = await getAppCredentials(); - const authOptions: { appId: number; privateKey: string; installationId: number; request?: typeof request } = { - appId, - privateKey, - installationId, - }; - if (ghesApiUrl) { - authOptions.request = request.defaults({ baseUrl: ghesApiUrl }); - } - const auth = createAppAuth(authOptions); - const installationAuth = await auth({ type: 'installation' }); - return createOctokitInstance(installationAuth.token, ghesApiUrl); -} - async function findRunnerByInstanceId( octokit: Octokit, owner: string, @@ -206,8 +98,7 @@ export async function deregisterRunner(instance: Instance, config: Config): Prom try { logger.info('Attempting to deregister runner from GitHub', { instanceId, owner, runnerType }); - const appOctokit = await createAuthenticatedClient(config.ghesApiUrl); - const installationOctokit = await createInstallationClient(appOctokit, owner, runnerType, config.ghesApiUrl); + const installationOctokit = await createRunnerInstallationClient(owner, runnerType, config.ghesApiUrl); const runner = await findRunnerByInstanceId(installationOctokit, owner, instanceId, runnerType); if (!runner) { @@ -261,8 +152,7 @@ export async function handleDeregisterRetry(queueUrl: string, message: Deregiste logger.info('Processing deregistration retry from SQS', { instanceId, owner, runnerType, retryCount }); try { - const appOctokit = await createAuthenticatedClient(''); - const installationOctokit = await createInstallationClient(appOctokit, owner, runnerType, ''); + const installationOctokit = await createRunnerInstallationClient(owner, runnerType, process.env.GHES_URL ?? ''); const runner = await findRunnerByInstanceId(installationOctokit, owner, instanceId, runnerType); if (!runner) { diff --git a/lambdas/functions/termination-watcher/src/github-app-client.test.ts b/lambdas/functions/termination-watcher/src/github-app-client.test.ts new file mode 100644 index 0000000000..17d278221e --- /dev/null +++ b/lambdas/functions/termination-watcher/src/github-app-client.test.ts @@ -0,0 +1,161 @@ +import { createCommonStorage, type GitHubAppCredentialsStore } from '@aws-github-runner/storage-providers'; +import { describe, it, expect, beforeEach, vi } from 'vitest'; +import { createThrottleOptions, resetAppCredentialsCache, createRunnerInstallationClient } from './github-app-client'; +import type { EndpointDefaults } from '@octokit/types'; + +vi.mock('@aws-github-runner/storage-providers', () => ({ + createCommonStorage: vi.fn(), +})); + +const mockedCreateCommonStorage = vi.mocked(createCommonStorage); +const mockGetCredentials = vi.fn(); +const credentialsStore = { get: mockGetCredentials } satisfies GitHubAppCredentialsStore; + +const mockCreateAppAuth = vi.fn(); +vi.mock('@octokit/auth-app', () => ({ + createAppAuth: (...args: unknown[]) => mockCreateAppAuth(...args), +})); + +const mockPaginate = { + iterator: vi.fn(), +}; + +const mockActions = { + listSelfHostedRunnersForOrg: vi.fn(), + listSelfHostedRunnersForRepo: vi.fn(), + deleteSelfHostedRunnerFromOrg: vi.fn(), + deleteSelfHostedRunnerFromRepo: vi.fn(), +}; + +const mockApps = { + getOrgInstallation: vi.fn(), + getRepoInstallation: vi.fn(), +}; + +const mockHookAfter = vi.fn(); + +function MockOctokit() { + return { + hook: { after: mockHookAfter }, + actions: mockActions, + apps: mockApps, + paginate: mockPaginate, + }; +} +MockOctokit.plugin = vi.fn().mockReturnValue(MockOctokit); + +vi.mock('@octokit/rest', () => ({ + Octokit: MockOctokit, +})); + +vi.mock('@octokit/plugin-throttling', () => ({ + throttling: vi.fn(), +})); + +vi.mock('@octokit/request', () => ({ + request: { + defaults: vi.fn().mockReturnValue(vi.fn()), + }, +})); + +function setupAuthMocks() { + mockGetCredentials.mockResolvedValue([{ appId: 12345, privateKey: 'fake-private-key' }]); + + // App auth returns app token + const mockAuth = vi.fn(); + mockAuth.mockImplementation((opts: { type: string }) => { + if (opts.type === 'app') { + return Promise.resolve({ token: 'app-token' }); + } + return Promise.resolve({ token: 'installation-token' }); + }); + mockCreateAppAuth.mockReturnValue(mockAuth); +} + +describe('multi-App installation clients', () => { + beforeEach(() => { + vi.clearAllMocks(); + resetAppCredentialsCache(); + mockedCreateCommonStorage.mockReturnValue({ githubAppCredentials: credentialsStore }); + setupAuthMocks(); + }); + + it('creates an organization installation client for scheduled registration cleanup', async () => { + mockApps.getOrgInstallation.mockResolvedValue({ data: { id: 999 } }); + const client = await createRunnerInstallationClient('test-org', 'Org', ''); + expect(client.actions).toBe(mockActions); + expect(mockApps.getOrgInstallation).toHaveBeenCalledWith({ org: 'test-org' }); + expect(mockCreateAppAuth).toHaveBeenCalledWith({ + appId: 12345, + privateKey: 'fake-private-key', + installationId: 999, + }); + }); + + it('uses additional credentials and keeps App and installation auth paired', async () => { + mockGetCredentials.mockResolvedValue([ + { appId: 1, privateKey: 'one' }, + { appId: 2, privateKey: 'two' }, + ]); + mockApps.getOrgInstallation.mockResolvedValue({ data: { id: 222 } }); + const random = vi.spyOn(Math, 'random').mockReturnValue(0.99); + try { + await createRunnerInstallationClient('test-org', 'Org', ''); + expect(mockCreateAppAuth).toHaveBeenNthCalledWith(1, { appId: 2, privateKey: 'two' }); + expect(mockCreateAppAuth).toHaveBeenNthCalledWith(2, { appId: 2, privateKey: 'two', installationId: 222 }); + } finally { + random.mockRestore(); + } + }); + + it('selects another App after an exhausted installation response', async () => { + mockGetCredentials.mockResolvedValue([ + { appId: 1, privateKey: 'one' }, + { appId: 2, privateKey: 'two' }, + ]); + mockApps.getOrgInstallation.mockResolvedValue({ data: { id: 222 } }); + const random = vi.spyOn(Math, 'random').mockReturnValue(0); + try { + await createRunnerInstallationClient('test-org', 'Org', ''); + mockHookAfter.mock.calls[0][1]({ headers: { 'x-ratelimit-remaining': '0' } }); + mockCreateAppAuth.mockClear(); + await createRunnerInstallationClient('test-org', 'Org', ''); + expect(mockCreateAppAuth).toHaveBeenNthCalledWith(1, { appId: 2, privateKey: 'two' }); + } finally { + random.mockRestore(); + } + }); + + it('skips an App in secondary-limit cooldown', async () => { + mockGetCredentials.mockResolvedValue([ + { appId: 1, privateKey: 'one' }, + { appId: 2, privateKey: 'two' }, + ]); + mockApps.getOrgInstallation.mockResolvedValue({ data: { id: 222 } }); + createThrottleOptions(1).onSecondaryRateLimit(60, { method: 'GET', url: '/runners' } as Required); + const random = vi.spyOn(Math, 'random').mockReturnValue(0); + try { + await createRunnerInstallationClient('test-org', 'Org', ''); + expect(mockCreateAppAuth).toHaveBeenNthCalledWith(1, { appId: 2, privateKey: 'two' }); + } finally { + random.mockRestore(); + } + }); + + it('tries another configured App when installation authentication fails', async () => { + mockGetCredentials.mockResolvedValue([ + { appId: 1, privateKey: 'one' }, + { appId: 2, privateKey: 'two' }, + ]); + mockApps.getOrgInstallation + .mockRejectedValueOnce(new Error('not installed')) + .mockResolvedValue({ data: { id: 222 } }); + const random = vi.spyOn(Math, 'random').mockReturnValue(0); + try { + await createRunnerInstallationClient('test-org', 'Org', ''); + expect(mockCreateAppAuth).toHaveBeenLastCalledWith({ appId: 2, privateKey: 'two', installationId: 222 }); + } finally { + random.mockRestore(); + } + }); +}); diff --git a/lambdas/functions/termination-watcher/src/github-app-client.ts b/lambdas/functions/termination-watcher/src/github-app-client.ts new file mode 100644 index 0000000000..ab10f73643 --- /dev/null +++ b/lambdas/functions/termination-watcher/src/github-app-client.ts @@ -0,0 +1,167 @@ +import { createAppAuth } from '@octokit/auth-app'; +import { Octokit } from '@octokit/rest'; +import { throttling } from '@octokit/plugin-throttling'; +import { request } from '@octokit/request'; +import { createChildLogger } from '@aws-github-runner/aws-powertools-util'; +import { createCommonStorage, type GitHubAppCredential } from '@aws-github-runner/storage-providers'; +import type { EndpointDefaults } from '@octokit/types'; + +const logger = createChildLogger('github-app-client'); + +let appCredentialsPromise: Promise | undefined; + +const appBudgets = new Map(); + +function coolDown(appId?: number): void { + if (appId !== undefined) appBudgets.set(appId, { remaining: 0, cooldownUntil: Date.now() + 60000 }); +} + +function selectCredential(credentials: GitHubAppCredential[]): GitHubAppCredential { + const offset = Math.floor(Math.random() * credentials.length); + const rotated = [...credentials.slice(offset), ...credentials.slice(0, offset)]; + const available = rotated.filter( + (credential) => (appBudgets.get(credential.appId)?.cooldownUntil ?? 0) <= Date.now(), + ); + return (available.length ? available : rotated).reduce((best, credential) => + (appBudgets.get(credential.appId)?.remaining ?? Infinity) > (appBudgets.get(best.appId)?.remaining ?? Infinity) + ? credential + : best, + ); +} + +export function createThrottleOptions(appId?: number) { + return { + onRateLimit: (_retryAfter: number, options: Required) => { + coolDown(appId); + logger.warn(`Rate limit hit for ${options.method} ${options.url}`); + return false; + }, + onSecondaryRateLimit: (_retryAfter: number, options: Required) => { + coolDown(appId); + logger.warn(`Secondary rate limit hit for ${options.method} ${options.url}`); + return false; + }, + }; +} + +async function loadAppCredentials(): Promise { + const credentials = await createCommonStorage().githubAppCredentials.get(); + if (credentials.length === 0) { + throw new Error('No GitHub App credentials found'); + } + return credentials; +} + +function getAppCredentials(): Promise { + if (!appCredentialsPromise) { + appCredentialsPromise = loadAppCredentials().catch((error: unknown) => { + appCredentialsPromise = undefined; + throw error; + }); + } + return appCredentialsPromise; +} + +export function resetAppCredentialsCache(): void { + appCredentialsPromise = undefined; + appBudgets.clear(); +} + +function createOctokitInstance(token: string, ghesApiUrl: string, appId?: number): Octokit { + const CustomOctokit = Octokit.plugin(throttling); + const octokitOptions: ConstructorParameters[0] = { + auth: token, + }; + if (ghesApiUrl) { + octokitOptions.baseUrl = ghesApiUrl; + } + const client = new CustomOctokit({ + ...octokitOptions, + userAgent: 'github-aws-runners-termination-watcher', + throttle: createThrottleOptions(appId), + }); + if (appId !== undefined) + client.hook.after('request', (response) => { + const remaining = Number.parseInt(String(response.headers['x-ratelimit-remaining']), 10); + if (Number.isFinite(remaining)) + appBudgets.set(appId, { + remaining, + cooldownUntil: remaining === 0 ? Date.now() + 60000 : 0, + }); + }); + return client; +} + +async function createAuthenticatedClient(ghesApiUrl: string, credential: GitHubAppCredential): Promise { + const { appId, privateKey } = credential; + const authOptions: { appId: number; privateKey: string; request?: typeof request } = { + appId, + privateKey, + }; + if (ghesApiUrl) { + authOptions.request = request.defaults({ baseUrl: ghesApiUrl }); + } + const auth = createAppAuth(authOptions); + const appAuth = await auth({ type: 'app' }); + return createOctokitInstance(appAuth.token, ghesApiUrl); +} + +async function getInstallationId(octokit: Octokit, owner: string): Promise { + const { data: installation } = await octokit.apps.getOrgInstallation({ org: owner }); + return installation.id; +} + +async function getInstallationIdForRepo(octokit: Octokit, owner: string, repo: string): Promise { + const { data: installation } = await octokit.apps.getRepoInstallation({ owner, repo }); + return installation.id; +} + +async function createInstallationClient( + appOctokit: Octokit, + owner: string, + runnerType: string, + ghesApiUrl: string, + credential: GitHubAppCredential, +): Promise { + let installationId: number; + if (runnerType === 'Repo') { + const [repoOwner, repo] = owner.split('/'); + installationId = await getInstallationIdForRepo(appOctokit, repoOwner, repo); + } else { + installationId = await getInstallationId(appOctokit, owner); + } + + const { appId, privateKey } = credential; + const authOptions: { appId: number; privateKey: string; installationId: number; request?: typeof request } = { + appId, + privateKey, + installationId, + }; + if (ghesApiUrl) { + authOptions.request = request.defaults({ baseUrl: ghesApiUrl }); + } + const auth = createAppAuth(authOptions); + const installationAuth = await auth({ type: 'installation' }); + return createOctokitInstance(installationAuth.token, ghesApiUrl, appId); +} + +export async function createRunnerInstallationClient( + owner: string, + runnerType: string, + ghesApiUrl: string, +): Promise { + const remaining = [...(await getAppCredentials())]; + while (remaining.length) { + const credential = selectCredential(remaining); + remaining.splice(remaining.indexOf(credential), 1); + try { + const appClient = await createAuthenticatedClient(ghesApiUrl, credential); + return await createInstallationClient(appClient, owner, runnerType, ghesApiUrl, credential); + } catch (error) { + coolDown(credential.appId); + if (!remaining.length) throw error; + logger.warn('GitHub App authentication failed; trying another configured app', { appId: credential.appId }); + } + } + throw new Error('No GitHub App credentials found'); +} diff --git a/modules/multi-runner/termination-watcher.tf b/modules/multi-runner/termination-watcher.tf index 45e30373f7..bd998f9fe1 100644 --- a/modules/multi-runner/termination-watcher.tf +++ b/modules/multi-runner/termination-watcher.tf @@ -35,8 +35,10 @@ locals { } enable_runner_deregistration = local.effective_config.compute_provider.aws.ec2.instance_termination_watcher.features.runner_deregistration.enabled github_app_parameters = local.effective_config.compute_provider.aws.ec2.instance_termination_watcher.features.runner_deregistration.enabled ? { - id = local.github_app_parameters.id - key_base64 = local.github_app_parameters.key_base64 + id = local.github_app_parameters.id + key_base64 = local.github_app_parameters.key_base64 + additional_apps_manifest = local.github_app_parameters.additional_apps_manifest + additional_app_parameter_arns = local.github_app_parameters.additional_app_parameter_arns } : null ghes_url = local.effective_config.github.enterprise_server.url environment_variables = local.effective_config.compute_provider.aws.ec2.instance_termination_watcher.environment_variables diff --git a/modules/termination-watcher/README.md b/modules/termination-watcher/README.md index 89a488658a..4a450cbee6 100644 --- a/modules/termination-watcher/README.md +++ b/modules/termination-watcher/README.md @@ -55,6 +55,8 @@ To compile all TypeScript/JavaScript sources in a single file [ncc](https://gith yarn run dist ``` +Deregistration and SQS retries can use additional GitHub Apps through `config.github_app_parameters.additional_apps_manifest` and `additional_app_parameter_arns`. The multi-runner module passes its configured additional Apps automatically. Selection uses observed installation quota, with a one-minute cooldown after throttling and fallback to another App when authentication fails. Each cleanup operation keeps App and installation authentication paired; existing single-App configurations remain supported. + ## Requirements @@ -94,7 +96,7 @@ yarn run dist | Name | Description | Type | Default | Required | |------|-------------|------|---------|:--------:| -| [config](#input\_config) | Configuration for the spot termination watcher.

`aws_partition`: Partition for the base arn if not 'aws'
`architecture`: AWS Lambda architecture. Lambda functions using Graviton processors ('arm64') tend to have better price/performance than 'x86\_64' functions.
`environment_variables`: Environment variables for the lambda.
'features': Features to enable the different lambda functions to handle spot termination events.
`lambda_principals`: Add extra principals to the role created for execution of the lambda, e.g. for local testing.
`lambda_tags`: Map of tags that will be added to created resources. By default resources will be tagged with name and environment.
`log_level`: Logging level for lambda logging. Valid values are 'silly', 'trace', 'debug', 'info', 'warn', 'error', 'fatal'.
`log_class`: The log class of the CloudWatch log group. Valid values are `STANDARD` or `INFREQUENT_ACCESS`.
`logging_kms_key_id`: Specifies the kms key id to encrypt the logs with
`logging_retention_in_days`: Specifies the number of days you want to retain log events for the lambda log group. Possible values are: 0, 1, 3, 5, 7, 14, 30, 60, 90, 120, 150, 180, 365, 400, 545, 731, 1827, and 3653.
`memory_size`: Memory size limit in MB of the lambda.
`prefix`: The prefix used for naming resources.
`role_path`: The path that will be added to the role, if not set the environment name will be used.
`role_permissions_boundary`: Permissions boundary that will be added to the created role for the lambda.
`runtime`: AWS Lambda runtime.
`s3_bucket`: S3 bucket from which to specify lambda functions. This is an alternative to providing local files directly.
`s3_key`: S3 key for syncer lambda function. Required if using S3 bucket to specify lambdas.
`s3_object_version`: S3 object version for syncer lambda function. Useful if S3 versioning is enabled on source bucket.
`security_group_ids`: List of security group IDs associated with the Lambda function.
`subnet_ids`: List of subnets in which the action runners will be launched, the subnets needs to be subnets in the `vpc_id`.
`tag_filters`: Map of tags that will be used to filter the resources to be tracked. Only for which all tags are present and starting with the same value as the value in the map will be tracked.
`tags`: Map of tags that will be added to created resources. By default resources will be tagged with name and environment.
`timeout`: Time out of the lambda in seconds.
`tracing_config`: Configuration for lambda tracing.
`zip`: File location of the lambda zip file.
`enable_runner_deregistration`: Enable or disable deregistering the runner from GitHub when its EC2 instance is terminated.
`github_app_parameters`: GitHub App SSM parameters (`id` and `key_base64`, each a map of `arn`/`name`) used to authenticate to GitHub when deregistering runners.
`ghes_url`: GitHub Enterprise Server URL used to target the GHES API when deregistering runners. Leave `null` for github.com. | 
object({
    aws_partition         = optional(string, null)
    architecture          = optional(string, null)
    environment_variables = optional(map(string), {})
    features = optional(object({
      enable_spot_termination_handler              = optional(bool, true)
      enable_spot_termination_notification_watcher = optional(bool, true)
    }), {})
    lambda_tags               = optional(map(string), {})
    log_level                 = optional(string, null)
    log_class                 = optional(string, "STANDARD")
    logging_kms_key_id        = optional(string, null)
    logging_retention_in_days = optional(number, null)
    memory_size               = optional(number, null)
    metrics = optional(object({
      enable    = optional(bool, false)
      namespace = optional(string, "GitHub Runners")
      metric = optional(object({
        enable_spot_termination         = optional(bool, true)
        enable_spot_termination_warning = optional(bool, true)
      }), {})
    }), {})
    prefix = optional(string, null)
    principals = optional(list(object({
      type        = string
      identifiers = list(string)
    })), [])
    role_path                 = optional(string, null)
    role_permissions_boundary = optional(string, null)
    runtime                   = optional(string, null)
    s3_bucket                 = optional(string, null)
    s3_key                    = optional(string, null)
    s3_object_version         = optional(string, null)
    security_group_ids        = optional(list(string), [])
    subnet_ids                = optional(list(string), [])
    tag_filters               = optional(map(string), null)
    tags                      = optional(map(string), {})
    timeout                   = optional(number, null)
    tracing_config = optional(object({
      mode                  = optional(string, null)
      capture_http_requests = optional(bool, false)
      capture_error         = optional(bool, false)
    }), {})
    zip                          = optional(string, null)
    enable_runner_deregistration = optional(bool, false)
    github_app_parameters = optional(object({
      id         = map(string)
      key_base64 = map(string)
    }), null)
    ghes_url = optional(string, null)
  })
| n/a | yes | +| [config](#input\_config) | Configuration for the spot termination watcher.

`aws_partition`: Partition for the base arn if not 'aws'
`architecture`: AWS Lambda architecture. Lambda functions using Graviton processors ('arm64') tend to have better price/performance than 'x86\_64' functions.
`environment_variables`: Environment variables for the lambda.
'features': Features to enable the different lambda functions to handle spot termination events.
`lambda_principals`: Add extra principals to the role created for execution of the lambda, e.g. for local testing.
`lambda_tags`: Map of tags that will be added to created resources. By default resources will be tagged with name and environment.
`log_level`: Logging level for lambda logging. Valid values are 'silly', 'trace', 'debug', 'info', 'warn', 'error', 'fatal'.
`log_class`: The log class of the CloudWatch log group. Valid values are `STANDARD` or `INFREQUENT_ACCESS`.
`logging_kms_key_id`: Specifies the kms key id to encrypt the logs with
`logging_retention_in_days`: Specifies the number of days you want to retain log events for the lambda log group. Possible values are: 0, 1, 3, 5, 7, 14, 30, 60, 90, 120, 150, 180, 365, 400, 545, 731, 1827, and 3653.
`memory_size`: Memory size limit in MB of the lambda.
`prefix`: The prefix used for naming resources.
`role_path`: The path that will be added to the role, if not set the environment name will be used.
`role_permissions_boundary`: Permissions boundary that will be added to the created role for the lambda.
`runtime`: AWS Lambda runtime.
`s3_bucket`: S3 bucket from which to specify lambda functions. This is an alternative to providing local files directly.
`s3_key`: S3 key for syncer lambda function. Required if using S3 bucket to specify lambdas.
`s3_object_version`: S3 object version for syncer lambda function. Useful if S3 versioning is enabled on source bucket.
`security_group_ids`: List of security group IDs associated with the Lambda function.
`subnet_ids`: List of subnets in which the action runners will be launched, the subnets needs to be subnets in the `vpc_id`.
`tag_filters`: Map of tags that will be used to filter the resources to be tracked. Only for which all tags are present and starting with the same value as the value in the map will be tracked.
`tags`: Map of tags that will be added to created resources. By default resources will be tagged with name and environment.
`timeout`: Time out of the lambda in seconds.
`tracing_config`: Configuration for lambda tracing.
`zip`: File location of the lambda zip file.
`enable_runner_deregistration`: Enable or disable deregistering the runner from GitHub when its EC2 instance is terminated.
`github_app_parameters`: GitHub App SSM parameters (`id` and `key_base64`, each a map of `arn`/`name`) used to authenticate to GitHub when deregistering runners. Optional `additional_apps_manifest` (`name`/`arn`) and `additional_app_parameter_arns` enable credentials from the multi-App manifest.
`ghes_url`: GitHub Enterprise Server URL used to target the GHES API when deregistering runners. Leave `null` for github.com. | 
object({
    aws_partition         = optional(string, null)
    architecture          = optional(string, null)
    environment_variables = optional(map(string), {})
    features = optional(object({
      enable_spot_termination_handler              = optional(bool, true)
      enable_spot_termination_notification_watcher = optional(bool, true)
    }), {})
    lambda_tags               = optional(map(string), {})
    log_level                 = optional(string, null)
    log_class                 = optional(string, "STANDARD")
    logging_kms_key_id        = optional(string, null)
    logging_retention_in_days = optional(number, null)
    memory_size               = optional(number, null)
    metrics = optional(object({
      enable    = optional(bool, false)
      namespace = optional(string, "GitHub Runners")
      metric = optional(object({
        enable_spot_termination         = optional(bool, true)
        enable_spot_termination_warning = optional(bool, true)
      }), {})
    }), {})
    prefix = optional(string, null)
    principals = optional(list(object({
      type        = string
      identifiers = list(string)
    })), [])
    role_path                 = optional(string, null)
    role_permissions_boundary = optional(string, null)
    runtime                   = optional(string, null)
    s3_bucket                 = optional(string, null)
    s3_key                    = optional(string, null)
    s3_object_version         = optional(string, null)
    security_group_ids        = optional(list(string), [])
    subnet_ids                = optional(list(string), [])
    tag_filters               = optional(map(string), null)
    tags                      = optional(map(string), {})
    timeout                   = optional(number, null)
    tracing_config = optional(object({
      mode                  = optional(string, null)
      capture_http_requests = optional(bool, false)
      capture_error         = optional(bool, false)
    }), {})
    zip                          = optional(string, null)
    enable_runner_deregistration = optional(bool, false)
    github_app_parameters = optional(object({
      id                            = map(string)
      key_base64                    = map(string)
      additional_apps_manifest      = optional(object({ name = string, arn = string }))
      additional_app_parameter_arns = optional(list(string), [])
    }), null)
    ghes_url = optional(string, null)
  })
| n/a | yes | ## Outputs diff --git a/modules/termination-watcher/deregister-retry.tf b/modules/termination-watcher/deregister-retry.tf index 921d7abf5e..b8c2cf9b19 100644 --- a/modules/termination-watcher/deregister-retry.tf +++ b/modules/termination-watcher/deregister-retry.tf @@ -99,7 +99,7 @@ resource "aws_iam_role_policy" "deregister_retry_ssm" { Statement = [ { Effect = "Allow" - Action = ["ssm:GetParameter"] + Action = ["ssm:GetParameter", "ssm:GetParameters"] Resource = local.ssm_parameter_arns } ] diff --git a/modules/termination-watcher/main.tf b/modules/termination-watcher/main.tf index 919ba3a3e5..a05d3ecb74 100644 --- a/modules/termination-watcher/main.tf +++ b/modules/termination-watcher/main.tf @@ -9,14 +9,16 @@ locals { PARAMETER_GITHUB_APP_ID_NAME = var.config.github_app_parameters.id.name PARAMETER_GITHUB_APP_KEY_BASE64_NAME = var.config.github_app_parameters.key_base64.name GHES_URL = var.config.ghes_url != null ? var.config.ghes_url : "" + }, var.config.github_app_parameters.additional_apps_manifest == null ? {} : { + PARAMETER_GITHUB_APPS_MANIFEST_NAME = var.config.github_app_parameters.additional_apps_manifest.name }, length(aws_sqs_queue.deregister_retry) > 0 ? { DEREGISTER_RETRY_QUEUE_URL = aws_sqs_queue.deregister_retry[0].url } : {}) : {} - ssm_parameter_arns = local.enable_runner_deregistration ? [ + ssm_parameter_arns = local.enable_runner_deregistration ? concat([ var.config.github_app_parameters.id.arn, var.config.github_app_parameters.key_base64.arn, - ] : [] + ], var.config.github_app_parameters.additional_apps_manifest == null ? [] : [var.config.github_app_parameters.additional_apps_manifest.arn], var.config.github_app_parameters.additional_app_parameter_arns) : [] environment_variables = { ENABLE_METRICS_SPOT_WARNING = var.config.metrics != null ? var.config.metrics.enable && var.config.metrics.metric.enable_spot_termination_warning : false diff --git a/modules/termination-watcher/notification/main.tf b/modules/termination-watcher/notification/main.tf index 735c34126b..5fca0589d1 100644 --- a/modules/termination-watcher/notification/main.tf +++ b/modules/termination-watcher/notification/main.tf @@ -100,7 +100,7 @@ resource "aws_iam_role_policy" "ssm_policy" { Statement = [ { Effect = "Allow" - Action = ["ssm:GetParameter"] + Action = ["ssm:GetParameter", "ssm:GetParameters"] Resource = var.config._ssm_parameter_arns } ] diff --git a/modules/termination-watcher/termination/main.tf b/modules/termination-watcher/termination/main.tf index f43b61775a..c6b980537f 100644 --- a/modules/termination-watcher/termination/main.tf +++ b/modules/termination-watcher/termination/main.tf @@ -64,7 +64,7 @@ resource "aws_iam_role_policy" "ssm_policy" { Statement = [ { Effect = "Allow" - Action = ["ssm:GetParameter"] + Action = ["ssm:GetParameter", "ssm:GetParameters"] Resource = var.config._ssm_parameter_arns } ] diff --git a/modules/termination-watcher/tests/multi-app.tftest.hcl b/modules/termination-watcher/tests/multi-app.tftest.hcl new file mode 100644 index 0000000000..0838b1480c --- /dev/null +++ b/modules/termination-watcher/tests/multi-app.tftest.hcl @@ -0,0 +1,78 @@ +mock_provider "aws" { + mock_data "aws_iam_policy_document" { defaults = { json = "{\"Version\":\"2012-10-17\",\"Statement\":[]}" } } + mock_resource "aws_iam_role" { defaults = { arn = "arn:aws:iam::123456789012:role/test" } } + mock_resource "aws_lambda_function" { defaults = { arn = "arn:aws:lambda:eu-west-1:123456789012:function:test" } } + mock_resource "aws_sqs_queue" { defaults = { arn = "arn:aws:sqs:eu-west-1:123456789012:test", url = "https://sqs.eu-west-1.amazonaws.com/123456789012/test" } } +} +variables { + config = { + prefix = "test" + s3_bucket = "artifacts" + s3_key = "termination-watcher.zip" + runtime = "nodejs24.x" + timeout = 60 + memory_size = 256 + architecture = "arm64" + tag_filters = { "ghr:environment" = "test" } + features = { enable_spot_termination_handler = false, enable_spot_termination_notification_watcher = false } + enable_runner_deregistration = true + github_app_parameters = { + id = { name = "/app/id", arn = "arn:aws:ssm:eu-west-1:123456789012:parameter/app/id" } + key_base64 = { name = "/app/key", arn = "arn:aws:ssm:eu-west-1:123456789012:parameter/app/key" } + additional_apps_manifest = { name = "/app/manifest", arn = "arn:aws:ssm:eu-west-1:123456789012:parameter/app/manifest" } + additional_app_parameter_arns = ["arn:aws:ssm:eu-west-1:123456789012:parameter/app/extra/key"] + } + } +} +run "additional_apps_reach_retry_lambda" { + command = apply + assert { + condition = output.deregister_retry.lambda.environment[0].variables["PARAMETER_GITHUB_APPS_MANIFEST_NAME"] == "/app/manifest" + error_message = "The retry Lambda must receive the manifest." + } + assert { + condition = toset(jsondecode(aws_iam_role_policy.deregister_retry_ssm[0].policy).Statement[0].Action) == toset(["ssm:GetParameter", "ssm:GetParameters"]) && length(local.ssm_parameter_arns) == 4 + error_message = "Credential reads must support manifest and batched credential loading." + } +} +run "disabled_deregistration" { + command = plan + variables { + config = { + prefix = "test" + features = { enable_spot_termination_handler = false, enable_spot_termination_notification_watcher = false } + enable_runner_deregistration = false + tag_filters = {} + } + } + assert { + condition = length(local.deregistration_env_vars) == 0 && length(local.ssm_parameter_arns) == 0 + error_message = "Disabled deregistration must not need credentials or grant access." + } +} + +run "single_app_defaults" { + command = apply + variables { + config = { + prefix = "test" + s3_bucket = "artifacts" + s3_key = "termination-watcher.zip" + runtime = "nodejs24.x" + timeout = 60 + memory_size = 256 + architecture = "arm64" + tag_filters = { "ghr:environment" = "test" } + features = { enable_spot_termination_handler = false, enable_spot_termination_notification_watcher = false } + enable_runner_deregistration = true + github_app_parameters = { + id = { name = "/app/id", arn = "arn:aws:ssm:eu-west-1:123456789012:parameter/app/id" } + key_base64 = { name = "/app/key", arn = "arn:aws:ssm:eu-west-1:123456789012:parameter/app/key" } + } + } + } + assert { + condition = !contains(keys(output.deregister_retry.lambda.environment[0].variables), "PARAMETER_GITHUB_APPS_MANIFEST_NAME") && length(local.ssm_parameter_arns) == 2 + error_message = "Single-App deployments retain primary credential access without an optional manifest key." + } +} diff --git a/modules/termination-watcher/variables.tf b/modules/termination-watcher/variables.tf index a72bf74916..c4f58b606c 100644 --- a/modules/termination-watcher/variables.tf +++ b/modules/termination-watcher/variables.tf @@ -28,7 +28,7 @@ variable "config" { `tracing_config`: Configuration for lambda tracing. `zip`: File location of the lambda zip file. `enable_runner_deregistration`: Enable or disable deregistering the runner from GitHub when its EC2 instance is terminated. - `github_app_parameters`: GitHub App SSM parameters (`id` and `key_base64`, each a map of `arn`/`name`) used to authenticate to GitHub when deregistering runners. + `github_app_parameters`: GitHub App SSM parameters (`id` and `key_base64`, each a map of `arn`/`name`) used to authenticate to GitHub when deregistering runners. Optional `additional_apps_manifest` (`name`/`arn`) and `additional_app_parameter_arns` enable credentials from the multi-App manifest. `ghes_url`: GitHub Enterprise Server URL used to target the GHES API when deregistering runners. Leave `null` for github.com. EOF type = object({ @@ -77,8 +77,10 @@ variable "config" { zip = optional(string, null) enable_runner_deregistration = optional(bool, false) github_app_parameters = optional(object({ - id = map(string) - key_base64 = map(string) + id = map(string) + key_base64 = map(string) + additional_apps_manifest = optional(object({ name = string, arn = string })) + additional_app_parameter_arns = optional(list(string), []) }), null) ghes_url = optional(string, null) })