From 3cfccf0052218e6dd4f3de8346c98458826c4ca4 Mon Sep 17 00:00:00 2001 From: Dimi Kot Date: Mon, 26 Aug 2024 09:57:49 -0700 Subject: [PATCH] Switch to ci-scaler and instance-to-ami-cdk --- .eslintrc.base.js | 4 + .npmrc | 1 - .prettierrc | 8 + README.md | 3 +- docs/README.md | 2 +- docs/classes/CiStorage.md | 81 +- docs/interfaces/CiStorageProps.md | 58 +- jest.config.base.js | 2 +- package.json | 23 +- src/CiStorage.ts | 690 +- src/__tests__/CiStorage.test.ts | 77 +- .../__snapshots__/CiStorage.test.ts.snap | 6303 +++++++++++++---- src/internal/__tests__/namer.test.ts | 1 + src/internal/cloudConfigBuild.ts | 313 +- src/internal/dedent.ts | 8 +- src/internal/namer.ts | 2 +- src/internal/userDataCausesReplacement.ts | 40 + 17 files changed, 5958 insertions(+), 1658 deletions(-) delete mode 100644 .npmrc create mode 100644 .prettierrc create mode 100644 src/internal/userDataCausesReplacement.ts diff --git a/.eslintrc.base.js b/.eslintrc.base.js index b4ce345..9eb958b 100644 --- a/.eslintrc.base.js +++ b/.eslintrc.base.js @@ -38,6 +38,7 @@ module.exports = (projectRoot, extraRules = {}) => ({ "typescript-enum", "typescript-sort-keys", "unused-imports", + "no-only-tests", ], settings: { react: { @@ -434,6 +435,9 @@ module.exports = (projectRoot, extraRules = {}) => ({ ], quotes: ["error", "double", { avoidEscape: true }], + + "no-only-tests/no-only-tests": "error", + ...extraRules, }, }); diff --git a/.npmrc b/.npmrc deleted file mode 100644 index c52ad5f..0000000 --- a/.npmrc +++ /dev/null @@ -1 +0,0 @@ -# Published to https://www.npmjs.com diff --git a/.prettierrc b/.prettierrc new file mode 100644 index 0000000..f5688b5 --- /dev/null +++ b/.prettierrc @@ -0,0 +1,8 @@ +{ + "overrides": [ + { + "files": ["*.ts", "*.tsx"], + "options": { "parser": "typescript" } + } + ] +} diff --git a/README.md b/README.md index 760a77d..e349585 100644 --- a/README.md +++ b/README.md @@ -1,8 +1,7 @@ -# ci-storage-cdk: a CDK construct to deploy ci-storage infrastructure +# ci-storage-cdk: A CDK construct to deploy ci-storage infrastructure ![CI run](https://github.com/clickup/ci-storage-cdk/actions/workflows/ci.yml/badge.svg?branch=main) See also: - [full API documentation](https://github.com/clickup/ci-storage-cdk/blob/master/docs/modules.md) - [ci-storage tool documentation](https://github.com/dimikot/ci-storage) - diff --git a/docs/README.md b/docs/README.md index 4b8bff2..b3c7acd 100644 --- a/docs/README.md +++ b/docs/README.md @@ -1,6 +1,6 @@ @clickup/ci-storage-cdk / [Exports](modules.md) -# ci-storage-cdk: a CDK construct to deploy ci-storage infrastructure +# ci-storage-cdk: A CDK construct to deploy ci-storage infrastructure ![CI run](https://github.com/clickup/ci-storage-cdk/actions/workflows/ci.yml/badge.svg?branch=main) diff --git a/docs/classes/CiStorage.md b/docs/classes/CiStorage.md index 4b4c8a9..3bdbb84 100644 --- a/docs/classes/CiStorage.md +++ b/docs/classes/CiStorage.md @@ -5,23 +5,25 @@ A reusable Construct to launch ci-storage infra in some other stack. This class is meant to be put in a public domain and then used in any project. -- The construct launches a pool of self-hosted runners plus a number of - central "host" instances. +- The construct launches a pool of self-hosted runners plus a central "host" + instance. - On each instance, a corresponding GitHub repo is pulled (possibly using a sparse checkout), and then, `docker compose` is run. There is no need to - pre-build any images or publish them anywhere, it's all on the fly. + pre-build any images or publish them anywhere, it's all done on the fly. Why vanilla EC2 instances + docker-compose and not ECS or Fargate? -1. Because for ECS and Fargate, in 2 minutes after the termination warning, - we only have more 2 minutes to shutdown the OS (it's documented, i.e. 4 - minutes in total to cleanly shutdown). And for vanilla instances, people - claim that the second timeout is way higher (although undocumented). We - need more time to finish running CI jobs, and 4 minutes are not enough. +1. For ECS and Fargate, in 2 minutes after the termination warning, we only + have more 2 minutes to shutdown the OS (it's documented, i.e. 4 minutes in + total to cleanly shutdown). And for vanilla instances, people claim that + the second timeout is way higher (although undocumented). We need more + time to finish running CI jobs, and 4 minutes are not enough. 2. We anyways need to run tests locally on Mac, and to do this, we use docker-compose. Which means that in the CI environment, we'd better use exactly the same configuration (same docker-compose), otherwise the environments diverge and are hard to debug/support. +3. Tests often times need to run "Docker in Docker", which is problematic in + ECS and Fargate environment. ## Hierarchy @@ -53,7 +55,7 @@ Construct.constructor #### Defined in -[src/CiStorage.ts:198](https://github.com/clickup/ci-storage-cdk/blob/master/src/CiStorage.ts#L198) +[src/CiStorage.ts:203](https://github.com/clickup/ci-storage-cdk/blob/master/src/CiStorage.ts#L203) ## Properties @@ -63,17 +65,17 @@ Construct.constructor #### Defined in -[src/CiStorage.ts:185](https://github.com/clickup/ci-storage-cdk/blob/master/src/CiStorage.ts#L185) +[src/CiStorage.ts:186](https://github.com/clickup/ci-storage-cdk/blob/master/src/CiStorage.ts#L186) ___ -### securityGroup +### hostedZone -• `Readonly` **securityGroup**: `SecurityGroup` +• `Optional` `Readonly` **hostedZone**: `IHostedZone` #### Defined in -[src/CiStorage.ts:186](https://github.com/clickup/ci-storage-cdk/blob/master/src/CiStorage.ts#L186) +[src/CiStorage.ts:187](https://github.com/clickup/ci-storage-cdk/blob/master/src/CiStorage.ts#L187) ___ @@ -83,7 +85,7 @@ ___ #### Defined in -[src/CiStorage.ts:187](https://github.com/clickup/ci-storage-cdk/blob/master/src/CiStorage.ts#L187) +[src/CiStorage.ts:188](https://github.com/clickup/ci-storage-cdk/blob/master/src/CiStorage.ts#L188) ___ @@ -93,7 +95,7 @@ ___ #### Defined in -[src/CiStorage.ts:188](https://github.com/clickup/ci-storage-cdk/blob/master/src/CiStorage.ts#L188) +[src/CiStorage.ts:189](https://github.com/clickup/ci-storage-cdk/blob/master/src/CiStorage.ts#L189) ___ @@ -110,47 +112,64 @@ ___ #### Defined in -[src/CiStorage.ts:189](https://github.com/clickup/ci-storage-cdk/blob/master/src/CiStorage.ts#L189) +[src/CiStorage.ts:190](https://github.com/clickup/ci-storage-cdk/blob/master/src/CiStorage.ts#L190) ___ -### launchTemplate +### securityGroup -• `Readonly` **launchTemplate**: `LaunchTemplate` +• `Readonly` **securityGroup**: `SecurityGroup` #### Defined in -[src/CiStorage.ts:190](https://github.com/clickup/ci-storage-cdk/blob/master/src/CiStorage.ts#L190) +[src/CiStorage.ts:191](https://github.com/clickup/ci-storage-cdk/blob/master/src/CiStorage.ts#L191) ___ -### autoScalingGroup +### vpcLink -• `Readonly` **autoScalingGroup**: `AutoScalingGroup` +• `Readonly` **vpcLink**: `VpcLink` #### Defined in -[src/CiStorage.ts:191](https://github.com/clickup/ci-storage-cdk/blob/master/src/CiStorage.ts#L191) +[src/CiStorage.ts:192](https://github.com/clickup/ci-storage-cdk/blob/master/src/CiStorage.ts#L192) ___ -### hostedZone +### host -• `Optional` `Readonly` **hostedZone**: `IHostedZone` +• `Readonly` **host**: `Object` + +#### Type declaration + +| Name | Type | +| :------ | :------ | +| `fqdn` | `undefined` \| `string` | +| `instance` | `CfnInstance` | #### Defined in -[src/CiStorage.ts:192](https://github.com/clickup/ci-storage-cdk/blob/master/src/CiStorage.ts#L192) +[src/CiStorage.ts:193](https://github.com/clickup/ci-storage-cdk/blob/master/src/CiStorage.ts#L193) ___ -### hosts +### autoScalingGroups -• `Readonly` **hosts**: \{ `fqdn`: `undefined` \| `string` ; `instance`: `Instance` }[] = `[]` +• `Readonly` **autoScalingGroups**: \{ `autoScalingGroup`: `AutoScalingGroup` ; `launchTemplate`: `LaunchTemplate` }[] = `[]` #### Defined in -[src/CiStorage.ts:193](https://github.com/clickup/ci-storage-cdk/blob/master/src/CiStorage.ts#L193) +[src/CiStorage.ts:197](https://github.com/clickup/ci-storage-cdk/blob/master/src/CiStorage.ts#L197) + +___ + +### instanceToAmi + +• `Readonly` **instanceToAmi**: `InstanceToAmi` + +#### Defined in + +[src/CiStorage.ts:201](https://github.com/clickup/ci-storage-cdk/blob/master/src/CiStorage.ts#L201) ___ @@ -160,7 +179,7 @@ ___ #### Defined in -[src/CiStorage.ts:199](https://github.com/clickup/ci-storage-cdk/blob/master/src/CiStorage.ts#L199) +[src/CiStorage.ts:204](https://github.com/clickup/ci-storage-cdk/blob/master/src/CiStorage.ts#L204) ___ @@ -170,7 +189,7 @@ ___ #### Defined in -[src/CiStorage.ts:200](https://github.com/clickup/ci-storage-cdk/blob/master/src/CiStorage.ts#L200) +[src/CiStorage.ts:205](https://github.com/clickup/ci-storage-cdk/blob/master/src/CiStorage.ts#L205) ___ @@ -180,4 +199,4 @@ ___ #### Defined in -[src/CiStorage.ts:201](https://github.com/clickup/ci-storage-cdk/blob/master/src/CiStorage.ts#L201) +[src/CiStorage.ts:206](https://github.com/clickup/ci-storage-cdk/blob/master/src/CiStorage.ts#L206) diff --git a/docs/interfaces/CiStorageProps.md b/docs/interfaces/CiStorageProps.md index 3a80d07..f657b0d 100644 --- a/docs/interfaces/CiStorageProps.md +++ b/docs/interfaces/CiStorageProps.md @@ -17,7 +17,7 @@ VPC to use by this construct. #### Defined in -[src/CiStorage.ts:57](https://github.com/clickup/ci-storage-cdk/blob/master/src/CiStorage.ts#L57) +[src/CiStorage.ts:62](https://github.com/clickup/ci-storage-cdk/blob/master/src/CiStorage.ts#L62) ___ @@ -25,12 +25,11 @@ ___ • **inlinePolicies**: `undefined` \| \{ `[name: string]`: `PolicyDocument`; } -Instance Profile Role inline policies to be used for all created -instances. +Instance Profile Role inline policies for all created instances. #### Defined in -[src/CiStorage.ts:60](https://github.com/clickup/ci-storage-cdk/blob/master/src/CiStorage.ts#L60) +[src/CiStorage.ts:64](https://github.com/clickup/ci-storage-cdk/blob/master/src/CiStorage.ts#L64) ___ @@ -43,7 +42,7 @@ with that value, separated by "-". #### Defined in -[src/CiStorage.ts:63](https://github.com/clickup/ci-storage-cdk/blob/master/src/CiStorage.ts#L63) +[src/CiStorage.ts:67](https://github.com/clickup/ci-storage-cdk/blob/master/src/CiStorage.ts#L67) ___ @@ -62,7 +61,7 @@ A Hosted Zone to register the host instances in. #### Defined in -[src/CiStorage.ts:65](https://github.com/clickup/ci-storage-cdk/blob/master/src/CiStorage.ts#L65) +[src/CiStorage.ts:69](https://github.com/clickup/ci-storage-cdk/blob/master/src/CiStorage.ts#L69) ___ @@ -75,7 +74,7 @@ must pre-exist. #### Defined in -[src/CiStorage.ts:73](https://github.com/clickup/ci-storage-cdk/blob/master/src/CiStorage.ts#L73) +[src/CiStorage.ts:77](https://github.com/clickup/ci-storage-cdk/blob/master/src/CiStorage.ts#L77) ___ @@ -87,40 +86,19 @@ Time zone for instances, example: America/Los_Angeles. #### Defined in -[src/CiStorage.ts:75](https://github.com/clickup/ci-storage-cdk/blob/master/src/CiStorage.ts#L75) +[src/CiStorage.ts:79](https://github.com/clickup/ci-storage-cdk/blob/master/src/CiStorage.ts#L79) ___ -### runner +### runners -• **runner**: `Object` +• **runners**: \{ `label`: `string` ; `ghRepository`: `string` ; `ghDockerComposeDirectoryUrl`: `string` ; `imageSsmName`: `string` ; `volumeRootGb`: `number` ; `volumeRootIops?`: `number` ; `volumeRootThroughput?`: `number` ; `volumeLogsGb`: `number` ; `swapSizeGb?`: `number` ; `instanceRequirements`: [`InstanceRequirementsProperty`, ...InstanceRequirementsProperty[]] ; `onDemandPercentageAboveBaseCapacity`: `number` ; `minCapacity`: \{ `id`: `string` ; `value`: `number` ; `cron`: \{ `timeZone?`: `string` } & `CronOptions` }[] ; `maxCapacity`: `number` ; `maxInstanceLifetime`: `Duration` }[] Configuration for self-hosted runner instances in the pool. -#### Type declaration - -| Name | Type | Description | -| :------ | :------ | :------ | -| `ghRepository` | `string` | "{owner}/{repository}" which this self-hosted runners pool serves. | -| `ghDockerComposeDirectoryUrl` | `string` | URL of docker-compose.yml (or compose.yml) directory. The tool will sparse-checkout that directory. The format is Dockerfile-compatible: https://github.com/owner/repo[#[branch]:/directory/with/compose/] | -| `imageSsmName` | `string` | SSM parameter name which holds the reference to an instance image. | -| `volumeGb` | `number` | Size of the root volume. | -| `swapSizeGb?` | `number` | Size of swap file (if you need it). | -| `tmpfsMaxSizeGb?` | `number` | If set, mounts /var/lib/docker to tmpfs with the provided max size. | -| `instanceRequirements` | [`InstanceRequirementsProperty`, ...InstanceRequirementsProperty[]] | The list of requirements to choose Spot Instances. | -| `scale` | \{ `onDemandPercentageAboveBaseCapacity`: `number` ; `maxActiveRunnersPercent`: \{ `periodSec`: `number` ; `value`: `number` ; `scalingSteps?`: `number` } ; `minCapacity`: \{ `id`: `string` ; `value`: `number` ; `cron`: \{ `timeZone?`: `string` } & `CronOptions` }[] ; `maxCapacity`: `number` ; `maxInstanceLifetime`: `Duration` } | Scaling options. | -| `scale.onDemandPercentageAboveBaseCapacity` | `number` | The percentages of On-Demand Instances and Spot Instances for your additional capacity. | -| `scale.maxActiveRunnersPercent` | \{ `periodSec`: `number` ; `value`: `number` ; `scalingSteps?`: `number` } | Maximum percentage of active runners. If the MAX metric of number of active runners within the recent periodSec interval grows beyond this threshold, the autoscaling group will launch new instances until the percentage drops, or maxCapacity is reached. | -| `scale.maxActiveRunnersPercent.periodSec` | `number` | Calculate MAX metric within that period. The higher is the value, the slower will the capacity lower (but it doesn't affect how fast will it increase). | -| `scale.maxActiveRunnersPercent.value` | `number` | Value to use for the target percentage of active (busy) runners. | -| `scale.maxActiveRunnersPercent.scalingSteps?` | `number` | Desired number of ScalingInterval items in scalingSteps. | -| `scale.minCapacity` | \{ `id`: `string` ; `value`: `number` ; `cron`: \{ `timeZone?`: `string` } & `CronOptions` }[] | Minimal number of idle runners to keep, depending on the daytime. If the auto scaling group has less than this number of instances, the new instances will be created. | -| `scale.maxCapacity` | `number` | Maximum total number of instances. | -| `scale.maxInstanceLifetime` | `Duration` | Re-create instances time to time. | - #### Defined in -[src/CiStorage.ts:77](https://github.com/clickup/ci-storage-cdk/blob/master/src/CiStorage.ts#L77) +[src/CiStorage.ts:81](https://github.com/clickup/ci-storage-cdk/blob/master/src/CiStorage.ts#L81) ___ @@ -128,9 +106,10 @@ ___ • **host**: `Object` -Configuration for ci-storage host instance in the pool. This instance also -runs common services reusable by self-hosted runners. Each self-hosted -runner has its localhost ports redirected to that instance. +Configuration for ci-storage host instance in the pool. This instance runs +ci-storage container, ci-scaler container and also common database services +reusable by self-hosted runners. Each self-hosted runner has its localhost +ports redirected to those instance's services. #### Type declaration @@ -139,12 +118,11 @@ runner has its localhost ports redirected to that instance. | `ghDockerComposeDirectoryUrl` | `string` | URL of docker-compose.yml (or compose.yml) directory. The tool will sparse-checkout that directory. The format is Dockerfile-compatible: https://github.com/owner/repo[#[branch]:/directory/with/compose/] | | `dockerComposeProfiles?` | `string`[] | List of profiles from docker-compose to additionally start. | | `imageSsmName` | `string` | SSM parameter name which holds the reference to an instance image. | -| `swapSizeGb?` | `number` | Size of swap file (if you need it). | -| `tmpfsMaxSizeGb?` | `number` | If set, mounts /var/lib/docker to tmpfs with the provided max size and copies it from the old instance when the instance gets replaced. | +| `swapSizeGb?` | `number` | Size of swap file (if you need it). The swapfile will be placed to /var/swapfile on the root volume. | +| `varLibDockerOnTmpfsMaxSizeGb?` | `number` | If set, mounts the entire /var/lib/docker host instance directory to tmpfs with the provided max size, and also copies it from the old instance when the instance gets replaced. | | `instanceType` | `string` | Full name of the Instance type. | -| `machines` | `number` | Number of instances to create. | -| `ports` | \{ `port`: `Port` ; `description`: `string` }[] | Ports to be open in the security group for connection from all runners to the host. | +| `ports` | \{ `port`: `Port` ; `description`: `string` ; `isWebhook?`: ``true`` }[] | Ports to be open in the security group for connection from any CI resources to the host (private IP addresses only). For a port marked as isWebhook=true, the following AWS resources will also be created: HttpApi->NLB->TargetGroup, where HttpApi will have an auto-generated AWS domain with SSL enabled, and the whole chain will proxy all POST requests (presumably, sent by GitHub webhooks) to the given host's port. | #### Defined in -[src/CiStorage.ts:138](https://github.com/clickup/ci-storage-cdk/blob/master/src/CiStorage.ts#L138) +[src/CiStorage.ts:133](https://github.com/clickup/ci-storage-cdk/blob/master/src/CiStorage.ts#L133) diff --git a/jest.config.base.js b/jest.config.base.js index c0ea465..cee966c 100644 --- a/jest.config.base.js +++ b/jest.config.base.js @@ -6,7 +6,7 @@ module.exports = { restoreMocks: true, ...(process.env.IN_JEST_PROJECT ? {} - : { forceExit: true, testTimeout: 30000, forceExit: true }), + : { forceExit: true, testTimeout: 30000 }), transform: { "\\.ts$": "ts-jest", }, diff --git a/package.json b/package.json index 2b32298..7f7f121 100644 --- a/package.json +++ b/package.json @@ -1,7 +1,7 @@ { "name": "@clickup/ci-storage-cdk", "description": "A CDK construct to deploy ci-storage infrastructure", - "version": "2.10.294", + "version": "2.11.1", "license": "MIT", "keywords": [ "cdk", @@ -12,15 +12,15 @@ "types": "./dist/index.d.ts", "exports": "./dist/index.js", "scripts": { - "build": "$npm_execpath run clean; tsc", + "build": "tsc", "dev": "tsc --watch --preserveWatchOutput", "lint": "bash internal/lint.sh", - "test": "$npm_execpath run build && jest", + "test": "tsc && jest", "docs": "bash internal/docs.sh", - "clean": "rm -rf dist yarn.lock package-lock.json pnpm-lock.yaml *.log", + "clean": "bash internal/clean.sh", "copy-package-to-public-dir": "copy-package-to-public-dir.sh", "backport-package-from-public-dir": "backport-package-from-public-dir.sh", - "deploy": "$npm_execpath run build && $npm_execpath run test && $npm_execpath publish --access=public" + "deploy": "$npm_execpath run clean && $npm_execpath install && $npm_execpath run test && $npm_execpath publish --access=public" }, "devDependencies": { "@types/jest": "^29.5.5", @@ -32,6 +32,7 @@ "eslint-import-resolver-typescript": "^3.5.5", "eslint-plugin-import": "^2.27.5", "eslint-plugin-lodash": "^7.4.0", + "eslint-plugin-no-only-tests": "^3.1.0", "eslint-plugin-node": "^11.1.0", "eslint-plugin-react-hooks": "^4.6.0", "eslint-plugin-react": "^7.32.2", @@ -48,17 +49,17 @@ "typescript": "^5.2.2" }, "dependencies": { - "aws-cdk": "^2.133.0", - "aws-cdk-lib": "^2.133.0", + "@clickup/instance-to-ami-cdk": "^2.10.295", + "aws-cdk-lib": "^2.154.1", + "aws-cdk": "^2.154.1", "cdk-ec2-key-pair": "^3.3.3", "constructs": "^10.3.0", + "js-yaml": "^4.1.0", "lodash": "^4.17.21", - "multi-convention-namer": "^0.1.12", - "js-yaml": "^4.1.0" + "multi-convention-namer": "^0.1.12" }, "repository": { "type": "git", - "url": "git://github.com/clickup/ci-storage-cdk", - "directory": "ci-storage-cdk" + "url": "git://github.com/clickup/ci-storage-cdk" } } diff --git a/src/CiStorage.ts b/src/CiStorage.ts index f2a897e..a465235 100644 --- a/src/CiStorage.ts +++ b/src/CiStorage.ts @@ -1,32 +1,37 @@ +import { InstanceToAmi } from "@clickup/instance-to-ami-cdk"; import { ArnFormat, Duration, Stack, Tags } from "aws-cdk-lib"; -import type { - CfnAutoScalingGroup, - CronOptions, -} from "aws-cdk-lib/aws-autoscaling"; +import { HttpApi, HttpMethod, VpcLink } from "aws-cdk-lib/aws-apigatewayv2"; +import { HttpNlbIntegration } from "aws-cdk-lib/aws-apigatewayv2-integrations"; import { - AdjustmentType, AutoScalingGroup, + BlockDeviceVolume, + EbsDeviceVolumeType, GroupMetrics, OnDemandAllocationStrategy, Schedule, SpotAllocationStrategy, UpdatePolicy, } from "aws-cdk-lib/aws-autoscaling"; -import { Metric } from "aws-cdk-lib/aws-cloudwatch"; -import type { IKeyPair, IVpc, CfnInstance } from "aws-cdk-lib/aws-ec2"; +import type { + CfnAutoScalingGroup, + CronOptions, +} from "aws-cdk-lib/aws-autoscaling"; +import type { IKeyPair, IVpc } from "aws-cdk-lib/aws-ec2"; import { + CfnInstance, MachineImage, - OperatingSystemType, UserData, LaunchTemplate, KeyPair as Ec2KeyPair, - BlockDeviceVolume, - EbsDeviceVolumeType, - Instance, - InstanceType, SecurityGroup, Port, } from "aws-cdk-lib/aws-ec2"; +import { + NetworkLoadBalancer, + NetworkTargetGroup, + Protocol, +} from "aws-cdk-lib/aws-elasticloadbalancingv2"; +import { InstanceIdTarget } from "aws-cdk-lib/aws-elasticloadbalancingv2-targets"; import type { RoleProps } from "aws-cdk-lib/aws-iam"; import { ManagedPolicy, @@ -40,11 +45,11 @@ import { ARecord, RecordTarget, HostedZone } from "aws-cdk-lib/aws-route53"; import { KeyPair } from "cdk-ec2-key-pair"; import { Construct } from "constructs"; import padStart from "lodash/padStart"; -import range from "lodash/range"; -import { buildPercentScalingSteps } from "./internal/buildPercentScalingSteps"; import { cloudConfigBuild } from "./internal/cloudConfigBuild"; import { cloudConfigYamlDump } from "./internal/cloudConfigYamlDump"; +import { dedent } from "./internal/dedent"; import { namer } from "./internal/namer"; +import { userDataCausesReplacement } from "./internal/userDataCausesReplacement"; /** * Only plain primitive typed properties are allowed to simplify usage of this @@ -55,8 +60,7 @@ import { namer } from "./internal/namer"; export interface CiStorageProps { /** VPC to use by this construct. */ vpc: IVpc; - /** Instance Profile Role inline policies to be used for all created - * instances. */ + /** Instance Profile Role inline policies for all created instances. */ inlinePolicies: RoleProps["inlinePolicies"]; /** All instance names (and hostname for the host instances) will be prefixed * with that value, separated by "-". */ @@ -74,8 +78,10 @@ export interface CiStorageProps { /** Time zone for instances, example: America/Los_Angeles. */ timeZone?: string; /** Configuration for self-hosted runner instances in the pool. */ - runner: { - /** "{owner}/{repository}" which this self-hosted runners pool serves. */ + runners: Array<{ + /** Primary label of this runner group. */ + label: string; + /** "{owner}/{repo}" which this self-hosted runners pool serves. */ ghRepository: string; /** URL of docker-compose.yml (or compose.yml) directory. The tool will * sparse-checkout that directory. The format is Dockerfile-compatible: @@ -84,57 +90,46 @@ export interface CiStorageProps { /** SSM parameter name which holds the reference to an instance image. */ imageSsmName: string; /** Size of the root volume. */ - volumeGb: number; - /** Size of swap file (if you need it). */ + volumeRootGb: number; + /** If set, IOPS for the root volume. */ + volumeRootIops?: number; + /** If set, throughput (MB/s) for the root volume. */ + volumeRootThroughput?: number; + /** Size of /var/log volume. */ + volumeLogsGb: number; + /** Size of swap file (if you need it). The swapfile will be placed on the + * logs volume and increase its size (added to volumeLogsGb). */ swapSizeGb?: number; - /** If set, mounts /var/lib/docker to tmpfs with the provided max size. */ - tmpfsMaxSizeGb?: number; /** The list of requirements to choose Spot Instances. */ instanceRequirements: [ CfnAutoScalingGroup.InstanceRequirementsProperty, ...CfnAutoScalingGroup.InstanceRequirementsProperty[], ]; - /** Scaling options. */ - scale: { - /** The percentages of On-Demand Instances and Spot Instances for your - * additional capacity. */ - onDemandPercentageAboveBaseCapacity: number; - /** Maximum percentage of active runners. If the MAX metric of number of - * active runners within the recent periodSec interval grows beyond this - * threshold, the autoscaling group will launch new instances until the - * percentage drops, or maxCapacity is reached. */ - maxActiveRunnersPercent: { - /** Calculate MAX metric within that period. The higher is the value, - * the slower will the capacity lower (but it doesn't affect how fast - * will it increase). */ - periodSec: number; - /** Value to use for the target percentage of active (busy) runners. */ - value: number; - /** Desired number of ScalingInterval items in scalingSteps. */ - scalingSteps?: number; - }; - /** Minimal number of idle runners to keep, depending on the daytime. If - * the auto scaling group has less than this number of instances, the new - * instances will be created. */ - minCapacity: Array<{ - /** Alpha-numeric id of this schedule. */ - id: string; - /** Value to assign to minCapacity when reaching the schedule time. Note - * that it doesn't apply retrospectively, i.e. there is no processing of - * past-due schedules in AWS. */ - value: number; - /** Schedule info. Time zone example: America/Los_Angeles. */ - cron: { timeZone?: string } & CronOptions; - }>; - /** Maximum total number of instances. */ - maxCapacity: number; - /** Re-create instances time to time. */ - maxInstanceLifetime: Duration; - }; - }; - /** Configuration for ci-storage host instance in the pool. This instance also - * runs common services reusable by self-hosted runners. Each self-hosted - * runner has its localhost ports redirected to that instance. */ + /** The percentages of On-Demand Instances and Spot Instances for your + * additional capacity. */ + onDemandPercentageAboveBaseCapacity: number; + /** Minimal number of idle runners to keep, depending on the daytime. If + * the auto scaling group has less than this number of instances, the new + * instances will be created. */ + minCapacity: Array<{ + /** Alpha-numeric id of this schedule. */ + id: string; + /** Value to assign to minCapacity when reaching the schedule time. Note + * that it doesn't apply retrospectively, i.e. there is no processing of + * past-due schedules in AWS. */ + value: number; + /** Schedule info. Time zone example: America/Los_Angeles. */ + cron: { timeZone?: string } & CronOptions; + }>; + /** Maximum total number of instances. */ + maxCapacity: number; + /** Re-create instances time to time. */ + maxInstanceLifetime: Duration; + }>; + /** Configuration for ci-storage host instance in the pool. This instance runs + * ci-storage container, ci-scaler container and also common database services + * reusable by self-hosted runners. Each self-hosted runner has its localhost + * ports redirected to those instance's services. */ host: { /** URL of docker-compose.yml (or compose.yml) directory. The tool will * sparse-checkout that directory. The format is Dockerfile-compatible: @@ -144,18 +139,22 @@ export interface CiStorageProps { dockerComposeProfiles?: string[]; /** SSM parameter name which holds the reference to an instance image. */ imageSsmName: string; - /** Size of swap file (if you need it). */ + /** Size of swap file (if you need it). The swapfile will be placed to + * /var/swapfile on the root volume. */ swapSizeGb?: number; - /** If set, mounts /var/lib/docker to tmpfs with the provided max size and - * copies it from the old instance when the instance gets replaced. */ - tmpfsMaxSizeGb?: number; + /** If set, mounts the entire /var/lib/docker host instance directory to + * tmpfs with the provided max size, and also copies it from the old + * instance when the instance gets replaced. */ + varLibDockerOnTmpfsMaxSizeGb?: number; /** Full name of the Instance type. */ instanceType: string; - /** Number of instances to create. */ - machines: number; - /** Ports to be open in the security group for connection from all runners - * to the host. */ - ports: Array<{ port: Port; description: string }>; + /** Ports to be open in the security group for connection from any CI + * resources to the host (private IP addresses only). For a port marked as + * isWebhook=true, the following AWS resources will also be created: + * HttpApi->NLB->TargetGroup, where HttpApi will have an auto-generated AWS + * domain with SSL enabled, and the whole chain will proxy all POST requests + * (presumably, sent by GitHub webhooks) to the given host's port. */ + ports: Array<{ port: Port; description: string; isWebhook?: true }>; }; } @@ -163,37 +162,43 @@ export interface CiStorageProps { * A reusable Construct to launch ci-storage infra in some other stack. This * class is meant to be put in a public domain and then used in any project. * - * - The construct launches a pool of self-hosted runners plus a number of - * central "host" instances. + * - The construct launches a pool of self-hosted runners plus a central "host" + * instance. * - On each instance, a corresponding GitHub repo is pulled (possibly using a * sparse checkout), and then, `docker compose` is run. There is no need to - * pre-build any images or publish them anywhere, it's all on the fly. + * pre-build any images or publish them anywhere, it's all done on the fly. * * Why vanilla EC2 instances + docker-compose and not ECS or Fargate? * - * 1. Because for ECS and Fargate, in 2 minutes after the termination warning, - * we only have more 2 minutes to shutdown the OS (it's documented, i.e. 4 - * minutes in total to cleanly shutdown). And for vanilla instances, people - * claim that the second timeout is way higher (although undocumented). We - * need more time to finish running CI jobs, and 4 minutes are not enough. + * 1. For ECS and Fargate, in 2 minutes after the termination warning, we only + * have more 2 minutes to shutdown the OS (it's documented, i.e. 4 minutes in + * total to cleanly shutdown). And for vanilla instances, people claim that + * the second timeout is way higher (although undocumented). We need more + * time to finish running CI jobs, and 4 minutes are not enough. * 2. We anyways need to run tests locally on Mac, and to do this, we use * docker-compose. Which means that in the CI environment, we'd better use * exactly the same configuration (same docker-compose), otherwise the * environments diverge and are hard to debug/support. + * 3. Tests often times need to run "Docker in Docker", which is problematic in + * ECS and Fargate environment. */ export class CiStorage extends Construct { public readonly vpc: IVpc; - public readonly securityGroup: SecurityGroup; + public readonly hostedZone?: IHostedZone; public readonly keyPair: IKeyPair; public readonly keyPairPrivateKeySecretName: string; public readonly roles: { runner: Role; host: Role }; - public readonly launchTemplate: LaunchTemplate; - public readonly autoScalingGroup: AutoScalingGroup; - public readonly hostedZone?: IHostedZone; - public readonly hosts: Array<{ + public readonly securityGroup: SecurityGroup; + public readonly vpcLink: VpcLink; + public readonly host: { fqdn: string | undefined; - instance: Instance; + instance: CfnInstance; + }; + public readonly autoScalingGroups: Array<{ + autoScalingGroup: AutoScalingGroup; + launchTemplate: LaunchTemplate; }> = []; + public readonly instanceToAmi: InstanceToAmi; constructor( public readonly scope: Construct, @@ -203,8 +208,32 @@ export class CiStorage extends Construct { super(scope, namer(key as any).pascal); const instanceNamePrefix = namer(props.instanceNamePrefix as any); + const webhookHttpPort = parseInt( + props.host.ports.find((p) => p.isWebhook)?.port.toString() || "0", + ); + if (!webhookHttpPort) { + throw Error( + "One item in props.host.ports list must have isWebhook=true and be a single TCP port", + ); + } + + const asgSpecs = props.runners.map((runner) => { + const id = namer( + "runner", + runner.label.toLowerCase() as Lowercase, + ); + return { id, runner, asgName: id.pathKebabFrom(this) }; + }); + this.vpc = props.vpc; + { + const id = namer("zone"); + this.hostedZone = props.hostedZone + ? HostedZone.fromHostedZoneAttributes(this, id.pascal, props.hostedZone) + : undefined; + } + { const id = namer("ssh", "id", "rsa"); const keyPair = new KeyPair(this, id.pascal, { @@ -214,7 +243,7 @@ export class CiStorage extends Construct { }); this.keyPair = Ec2KeyPair.fromKeyPairName( this, - namer("key", "pair").pascal, + namer(id, "key", "pair").pascal, keyPair.keyPairName, ); this.keyPairPrivateKeySecretName = `ec2-ssh-key/${this.keyPair.keyPairName}/private`; @@ -288,6 +317,59 @@ export class CiStorage extends Construct { }), ], }), + [namer("tags", "policy").pascal]: new PolicyDocument({ + statements: [ + new PolicyStatement({ + actions: ["ec2:DescribeTags"], + resources: ["*"], + // Describe* don't support resource-level permissions. + }), + new PolicyStatement({ + actions: ["ec2:CreateTags"], + resources: [ + Stack.of(this).formatArn({ + service: "ec2", + resource: "instance", + resourceName: "*", + arnFormat: ArnFormat.SLASH_RESOURCE_NAME, + }), + ], + conditions: { + StringEquals: { + "ec2:ResourceTag/aws:autoscaling:groupName": + asgSpecs.map(({ asgName }) => asgName), + }, + }, + }), + ], + }), + ...(kind === "host" + ? { + [namer("scaler", "policy").pascal]: new PolicyDocument({ + statements: [ + new PolicyStatement({ + actions: ["autoscaling:DescribeAutoScalingGroups"], + resources: ["*"], + // Describe* don't support resource-level permissions. + }), + new PolicyStatement({ + actions: [ + "autoscaling:SetDesiredCapacity", + "autoscaling:TerminateInstanceInAutoScalingGroup", + ], + resources: [ + Stack.of(this).formatArn({ + service: "autoscaling", + resource: "autoScalingGroup", + resourceName: `*:autoScalingGroupName/${namer("*").pathKebabFrom(this)}`, + arnFormat: ArnFormat.COLON_RESOURCE_NAME, + }), + ], + }), + ], + }), + } + : {}), }, }), ]), @@ -309,228 +391,328 @@ export class CiStorage extends Construct { this.securityGroup.addIngressRule( this.securityGroup, port, - `from runners and host to ${description}`, + `from ${namer(this.key as any).pathKebabFrom(scope)} to ${description}`, ); } } { - const id = namer("zone"); - if (props.hostedZone) { - this.hostedZone = HostedZone.fromHostedZoneAttributes( - this, - id.pascal, - props.hostedZone, - ); - } + const id = namer("link"); + this.vpcLink = new VpcLink(this, id.pascal, { + vpcLinkName: id.pathKebabFrom(this), + vpc: this.vpc, + securityGroups: [this.securityGroup], + }); } + // + // Host resources. + // + { - const machineImage = MachineImage.fromSsmParameter( - props.host.imageSsmName, - { os: OperatingSystemType.LINUX }, + const id = namer( + instanceNamePrefix, + "host", + padStart("1", 3, "0").toString() as Lowercase, ); - for (const i in range(props.host.machines)) { - const id = namer( - instanceNamePrefix, - "host", - padStart(i + 1, 3, "0").toString() as Lowercase, - ); - const recordName = id.kebab; - const fqdn = this.hostedZone - ? recordName + "." + this.hostedZone.zoneName.replace(/\.$/, "") - : undefined; - const userData = UserData.custom( - cloudConfigYamlDump( - cloudConfigBuild({ - fqdn, - ghTokenSecretName: props.ghTokenSecretName, - ghDockerComposeDirectoryUrl: - props.host.ghDockerComposeDirectoryUrl, - dockerComposeEnv: {}, - dockerComposeProfiles: props.host.dockerComposeProfiles ?? [], - keyPairPrivateKeySecretName: this.keyPairPrivateKeySecretName, - timeZone: props.timeZone, - tmpfs: props.host.tmpfsMaxSizeGb - ? { - path: "/var/lib/docker", - maxSizeGb: props.host.tmpfsMaxSizeGb, - } - : undefined, - swapSizeGb: props.host.swapSizeGb, - }), - ), - ); + const recordName = id.kebab; + const fqdn = this.hostedZone + ? recordName + "." + this.hostedZone.zoneName.replace(/\.$/, "") + : undefined; - const instance = new Instance( - this, - // As opposed to all other places, here we MUST prepend the instance - // construct id with the FULL scope (typically owning stack name) due - // to this bug: https://github.com/aws/aws-cdk/issues/22695 - the full - // instance id must be globally unique across all stacks, otherwise - // CDK fails to create automatic launch templates for them. - // - // With the current code, the auto-created launch template name is: - // - ..........Stk + Cnstrct + MyCiHost001 + Instance + LaunchTemplate - // - // But the instance id itself is ugly: - // - Cnstrct + Stk + Cnstrct + MyCiHost001 + Instance - namer(id, "instance").pathPascalFrom(this), - { - vpc: this.vpc, - securityGroup: this.securityGroup, - availabilityZone: this.vpc.availabilityZones[0], - instanceType: new InstanceType(props.host.instanceType), - machineImage, - role: this.roles.host, - keyPair: this.keyPair, - userData, - blockDevices: [ + // HttpApi is needed to get an auto-generated AWS domain with SSL. + const httpApi = new HttpApi(this, namer(id, "api").pascal, { + apiName: id.pathKebabFrom(this), + }); + + const userData = UserData.custom( + cloudConfigYamlDump( + cloudConfigBuild({ + fqdn, + ghTokenSecretName: props.ghTokenSecretName, + ghDockerComposeDirectoryUrl: props.host.ghDockerComposeDirectoryUrl, + dockerComposeEnv: { + DOMAIN: httpApi.apiEndpoint, + ASGS: asgSpecs + .map( + ({ runner, asgName }) => + `${runner.ghRepository}:${runner.label}:${asgName}`, + ) + .join(" "), + }, + dockerComposeProfiles: props.host.dockerComposeProfiles ?? [], + dockerComposePrePullImages: [ { - deviceName: "/dev/sda1", - volume: BlockDeviceVolume.ebs(20, { - encrypted: true, - volumeType: EbsDeviceVolumeType.GP2, - deleteOnTermination: true, - }), + repo: "ghcr.io", + image: "dimikot/ci-storage", + tags: ["main", "latest"], + }, + { + repo: "ghcr.io", + image: "dimikot/ci-scaler", + tags: ["main", "latest"], }, ], - userDataCausesReplacement: true, - requireImdsv2: true, - detailedMonitoring: true, + dockerComposeCmdAfter: null, + keyPairPrivateKeySecretName: this.keyPairPrivateKeySecretName, + timeZone: props.timeZone, + ephemeral: undefined, + tmpfs: props.host.varLibDockerOnTmpfsMaxSizeGb + ? { + path: "/var/lib/docker", + chmod: "710", + maxSizeGb: props.host.varLibDockerOnTmpfsMaxSizeGb, + } + : undefined, + swapSizeGb: props.host.swapSizeGb, + }), + ), + ); + + const launchTemplate = new LaunchTemplate(this, namer(id, "lt").pascal, { + launchTemplateName: id.pathKebabFrom(this), + role: this.roles.host, // LaunchTemplate creates InstanceProfile internally + requireImdsv2: true, + httpPutResponseHopLimit: 2, // LaunchTemplate is the ONLY way to set it + detailedMonitoring: true, + // The properties below are set at LaunchTemplate level, since it's the + // only way to set them when using AutoScalingGroup. + machineImage: MachineImage.fromSsmParameter(props.host.imageSsmName), + securityGroup: this.securityGroup, + keyPair: this.keyPair, + blockDevices: [ + { + deviceName: "/dev/sda1", + volume: BlockDeviceVolume.ebs(20, { + encrypted: true, + volumeType: EbsDeviceVolumeType.GP2, + deleteOnTermination: true, + }), }, - ); - (instance.node.defaultChild as CfnInstance).cfnOptions.creationPolicy = - { resourceSignal: { count: 1, timeout: "PT15M" } }; - Tags.of(instance.instance).add("Name", fqdn ?? recordName); - this.hosts.push({ fqdn, instance }); + ], + userData, + }); + + const instance = new CfnInstance(this, namer(id, "instance").pascal, { + launchTemplate: { + launchTemplateId: launchTemplate.launchTemplateId, + version: launchTemplate.versionNumber, + }, + instanceType: props.host.instanceType, + subnetId: this.vpc.privateSubnets[0].subnetId, + availabilityZone: this.vpc.availabilityZones[0], + tags: [{ key: "Name", value: fqdn ?? recordName }], + }); + userDataCausesReplacement(instance, userData); + instance.cfnOptions.creationPolicy = { + resourceSignal: { count: 1, timeout: "PT15M" }, + }; - if (this.hostedZone) { - new ARecord(this, namer(id, namer("a")).pascal, { - zone: this.hostedZone, - recordName, - target: RecordTarget.fromIpAddresses(instance.instancePrivateIp), - ttl: Duration.minutes(1), - }); - } + if (this.hostedZone) { + new ARecord(this, namer(id, "a").pascal, { + zone: this.hostedZone, + recordName, + target: RecordTarget.fromIpAddresses(instance.attrPrivateIp), + ttl: Duration.minutes(1), + }); } + + // NLB is needed, since it's the only reliable way for HttpApi to reach an + // individual instance (HttpUrlIntegration doesn't work with private IPs + // since there is no easy way to represent an HttpApi or VpcLink in a + // security group's source/peer). + const nlb = new NetworkLoadBalancer(this, namer(id, "nlb").pascal, { + loadBalancerName: id.pathKebabFrom(this), + vpc: this.vpc, + internetFacing: false, + securityGroups: [this.securityGroup], + }); + const nlbTargetGroup = new NetworkTargetGroup( + this, + namer(id, "tg").pascal, + { + targetGroupName: id.pathKebabFrom(this), + vpc: this.vpc, + port: webhookHttpPort, + targets: [new InstanceIdTarget(instance.ref)], + healthCheck: { + interval: Duration.seconds(5), // minimal possible + timeout: Duration.seconds(2), // minimal possible + healthyThresholdCount: 2, // minimal possible; appear healthy ASAP + unhealthyThresholdCount: 10, // maximal possible; appear unhealthy after a long time + }, + }, + ); + const nlbListener = nlb.addListener(namer("listener").pascal, { + port: webhookHttpPort, + protocol: Protocol.TCP, + defaultTargetGroups: [nlbTargetGroup], + }); + httpApi.addRoutes({ + path: "/{proxy+}", + methods: [HttpMethod.POST], + integration: new HttpNlbIntegration( + namer("integration").pascal, + nlbListener, + { vpcLink: this.vpcLink }, + ), + }); + + this.host = { fqdn, instance }; } - { + // + // Runner resources. + // + + const instanceToAmiName = namer("instancetoami").pathKebabFrom(this); + + for (const { id, runner, asgName } of asgSpecs) { const userData = UserData.custom( cloudConfigYamlDump( cloudConfigBuild({ fqdn: undefined, // no way to assign an unique hostname via LaunchTemplate ghTokenSecretName: props.ghTokenSecretName, - ghDockerComposeDirectoryUrl: - props.runner.ghDockerComposeDirectoryUrl, + ghDockerComposeDirectoryUrl: runner.ghDockerComposeDirectoryUrl, dockerComposeEnv: { - GH_REPOSITORY: props.runner.ghRepository, - GH_LABELS: `${instanceNamePrefix.kebab},ci-storage`, - // Future idea: each runner should know its host (for load - // balancing purposes); for now, we just hardcode the 1st one. - FORWARD_HOST: this.hosts[0].fqdn ?? "", + // - GH_TOKEN: passed by cloudConfigBuild() + // - TZ: passed by cloudConfigBuild() + // - FORWARD_PORTS: implied to be set in ci-runner's compose.yml + // - CI_STORAGE_HOST: implied to be set in ci-runner's compose.yml + GH_REPOSITORY: runner.ghRepository, + GH_LABELS: `${instanceNamePrefix.kebab},${runner.label}`, + FORWARD_HOST: this.host.fqdn || this.host.instance.attrPrivateIp, }, dockerComposeProfiles: [], + dockerComposePrePullImages: [ + { + repo: "ghcr.io", + image: "dimikot/ci-runner", + tags: ["main", "latest"], + }, + ], + dockerComposeCmdAfter: dedent(` + export deps=$(docker image ls --format "{{.Repository}}:{{.ID}}:{{.Tag}}" | grep dimikot/ci-runner) + export instanceId=$(cloud-init query ds.meta_data.instance_id) + aws lambda invoke --function-name "${instanceToAmiName}" \\ + --payload "$(jq -nc '{"instanceId":$ENV.instanceId,"deps":$ENV.deps}')" \\ + /dev/stdout | jq -s '.[0]' + `), keyPairPrivateKeySecretName: this.keyPairPrivateKeySecretName, timeZone: props.timeZone, - tmpfs: props.runner.tmpfsMaxSizeGb - ? { - path: "/var/lib/docker", - maxSizeGb: props.runner.tmpfsMaxSizeGb, - } - : undefined, - swapSizeGb: props.runner.swapSizeGb, + ephemeral: { + path: "/var/log", + chown: "0:syslog", + chmod: "775", + }, + tmpfs: undefined, // compose.yml will mount /mnt on tmpfs by itself + swapSizeGb: runner.swapSizeGb, }), ), ); - const id = namer("lt"); - this.launchTemplate = new LaunchTemplate(this, id.pascal, { + + const launchTemplate = new LaunchTemplate(this, namer(id, "lt").pascal, { launchTemplateName: id.pathKebabFrom(this), - machineImage: MachineImage.fromSsmParameter(props.runner.imageSsmName, { - os: OperatingSystemType.LINUX, - }), - keyPair: this.keyPair, role: this.roles.runner, // LaunchTemplate creates InstanceProfile internally + requireImdsv2: true, + httpPutResponseHopLimit: 2, // LaunchTemplate is the ONLY way to set it + detailedMonitoring: true, + // The properties below are set at LaunchTemplate level, since it's the + // only way to set them when using AutoScalingGroup. + machineImage: MachineImage.fromSsmParameter(props.host.imageSsmName), + securityGroup: this.securityGroup, + keyPair: this.keyPair, blockDevices: [ { deviceName: "/dev/sda1", - volume: BlockDeviceVolume.ebs(props.runner.volumeGb, { + volume: BlockDeviceVolume.ebs(runner.volumeRootGb, { encrypted: true, - volumeType: EbsDeviceVolumeType.GP2, + volumeType: EbsDeviceVolumeType.GP3, deleteOnTermination: true, + iops: runner.volumeRootIops, + throughput: runner.volumeRootThroughput, }), }, + { + deviceName: "/dev/sdb", // doesn't matter, AWS renames them unpredictably + volume: BlockDeviceVolume.ebs( + runner.volumeLogsGb + (runner.swapSizeGb ?? 0), + { + encrypted: true, + volumeType: EbsDeviceVolumeType.GP3, + deleteOnTermination: true, + }, + ), + }, ], userData, - securityGroup: this.securityGroup, - requireImdsv2: true, - httpPutResponseHopLimit: 2, - detailedMonitoring: true, }); - this.launchTemplate.node.addDependency(this.keyPair); - } - { - const id = namer("asg", "runner"); - this.autoScalingGroup = new AutoScalingGroup(this, id.pascal, { - autoScalingGroupName: id.pathKebabFrom(this), - vpc: this.vpc, - maxCapacity: props.runner.scale.maxCapacity, - maxInstanceLifetime: props.runner.scale.maxInstanceLifetime, - mixedInstancesPolicy: { - instancesDistribution: { - onDemandAllocationStrategy: OnDemandAllocationStrategy.LOWEST_PRICE, - onDemandPercentageAboveBaseCapacity: - props.runner.scale.onDemandPercentageAboveBaseCapacity, - spotAllocationStrategy: - SpotAllocationStrategy.PRICE_CAPACITY_OPTIMIZED, - }, - launchTemplate: this.launchTemplate, - launchTemplateOverrides: props.runner.instanceRequirements.map( - (req) => ({ - instanceRequirements: req, - }), + const autoScalingGroup = new AutoScalingGroup( + this, + namer(id, "asg").pascal, + { + autoScalingGroupName: asgName, + vpc: this.vpc, + vpcSubnets: this.vpc.selectSubnets({ + // Create all instances in the same (first) AZ. This is needed to + // save money on "fast snapshot restore" feature (see InstanceToAmi) + // which charges ~$0.75 per snapshot per AZ per hour. + availabilityZones: [this.vpc.availabilityZones.sort()[0]], + }), + minCapacity: Math.min( + ...runner.minCapacity.map(({ value }) => value), ), + maxCapacity: runner.maxCapacity, + maxInstanceLifetime: runner.maxInstanceLifetime, + capacityRebalance: true, + mixedInstancesPolicy: { + instancesDistribution: { + onDemandAllocationStrategy: + OnDemandAllocationStrategy.LOWEST_PRICE, + onDemandPercentageAboveBaseCapacity: + runner.onDemandPercentageAboveBaseCapacity, + spotAllocationStrategy: + SpotAllocationStrategy.PRICE_CAPACITY_OPTIMIZED, + }, + launchTemplate, + launchTemplateOverrides: runner.instanceRequirements.map((req) => ({ + instanceRequirements: req, + })), + }, + cooldown: Duration.seconds(30), + defaultInstanceWarmup: Duration.seconds(60), + groupMetrics: [GroupMetrics.all()], + updatePolicy: UpdatePolicy.rollingUpdate(), }, - cooldown: Duration.seconds(30), - defaultInstanceWarmup: Duration.seconds(60), - groupMetrics: [GroupMetrics.all()], - updatePolicy: UpdatePolicy.rollingUpdate(), - }); - Tags.of(this.autoScalingGroup).add( + ); + Tags.of(autoScalingGroup).add( "Name", - namer(instanceNamePrefix, "runner").kebab, + namer(instanceNamePrefix, id).kebab, ); - const metric = new Metric({ - namespace: "ci-storage/metrics", - metricName: "ActiveRunnersPercent", - dimensionsMap: { GH_REPOSITORY: props.runner.ghRepository }, - period: Duration.seconds( - props.runner.scale.maxActiveRunnersPercent.periodSec, - ), - statistic: "max", - }); - const scalingSteps = buildPercentScalingSteps( - props.runner.scale.maxActiveRunnersPercent.value, - props.runner.scale.maxActiveRunnersPercent.scalingSteps ?? 6, - ); - this.autoScalingGroup.scaleOnMetric("ActiveRunnersPercent", { - metric, - adjustmentType: AdjustmentType.PERCENT_CHANGE_IN_CAPACITY, - scalingSteps, - evaluationPeriods: 1, - datapointsToAlarm: 1, - }); - for (const { id, value, cron } of props.runner.scale.minCapacity) { - this.autoScalingGroup.scaleOnSchedule(id, { + for (const { id, value, cron } of runner.minCapacity) { + autoScalingGroup.scaleOnSchedule(id, { minCapacity: value, timeZone: cron.timeZone ?? props.timeZone, schedule: Schedule.cron({ minute: "0", ...cron }), }); } + + this.autoScalingGroups.push({ autoScalingGroup, launchTemplate }); + } + + // + // InstanceToAmi + // + { + const id = namer("instance", "to", "ami"); + this.instanceToAmi = new InstanceToAmi(this, id.pascal, { + name: instanceToAmiName, + autoScalingGroups: this.autoScalingGroups, + addToRoles: Object.values(this.roles), + }); } } } diff --git a/src/__tests__/CiStorage.test.ts b/src/__tests__/CiStorage.test.ts index 2bbcee1..730a565 100644 --- a/src/__tests__/CiStorage.test.ts +++ b/src/__tests__/CiStorage.test.ts @@ -30,53 +30,64 @@ class CiStorageStack extends Stack { }, ghTokenSecretName: "ci-storage/gh-token", timeZone: "America/Los_Angeles", - runner: { - ghRepository: "time-loop/slapdash", - ghDockerComposeDirectoryUrl: - "https://github.com/dimikot/ci-storage#:docker", - imageSsmName: "test-imageSsmName", - volumeGb: 50, - swapSizeGb: 8, - tmpfsMaxSizeGb: 4, - instanceRequirements: [ - { - memoryMiB: { min: 8192, max: 16384 }, - vCpuCount: { min: 4, max: 8 }, - }, - ], - scale: { - onDemandPercentageAboveBaseCapacity: 10, - maxActiveRunnersPercent: { - periodSec: 600, - value: 70, - scalingSteps: 10, - }, - minCapacity: [ + runners: [ + { + label: "ci-small", + ghRepository: "time-loop/slapdash", + ghDockerComposeDirectoryUrl: + "https://github.com/dimikot/ci-storage#:docker", + imageSsmName: "/test-image/ssm/name", + volumeRootGb: 20, + volumeLogsGb: 5, + swapSizeGb: 4, + instanceRequirements: [ { - id: "CaWorkDayStarts", - value: 10, - cron: { hour: "8" }, + memoryMiB: { min: 4096, max: 8192 }, + vCpuCount: { min: 2, max: 4 }, }, + ], + onDemandPercentageAboveBaseCapacity: 10, + minCapacity: [ + { id: "CaWorkDayStarts", value: 10, cron: { hour: "8" } }, + { id: "CaWorkDayEnds", value: 5, cron: { hour: "18" } }, + ], + maxCapacity: 10, + maxInstanceLifetime: Duration.days(1), + }, + { + label: "ci-large", + ghRepository: "time-loop/slapdash", + ghDockerComposeDirectoryUrl: + "https://github.com/dimikot/ci-storage#:docker", + imageSsmName: "/test-image/ssm/name", + volumeRootGb: 40, + volumeLogsGb: 5, + swapSizeGb: 8, + instanceRequirements: [ { - id: "CaWorkDayEnds", - value: 5, - cron: { hour: "18" }, + memoryMiB: { min: 8192, max: 16384 }, + vCpuCount: { min: 4, max: 8 }, }, ], + onDemandPercentageAboveBaseCapacity: 10, + minCapacity: [ + { id: "CaWorkDayStarts", value: 10, cron: { hour: "8" } }, + { id: "CaWorkDayEnds", value: 5, cron: { hour: "18" } }, + ], maxCapacity: 20, maxInstanceLifetime: Duration.days(1), }, - }, + ], host: { ghDockerComposeDirectoryUrl: "https://github.com/dimikot/ci-storage#:docker", dockerComposeProfiles: ["ci"], - imageSsmName: "test-imageSsmName", - tmpfsMaxSizeGb: 4, + imageSsmName: "/test-image/ssm/name", + varLibDockerOnTmpfsMaxSizeGb: 4, instanceType: "t3.large", - machines: 1, ports: [ - { port: Port.tcp(10022), description: "ci-storage container" }, + { port: Port.tcp(26022), description: "ci-storage" }, + { port: Port.tcp(28088), description: "ci-scaler", isWebhook: true }, { port: Port.tcpRange(42000, 42042), description: "test ports" }, ], }, diff --git a/src/__tests__/__snapshots__/CiStorage.test.ts.snap b/src/__tests__/__snapshots__/CiStorage.test.ts.snap index 1a6b80e..a7708b5 100644 --- a/src/__tests__/__snapshots__/CiStorage.test.ts.snap +++ b/src/__tests__/__snapshots__/CiStorage.test.ts.snap @@ -3,300 +3,203 @@ exports[`CiStorage 1`] = ` { "Description": "A stack to generate unit tests for CiStorage construct", + "Mappings": { + "LatestNodeRuntimeMap": { + "af-south-1": { + "value": "nodejs20.x", + }, + "ap-east-1": { + "value": "nodejs20.x", + }, + "ap-northeast-1": { + "value": "nodejs20.x", + }, + "ap-northeast-2": { + "value": "nodejs20.x", + }, + "ap-northeast-3": { + "value": "nodejs20.x", + }, + "ap-south-1": { + "value": "nodejs20.x", + }, + "ap-south-2": { + "value": "nodejs20.x", + }, + "ap-southeast-1": { + "value": "nodejs20.x", + }, + "ap-southeast-2": { + "value": "nodejs20.x", + }, + "ap-southeast-3": { + "value": "nodejs20.x", + }, + "ap-southeast-4": { + "value": "nodejs20.x", + }, + "ap-southeast-5": { + "value": "nodejs20.x", + }, + "ap-southeast-7": { + "value": "nodejs20.x", + }, + "ca-central-1": { + "value": "nodejs20.x", + }, + "ca-west-1": { + "value": "nodejs20.x", + }, + "cn-north-1": { + "value": "nodejs18.x", + }, + "cn-northwest-1": { + "value": "nodejs18.x", + }, + "eu-central-1": { + "value": "nodejs20.x", + }, + "eu-central-2": { + "value": "nodejs20.x", + }, + "eu-isoe-west-1": { + "value": "nodejs18.x", + }, + "eu-north-1": { + "value": "nodejs20.x", + }, + "eu-south-1": { + "value": "nodejs20.x", + }, + "eu-south-2": { + "value": "nodejs20.x", + }, + "eu-west-1": { + "value": "nodejs20.x", + }, + "eu-west-2": { + "value": "nodejs20.x", + }, + "eu-west-3": { + "value": "nodejs20.x", + }, + "il-central-1": { + "value": "nodejs20.x", + }, + "me-central-1": { + "value": "nodejs20.x", + }, + "me-south-1": { + "value": "nodejs20.x", + }, + "mx-central-1": { + "value": "nodejs20.x", + }, + "sa-east-1": { + "value": "nodejs20.x", + }, + "us-east-1": { + "value": "nodejs20.x", + }, + "us-east-2": { + "value": "nodejs20.x", + }, + "us-gov-east-1": { + "value": "nodejs18.x", + }, + "us-gov-west-1": { + "value": "nodejs18.x", + }, + "us-iso-east-1": { + "value": "nodejs18.x", + }, + "us-iso-west-1": { + "value": "nodejs18.x", + }, + "us-isob-east-1": { + "value": "nodejs18.x", + }, + "us-west-1": { + "value": "nodejs20.x", + }, + "us-west-2": { + "value": "nodejs20.x", + }, + }, + }, "Parameters": { "BootstrapVersion": { "Default": "/cdk-bootstrap/hnb659fds/version", "Description": "Version of the CDK Bootstrap resources in this environment, automatically retrieved from SSM Parameter Store. [cdk:skip]", "Type": "AWS::SSM::Parameter::Value", }, - "SsmParameterValuetestimageSsmNameC96584B6F00A464EAD1953AFF4B05118Parameter": { - "Default": "test-imageSsmName", + "SsmParameterValuetestimagessmnameC96584B6F00A464EAD1953AFF4B05118Parameter": { + "Default": "/test-image/ssm/name", "Type": "AWS::SSM::Parameter::Value", }, }, "Resources": { - "CnstrctAsgRunnerASG489B3A08": { + "AWS679f53fac002430cb0da5b7982bd22872D164C4C": { + "DependsOn": [ + "AWS679f53fac002430cb0da5b7982bd2287ServiceRoleC1EA0FF2", + ], "Properties": { - "AutoScalingGroupName": "stk-cnstrct-asgrunner", - "Cooldown": "30", - "DefaultInstanceWarmup": 60, - "MaxInstanceLifetime": 86400, - "MaxSize": "20", - "MetricsCollection": [ - { - "Granularity": "1Minute", - }, - ], - "MinSize": "1", - "MixedInstancesPolicy": { - "InstancesDistribution": { - "OnDemandAllocationStrategy": "lowest-price", - "OnDemandPercentageAboveBaseCapacity": 10, - "SpotAllocationStrategy": "price-capacity-optimized", - }, - "LaunchTemplate": { - "LaunchTemplateSpecification": { - "LaunchTemplateId": { - "Ref": "CnstrctLtAEEE5196", - }, - "Version": { - "Fn::GetAtt": [ - "CnstrctLtAEEE5196", - "LatestVersionNumber", - ], - }, - }, - "Overrides": [ - { - "InstanceRequirements": { - "MemoryMiB": { - "Max": 16384, - "Min": 8192, - }, - "VCpuCount": { - "Max": 8, - "Min": 4, - }, - }, - }, - ], + "Code": { + "S3Bucket": { + "Fn::Sub": "cdk-hnb659fds-assets-\${AWS::AccountId}-\${AWS::Region}", }, + "S3Key": "97f30e67419a1676a2215492723e5add1aa491caf0cbe2dd878fc4fab0468cd4.zip", }, - "Tags": [ - { - "Key": "Name", - "PropagateAtLaunch": true, - "Value": "my-ci-runner", - }, - ], - "VPCZoneIdentifier": [ - { - "Ref": "VpcPrivateSubnet1Subnet536B997A", - }, - { - "Ref": "VpcPrivateSubnet2Subnet3788AAA1", - }, - ], - }, - "Type": "AWS::AutoScaling::AutoScalingGroup", - "UpdatePolicy": { - "AutoScalingRollingUpdate": { - "SuspendProcesses": [ - "HealthCheck", - "ReplaceUnhealthy", - "AZRebalance", - "AlarmNotification", - "ScheduledActions", - "InstanceRefresh", + "Handler": "index.handler", + "Role": { + "Fn::GetAtt": [ + "AWS679f53fac002430cb0da5b7982bd2287ServiceRoleC1EA0FF2", + "Arn", ], }, - "AutoScalingScheduledAction": { - "IgnoreUnmodifiedGroupSizeProperties": true, - }, - }, - }, - "CnstrctAsgRunnerActiveRunnersPercentLowerAlarm3D348E7B": { - "Properties": { - "AlarmActions": [ - { - "Ref": "CnstrctAsgRunnerActiveRunnersPercentLowerPolicy9E822646", - }, - ], - "AlarmDescription": "Lower threshold scaling alarm", - "ComparisonOperator": "LessThanOrEqualToThreshold", - "DatapointsToAlarm": 1, - "Dimensions": [ - { - "Name": "GH_REPOSITORY", - "Value": "time-loop/slapdash", - }, - ], - "EvaluationPeriods": 1, - "MetricName": "ActiveRunnersPercent", - "Namespace": "ci-storage/metrics", - "Period": 600, - "Statistic": "Maximum", - "Tags": [ - { - "Key": "Name", - "Value": "my-ci-runner", - }, - ], - "Threshold": 63, - }, - "Type": "AWS::CloudWatch::Alarm", - }, - "CnstrctAsgRunnerActiveRunnersPercentLowerPolicy9E822646": { - "Properties": { - "AdjustmentType": "PercentChangeInCapacity", - "AutoScalingGroupName": { - "Ref": "CnstrctAsgRunnerASG489B3A08", + "Runtime": { + "Fn::FindInMap": [ + "LatestNodeRuntimeMap", + { + "Ref": "AWS::Region", + }, + "value", + ], }, - "MetricAggregationType": "Maximum", - "PolicyType": "StepScaling", - "StepAdjustments": [ - { - "MetricIntervalLowerBound": -7, - "MetricIntervalUpperBound": 0, - "ScalingAdjustment": -10, - }, - { - "MetricIntervalLowerBound": -14, - "MetricIntervalUpperBound": -7, - "ScalingAdjustment": -20, - }, - { - "MetricIntervalLowerBound": -21, - "MetricIntervalUpperBound": -14, - "ScalingAdjustment": -30, - }, - { - "MetricIntervalLowerBound": -28, - "MetricIntervalUpperBound": -21, - "ScalingAdjustment": -40, - }, - { - "MetricIntervalLowerBound": -35, - "MetricIntervalUpperBound": -28, - "ScalingAdjustment": -50, - }, - { - "MetricIntervalLowerBound": -42, - "MetricIntervalUpperBound": -35, - "ScalingAdjustment": -60, - }, - { - "MetricIntervalLowerBound": -49, - "MetricIntervalUpperBound": -42, - "ScalingAdjustment": -70, - }, - { - "MetricIntervalLowerBound": -56, - "MetricIntervalUpperBound": -49, - "ScalingAdjustment": -80, - }, - { - "MetricIntervalUpperBound": -56, - "ScalingAdjustment": -90, - }, - ], - }, - "Type": "AWS::AutoScaling::ScalingPolicy", - }, - "CnstrctAsgRunnerActiveRunnersPercentUpperAlarm9D4818B9": { - "Properties": { - "AlarmActions": [ - { - "Ref": "CnstrctAsgRunnerActiveRunnersPercentUpperPolicy07F985DA", - }, - ], - "AlarmDescription": "Upper threshold scaling alarm", - "ComparisonOperator": "GreaterThanOrEqualToThreshold", - "DatapointsToAlarm": 1, - "Dimensions": [ - { - "Name": "GH_REPOSITORY", - "Value": "time-loop/slapdash", - }, - ], - "EvaluationPeriods": 1, - "MetricName": "ActiveRunnersPercent", - "Namespace": "ci-storage/metrics", - "Period": 600, - "Statistic": "Maximum", - "Tags": [ - { - "Key": "Name", - "Value": "my-ci-runner", - }, - ], - "Threshold": 70, + "Timeout": 120, }, - "Type": "AWS::CloudWatch::Alarm", + "Type": "AWS::Lambda::Function", }, - "CnstrctAsgRunnerActiveRunnersPercentUpperPolicy07F985DA": { + "AWS679f53fac002430cb0da5b7982bd2287ServiceRoleC1EA0FF2": { "Properties": { - "AdjustmentType": "PercentChangeInCapacity", - "AutoScalingGroupName": { - "Ref": "CnstrctAsgRunnerASG489B3A08", + "AssumeRolePolicyDocument": { + "Statement": [ + { + "Action": "sts:AssumeRole", + "Effect": "Allow", + "Principal": { + "Service": "lambda.amazonaws.com", + }, + }, + ], + "Version": "2012-10-17", }, - "MetricAggregationType": "Maximum", - "PolicyType": "StepScaling", - "StepAdjustments": [ - { - "MetricIntervalLowerBound": 0, - "MetricIntervalUpperBound": 3, - "ScalingAdjustment": 4, - }, - { - "MetricIntervalLowerBound": 3, - "MetricIntervalUpperBound": 6, - "ScalingAdjustment": 9, - }, - { - "MetricIntervalLowerBound": 6, - "MetricIntervalUpperBound": 9, - "ScalingAdjustment": 13, - }, - { - "MetricIntervalLowerBound": 9, - "MetricIntervalUpperBound": 12, - "ScalingAdjustment": 17, - }, - { - "MetricIntervalLowerBound": 12, - "MetricIntervalUpperBound": 15, - "ScalingAdjustment": 21, - }, - { - "MetricIntervalLowerBound": 15, - "MetricIntervalUpperBound": 18, - "ScalingAdjustment": 26, - }, - { - "MetricIntervalLowerBound": 18, - "MetricIntervalUpperBound": 21, - "ScalingAdjustment": 30, - }, - { - "MetricIntervalLowerBound": 21, - "MetricIntervalUpperBound": 24, - "ScalingAdjustment": 34, - }, - { - "MetricIntervalLowerBound": 24, - "MetricIntervalUpperBound": 27, - "ScalingAdjustment": 39, - }, + "ManagedPolicyArns": [ { - "MetricIntervalLowerBound": 27, - "ScalingAdjustment": 43, + "Fn::Join": [ + "", + [ + "arn:", + { + "Ref": "AWS::Partition", + }, + ":iam::aws:policy/service-role/AWSLambdaBasicExecutionRole", + ], + ], }, ], }, - "Type": "AWS::AutoScaling::ScalingPolicy", - }, - "CnstrctAsgRunnerScheduledActionCaWorkDayEnds81158DAD": { - "Properties": { - "AutoScalingGroupName": { - "Ref": "CnstrctAsgRunnerASG489B3A08", - }, - "MinSize": 5, - "Recurrence": "0 18 * * *", - "TimeZone": "America/Los_Angeles", - }, - "Type": "AWS::AutoScaling::ScheduledAction", - }, - "CnstrctAsgRunnerScheduledActionCaWorkDayStarts6A1CC23B": { - "Properties": { - "AutoScalingGroupName": { - "Ref": "CnstrctAsgRunnerASG489B3A08", - }, - "MinSize": 10, - "Recurrence": "0 8 * * *", - "TimeZone": "America/Los_Angeles", - }, - "Type": "AWS::AutoScaling::ScheduledAction", + "Type": "AWS::IAM::Role", }, "CnstrctHostRole5DD9F366": { "Properties": { @@ -447,388 +350,447 @@ exports[`CiStorage 1`] = ` }, "PolicyName": "SignalResourcePolicy", }, - ], - "RoleName": "StkCnstrctHostRole", - }, - "Type": "AWS::IAM::Role", - }, - "CnstrctLtAEEE5196": { - "DependsOn": [ - "CnstrctRunnerRole341E54FA", - ], - "Properties": { - "LaunchTemplateData": { - "BlockDeviceMappings": [ - { - "DeviceName": "/dev/sda1", - "Ebs": { - "DeleteOnTermination": true, - "Encrypted": true, - "VolumeSize": 50, - "VolumeType": "gp2", - }, - }, - ], - "IamInstanceProfile": { - "Arn": { - "Fn::GetAtt": [ - "CnstrctLtProfileE3059BA8", - "Arn", + { + "PolicyDocument": { + "Statement": [ + { + "Action": "ec2:DescribeTags", + "Effect": "Allow", + "Resource": "*", + }, + { + "Action": "ec2:CreateTags", + "Condition": { + "StringEquals": { + "ec2:ResourceTag/aws:autoscaling:groupName": [ + "stk-cnstrct-runnercismall", + "stk-cnstrct-runnercilarge", + ], + }, + }, + "Effect": "Allow", + "Resource": { + "Fn::Join": [ + "", + [ + "arn:", + { + "Ref": "AWS::Partition", + }, + ":ec2:", + { + "Ref": "AWS::Region", + }, + ":", + { + "Ref": "AWS::AccountId", + }, + ":instance/*", + ], + ], + }, + }, ], + "Version": "2012-10-17", }, + "PolicyName": "TagsPolicy", }, - "ImageId": { - "Ref": "SsmParameterValuetestimageSsmNameC96584B6F00A464EAD1953AFF4B05118Parameter", - }, - "KeyName": { - "Fn::GetAtt": [ - "CnstrctSshIdRsaEC2KeyPairstkcnstrctsshidrsaF4F38F69", - "KeyPairName", - ], + { + "PolicyDocument": { + "Statement": [ + { + "Action": "autoscaling:DescribeAutoScalingGroups", + "Effect": "Allow", + "Resource": "*", + }, + { + "Action": [ + "autoscaling:SetDesiredCapacity", + "autoscaling:TerminateInstanceInAutoScalingGroup", + ], + "Effect": "Allow", + "Resource": { + "Fn::Join": [ + "", + [ + "arn:", + { + "Ref": "AWS::Partition", + }, + ":autoscaling:", + { + "Ref": "AWS::Region", + }, + ":", + { + "Ref": "AWS::AccountId", + }, + ":autoScalingGroup:*:autoScalingGroupName/stk-cnstrct-*", + ], + ], + }, + }, + ], + "Version": "2012-10-17", + }, + "PolicyName": "ScalerPolicy", }, - "MetadataOptions": { - "HttpPutResponseHopLimit": 2, - "HttpTokens": "required", + ], + "RoleName": "StkCnstrctHostRole", + }, + "Type": "AWS::IAM::Role", + }, + "CnstrctHostRoleDefaultPolicy8426BDC1": { + "Properties": { + "PolicyDocument": { + "Statement": [ + { + "Action": "lambda:InvokeFunction", + "Effect": "Allow", + "Resource": { + "Fn::GetAtt": [ + "CnstrctInstanceToAmiStartExecution3968560D", + "Arn", + ], + }, + }, + ], + "Version": "2012-10-17", + }, + "PolicyName": "CnstrctHostRoleDefaultPolicy8426BDC1", + "Roles": [ + { + "Ref": "CnstrctHostRole5DD9F366", }, - "Monitoring": { - "Enabled": true, + ], + }, + "Type": "AWS::IAM::Policy", + }, + "CnstrctInstanceToAmiAutoScalingGroupGetNameFromInstance27F9175A": { + "DependsOn": [ + "CnstrctInstanceToAmiAutoScalingGroupGetNameFromInstanceServiceRoleDefaultPolicy2852B3B3", + "CnstrctInstanceToAmiAutoScalingGroupGetNameFromInstanceServiceRole930D7FC4", + ], + "Properties": { + "Code": { + "S3Bucket": { + "Fn::Sub": "cdk-hnb659fds-assets-\${AWS::AccountId}-\${AWS::Region}", }, - "SecurityGroupIds": [ + "S3Key": "24889f5dc169825cb8dd9c7076e6f6886d38a18a2db3fd36a2431314d5711df1.zip", + }, + "Handler": "AutoScalingGroupGetNameFromInstance.handler", + "Role": { + "Fn::GetAtt": [ + "CnstrctInstanceToAmiAutoScalingGroupGetNameFromInstanceServiceRole930D7FC4", + "Arn", + ], + }, + "Runtime": "nodejs20.x", + "Timeout": 180, + }, + "Type": "AWS::Lambda::Function", + }, + "CnstrctInstanceToAmiAutoScalingGroupGetNameFromInstanceServiceRole930D7FC4": { + "Properties": { + "AssumeRolePolicyDocument": { + "Statement": [ { - "Fn::GetAtt": [ - "CnstrctSgF5C70BA4", - "GroupId", - ], + "Action": "sts:AssumeRole", + "Effect": "Allow", + "Principal": { + "Service": "lambda.amazonaws.com", + }, }, ], - "TagSpecifications": [ - { - "ResourceType": "instance", - "Tags": [ + "Version": "2012-10-17", + }, + "ManagedPolicyArns": [ + { + "Fn::Join": [ + "", + [ + "arn:", { - "Key": "Name", - "Value": "Stk/Cnstrct/Lt", + "Ref": "AWS::Partition", }, + ":iam::aws:policy/service-role/AWSLambdaBasicExecutionRole", + ], + ], + }, + ], + }, + "Type": "AWS::IAM::Role", + }, + "CnstrctInstanceToAmiAutoScalingGroupGetNameFromInstanceServiceRoleDefaultPolicy2852B3B3": { + "Properties": { + "PolicyDocument": { + "Statement": [ + { + "Action": [ + "autoscaling:DescribeAutoScalingGroups", + "autoscaling:DescribeAutoScalingInstances", + "autoscaling:UpdateAutoScalingGroup", + "ec2:CreateLaunchTemplateVersion", + "ec2:CreateSnapshot", + "ec2:CreateTags", + "ec2:DeleteLaunchTemplateVersions", + "ec2:DeleteSnapshot", + "ec2:DeregisterImage", + "ec2:DescribeFastSnapshotRestores", + "ec2:DescribeImages", + "ec2:DescribeInstances", + "ec2:DescribeLaunchTemplates", + "ec2:DescribeLaunchTemplateVersions", + "ec2:DescribeSnapshots", + "ec2:DisableFastSnapshotRestores", + "ec2:EnableFastSnapshotRestores", + "ec2:RegisterImage", + "ec2:RunInstances", + "iam:PassRole", + "logs:CreateLogGroup", + "logs:CreateLogStream", + "logs:PutLogEvents", ], + "Effect": "Allow", + "Resource": "*", }, + ], + "Version": "2012-10-17", + }, + "PolicyName": "CnstrctInstanceToAmiAutoScalingGroupGetNameFromInstanceServiceRoleDefaultPolicy2852B3B3", + "Roles": [ + { + "Ref": "CnstrctInstanceToAmiAutoScalingGroupGetNameFromInstanceServiceRole930D7FC4", + }, + ], + }, + "Type": "AWS::IAM::Policy", + }, + "CnstrctInstanceToAmiAutoScalingGroupUpdateLaunchTemplateVersion08E6BA51": { + "DependsOn": [ + "CnstrctInstanceToAmiAutoScalingGroupUpdateLaunchTemplateVersionServiceRoleDefaultPolicy0B2C2009", + "CnstrctInstanceToAmiAutoScalingGroupUpdateLaunchTemplateVersionServiceRole2CD44286", + ], + "Properties": { + "Code": { + "S3Bucket": { + "Fn::Sub": "cdk-hnb659fds-assets-\${AWS::AccountId}-\${AWS::Region}", + }, + "S3Key": "24889f5dc169825cb8dd9c7076e6f6886d38a18a2db3fd36a2431314d5711df1.zip", + }, + "Handler": "AutoScalingGroupUpdateLaunchTemplateVersion.handler", + "Role": { + "Fn::GetAtt": [ + "CnstrctInstanceToAmiAutoScalingGroupUpdateLaunchTemplateVersionServiceRole2CD44286", + "Arn", + ], + }, + "Runtime": "nodejs20.x", + "Timeout": 180, + }, + "Type": "AWS::Lambda::Function", + }, + "CnstrctInstanceToAmiAutoScalingGroupUpdateLaunchTemplateVersionServiceRole2CD44286": { + "Properties": { + "AssumeRolePolicyDocument": { + "Statement": [ { - "ResourceType": "volume", - "Tags": [ + "Action": "sts:AssumeRole", + "Effect": "Allow", + "Principal": { + "Service": "lambda.amazonaws.com", + }, + }, + ], + "Version": "2012-10-17", + }, + "ManagedPolicyArns": [ + { + "Fn::Join": [ + "", + [ + "arn:", { - "Key": "Name", - "Value": "Stk/Cnstrct/Lt", + "Ref": "AWS::Partition", }, + ":iam::aws:policy/service-role/AWSLambdaBasicExecutionRole", + ], + ], + }, + ], + }, + "Type": "AWS::IAM::Role", + }, + "CnstrctInstanceToAmiAutoScalingGroupUpdateLaunchTemplateVersionServiceRoleDefaultPolicy0B2C2009": { + "Properties": { + "PolicyDocument": { + "Statement": [ + { + "Action": [ + "autoscaling:DescribeAutoScalingGroups", + "autoscaling:DescribeAutoScalingInstances", + "autoscaling:UpdateAutoScalingGroup", + "ec2:CreateLaunchTemplateVersion", + "ec2:CreateSnapshot", + "ec2:CreateTags", + "ec2:DeleteLaunchTemplateVersions", + "ec2:DeleteSnapshot", + "ec2:DeregisterImage", + "ec2:DescribeFastSnapshotRestores", + "ec2:DescribeImages", + "ec2:DescribeInstances", + "ec2:DescribeLaunchTemplates", + "ec2:DescribeLaunchTemplateVersions", + "ec2:DescribeSnapshots", + "ec2:DisableFastSnapshotRestores", + "ec2:EnableFastSnapshotRestores", + "ec2:RegisterImage", + "ec2:RunInstances", + "iam:PassRole", + "logs:CreateLogGroup", + "logs:CreateLogStream", + "logs:PutLogEvents", ], + "Effect": "Allow", + "Resource": "*", }, ], - "UserData": { - "Fn::Base64": { - "Fn::Join": [ - "", - [ - "#cloud-config -timezone: America/Los_Angeles -swap: - filename: /var/swapfile - size: auto - maxsize: 8589934592 -mounts: - - - tmpfs - - /var/lib/docker - - tmpfs - - defaults,noatime,exec,mode=0710,nr_inodes=0,size=4G - - "0" - - "0" -apt_sources: - - source: deb https://cli.github.com/packages stable main - keyid: 23F3D4EA75716059 - filename: github-cli.list - - source: deb https://download.docker.com/linux/ubuntu $RELEASE stable - keyid: 9DC858229FC7DD38854AE2D88D81803C0EBFCD88 - filename: docker.list -packages: - - awscli - - gh - - docker-ce - - docker-ce-cli - - containerd.io - - docker-compose-plugin - - qemu - - qemu-user-static - - binfmt-support - - git - - gosu - - mc - - curl - - apt-transport-https - - ca-certificates - - tzdata - - atop - - iotop - - htop - - bwm-ng - - jq -write_files: - - path: /etc/sysctl.d/enable-ipv4-forwarding.conf - content: | - net.ipv4.conf.all.forwarding=1 - - path: /etc/default/atop - content: | - LOGOPTS="-R" - LOGINTERVAL=15 - LOGGENERATIONS=4 - - path: /etc/environment - append: true - content: | - TZ="America/Los_Angeles" - - path: /etc/environment - append: true - content: | - LESS="RS" - - path: /etc/docker/daemon.json - permissions: "0644" - content: | - { - "log-driver": "syslog", - "log-opts": { - "tag": "docker/{{.Name}}" + "Version": "2012-10-17", }, - "runtimes": { - "sysbox-runc": { - "path": "/usr/bin/sysbox-runc" - } + "PolicyName": "CnstrctInstanceToAmiAutoScalingGroupUpdateLaunchTemplateVersionServiceRoleDefaultPolicy0B2C2009", + "Roles": [ + { + "Ref": "CnstrctInstanceToAmiAutoScalingGroupUpdateLaunchTemplateVersionServiceRole2CD44286", + }, + ], + }, + "Type": "AWS::IAM::Policy", + }, + "CnstrctInstanceToAmiFastSnapshotRestoreEnable8C7FE336": { + "DependsOn": [ + "CnstrctInstanceToAmiFastSnapshotRestoreEnableServiceRoleDefaultPolicyE06BF178", + "CnstrctInstanceToAmiFastSnapshotRestoreEnableServiceRole985CCAE9", + ], + "Properties": { + "Code": { + "S3Bucket": { + "Fn::Sub": "cdk-hnb659fds-assets-\${AWS::AccountId}-\${AWS::Region}", + }, + "S3Key": "24889f5dc169825cb8dd9c7076e6f6886d38a18a2db3fd36a2431314d5711df1.zip", }, - "default-runtime": "sysbox-runc", - "userns-remap": "sysbox" - } - - path: /var/lib/cloud/scripts/per-once/apply-services-configs.sh - permissions: "0755" - content: | - #!/bin/bash - set -e -o pipefail && echo ================ && echo "Running $BASH_SOURCE as $(whoami)" && set -o xtrace && export AWS_DEFAULT_REGION=$(ec2metadata --availability-zone | sed "s/[a-z]$//") - service atop restart || true - sysctl --system - - path: /var/lib/cloud/scripts/per-once/increase-docker-shutdown-timeout.sh - permissions: "0755" - content: | - #!/bin/bash - set -e -o pipefail && echo ================ && echo "Running $BASH_SOURCE as $(whoami)" && set -o xtrace && export AWS_DEFAULT_REGION=$(ec2metadata --availability-zone | sed "s/[a-z]$//") - sed -i -E '/TimeoutStartSec=.*/a TimeoutStopSec=3600' /usr/lib/systemd/system/docker.service - systemctl daemon-reload - - path: /var/lib/cloud/scripts/per-once/add-ubuntu-user-to-docker-group-to-access-socket.sh - permissions: "0755" - content: | - #!/bin/bash - set -e -o pipefail && echo ================ && echo "Running $BASH_SOURCE as $(whoami)" && set -o xtrace && export AWS_DEFAULT_REGION=$(ec2metadata --availability-zone | sed "s/[a-z]$//") - usermod -aG docker ubuntu - - path: /var/lib/cloud/scripts/per-once/install-sysbox-for-docker-in-docker.sh - permissions: "0755" - content: | - #!/bin/bash - set -e -o pipefail && echo ================ && echo "Running $BASH_SOURCE as $(whoami)" && set -o xtrace && export AWS_DEFAULT_REGION=$(ec2metadata --availability-zone | sed "s/[a-z]$//") - systemctl stop docker docker.socket || true - wget -nv -O /tmp/sysbox-ce.deb "https://downloads.nestybox.com/sysbox/releases/v0.6.4/sysbox-ce_0.6.4-0.linux_$(dpkg --print-architecture).deb" - dpkg -i /tmp/sysbox-ce.deb - rm -f /tmp/sysbox-ce.deb - - path: /var/lib/cloud/scripts/per-once/switch-ssm-user-to-ubuntu-on-login.sh - permissions: "0755" - content: | - #!/bin/bash - set -e -o pipefail && echo ================ && echo "Running $BASH_SOURCE as $(whoami)" && set -o xtrace && export AWS_DEFAULT_REGION=$(ec2metadata --availability-zone | sed "s/[a-z]$//") - echo '[ "$0$@" = "sh" ] && ENV= sudo -u ubuntu -i' > /etc/profile.ssm-user - mkdir -p /etc/systemd/system/snap.amazon-ssm-agent.amazon-ssm-agent.service.d/ - ( - echo '[Service]' - echo 'Environment="ENV=/etc/profile.ssm-user"' - ) > /etc/systemd/system/snap.amazon-ssm-agent.amazon-ssm-agent.service.d/sh-env.conf - systemctl daemon-reload - systemctl restart snap.amazon-ssm-agent.amazon-ssm-agent.service || true - - path: /etc/rsyslog.d/01-docker-tag-to-serial-console.conf - permissions: "0644" - content: | - if $syslogtag startswith 'docker/' then -/dev/console - # It will also write to /var/log/syslog as usual. - - path: /var/lib/cloud/scripts/per-once/allow-rsyslog-write-to-serial-console.sh - permissions: "0755" - content: | - #!/bin/bash - set -e -o pipefail && echo ================ && echo "Running $BASH_SOURCE as $(whoami)" && set -o xtrace && export AWS_DEFAULT_REGION=$(ec2metadata --availability-zone | sed "s/[a-z]$//") - usermod -a -G tty syslog - systemctl restart rsyslog - - path: /var/lib/cloud/scripts/per-boot/run-docker-compose-on-boot.sh - permissions: "0755" - content: | - #!/bin/bash - set -e -o pipefail && echo ================ && echo "Running $BASH_SOURCE as $(whoami)" && set -o xtrace && export AWS_DEFAULT_REGION=$(ec2metadata --availability-zone | sed "s/[a-z]$//") - echo "*/2 * * * * ubuntu /home/ubuntu/run-docker-compose.sh --no-logs 2>&1 | logger -t docker/run-docker-compose" > /etc/cron.d/run-docker-compose - exec /home/ubuntu/run-docker-compose.sh - - path: /home/ubuntu/run-docker-compose.sh - owner: ubuntu:ubuntu - permissions: "0755" - defer: true - content: | - #!/bin/bash - - # Switch to non-privileged user if running as root. - if [[ $(whoami) != "ubuntu" ]]; then - exec gosu ubuntu "$BASH_SOURCE" "$@" - fi - - # Ensure there is only one instance of this script running. - exec {FD}<$BASH_SOURCE - flock -n "$FD" || { echo "Already running."; exit 0; } - set -e -o pipefail && echo ================ && echo "Running $BASH_SOURCE as $(whoami)" && set -o xtrace && export AWS_DEFAULT_REGION=$(ec2metadata --availability-zone | sed "s/[a-z]$//") - - # Make sure we're using the right timezone; it may be not up - # to date in the current environment during the very 1st run - # from run-docker-compose-on-boot.sh. - source /etc/environment - export TZ - - # Load private and public keys from Secrets Manager to ~/.ssh. - mkdir -p ~/.ssh && chmod 700 ~/.ssh - aws secretsmanager get-secret-value \\ - --secret-id "ec2-ssh-key/", - { - "Fn::GetAtt": [ - "CnstrctSshIdRsaEC2KeyPairstkcnstrctsshidrsaF4F38F69", - "KeyPairName", - ], - }, - "/private" \\ - --query SecretString --output text \\ - > ~/.ssh/ci-storage - chmod 600 ~/.ssh/ci-storage - ssh-keygen -f ~/.ssh/ci-storage -y > ~/.ssh/ci-storage.pub - - # Load GitHub PAT from Secrets Manager and log in to GitHub. - aws secretsmanager get-secret-value \\ - --secret-id "ci-storage/gh-token" \\ - --query SecretString --output text \\ - | gh auth login --with-token - gh auth setup-git - - # Log in to ghcr.io every hour. - config=~/.docker/config.json - if [[ ! -f $config ]] || find "$config" -type f -mmin +60 | grep -q .; then - gh auth token | docker login ghcr.io -u "$(gh api user -q .login)" --password-stdin - fi - - # Pull the repository. - mkdir -p ~/git && cd ~/git - if [[ ! -d .git ]]; then - git clone -n --depth=1 --filter=tree:0 "https://github.com/dimikot/ci-storage" . - if [[ "docker" != "." ]]; then - git sparse-checkout set --no-cone "docker" - fi - git checkout - else - git pull --rebase - fi - - # Process some tokens and print rate limits without xtrace. - set +o xtrace - GH_TOKEN=$(gh auth token) - echo "Docker Hub Rate Limits:" - docker_hub_token=$(curl -s "https://auth.docker.io/token?service=registry.docker.io&scope=repository:ratelimitpreview/test:pull" | jq -r .token || true) - curl -s --head -H "Authorization: Bearer $docker_hub_token" https://registry-1.docker.io/v2/ratelimitpreview/test/manifests/latest | grep ratelimit || true - echo "GitHub Core Rate Limits:" - gh api -i -X HEAD /rate_limit | grep Ratelimit - set -o xtrace - - # Export env vars for docker compose. - export GH_TOKEN - export GH_REPOSITORY="time-loop/slapdash" - export GH_LABELS="my-ci,ci-storage" - export FORWARD_HOST="my-ci-host-001.test-zoneName" - - # It it's the very 1st run, start Docker service. We do not start it every run, - # because otherwise we wouldn't be able to "systemctl stop docker docker.socket" - # manually or while copying files from the old host. - file=~/.docker-started-after-first-git-clone - if [[ ! -f $file ]]; then - sudo systemctl start docker docker.socket - touch $file - fi - - # Run docker compose. - cd "docker" - docker pull ghcr.io/dimikot/ci-storage:main || true - docker pull ghcr.io/dimikot/ci-runner:main || true - docker compose up --build --remove-orphans -d - sleep 5 - if [[ "$1" != "--no-logs" ]]; then - docker compose logs -n 10 - fi - docker system prune --volumes -f - - path: /home/ubuntu/.bash_profile - owner: ubuntu:ubuntu - permissions: "0644" - defer: true - content: | - #!/bin/bash - C_CMD="\\033[0;36m" - C_NO="\\033[0m" - if [[ -d ~/git/"docker" ]]; then - cd ~/git/"docker" - echo -e "$C_CMD\\$ docker compose ps$C_NO" - docker --log-level=ERROR compose ps --format="table {{.Service}}\\t{{.Status}}\\t{{.Ports}}" - services=$(docker compose ps --format '{{.Service}}' 2>/dev/null) - if [[ "$services" != "" && $(echo "$services" | wc -l) -eq 1 ]]; then - cmd="docker compose exec $services bash -l" - echo -e "$C_CMD\\$ $cmd$C_NO" - eval "$cmd" - fi - fi -", - ], - ], + "Handler": "FastSnapshotRestoreEnable.handler", + "Role": { + "Fn::GetAtt": [ + "CnstrctInstanceToAmiFastSnapshotRestoreEnableServiceRole985CCAE9", + "Arn", + ], + }, + "Runtime": "nodejs20.x", + "Timeout": 180, + }, + "Type": "AWS::Lambda::Function", + }, + "CnstrctInstanceToAmiFastSnapshotRestoreEnableServiceRole985CCAE9": { + "Properties": { + "AssumeRolePolicyDocument": { + "Statement": [ + { + "Action": "sts:AssumeRole", + "Effect": "Allow", + "Principal": { + "Service": "lambda.amazonaws.com", + }, }, - }, + ], + "Version": "2012-10-17", }, - "LaunchTemplateName": "stk-cnstrct-lt", - "TagSpecifications": [ + "ManagedPolicyArns": [ { - "ResourceType": "launch-template", - "Tags": [ - { - "Key": "Name", - "Value": "Stk/Cnstrct/Lt", - }, + "Fn::Join": [ + "", + [ + "arn:", + { + "Ref": "AWS::Partition", + }, + ":iam::aws:policy/service-role/AWSLambdaBasicExecutionRole", + ], ], }, ], }, - "Type": "AWS::EC2::LaunchTemplate", + "Type": "AWS::IAM::Role", }, - "CnstrctLtProfileE3059BA8": { + "CnstrctInstanceToAmiFastSnapshotRestoreEnableServiceRoleDefaultPolicyE06BF178": { "Properties": { + "PolicyDocument": { + "Statement": [ + { + "Action": [ + "autoscaling:DescribeAutoScalingGroups", + "autoscaling:DescribeAutoScalingInstances", + "autoscaling:UpdateAutoScalingGroup", + "ec2:CreateLaunchTemplateVersion", + "ec2:CreateSnapshot", + "ec2:CreateTags", + "ec2:DeleteLaunchTemplateVersions", + "ec2:DeleteSnapshot", + "ec2:DeregisterImage", + "ec2:DescribeFastSnapshotRestores", + "ec2:DescribeImages", + "ec2:DescribeInstances", + "ec2:DescribeLaunchTemplates", + "ec2:DescribeLaunchTemplateVersions", + "ec2:DescribeSnapshots", + "ec2:DisableFastSnapshotRestores", + "ec2:EnableFastSnapshotRestores", + "ec2:RegisterImage", + "ec2:RunInstances", + "iam:PassRole", + "logs:CreateLogGroup", + "logs:CreateLogStream", + "logs:PutLogEvents", + ], + "Effect": "Allow", + "Resource": "*", + }, + ], + "Version": "2012-10-17", + }, + "PolicyName": "CnstrctInstanceToAmiFastSnapshotRestoreEnableServiceRoleDefaultPolicyE06BF178", "Roles": [ { - "Ref": "CnstrctRunnerRole341E54FA", + "Ref": "CnstrctInstanceToAmiFastSnapshotRestoreEnableServiceRole985CCAE9", }, ], }, - "Type": "AWS::IAM::InstanceProfile", + "Type": "AWS::IAM::Policy", }, - "CnstrctMyCiHost001A9DE30324": { + "CnstrctInstanceToAmiFastSnapshotRestoreGC871594F1": { + "DependsOn": [ + "CnstrctInstanceToAmiFastSnapshotRestoreGCServiceRoleDefaultPolicyD7899CD2", + "CnstrctInstanceToAmiFastSnapshotRestoreGCServiceRoleCC7B2E07", + ], "Properties": { - "HostedZoneId": "test-hostedZoneId", - "Name": "my-ci-host-001.test-zoneName.", - "ResourceRecords": [ - { - "Fn::GetAtt": [ - "CnstrctStkCnstrctMyCiHost001InstanceCCA3759A90e3e9f8dfbab5f0", - "PrivateIp", - ], + "Code": { + "S3Bucket": { + "Fn::Sub": "cdk-hnb659fds-assets-\${AWS::AccountId}-\${AWS::Region}", }, - ], - "TTL": "60", - "Type": "A", + "S3Key": "24889f5dc169825cb8dd9c7076e6f6886d38a18a2db3fd36a2431314d5711df1.zip", + }, + "Handler": "FastSnapshotRestoreGC.handler", + "Role": { + "Fn::GetAtt": [ + "CnstrctInstanceToAmiFastSnapshotRestoreGCServiceRoleCC7B2E07", + "Arn", + ], + }, + "Runtime": "nodejs20.x", + "Timeout": 180, }, - "Type": "AWS::Route53::RecordSet", + "Type": "AWS::Lambda::Function", }, - "CnstrctRunnerRole341E54FA": { + "CnstrctInstanceToAmiFastSnapshotRestoreGCServiceRoleCC7B2E07": { "Properties": { "AssumeRolePolicyDocument": { "Statement": [ @@ -836,7 +798,7 @@ write_files: "Action": "sts:AssumeRole", "Effect": "Allow", "Principal": { - "Service": "ec2.amazonaws.com", + "Service": "lambda.amazonaws.com", }, }, ], @@ -851,671 +813,4646 @@ write_files: { "Ref": "AWS::Partition", }, - ":iam::aws:policy/service-role/AmazonEC2RoleforSSM", - ], - ], - }, - { - "Fn::Join": [ - "", - [ - "arn:", - { - "Ref": "AWS::Partition", - }, - ":iam::aws:policy/CloudWatchAgentServerPolicy", + ":iam::aws:policy/service-role/AWSLambdaBasicExecutionRole", ], ], }, ], - "Policies": [ - { - "PolicyDocument": { - "Statement": [ - { - "Action": "secretsmanager:GetSecretValue", - "Effect": "Allow", - "Resource": { - "Fn::Join": [ - "", - [ - "arn:", - { - "Ref": "AWS::Partition", - }, - ":secretsmanager:", - { - "Ref": "AWS::Region", - }, - ":", - { - "Ref": "AWS::AccountId", - }, - ":secret:ec2-ssh-key/", - { - "Fn::GetAtt": [ - "CnstrctSshIdRsaEC2KeyPairstkcnstrctsshidrsaF4F38F69", - "KeyPairName", - ], - }, - "/private*", - ], - ], - }, - }, + }, + "Type": "AWS::IAM::Role", + }, + "CnstrctInstanceToAmiFastSnapshotRestoreGCServiceRoleDefaultPolicyD7899CD2": { + "Properties": { + "PolicyDocument": { + "Statement": [ + { + "Action": [ + "autoscaling:DescribeAutoScalingGroups", + "autoscaling:DescribeAutoScalingInstances", + "autoscaling:UpdateAutoScalingGroup", + "ec2:CreateLaunchTemplateVersion", + "ec2:CreateSnapshot", + "ec2:CreateTags", + "ec2:DeleteLaunchTemplateVersions", + "ec2:DeleteSnapshot", + "ec2:DeregisterImage", + "ec2:DescribeFastSnapshotRestores", + "ec2:DescribeImages", + "ec2:DescribeInstances", + "ec2:DescribeLaunchTemplates", + "ec2:DescribeLaunchTemplateVersions", + "ec2:DescribeSnapshots", + "ec2:DisableFastSnapshotRestores", + "ec2:EnableFastSnapshotRestores", + "ec2:RegisterImage", + "ec2:RunInstances", + "iam:PassRole", + "logs:CreateLogGroup", + "logs:CreateLogStream", + "logs:PutLogEvents", ], - "Version": "2012-10-17", + "Effect": "Allow", + "Resource": "*", }, - "PolicyName": "KeyPairPolicy", - }, + ], + "Version": "2012-10-17", + }, + "PolicyName": "CnstrctInstanceToAmiFastSnapshotRestoreGCServiceRoleDefaultPolicyD7899CD2", + "Roles": [ { - "PolicyDocument": { - "Statement": [ - { - "Action": "secretsmanager:GetSecretValue", - "Effect": "Allow", - "Resource": { - "Fn::Join": [ - "", - [ - "arn:", - { - "Ref": "AWS::Partition", - }, - ":secretsmanager:", - { - "Ref": "AWS::Region", - }, - ":", - { - "Ref": "AWS::AccountId", - }, - ":secret:ci-storage/gh-token*", - ], - ], - }, - }, - ], - "Version": "2012-10-17", - }, - "PolicyName": "GhTokenPolicy", + "Ref": "CnstrctInstanceToAmiFastSnapshotRestoreGCServiceRoleCC7B2E07", + }, + ], + }, + "Type": "AWS::IAM::Policy", + }, + "CnstrctInstanceToAmiFastSnapshotRestoreGetAction81EE8A35": { + "DependsOn": [ + "CnstrctInstanceToAmiFastSnapshotRestoreGetActionServiceRoleDefaultPolicy741610CF", + "CnstrctInstanceToAmiFastSnapshotRestoreGetActionServiceRole98E3327A", + ], + "Properties": { + "Code": { + "S3Bucket": { + "Fn::Sub": "cdk-hnb659fds-assets-\${AWS::AccountId}-\${AWS::Region}", }, + "S3Key": "24889f5dc169825cb8dd9c7076e6f6886d38a18a2db3fd36a2431314d5711df1.zip", + }, + "Handler": "FastSnapshotRestoreGetAction.handler", + "Role": { + "Fn::GetAtt": [ + "CnstrctInstanceToAmiFastSnapshotRestoreGetActionServiceRole98E3327A", + "Arn", + ], + }, + "Runtime": "nodejs20.x", + "Timeout": 180, + }, + "Type": "AWS::Lambda::Function", + }, + "CnstrctInstanceToAmiFastSnapshotRestoreGetActionServiceRole98E3327A": { + "Properties": { + "AssumeRolePolicyDocument": { + "Statement": [ + { + "Action": "sts:AssumeRole", + "Effect": "Allow", + "Principal": { + "Service": "lambda.amazonaws.com", + }, + }, + ], + "Version": "2012-10-17", + }, + "ManagedPolicyArns": [ { - "PolicyDocument": { - "Statement": [ - { - "Action": "ec2:DescribeInstances", - "Effect": "Allow", - "Resource": "*", - }, + "Fn::Join": [ + "", + [ + "arn:", { - "Action": "cloudformation:SignalResource", - "Effect": "Allow", - "Resource": { - "Fn::Join": [ - "", - [ - "arn:", - { - "Ref": "AWS::Partition", - }, - ":cloudformation:", - { - "Ref": "AWS::Region", - }, - ":", - { - "Ref": "AWS::AccountId", - }, - ":stack/Stk/*", - ], - ], - }, + "Ref": "AWS::Partition", }, + ":iam::aws:policy/service-role/AWSLambdaBasicExecutionRole", ], - "Version": "2012-10-17", - }, - "PolicyName": "SignalResourcePolicy", + ], }, ], - "RoleName": "StkCnstrctRunnerRole", }, "Type": "AWS::IAM::Role", }, - "CnstrctSgF5C70BA4": { + "CnstrctInstanceToAmiFastSnapshotRestoreGetActionServiceRoleDefaultPolicy741610CF": { "Properties": { - "GroupDescription": "stk-cnstrct-sg", - "GroupName": "stk-cnstrct-sg", - "SecurityGroupEgress": [ - { - "CidrIp": "0.0.0.0/0", - "Description": "Allow all outbound traffic by default", - "IpProtocol": "-1", - }, - ], - "Tags": [ + "PolicyDocument": { + "Statement": [ + { + "Action": [ + "autoscaling:DescribeAutoScalingGroups", + "autoscaling:DescribeAutoScalingInstances", + "autoscaling:UpdateAutoScalingGroup", + "ec2:CreateLaunchTemplateVersion", + "ec2:CreateSnapshot", + "ec2:CreateTags", + "ec2:DeleteLaunchTemplateVersions", + "ec2:DeleteSnapshot", + "ec2:DeregisterImage", + "ec2:DescribeFastSnapshotRestores", + "ec2:DescribeImages", + "ec2:DescribeInstances", + "ec2:DescribeLaunchTemplates", + "ec2:DescribeLaunchTemplateVersions", + "ec2:DescribeSnapshots", + "ec2:DisableFastSnapshotRestores", + "ec2:EnableFastSnapshotRestores", + "ec2:RegisterImage", + "ec2:RunInstances", + "iam:PassRole", + "logs:CreateLogGroup", + "logs:CreateLogStream", + "logs:PutLogEvents", + ], + "Effect": "Allow", + "Resource": "*", + }, + ], + "Version": "2012-10-17", + }, + "PolicyName": "CnstrctInstanceToAmiFastSnapshotRestoreGetActionServiceRoleDefaultPolicy741610CF", + "Roles": [ { - "Key": "Name", - "Value": "stk-cnstrct-sg", + "Ref": "CnstrctInstanceToAmiFastSnapshotRestoreGetActionServiceRole98E3327A", }, ], - "VpcId": { - "Ref": "Vpc8378EB38", - }, }, - "Type": "AWS::EC2::SecurityGroup", + "Type": "AWS::IAM::Policy", }, - "CnstrctSgfromStkCnstrctSgCE9C2C74100223764E466": { + "CnstrctInstanceToAmiImageCreateE181B0A7": { + "DependsOn": [ + "CnstrctInstanceToAmiImageCreateServiceRoleDefaultPolicy4EC495B0", + "CnstrctInstanceToAmiImageCreateServiceRole018DA59A", + ], "Properties": { - "Description": "from runners and host to ci-storage container", - "FromPort": 10022, - "GroupId": { - "Fn::GetAtt": [ - "CnstrctSgF5C70BA4", - "GroupId", - ], + "Code": { + "S3Bucket": { + "Fn::Sub": "cdk-hnb659fds-assets-\${AWS::AccountId}-\${AWS::Region}", + }, + "S3Key": "24889f5dc169825cb8dd9c7076e6f6886d38a18a2db3fd36a2431314d5711df1.zip", }, - "IpProtocol": "tcp", - "SourceSecurityGroupId": { + "Handler": "ImageCreate.handler", + "Role": { "Fn::GetAtt": [ - "CnstrctSgF5C70BA4", - "GroupId", + "CnstrctInstanceToAmiImageCreateServiceRole018DA59A", + "Arn", ], }, - "ToPort": 10022, + "Runtime": "nodejs20.x", + "Timeout": 180, }, - "Type": "AWS::EC2::SecurityGroupIngress", + "Type": "AWS::Lambda::Function", }, - "CnstrctSgfromStkCnstrctSgCE9C2C74226B452137": { + "CnstrctInstanceToAmiImageCreateServiceRole018DA59A": { "Properties": { - "Description": "from runners and host to SSH", - "FromPort": 22, - "GroupId": { - "Fn::GetAtt": [ - "CnstrctSgF5C70BA4", - "GroupId", - ], - }, - "IpProtocol": "tcp", - "SourceSecurityGroupId": { - "Fn::GetAtt": [ - "CnstrctSgF5C70BA4", - "GroupId", + "AssumeRolePolicyDocument": { + "Statement": [ + { + "Action": "sts:AssumeRole", + "Effect": "Allow", + "Principal": { + "Service": "lambda.amazonaws.com", + }, + }, ], + "Version": "2012-10-17", }, - "ToPort": 22, + "ManagedPolicyArns": [ + { + "Fn::Join": [ + "", + [ + "arn:", + { + "Ref": "AWS::Partition", + }, + ":iam::aws:policy/service-role/AWSLambdaBasicExecutionRole", + ], + ], + }, + ], }, - "Type": "AWS::EC2::SecurityGroupIngress", + "Type": "AWS::IAM::Role", }, - "CnstrctSgfromStkCnstrctSgCE9C2C744200042042B0BB20A9": { + "CnstrctInstanceToAmiImageCreateServiceRoleDefaultPolicy4EC495B0": { "Properties": { - "Description": "from runners and host to test ports", - "FromPort": 42000, - "GroupId": { - "Fn::GetAtt": [ - "CnstrctSgF5C70BA4", - "GroupId", - ], - }, - "IpProtocol": "tcp", - "SourceSecurityGroupId": { - "Fn::GetAtt": [ - "CnstrctSgF5C70BA4", - "GroupId", + "PolicyDocument": { + "Statement": [ + { + "Action": [ + "autoscaling:DescribeAutoScalingGroups", + "autoscaling:DescribeAutoScalingInstances", + "autoscaling:UpdateAutoScalingGroup", + "ec2:CreateLaunchTemplateVersion", + "ec2:CreateSnapshot", + "ec2:CreateTags", + "ec2:DeleteLaunchTemplateVersions", + "ec2:DeleteSnapshot", + "ec2:DeregisterImage", + "ec2:DescribeFastSnapshotRestores", + "ec2:DescribeImages", + "ec2:DescribeInstances", + "ec2:DescribeLaunchTemplates", + "ec2:DescribeLaunchTemplateVersions", + "ec2:DescribeSnapshots", + "ec2:DisableFastSnapshotRestores", + "ec2:EnableFastSnapshotRestores", + "ec2:RegisterImage", + "ec2:RunInstances", + "iam:PassRole", + "logs:CreateLogGroup", + "logs:CreateLogStream", + "logs:PutLogEvents", + ], + "Effect": "Allow", + "Resource": "*", + }, ], + "Version": "2012-10-17", }, - "ToPort": 42042, + "PolicyName": "CnstrctInstanceToAmiImageCreateServiceRoleDefaultPolicy4EC495B0", + "Roles": [ + { + "Ref": "CnstrctInstanceToAmiImageCreateServiceRole018DA59A", + }, + ], }, - "Type": "AWS::EC2::SecurityGroupIngress", + "Type": "AWS::IAM::Policy", }, - "CnstrctSshIdRsaEC2KeyPairstkcnstrctsshidrsaF4F38F69": { - "DeletionPolicy": "Delete", + "CnstrctInstanceToAmiImageGCC91F1072": { + "DependsOn": [ + "CnstrctInstanceToAmiImageGCServiceRoleDefaultPolicy1D2A6024", + "CnstrctInstanceToAmiImageGCServiceRoleFCDB1C6D", + ], "Properties": { - "Description": "Used to access ci-storage host from self-hosted runner nodes.", - "ExposePublicKey": false, - "KmsPrivate": "alias/aws/secretsmanager", - "KmsPublic": "alias/aws/secretsmanager", - "Name": "stk-cnstrct-sshidrsa", - "PublicKey": "", - "PublicKeyFormat": "OPENSSH", - "RemoveKeySecretsAfterDays": 0, - "SecretPrefix": "ec2-ssh-key/", - "ServiceToken": { + "Code": { + "S3Bucket": { + "Fn::Sub": "cdk-hnb659fds-assets-\${AWS::AccountId}-\${AWS::Region}", + }, + "S3Key": "24889f5dc169825cb8dd9c7076e6f6886d38a18a2db3fd36a2431314d5711df1.zip", + }, + "Handler": "ImageGC.handler", + "Role": { "Fn::GetAtt": [ - "EC2KeyNameManagerLambdaBE629145", + "CnstrctInstanceToAmiImageGCServiceRoleFCDB1C6D", "Arn", ], }, - "StackName": "Stk", - "StorePublicKey": false, - "Tags": { - "CreatedByCfnCustomResource": "CFN::Resource::Custom::EC2-Key-Pair", - }, + "Runtime": "nodejs20.x", + "Timeout": 180, }, - "Type": "Custom::EC2-Key-Pair", - "UpdateReplacePolicy": "Delete", + "Type": "AWS::Lambda::Function", }, - "CnstrctStkCnstrctMyCiHost001InstanceCCA3759A90e3e9f8dfbab5f0": { - "CreationPolicy": { - "ResourceSignal": { - "Count": 1, - "Timeout": "PT15M", + "CnstrctInstanceToAmiImageGCServiceRoleDefaultPolicy1D2A6024": { + "Properties": { + "PolicyDocument": { + "Statement": [ + { + "Action": [ + "autoscaling:DescribeAutoScalingGroups", + "autoscaling:DescribeAutoScalingInstances", + "autoscaling:UpdateAutoScalingGroup", + "ec2:CreateLaunchTemplateVersion", + "ec2:CreateSnapshot", + "ec2:CreateTags", + "ec2:DeleteLaunchTemplateVersions", + "ec2:DeleteSnapshot", + "ec2:DeregisterImage", + "ec2:DescribeFastSnapshotRestores", + "ec2:DescribeImages", + "ec2:DescribeInstances", + "ec2:DescribeLaunchTemplates", + "ec2:DescribeLaunchTemplateVersions", + "ec2:DescribeSnapshots", + "ec2:DisableFastSnapshotRestores", + "ec2:EnableFastSnapshotRestores", + "ec2:RegisterImage", + "ec2:RunInstances", + "iam:PassRole", + "logs:CreateLogGroup", + "logs:CreateLogStream", + "logs:PutLogEvents", + ], + "Effect": "Allow", + "Resource": "*", + }, + ], + "Version": "2012-10-17", }, + "PolicyName": "CnstrctInstanceToAmiImageGCServiceRoleDefaultPolicy1D2A6024", + "Roles": [ + { + "Ref": "CnstrctInstanceToAmiImageGCServiceRoleFCDB1C6D", + }, + ], }, - "DependsOn": [ - "CnstrctHostRole5DD9F366", - ], + "Type": "AWS::IAM::Policy", + }, + "CnstrctInstanceToAmiImageGCServiceRoleFCDB1C6D": { "Properties": { - "AvailabilityZone": { - "Fn::Select": [ - 0, + "AssumeRolePolicyDocument": { + "Statement": [ { - "Fn::GetAZs": "", + "Action": "sts:AssumeRole", + "Effect": "Allow", + "Principal": { + "Service": "lambda.amazonaws.com", + }, }, ], + "Version": "2012-10-17", }, - "BlockDeviceMappings": [ + "ManagedPolicyArns": [ { - "DeviceName": "/dev/sda1", - "Ebs": { - "DeleteOnTermination": true, - "Encrypted": true, - "VolumeSize": 20, - "VolumeType": "gp2", - }, + "Fn::Join": [ + "", + [ + "arn:", + { + "Ref": "AWS::Partition", + }, + ":iam::aws:policy/service-role/AWSLambdaBasicExecutionRole", + ], + ], }, ], - "IamInstanceProfile": { - "Ref": "CnstrctStkCnstrctMyCiHost001InstanceInstanceProfile2045E5AE", - }, - "ImageId": { - "Ref": "SsmParameterValuetestimageSsmNameC96584B6F00A464EAD1953AFF4B05118Parameter", + }, + "Type": "AWS::IAM::Role", + }, + "CnstrctInstanceToAmiImageGetActionF356B2EA": { + "DependsOn": [ + "CnstrctInstanceToAmiImageGetActionServiceRoleDefaultPolicy240980FD", + "CnstrctInstanceToAmiImageGetActionServiceRole3E92E2AF", + ], + "Properties": { + "Code": { + "S3Bucket": { + "Fn::Sub": "cdk-hnb659fds-assets-\${AWS::AccountId}-\${AWS::Region}", + }, + "S3Key": "24889f5dc169825cb8dd9c7076e6f6886d38a18a2db3fd36a2431314d5711df1.zip", }, - "InstanceType": "t3.large", - "KeyName": { + "Handler": "ImageGetAction.handler", + "Role": { "Fn::GetAtt": [ - "CnstrctSshIdRsaEC2KeyPairstkcnstrctsshidrsaF4F38F69", - "KeyPairName", + "CnstrctInstanceToAmiImageGetActionServiceRole3E92E2AF", + "Arn", ], }, - "LaunchTemplate": { - "LaunchTemplateName": "StkCnstrctMyCiHost001InstanceLaunchTemplate", - "Version": { - "Fn::GetAtt": [ - "CnstrctStkCnstrctMyCiHost001InstanceLaunchTemplateD3F24170", - "LatestVersionNumber", + "Runtime": "nodejs20.x", + "Timeout": 180, + }, + "Type": "AWS::Lambda::Function", + }, + "CnstrctInstanceToAmiImageGetActionServiceRole3E92E2AF": { + "Properties": { + "AssumeRolePolicyDocument": { + "Statement": [ + { + "Action": "sts:AssumeRole", + "Effect": "Allow", + "Principal": { + "Service": "lambda.amazonaws.com", + }, + }, + ], + "Version": "2012-10-17", + }, + "ManagedPolicyArns": [ + { + "Fn::Join": [ + "", + [ + "arn:", + { + "Ref": "AWS::Partition", + }, + ":iam::aws:policy/service-role/AWSLambdaBasicExecutionRole", + ], ], }, + ], + }, + "Type": "AWS::IAM::Role", + }, + "CnstrctInstanceToAmiImageGetActionServiceRoleDefaultPolicy240980FD": { + "Properties": { + "PolicyDocument": { + "Statement": [ + { + "Action": [ + "autoscaling:DescribeAutoScalingGroups", + "autoscaling:DescribeAutoScalingInstances", + "autoscaling:UpdateAutoScalingGroup", + "ec2:CreateLaunchTemplateVersion", + "ec2:CreateSnapshot", + "ec2:CreateTags", + "ec2:DeleteLaunchTemplateVersions", + "ec2:DeleteSnapshot", + "ec2:DeregisterImage", + "ec2:DescribeFastSnapshotRestores", + "ec2:DescribeImages", + "ec2:DescribeInstances", + "ec2:DescribeLaunchTemplates", + "ec2:DescribeLaunchTemplateVersions", + "ec2:DescribeSnapshots", + "ec2:DisableFastSnapshotRestores", + "ec2:EnableFastSnapshotRestores", + "ec2:RegisterImage", + "ec2:RunInstances", + "iam:PassRole", + "logs:CreateLogGroup", + "logs:CreateLogStream", + "logs:PutLogEvents", + ], + "Effect": "Allow", + "Resource": "*", + }, + ], + "Version": "2012-10-17", }, - "Monitoring": true, - "SecurityGroupIds": [ + "PolicyName": "CnstrctInstanceToAmiImageGetActionServiceRoleDefaultPolicy240980FD", + "Roles": [ { - "Fn::GetAtt": [ - "CnstrctSgF5C70BA4", - "GroupId", - ], + "Ref": "CnstrctInstanceToAmiImageGetActionServiceRole3E92E2AF", }, ], - "SubnetId": { - "Ref": "VpcPrivateSubnet1Subnet536B997A", + }, + "Type": "AWS::IAM::Policy", + }, + "CnstrctInstanceToAmiLaunchTemplateGC1849B9F5": { + "DependsOn": [ + "CnstrctInstanceToAmiLaunchTemplateGCServiceRoleDefaultPolicy6A1A6014", + "CnstrctInstanceToAmiLaunchTemplateGCServiceRoleFD4989D4", + ], + "Properties": { + "Code": { + "S3Bucket": { + "Fn::Sub": "cdk-hnb659fds-assets-\${AWS::AccountId}-\${AWS::Region}", + }, + "S3Key": "24889f5dc169825cb8dd9c7076e6f6886d38a18a2db3fd36a2431314d5711df1.zip", }, - "Tags": [ + "Handler": "LaunchTemplateGC.handler", + "Role": { + "Fn::GetAtt": [ + "CnstrctInstanceToAmiLaunchTemplateGCServiceRoleFD4989D4", + "Arn", + ], + }, + "Runtime": "nodejs20.x", + "Timeout": 180, + }, + "Type": "AWS::Lambda::Function", + }, + "CnstrctInstanceToAmiLaunchTemplateGCServiceRoleDefaultPolicy6A1A6014": { + "Properties": { + "PolicyDocument": { + "Statement": [ + { + "Action": [ + "autoscaling:DescribeAutoScalingGroups", + "autoscaling:DescribeAutoScalingInstances", + "autoscaling:UpdateAutoScalingGroup", + "ec2:CreateLaunchTemplateVersion", + "ec2:CreateSnapshot", + "ec2:CreateTags", + "ec2:DeleteLaunchTemplateVersions", + "ec2:DeleteSnapshot", + "ec2:DeregisterImage", + "ec2:DescribeFastSnapshotRestores", + "ec2:DescribeImages", + "ec2:DescribeInstances", + "ec2:DescribeLaunchTemplates", + "ec2:DescribeLaunchTemplateVersions", + "ec2:DescribeSnapshots", + "ec2:DisableFastSnapshotRestores", + "ec2:EnableFastSnapshotRestores", + "ec2:RegisterImage", + "ec2:RunInstances", + "iam:PassRole", + "logs:CreateLogGroup", + "logs:CreateLogStream", + "logs:PutLogEvents", + ], + "Effect": "Allow", + "Resource": "*", + }, + ], + "Version": "2012-10-17", + }, + "PolicyName": "CnstrctInstanceToAmiLaunchTemplateGCServiceRoleDefaultPolicy6A1A6014", + "Roles": [ { - "Key": "Name", - "Value": "my-ci-host-001.test-zoneName", + "Ref": "CnstrctInstanceToAmiLaunchTemplateGCServiceRoleFD4989D4", }, ], - "UserData": { - "Fn::Base64": { + }, + "Type": "AWS::IAM::Policy", + }, + "CnstrctInstanceToAmiLaunchTemplateGCServiceRoleFD4989D4": { + "Properties": { + "AssumeRolePolicyDocument": { + "Statement": [ + { + "Action": "sts:AssumeRole", + "Effect": "Allow", + "Principal": { + "Service": "lambda.amazonaws.com", + }, + }, + ], + "Version": "2012-10-17", + }, + "ManagedPolicyArns": [ + { "Fn::Join": [ "", [ - "#cloud-config -timezone: America/Los_Angeles -fqdn: my-ci-host-001.test-zoneName -hostname: my-ci-host-001.test-zoneName -mounts: - - - tmpfs - - /var/lib/docker - - tmpfs - - defaults,noatime,exec,mode=0710,nr_inodes=0,size=4G - - "0" - - "0" -apt_sources: - - source: deb https://cli.github.com/packages stable main - keyid: 23F3D4EA75716059 - filename: github-cli.list - - source: deb https://download.docker.com/linux/ubuntu $RELEASE stable - keyid: 9DC858229FC7DD38854AE2D88D81803C0EBFCD88 - filename: docker.list -packages: - - awscli - - gh - - docker-ce - - docker-ce-cli - - containerd.io - - docker-compose-plugin - - qemu - - qemu-user-static - - binfmt-support - - git - - gosu - - mc - - curl - - apt-transport-https - - ca-certificates - - tzdata - - atop - - iotop - - htop - - bwm-ng - - jq -write_files: - - path: /etc/sysctl.d/enable-ipv4-forwarding.conf - content: | - net.ipv4.conf.all.forwarding=1 - - path: /etc/default/atop - content: | - LOGOPTS="-R" - LOGINTERVAL=15 - LOGGENERATIONS=4 - - path: /etc/environment - append: true - content: | - TZ="America/Los_Angeles" - - path: /etc/environment - append: true - content: | - LESS="RS" - - path: /etc/docker/daemon.json - permissions: "0644" - content: | - { - "log-driver": "syslog", - "log-opts": { - "tag": "docker/{{.Name}}" + "arn:", + { + "Ref": "AWS::Partition", + }, + ":iam::aws:policy/service-role/AWSLambdaBasicExecutionRole", + ], + ], + }, + ], + }, + "Type": "AWS::IAM::Role", + }, + "CnstrctInstanceToAmiLaunchTemplateGetNameFromInstance7067ACEF": { + "DependsOn": [ + "CnstrctInstanceToAmiLaunchTemplateGetNameFromInstanceServiceRoleDefaultPolicy54AB0253", + "CnstrctInstanceToAmiLaunchTemplateGetNameFromInstanceServiceRole7CC1BFBB", + ], + "Properties": { + "Code": { + "S3Bucket": { + "Fn::Sub": "cdk-hnb659fds-assets-\${AWS::AccountId}-\${AWS::Region}", + }, + "S3Key": "24889f5dc169825cb8dd9c7076e6f6886d38a18a2db3fd36a2431314d5711df1.zip", }, - "runtimes": { - "sysbox-runc": { - "path": "/usr/bin/sysbox-runc" - } + "Handler": "LaunchTemplateGetNameFromInstance.handler", + "Role": { + "Fn::GetAtt": [ + "CnstrctInstanceToAmiLaunchTemplateGetNameFromInstanceServiceRole7CC1BFBB", + "Arn", + ], }, - "default-runtime": "sysbox-runc", - "userns-remap": "sysbox" - } - - path: /var/lib/cloud/scripts/per-once/apply-services-configs.sh - permissions: "0755" - content: | - #!/bin/bash - set -e -o pipefail && echo ================ && echo "Running $BASH_SOURCE as $(whoami)" && set -o xtrace && export AWS_DEFAULT_REGION=$(ec2metadata --availability-zone | sed "s/[a-z]$//") - service atop restart || true - sysctl --system - - path: /var/lib/cloud/scripts/per-once/increase-docker-shutdown-timeout.sh - permissions: "0755" - content: | - #!/bin/bash - set -e -o pipefail && echo ================ && echo "Running $BASH_SOURCE as $(whoami)" && set -o xtrace && export AWS_DEFAULT_REGION=$(ec2metadata --availability-zone | sed "s/[a-z]$//") - sed -i -E '/TimeoutStartSec=.*/a TimeoutStopSec=3600' /usr/lib/systemd/system/docker.service - systemctl daemon-reload - - path: /var/lib/cloud/scripts/per-once/add-ubuntu-user-to-docker-group-to-access-socket.sh - permissions: "0755" - content: | - #!/bin/bash - set -e -o pipefail && echo ================ && echo "Running $BASH_SOURCE as $(whoami)" && set -o xtrace && export AWS_DEFAULT_REGION=$(ec2metadata --availability-zone | sed "s/[a-z]$//") - usermod -aG docker ubuntu - - path: /var/lib/cloud/scripts/per-once/install-sysbox-for-docker-in-docker.sh - permissions: "0755" - content: | - #!/bin/bash - set -e -o pipefail && echo ================ && echo "Running $BASH_SOURCE as $(whoami)" && set -o xtrace && export AWS_DEFAULT_REGION=$(ec2metadata --availability-zone | sed "s/[a-z]$//") - systemctl stop docker docker.socket || true - wget -nv -O /tmp/sysbox-ce.deb "https://downloads.nestybox.com/sysbox/releases/v0.6.4/sysbox-ce_0.6.4-0.linux_$(dpkg --print-architecture).deb" - dpkg -i /tmp/sysbox-ce.deb - rm -f /tmp/sysbox-ce.deb - - path: /var/lib/cloud/scripts/per-once/switch-ssm-user-to-ubuntu-on-login.sh - permissions: "0755" - content: | - #!/bin/bash - set -e -o pipefail && echo ================ && echo "Running $BASH_SOURCE as $(whoami)" && set -o xtrace && export AWS_DEFAULT_REGION=$(ec2metadata --availability-zone | sed "s/[a-z]$//") - echo '[ "$0$@" = "sh" ] && ENV= sudo -u ubuntu -i' > /etc/profile.ssm-user - mkdir -p /etc/systemd/system/snap.amazon-ssm-agent.amazon-ssm-agent.service.d/ - ( - echo '[Service]' - echo 'Environment="ENV=/etc/profile.ssm-user"' - ) > /etc/systemd/system/snap.amazon-ssm-agent.amazon-ssm-agent.service.d/sh-env.conf - systemctl daemon-reload - systemctl restart snap.amazon-ssm-agent.amazon-ssm-agent.service || true - - path: /var/lib/cloud/scripts/per-once/rsync-tmpfs-volume-from-old-instance.sh - permissions: "0755" - content: | - #!/bin/bash - set -e -o pipefail && echo ================ && echo "Running $BASH_SOURCE as $(whoami)" && set -o xtrace && export AWS_DEFAULT_REGION=$(ec2metadata --availability-zone | sed "s/[a-z]$//") - - instance_id=$(ec2metadata --instance-id) - stack_name=$( - aws ec2 describe-tags \\ - --filters "Name=resource-id,Values=$instance_id" "Name=key,Values=aws:cloudformation:stack-name" \\ - --query "Tags[0].Value" --output text - ) - logical_id=$( - aws ec2 describe-tags \\ - --filters "Name=resource-id,Values=$instance_id" "Name=key,Values=aws:cloudformation:logical-id" \\ - --query "Tags[0].Value" --output text - ) - old_instance_ip_addr=$( - aws ec2 describe-instances \\ - --filters "Name=tag:Name,Values=my-ci-host-001.test-zoneName" "Name=instance-state-name,Values=running" \\ - --query "Reservations[*].Instances[*].[InstanceId,PrivateIpAddress]" --output text \\ - | grep -v "$instance_id" | awk '{print $2}' | head -n1 || true - ) - - if [[ "$old_instance_ip_addr" != "" ]]; then - # Load private key from Secrets Manager to ~/.ssh, to access the old host. - mkdir -p ~/.ssh && chmod 700 ~/.ssh - aws secretsmanager get-secret-value \\ - --secret-id "ec2-ssh-key/", + "Runtime": "nodejs20.x", + "Timeout": 180, + }, + "Type": "AWS::Lambda::Function", + }, + "CnstrctInstanceToAmiLaunchTemplateGetNameFromInstanceServiceRole7CC1BFBB": { + "Properties": { + "AssumeRolePolicyDocument": { + "Statement": [ + { + "Action": "sts:AssumeRole", + "Effect": "Allow", + "Principal": { + "Service": "lambda.amazonaws.com", + }, + }, + ], + "Version": "2012-10-17", + }, + "ManagedPolicyArns": [ + { + "Fn::Join": [ + "", + [ + "arn:", { - "Fn::GetAtt": [ - "CnstrctSshIdRsaEC2KeyPairstkcnstrctsshidrsaF4F38F69", - "KeyPairName", - ], + "Ref": "AWS::Partition", }, - "/private" \\ - --query SecretString --output text \\ - > ~/.ssh/id_rsa - chmod 600 ~/.ssh/id_rsa - - # Stop Docker service on the current host. - systemctl stop docker docker.socket || true - - # Stop Docker service on the old (source) host. - ssh -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null \\ - "ubuntu@$old_instance_ip_addr" "sudo systemctl stop docker docker.socket || true" - - # 1. Surprisingly, it takes almost the same amount of time to rsync-init - # (if we would run it without stopping Docker on the old host first) - # as to the follow-up rsync-over (after we stopped Docker on the source). - # This is probably because of the RAM drive and large Docker volumes. So - # we skip rsync-init and just go with one full rsync run (with downtime). - # 2. Also, compression (even the fastest one) doesn't speed it up; probably - # because AWS network is faster than instances CPU still. - time rsync \\ - -aHXS --one-file-system --numeric-ids --delete $@ \\ - --rsh="ssh -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null" \\ - --rsync-path="sudo rsync" \\ - "ubuntu@$old_instance_ip_addr:/var/lib/docker/" "/var/lib/docker/" - - # We do NOT start Docker service here! Otherwise, it may auto-start some - # containers, those containers will expect the git directory to exist, - # although it may not exist yet. So, we start Docker service in - # run-docker-compose.sh (its 1st run), when we are sure that git is pulled. - fi - - aws cloudformation signal-resource \\ - --stack-name "$stack_name" --logical-resource-id "$logical_id" \\ - --unique-id "$instance_id" --status SUCCESS - - path: /etc/rsyslog.d/01-docker-tag-to-serial-console.conf - permissions: "0644" - content: | - if $syslogtag startswith 'docker/' then -/dev/console - # It will also write to /var/log/syslog as usual. - - path: /var/lib/cloud/scripts/per-once/allow-rsyslog-write-to-serial-console.sh - permissions: "0755" - content: | - #!/bin/bash - set -e -o pipefail && echo ================ && echo "Running $BASH_SOURCE as $(whoami)" && set -o xtrace && export AWS_DEFAULT_REGION=$(ec2metadata --availability-zone | sed "s/[a-z]$//") - usermod -a -G tty syslog - systemctl restart rsyslog - - path: /var/lib/cloud/scripts/per-boot/run-docker-compose-on-boot.sh - permissions: "0755" - content: | - #!/bin/bash - set -e -o pipefail && echo ================ && echo "Running $BASH_SOURCE as $(whoami)" && set -o xtrace && export AWS_DEFAULT_REGION=$(ec2metadata --availability-zone | sed "s/[a-z]$//") - echo "*/2 * * * * ubuntu /home/ubuntu/run-docker-compose.sh --no-logs 2>&1 | logger -t docker/run-docker-compose" > /etc/cron.d/run-docker-compose - exec /home/ubuntu/run-docker-compose.sh - - path: /home/ubuntu/run-docker-compose.sh - owner: ubuntu:ubuntu - permissions: "0755" - defer: true - content: | - #!/bin/bash - - # Switch to non-privileged user if running as root. - if [[ $(whoami) != "ubuntu" ]]; then - exec gosu ubuntu "$BASH_SOURCE" "$@" - fi - - # Ensure there is only one instance of this script running. - exec {FD}<$BASH_SOURCE - flock -n "$FD" || { echo "Already running."; exit 0; } - set -e -o pipefail && echo ================ && echo "Running $BASH_SOURCE as $(whoami)" && set -o xtrace && export AWS_DEFAULT_REGION=$(ec2metadata --availability-zone | sed "s/[a-z]$//") - - # Make sure we're using the right timezone; it may be not up - # to date in the current environment during the very 1st run - # from run-docker-compose-on-boot.sh. - source /etc/environment - export TZ - - # Load private and public keys from Secrets Manager to ~/.ssh. - mkdir -p ~/.ssh && chmod 700 ~/.ssh - aws secretsmanager get-secret-value \\ - --secret-id "ec2-ssh-key/", + ":iam::aws:policy/service-role/AWSLambdaBasicExecutionRole", + ], + ], + }, + ], + }, + "Type": "AWS::IAM::Role", + }, + "CnstrctInstanceToAmiLaunchTemplateGetNameFromInstanceServiceRoleDefaultPolicy54AB0253": { + "Properties": { + "PolicyDocument": { + "Statement": [ + { + "Action": [ + "autoscaling:DescribeAutoScalingGroups", + "autoscaling:DescribeAutoScalingInstances", + "autoscaling:UpdateAutoScalingGroup", + "ec2:CreateLaunchTemplateVersion", + "ec2:CreateSnapshot", + "ec2:CreateTags", + "ec2:DeleteLaunchTemplateVersions", + "ec2:DeleteSnapshot", + "ec2:DeregisterImage", + "ec2:DescribeFastSnapshotRestores", + "ec2:DescribeImages", + "ec2:DescribeInstances", + "ec2:DescribeLaunchTemplates", + "ec2:DescribeLaunchTemplateVersions", + "ec2:DescribeSnapshots", + "ec2:DisableFastSnapshotRestores", + "ec2:EnableFastSnapshotRestores", + "ec2:RegisterImage", + "ec2:RunInstances", + "iam:PassRole", + "logs:CreateLogGroup", + "logs:CreateLogStream", + "logs:PutLogEvents", + ], + "Effect": "Allow", + "Resource": "*", + }, + ], + "Version": "2012-10-17", + }, + "PolicyName": "CnstrctInstanceToAmiLaunchTemplateGetNameFromInstanceServiceRoleDefaultPolicy54AB0253", + "Roles": [ + { + "Ref": "CnstrctInstanceToAmiLaunchTemplateGetNameFromInstanceServiceRole7CC1BFBB", + }, + ], + }, + "Type": "AWS::IAM::Policy", + }, + "CnstrctInstanceToAmiLaunchTemplateUpdateImage99ABAA53": { + "DependsOn": [ + "CnstrctInstanceToAmiLaunchTemplateUpdateImageServiceRoleDefaultPolicy6FDF89CC", + "CnstrctInstanceToAmiLaunchTemplateUpdateImageServiceRole960A1622", + ], + "Properties": { + "Code": { + "S3Bucket": { + "Fn::Sub": "cdk-hnb659fds-assets-\${AWS::AccountId}-\${AWS::Region}", + }, + "S3Key": "24889f5dc169825cb8dd9c7076e6f6886d38a18a2db3fd36a2431314d5711df1.zip", + }, + "Handler": "LaunchTemplateUpdateImage.handler", + "Role": { + "Fn::GetAtt": [ + "CnstrctInstanceToAmiLaunchTemplateUpdateImageServiceRole960A1622", + "Arn", + ], + }, + "Runtime": "nodejs20.x", + "Timeout": 180, + }, + "Type": "AWS::Lambda::Function", + }, + "CnstrctInstanceToAmiLaunchTemplateUpdateImageServiceRole960A1622": { + "Properties": { + "AssumeRolePolicyDocument": { + "Statement": [ + { + "Action": "sts:AssumeRole", + "Effect": "Allow", + "Principal": { + "Service": "lambda.amazonaws.com", + }, + }, + ], + "Version": "2012-10-17", + }, + "ManagedPolicyArns": [ + { + "Fn::Join": [ + "", + [ + "arn:", + { + "Ref": "AWS::Partition", + }, + ":iam::aws:policy/service-role/AWSLambdaBasicExecutionRole", + ], + ], + }, + ], + }, + "Type": "AWS::IAM::Role", + }, + "CnstrctInstanceToAmiLaunchTemplateUpdateImageServiceRoleDefaultPolicy6FDF89CC": { + "Properties": { + "PolicyDocument": { + "Statement": [ + { + "Action": [ + "autoscaling:DescribeAutoScalingGroups", + "autoscaling:DescribeAutoScalingInstances", + "autoscaling:UpdateAutoScalingGroup", + "ec2:CreateLaunchTemplateVersion", + "ec2:CreateSnapshot", + "ec2:CreateTags", + "ec2:DeleteLaunchTemplateVersions", + "ec2:DeleteSnapshot", + "ec2:DeregisterImage", + "ec2:DescribeFastSnapshotRestores", + "ec2:DescribeImages", + "ec2:DescribeInstances", + "ec2:DescribeLaunchTemplates", + "ec2:DescribeLaunchTemplateVersions", + "ec2:DescribeSnapshots", + "ec2:DisableFastSnapshotRestores", + "ec2:EnableFastSnapshotRestores", + "ec2:RegisterImage", + "ec2:RunInstances", + "iam:PassRole", + "logs:CreateLogGroup", + "logs:CreateLogStream", + "logs:PutLogEvents", + ], + "Effect": "Allow", + "Resource": "*", + }, + ], + "Version": "2012-10-17", + }, + "PolicyName": "CnstrctInstanceToAmiLaunchTemplateUpdateImageServiceRoleDefaultPolicy6FDF89CC", + "Roles": [ + { + "Ref": "CnstrctInstanceToAmiLaunchTemplateUpdateImageServiceRole960A1622", + }, + ], + }, + "Type": "AWS::IAM::Policy", + }, + "CnstrctInstanceToAmiLogGroup607FDB05": { + "DeletionPolicy": "Delete", + "Properties": { + "LogGroupName": "/aws/vendedlogs/states/stk-cnstrct-instancetoami-Logs", + "RetentionInDays": 7, + }, + "Type": "AWS::Logs::LogGroup", + "UpdateReplacePolicy": "Delete", + }, + "CnstrctInstanceToAmiSnapshotCreate197B8B81": { + "DependsOn": [ + "CnstrctInstanceToAmiSnapshotCreateServiceRoleDefaultPolicy0D384822", + "CnstrctInstanceToAmiSnapshotCreateServiceRoleC41A6A8A", + ], + "Properties": { + "Code": { + "S3Bucket": { + "Fn::Sub": "cdk-hnb659fds-assets-\${AWS::AccountId}-\${AWS::Region}", + }, + "S3Key": "24889f5dc169825cb8dd9c7076e6f6886d38a18a2db3fd36a2431314d5711df1.zip", + }, + "Handler": "SnapshotCreate.handler", + "Role": { + "Fn::GetAtt": [ + "CnstrctInstanceToAmiSnapshotCreateServiceRoleC41A6A8A", + "Arn", + ], + }, + "Runtime": "nodejs20.x", + "Timeout": 180, + }, + "Type": "AWS::Lambda::Function", + }, + "CnstrctInstanceToAmiSnapshotCreateServiceRoleC41A6A8A": { + "Properties": { + "AssumeRolePolicyDocument": { + "Statement": [ + { + "Action": "sts:AssumeRole", + "Effect": "Allow", + "Principal": { + "Service": "lambda.amazonaws.com", + }, + }, + ], + "Version": "2012-10-17", + }, + "ManagedPolicyArns": [ + { + "Fn::Join": [ + "", + [ + "arn:", + { + "Ref": "AWS::Partition", + }, + ":iam::aws:policy/service-role/AWSLambdaBasicExecutionRole", + ], + ], + }, + ], + }, + "Type": "AWS::IAM::Role", + }, + "CnstrctInstanceToAmiSnapshotCreateServiceRoleDefaultPolicy0D384822": { + "Properties": { + "PolicyDocument": { + "Statement": [ + { + "Action": [ + "autoscaling:DescribeAutoScalingGroups", + "autoscaling:DescribeAutoScalingInstances", + "autoscaling:UpdateAutoScalingGroup", + "ec2:CreateLaunchTemplateVersion", + "ec2:CreateSnapshot", + "ec2:CreateTags", + "ec2:DeleteLaunchTemplateVersions", + "ec2:DeleteSnapshot", + "ec2:DeregisterImage", + "ec2:DescribeFastSnapshotRestores", + "ec2:DescribeImages", + "ec2:DescribeInstances", + "ec2:DescribeLaunchTemplates", + "ec2:DescribeLaunchTemplateVersions", + "ec2:DescribeSnapshots", + "ec2:DisableFastSnapshotRestores", + "ec2:EnableFastSnapshotRestores", + "ec2:RegisterImage", + "ec2:RunInstances", + "iam:PassRole", + "logs:CreateLogGroup", + "logs:CreateLogStream", + "logs:PutLogEvents", + ], + "Effect": "Allow", + "Resource": "*", + }, + ], + "Version": "2012-10-17", + }, + "PolicyName": "CnstrctInstanceToAmiSnapshotCreateServiceRoleDefaultPolicy0D384822", + "Roles": [ + { + "Ref": "CnstrctInstanceToAmiSnapshotCreateServiceRoleC41A6A8A", + }, + ], + }, + "Type": "AWS::IAM::Policy", + }, + "CnstrctInstanceToAmiSnapshotGC0E1F9932": { + "DependsOn": [ + "CnstrctInstanceToAmiSnapshotGCServiceRoleDefaultPolicyAA3AA903", + "CnstrctInstanceToAmiSnapshotGCServiceRoleC6251312", + ], + "Properties": { + "Code": { + "S3Bucket": { + "Fn::Sub": "cdk-hnb659fds-assets-\${AWS::AccountId}-\${AWS::Region}", + }, + "S3Key": "24889f5dc169825cb8dd9c7076e6f6886d38a18a2db3fd36a2431314d5711df1.zip", + }, + "Handler": "SnapshotGC.handler", + "Role": { + "Fn::GetAtt": [ + "CnstrctInstanceToAmiSnapshotGCServiceRoleC6251312", + "Arn", + ], + }, + "Runtime": "nodejs20.x", + "Timeout": 180, + }, + "Type": "AWS::Lambda::Function", + }, + "CnstrctInstanceToAmiSnapshotGCServiceRoleC6251312": { + "Properties": { + "AssumeRolePolicyDocument": { + "Statement": [ + { + "Action": "sts:AssumeRole", + "Effect": "Allow", + "Principal": { + "Service": "lambda.amazonaws.com", + }, + }, + ], + "Version": "2012-10-17", + }, + "ManagedPolicyArns": [ + { + "Fn::Join": [ + "", + [ + "arn:", + { + "Ref": "AWS::Partition", + }, + ":iam::aws:policy/service-role/AWSLambdaBasicExecutionRole", + ], + ], + }, + ], + }, + "Type": "AWS::IAM::Role", + }, + "CnstrctInstanceToAmiSnapshotGCServiceRoleDefaultPolicyAA3AA903": { + "Properties": { + "PolicyDocument": { + "Statement": [ + { + "Action": [ + "autoscaling:DescribeAutoScalingGroups", + "autoscaling:DescribeAutoScalingInstances", + "autoscaling:UpdateAutoScalingGroup", + "ec2:CreateLaunchTemplateVersion", + "ec2:CreateSnapshot", + "ec2:CreateTags", + "ec2:DeleteLaunchTemplateVersions", + "ec2:DeleteSnapshot", + "ec2:DeregisterImage", + "ec2:DescribeFastSnapshotRestores", + "ec2:DescribeImages", + "ec2:DescribeInstances", + "ec2:DescribeLaunchTemplates", + "ec2:DescribeLaunchTemplateVersions", + "ec2:DescribeSnapshots", + "ec2:DisableFastSnapshotRestores", + "ec2:EnableFastSnapshotRestores", + "ec2:RegisterImage", + "ec2:RunInstances", + "iam:PassRole", + "logs:CreateLogGroup", + "logs:CreateLogStream", + "logs:PutLogEvents", + ], + "Effect": "Allow", + "Resource": "*", + }, + ], + "Version": "2012-10-17", + }, + "PolicyName": "CnstrctInstanceToAmiSnapshotGCServiceRoleDefaultPolicyAA3AA903", + "Roles": [ + { + "Ref": "CnstrctInstanceToAmiSnapshotGCServiceRoleC6251312", + }, + ], + }, + "Type": "AWS::IAM::Policy", + }, + "CnstrctInstanceToAmiSnapshotGetAction89B3D7E9": { + "DependsOn": [ + "CnstrctInstanceToAmiSnapshotGetActionServiceRoleDefaultPolicy33FBE433", + "CnstrctInstanceToAmiSnapshotGetActionServiceRoleA9262A8D", + ], + "Properties": { + "Code": { + "S3Bucket": { + "Fn::Sub": "cdk-hnb659fds-assets-\${AWS::AccountId}-\${AWS::Region}", + }, + "S3Key": "24889f5dc169825cb8dd9c7076e6f6886d38a18a2db3fd36a2431314d5711df1.zip", + }, + "Handler": "SnapshotGetAction.handler", + "Role": { + "Fn::GetAtt": [ + "CnstrctInstanceToAmiSnapshotGetActionServiceRoleA9262A8D", + "Arn", + ], + }, + "Runtime": "nodejs20.x", + "Timeout": 180, + }, + "Type": "AWS::Lambda::Function", + }, + "CnstrctInstanceToAmiSnapshotGetActionServiceRoleA9262A8D": { + "Properties": { + "AssumeRolePolicyDocument": { + "Statement": [ + { + "Action": "sts:AssumeRole", + "Effect": "Allow", + "Principal": { + "Service": "lambda.amazonaws.com", + }, + }, + ], + "Version": "2012-10-17", + }, + "ManagedPolicyArns": [ + { + "Fn::Join": [ + "", + [ + "arn:", + { + "Ref": "AWS::Partition", + }, + ":iam::aws:policy/service-role/AWSLambdaBasicExecutionRole", + ], + ], + }, + ], + }, + "Type": "AWS::IAM::Role", + }, + "CnstrctInstanceToAmiSnapshotGetActionServiceRoleDefaultPolicy33FBE433": { + "Properties": { + "PolicyDocument": { + "Statement": [ + { + "Action": [ + "autoscaling:DescribeAutoScalingGroups", + "autoscaling:DescribeAutoScalingInstances", + "autoscaling:UpdateAutoScalingGroup", + "ec2:CreateLaunchTemplateVersion", + "ec2:CreateSnapshot", + "ec2:CreateTags", + "ec2:DeleteLaunchTemplateVersions", + "ec2:DeleteSnapshot", + "ec2:DeregisterImage", + "ec2:DescribeFastSnapshotRestores", + "ec2:DescribeImages", + "ec2:DescribeInstances", + "ec2:DescribeLaunchTemplates", + "ec2:DescribeLaunchTemplateVersions", + "ec2:DescribeSnapshots", + "ec2:DisableFastSnapshotRestores", + "ec2:EnableFastSnapshotRestores", + "ec2:RegisterImage", + "ec2:RunInstances", + "iam:PassRole", + "logs:CreateLogGroup", + "logs:CreateLogStream", + "logs:PutLogEvents", + ], + "Effect": "Allow", + "Resource": "*", + }, + ], + "Version": "2012-10-17", + }, + "PolicyName": "CnstrctInstanceToAmiSnapshotGetActionServiceRoleDefaultPolicy33FBE433", + "Roles": [ + { + "Ref": "CnstrctInstanceToAmiSnapshotGetActionServiceRoleA9262A8D", + }, + ], + }, + "Type": "AWS::IAM::Policy", + }, + "CnstrctInstanceToAmiStartExecution3968560D": { + "DependsOn": [ + "CnstrctInstanceToAmiStartExecutionServiceRoleDefaultPolicy7D2D308E", + "CnstrctInstanceToAmiStartExecutionServiceRole08358685", + ], + "Properties": { + "Code": { + "S3Bucket": { + "Fn::Sub": "cdk-hnb659fds-assets-\${AWS::AccountId}-\${AWS::Region}", + }, + "S3Key": "24889f5dc169825cb8dd9c7076e6f6886d38a18a2db3fd36a2431314d5711df1.zip", + }, + "Environment": { + "Variables": { + "STATE_MACHINE_ARN": { + "Fn::Join": [ + "", + [ + "arn:", + { + "Ref": "AWS::Partition", + }, + ":states:", + { + "Ref": "AWS::Region", + }, + ":", + { + "Ref": "AWS::AccountId", + }, + ":stateMachine:stk-cnstrct-instancetoami", + ], + ], + }, + "USER_DATA_HASHES": "stk-cnstrct-runnercismall=ac1bd295908bee9f&stk-cnstrct-runnercilarge=46cf1a6641f4d946", + }, + }, + "FunctionName": "stk-cnstrct-instancetoami", + "Handler": "StartExecution.handler", + "ReservedConcurrentExecutions": 1, + "Role": { + "Fn::GetAtt": [ + "CnstrctInstanceToAmiStartExecutionServiceRole08358685", + "Arn", + ], + }, + "Runtime": "nodejs20.x", + "Timeout": 180, + }, + "Type": "AWS::Lambda::Function", + }, + "CnstrctInstanceToAmiStartExecutionServiceRole08358685": { + "Properties": { + "AssumeRolePolicyDocument": { + "Statement": [ + { + "Action": "sts:AssumeRole", + "Effect": "Allow", + "Principal": { + "Service": "lambda.amazonaws.com", + }, + }, + ], + "Version": "2012-10-17", + }, + "ManagedPolicyArns": [ + { + "Fn::Join": [ + "", + [ + "arn:", + { + "Ref": "AWS::Partition", + }, + ":iam::aws:policy/service-role/AWSLambdaBasicExecutionRole", + ], + ], + }, + ], + }, + "Type": "AWS::IAM::Role", + }, + "CnstrctInstanceToAmiStartExecutionServiceRoleDefaultPolicy7D2D308E": { + "Properties": { + "PolicyDocument": { + "Statement": [ + { + "Action": [ + "states:ListExecutions", + "states:StartExecution", + ], + "Effect": "Allow", + "Resource": { + "Fn::Join": [ + "", + [ + "arn:", + { + "Ref": "AWS::Partition", + }, + ":states:", + { + "Ref": "AWS::Region", + }, + ":", + { + "Ref": "AWS::AccountId", + }, + ":stateMachine:stk-cnstrct-instancetoami", + ], + ], + }, + }, + { + "Action": [ + "states:DescribeExecution", + "states:StopExecution", + ], + "Effect": "Allow", + "Resource": { + "Fn::Join": [ + "", + [ + "arn:", + { + "Ref": "AWS::Partition", + }, + ":states:", + { + "Ref": "AWS::Region", + }, + ":", + { + "Ref": "AWS::AccountId", + }, + ":execution:stk-cnstrct-instancetoami:*", + ], + ], + }, + }, + { + "Action": [ + "ec2:DescribeInstances", + "ec2:DescribeLaunchTemplates", + ], + "Effect": "Allow", + "Resource": "*", + }, + ], + "Version": "2012-10-17", + }, + "PolicyName": "CnstrctInstanceToAmiStartExecutionServiceRoleDefaultPolicy7D2D308E", + "Roles": [ + { + "Ref": "CnstrctInstanceToAmiStartExecutionServiceRole08358685", + }, + ], + }, + "Type": "AWS::IAM::Policy", + }, + "CnstrctInstanceToAmiStateMachine523C3BC9": { + "DeletionPolicy": "Delete", + "DependsOn": [ + "CnstrctInstanceToAmiStartExecution3968560D", + "CnstrctInstanceToAmiStartExecutionServiceRoleDefaultPolicy7D2D308E", + "CnstrctInstanceToAmiStartExecutionServiceRole08358685", + "CnstrctInstanceToAmiStateMachineRoleDefaultPolicy5C292EAF", + "CnstrctInstanceToAmiStateMachineRoleA9BBDE99", + ], + "Properties": { + "DefinitionString": { + "Fn::Join": [ + "", + [ + "{"StartAt":"LaunchTemplateGetNameFromInstanceStep","States":{"LaunchTemplateGetNameFromInstanceStep":{"Next":"AutoScalingGroupGetNameFromInstanceStep","Retry":[{"ErrorEquals":["Lambda.ClientExecutionTimeoutException","Lambda.ServiceException","Lambda.AWSLambdaException","Lambda.SdkClientException"],"IntervalSeconds":2,"MaxAttempts":6,"BackoffRate":2}],"Type":"Task","OutputPath":"$.Payload","Resource":"arn:", + { + "Ref": "AWS::Partition", + }, + ":states:::lambda:invoke","Parameters":{"FunctionName":"", + { + "Fn::GetAtt": [ + "CnstrctInstanceToAmiLaunchTemplateGetNameFromInstance7067ACEF", + "Arn", + ], + }, + "","Payload.$":"$"}},"AutoScalingGroupGetNameFromInstanceStep":{"Next":"SnapshotCreateStep","Retry":[{"ErrorEquals":["Lambda.ClientExecutionTimeoutException","Lambda.ServiceException","Lambda.AWSLambdaException","Lambda.SdkClientException"],"IntervalSeconds":2,"MaxAttempts":6,"BackoffRate":2}],"Type":"Task","OutputPath":"$.Payload","Resource":"arn:", + { + "Ref": "AWS::Partition", + }, + ":states:::lambda:invoke","Parameters":{"FunctionName":"", + { + "Fn::GetAtt": [ + "CnstrctInstanceToAmiAutoScalingGroupGetNameFromInstance27F9175A", + "Arn", + ], + }, + "","Payload.$":"$"}},"SnapshotCreateStep":{"Next":"SnapshotWaitStep","Retry":[{"ErrorEquals":["Lambda.ClientExecutionTimeoutException","Lambda.ServiceException","Lambda.AWSLambdaException","Lambda.SdkClientException"],"IntervalSeconds":2,"MaxAttempts":6,"BackoffRate":2}],"Type":"Task","OutputPath":"$.Payload","Resource":"arn:", + { + "Ref": "AWS::Partition", + }, + ":states:::lambda:invoke","Parameters":{"FunctionName":"", + { + "Fn::GetAtt": [ + "CnstrctInstanceToAmiSnapshotCreate197B8B81", + "Arn", + ], + }, + "","Payload.$":"$"}},"SnapshotWaitStep":{"Type":"Wait","Seconds":10,"Next":"SnapshotGetActionStep"},"SnapshotGetActionStep":{"Next":"SnapshotGetActionChoice","Retry":[{"ErrorEquals":["Lambda.ClientExecutionTimeoutException","Lambda.ServiceException","Lambda.AWSLambdaException","Lambda.SdkClientException"],"IntervalSeconds":2,"MaxAttempts":6,"BackoffRate":2}],"Type":"Task","OutputPath":"$.Payload","Resource":"arn:", + { + "Ref": "AWS::Partition", + }, + ":states:::lambda:invoke","Parameters":{"FunctionName":"", + { + "Fn::GetAtt": [ + "CnstrctInstanceToAmiSnapshotGetAction89B3D7E9", + "Arn", + ], + }, + "","Payload.$":"$"}},"SnapshotGetActionChoice":{"Type":"Choice","Choices":[{"Variable":"$.action","StringEquals":"continue","Next":"FastSnapshotRestoreEnableStep"}],"Default":"SnapshotWaitStep"},"FastSnapshotRestoreEnableStep":{"Next":"FastSnapshotRestoreWaitStep","Retry":[{"ErrorEquals":["Lambda.ClientExecutionTimeoutException","Lambda.ServiceException","Lambda.AWSLambdaException","Lambda.SdkClientException"],"IntervalSeconds":2,"MaxAttempts":6,"BackoffRate":2}],"Type":"Task","OutputPath":"$.Payload","Resource":"arn:", + { + "Ref": "AWS::Partition", + }, + ":states:::lambda:invoke","Parameters":{"FunctionName":"", + { + "Fn::GetAtt": [ + "CnstrctInstanceToAmiFastSnapshotRestoreEnable8C7FE336", + "Arn", + ], + }, + "","Payload.$":"$"}},"FastSnapshotRestoreWaitStep":{"Type":"Wait","Seconds":5,"Next":"FastSnapshotRestoreGetActionStep"},"FastSnapshotRestoreGetActionStep":{"Next":"FastSnapshotRestoreGetActionChoice","Retry":[{"ErrorEquals":["Lambda.ClientExecutionTimeoutException","Lambda.ServiceException","Lambda.AWSLambdaException","Lambda.SdkClientException"],"IntervalSeconds":2,"MaxAttempts":6,"BackoffRate":2}],"Type":"Task","OutputPath":"$.Payload","Resource":"arn:", + { + "Ref": "AWS::Partition", + }, + ":states:::lambda:invoke","Parameters":{"FunctionName":"", + { + "Fn::GetAtt": [ + "CnstrctInstanceToAmiFastSnapshotRestoreGetAction81EE8A35", + "Arn", + ], + }, + "","Payload.$":"$"}},"FastSnapshotRestoreGetActionChoice":{"Type":"Choice","Choices":[{"Variable":"$.action","StringEquals":"continue","Next":"ImageCreateStep"}],"Default":"FastSnapshotRestoreWaitStep"},"ImageCreateStep":{"Next":"ImageWaitStep","Retry":[{"ErrorEquals":["Lambda.ClientExecutionTimeoutException","Lambda.ServiceException","Lambda.AWSLambdaException","Lambda.SdkClientException"],"IntervalSeconds":2,"MaxAttempts":6,"BackoffRate":2}],"Type":"Task","OutputPath":"$.Payload","Resource":"arn:", + { + "Ref": "AWS::Partition", + }, + ":states:::lambda:invoke","Parameters":{"FunctionName":"", + { + "Fn::GetAtt": [ + "CnstrctInstanceToAmiImageCreateE181B0A7", + "Arn", + ], + }, + "","Payload.$":"$"}},"ImageWaitStep":{"Type":"Wait","Seconds":1,"Next":"ImageGetActionStep"},"ImageGetActionStep":{"Next":"ImageGetActionChoice","Retry":[{"ErrorEquals":["Lambda.ClientExecutionTimeoutException","Lambda.ServiceException","Lambda.AWSLambdaException","Lambda.SdkClientException"],"IntervalSeconds":2,"MaxAttempts":6,"BackoffRate":2}],"Type":"Task","OutputPath":"$.Payload","Resource":"arn:", + { + "Ref": "AWS::Partition", + }, + ":states:::lambda:invoke","Parameters":{"FunctionName":"", + { + "Fn::GetAtt": [ + "CnstrctInstanceToAmiImageGetActionF356B2EA", + "Arn", + ], + }, + "","Payload.$":"$"}},"ImageGetActionChoice":{"Type":"Choice","Choices":[{"Variable":"$.action","StringEquals":"continue","Next":"LaunchTemplateUpdateImageStep"}],"Default":"ImageWaitStep"},"LaunchTemplateUpdateImageStep":{"Next":"AutoScalingGroupUpdateLaunchTemplateVersionStep","Retry":[{"ErrorEquals":["Lambda.ClientExecutionTimeoutException","Lambda.ServiceException","Lambda.AWSLambdaException","Lambda.SdkClientException"],"IntervalSeconds":2,"MaxAttempts":6,"BackoffRate":2},{"ErrorEquals":["States.ALL"]}],"Type":"Task","OutputPath":"$.Payload","Resource":"arn:", + { + "Ref": "AWS::Partition", + }, + ":states:::lambda:invoke","Parameters":{"FunctionName":"", + { + "Fn::GetAtt": [ + "CnstrctInstanceToAmiLaunchTemplateUpdateImage99ABAA53", + "Arn", + ], + }, + "","Payload.$":"$"}},"AutoScalingGroupUpdateLaunchTemplateVersionStep":{"Next":"ImageGCStep","Retry":[{"ErrorEquals":["Lambda.ClientExecutionTimeoutException","Lambda.ServiceException","Lambda.AWSLambdaException","Lambda.SdkClientException"],"IntervalSeconds":2,"MaxAttempts":6,"BackoffRate":2},{"ErrorEquals":["States.ALL"]}],"Type":"Task","OutputPath":"$.Payload","Resource":"arn:", + { + "Ref": "AWS::Partition", + }, + ":states:::lambda:invoke","Parameters":{"FunctionName":"", + { + "Fn::GetAtt": [ + "CnstrctInstanceToAmiAutoScalingGroupUpdateLaunchTemplateVersion08E6BA51", + "Arn", + ], + }, + "","Payload.$":"$"}},"ImageGCStep":{"Next":"SnapshotGCStep","Retry":[{"ErrorEquals":["Lambda.ClientExecutionTimeoutException","Lambda.ServiceException","Lambda.AWSLambdaException","Lambda.SdkClientException"],"IntervalSeconds":2,"MaxAttempts":6,"BackoffRate":2},{"ErrorEquals":["States.ALL"]}],"Type":"Task","OutputPath":"$.Payload","Resource":"arn:", + { + "Ref": "AWS::Partition", + }, + ":states:::lambda:invoke","Parameters":{"FunctionName":"", + { + "Fn::GetAtt": [ + "CnstrctInstanceToAmiImageGCC91F1072", + "Arn", + ], + }, + "","Payload.$":"$"}},"SnapshotGCStep":{"Next":"FastSnapshotRestoreGCStep","Retry":[{"ErrorEquals":["Lambda.ClientExecutionTimeoutException","Lambda.ServiceException","Lambda.AWSLambdaException","Lambda.SdkClientException"],"IntervalSeconds":2,"MaxAttempts":6,"BackoffRate":2},{"ErrorEquals":["States.ALL"]}],"Type":"Task","OutputPath":"$.Payload","Resource":"arn:", + { + "Ref": "AWS::Partition", + }, + ":states:::lambda:invoke","Parameters":{"FunctionName":"", + { + "Fn::GetAtt": [ + "CnstrctInstanceToAmiSnapshotGC0E1F9932", + "Arn", + ], + }, + "","Payload.$":"$"}},"FastSnapshotRestoreGCStep":{"Next":"LaunchTemplateGCStep","Retry":[{"ErrorEquals":["Lambda.ClientExecutionTimeoutException","Lambda.ServiceException","Lambda.AWSLambdaException","Lambda.SdkClientException"],"IntervalSeconds":2,"MaxAttempts":6,"BackoffRate":2},{"ErrorEquals":["States.ALL"]}],"Type":"Task","OutputPath":"$.Payload","Resource":"arn:", + { + "Ref": "AWS::Partition", + }, + ":states:::lambda:invoke","Parameters":{"FunctionName":"", + { + "Fn::GetAtt": [ + "CnstrctInstanceToAmiFastSnapshotRestoreGC871594F1", + "Arn", + ], + }, + "","Payload.$":"$"}},"LaunchTemplateGCStep":{"End":true,"Retry":[{"ErrorEquals":["Lambda.ClientExecutionTimeoutException","Lambda.ServiceException","Lambda.AWSLambdaException","Lambda.SdkClientException"],"IntervalSeconds":2,"MaxAttempts":6,"BackoffRate":2},{"ErrorEquals":["States.ALL"]}],"Type":"Task","OutputPath":"$.Payload","Resource":"arn:", + { + "Ref": "AWS::Partition", + }, + ":states:::lambda:invoke","Parameters":{"FunctionName":"", + { + "Fn::GetAtt": [ + "CnstrctInstanceToAmiLaunchTemplateGC1849B9F5", + "Arn", + ], + }, + "","Payload.$":"$"}}},"TimeoutSeconds":1800}", + ], + ], + }, + "LoggingConfiguration": { + "Destinations": [ + { + "CloudWatchLogsLogGroup": { + "LogGroupArn": { + "Fn::GetAtt": [ + "CnstrctInstanceToAmiLogGroup607FDB05", + "Arn", + ], + }, + }, + }, + ], + "IncludeExecutionData": true, + "Level": "ALL", + }, + "RoleArn": { + "Fn::GetAtt": [ + "CnstrctInstanceToAmiStateMachineRoleA9BBDE99", + "Arn", + ], + }, + "StateMachineName": "stk-cnstrct-instancetoami", + "TracingConfiguration": { + "Enabled": true, + }, + }, + "Type": "AWS::StepFunctions::StateMachine", + "UpdateReplacePolicy": "Delete", + }, + "CnstrctInstanceToAmiStateMachineRoleA9BBDE99": { + "Properties": { + "AssumeRolePolicyDocument": { + "Statement": [ + { + "Action": "sts:AssumeRole", + "Effect": "Allow", + "Principal": { + "Service": "states.amazonaws.com", + }, + }, + ], + "Version": "2012-10-17", + }, + "Policies": [ + { + "PolicyDocument": { + "Statement": [ + { + "Action": "lambda:InvokeFunction", + "Effect": "Allow", + "Resource": [ + { + "Fn::GetAtt": [ + "CnstrctInstanceToAmiAutoScalingGroupGetNameFromInstance27F9175A", + "Arn", + ], + }, + { + "Fn::GetAtt": [ + "CnstrctInstanceToAmiAutoScalingGroupUpdateLaunchTemplateVersion08E6BA51", + "Arn", + ], + }, + { + "Fn::GetAtt": [ + "CnstrctInstanceToAmiFastSnapshotRestoreEnable8C7FE336", + "Arn", + ], + }, + { + "Fn::GetAtt": [ + "CnstrctInstanceToAmiFastSnapshotRestoreGC871594F1", + "Arn", + ], + }, + { + "Fn::GetAtt": [ + "CnstrctInstanceToAmiFastSnapshotRestoreGetAction81EE8A35", + "Arn", + ], + }, + { + "Fn::GetAtt": [ + "CnstrctInstanceToAmiImageCreateE181B0A7", + "Arn", + ], + }, + { + "Fn::GetAtt": [ + "CnstrctInstanceToAmiImageGCC91F1072", + "Arn", + ], + }, + { + "Fn::GetAtt": [ + "CnstrctInstanceToAmiImageGetActionF356B2EA", + "Arn", + ], + }, + { + "Fn::GetAtt": [ + "CnstrctInstanceToAmiLaunchTemplateGC1849B9F5", + "Arn", + ], + }, + { + "Fn::GetAtt": [ + "CnstrctInstanceToAmiLaunchTemplateGetNameFromInstance7067ACEF", + "Arn", + ], + }, + { + "Fn::GetAtt": [ + "CnstrctInstanceToAmiLaunchTemplateUpdateImage99ABAA53", + "Arn", + ], + }, + { + "Fn::GetAtt": [ + "CnstrctInstanceToAmiSnapshotCreate197B8B81", + "Arn", + ], + }, + { + "Fn::GetAtt": [ + "CnstrctInstanceToAmiSnapshotGC0E1F9932", + "Arn", + ], + }, + { + "Fn::GetAtt": [ + "CnstrctInstanceToAmiSnapshotGetAction89B3D7E9", + "Arn", + ], + }, + ], + }, + ], + "Version": "2012-10-17", + }, + "PolicyName": "LambdaPolicy", + }, + { + "PolicyDocument": { + "Statement": [ + { + "Action": [ + "logs:CreateLogGroup", + "logs:CreateLogStream", + "logs:PutLogEvents", + ], + "Effect": "Allow", + "Resource": { + "Fn::Join": [ + "", + [ + "arn:", + { + "Ref": "AWS::Partition", + }, + ":logs:", + { + "Ref": "AWS::Region", + }, + ":", + { + "Ref": "AWS::AccountId", + }, + ":log-group:", + { + "Ref": "CnstrctInstanceToAmiLogGroup607FDB05", + }, + "*", + ], + ], + }, + }, + ], + "Version": "2012-10-17", + }, + "PolicyName": "LogPolicy", + }, + ], + }, + "Type": "AWS::IAM::Role", + }, + "CnstrctInstanceToAmiStateMachineRoleDefaultPolicy5C292EAF": { + "Properties": { + "PolicyDocument": { + "Statement": [ + { + "Action": "lambda:InvokeFunction", + "Effect": "Allow", + "Resource": [ + { + "Fn::GetAtt": [ + "CnstrctInstanceToAmiLaunchTemplateGetNameFromInstance7067ACEF", + "Arn", + ], + }, + { + "Fn::Join": [ + "", + [ + { + "Fn::GetAtt": [ + "CnstrctInstanceToAmiLaunchTemplateGetNameFromInstance7067ACEF", + "Arn", + ], + }, + ":*", + ], + ], + }, + ], + }, + { + "Action": "lambda:InvokeFunction", + "Effect": "Allow", + "Resource": [ + { + "Fn::GetAtt": [ + "CnstrctInstanceToAmiAutoScalingGroupGetNameFromInstance27F9175A", + "Arn", + ], + }, + { + "Fn::Join": [ + "", + [ + { + "Fn::GetAtt": [ + "CnstrctInstanceToAmiAutoScalingGroupGetNameFromInstance27F9175A", + "Arn", + ], + }, + ":*", + ], + ], + }, + ], + }, + { + "Action": "lambda:InvokeFunction", + "Effect": "Allow", + "Resource": [ + { + "Fn::GetAtt": [ + "CnstrctInstanceToAmiSnapshotCreate197B8B81", + "Arn", + ], + }, + { + "Fn::Join": [ + "", + [ + { + "Fn::GetAtt": [ + "CnstrctInstanceToAmiSnapshotCreate197B8B81", + "Arn", + ], + }, + ":*", + ], + ], + }, + ], + }, + { + "Action": "lambda:InvokeFunction", + "Effect": "Allow", + "Resource": [ + { + "Fn::GetAtt": [ + "CnstrctInstanceToAmiSnapshotGetAction89B3D7E9", + "Arn", + ], + }, + { + "Fn::Join": [ + "", + [ + { + "Fn::GetAtt": [ + "CnstrctInstanceToAmiSnapshotGetAction89B3D7E9", + "Arn", + ], + }, + ":*", + ], + ], + }, + ], + }, + { + "Action": "lambda:InvokeFunction", + "Effect": "Allow", + "Resource": [ + { + "Fn::GetAtt": [ + "CnstrctInstanceToAmiFastSnapshotRestoreEnable8C7FE336", + "Arn", + ], + }, + { + "Fn::Join": [ + "", + [ + { + "Fn::GetAtt": [ + "CnstrctInstanceToAmiFastSnapshotRestoreEnable8C7FE336", + "Arn", + ], + }, + ":*", + ], + ], + }, + ], + }, + { + "Action": "lambda:InvokeFunction", + "Effect": "Allow", + "Resource": [ + { + "Fn::GetAtt": [ + "CnstrctInstanceToAmiFastSnapshotRestoreGetAction81EE8A35", + "Arn", + ], + }, + { + "Fn::Join": [ + "", + [ + { + "Fn::GetAtt": [ + "CnstrctInstanceToAmiFastSnapshotRestoreGetAction81EE8A35", + "Arn", + ], + }, + ":*", + ], + ], + }, + ], + }, + { + "Action": "lambda:InvokeFunction", + "Effect": "Allow", + "Resource": [ + { + "Fn::GetAtt": [ + "CnstrctInstanceToAmiImageCreateE181B0A7", + "Arn", + ], + }, + { + "Fn::Join": [ + "", + [ + { + "Fn::GetAtt": [ + "CnstrctInstanceToAmiImageCreateE181B0A7", + "Arn", + ], + }, + ":*", + ], + ], + }, + ], + }, + { + "Action": "lambda:InvokeFunction", + "Effect": "Allow", + "Resource": [ + { + "Fn::GetAtt": [ + "CnstrctInstanceToAmiImageGetActionF356B2EA", + "Arn", + ], + }, + { + "Fn::Join": [ + "", + [ + { + "Fn::GetAtt": [ + "CnstrctInstanceToAmiImageGetActionF356B2EA", + "Arn", + ], + }, + ":*", + ], + ], + }, + ], + }, + { + "Action": "lambda:InvokeFunction", + "Effect": "Allow", + "Resource": [ + { + "Fn::GetAtt": [ + "CnstrctInstanceToAmiLaunchTemplateUpdateImage99ABAA53", + "Arn", + ], + }, + { + "Fn::Join": [ + "", + [ + { + "Fn::GetAtt": [ + "CnstrctInstanceToAmiLaunchTemplateUpdateImage99ABAA53", + "Arn", + ], + }, + ":*", + ], + ], + }, + ], + }, + { + "Action": "lambda:InvokeFunction", + "Effect": "Allow", + "Resource": [ + { + "Fn::GetAtt": [ + "CnstrctInstanceToAmiAutoScalingGroupUpdateLaunchTemplateVersion08E6BA51", + "Arn", + ], + }, + { + "Fn::Join": [ + "", + [ + { + "Fn::GetAtt": [ + "CnstrctInstanceToAmiAutoScalingGroupUpdateLaunchTemplateVersion08E6BA51", + "Arn", + ], + }, + ":*", + ], + ], + }, + ], + }, + { + "Action": "lambda:InvokeFunction", + "Effect": "Allow", + "Resource": [ + { + "Fn::GetAtt": [ + "CnstrctInstanceToAmiImageGCC91F1072", + "Arn", + ], + }, + { + "Fn::Join": [ + "", + [ + { + "Fn::GetAtt": [ + "CnstrctInstanceToAmiImageGCC91F1072", + "Arn", + ], + }, + ":*", + ], + ], + }, + ], + }, + { + "Action": "lambda:InvokeFunction", + "Effect": "Allow", + "Resource": [ + { + "Fn::GetAtt": [ + "CnstrctInstanceToAmiSnapshotGC0E1F9932", + "Arn", + ], + }, + { + "Fn::Join": [ + "", + [ + { + "Fn::GetAtt": [ + "CnstrctInstanceToAmiSnapshotGC0E1F9932", + "Arn", + ], + }, + ":*", + ], + ], + }, + ], + }, + { + "Action": "lambda:InvokeFunction", + "Effect": "Allow", + "Resource": [ + { + "Fn::GetAtt": [ + "CnstrctInstanceToAmiFastSnapshotRestoreGC871594F1", + "Arn", + ], + }, + { + "Fn::Join": [ + "", + [ + { + "Fn::GetAtt": [ + "CnstrctInstanceToAmiFastSnapshotRestoreGC871594F1", + "Arn", + ], + }, + ":*", + ], + ], + }, + ], + }, + { + "Action": "lambda:InvokeFunction", + "Effect": "Allow", + "Resource": [ + { + "Fn::GetAtt": [ + "CnstrctInstanceToAmiLaunchTemplateGC1849B9F5", + "Arn", + ], + }, + { + "Fn::Join": [ + "", + [ + { + "Fn::GetAtt": [ + "CnstrctInstanceToAmiLaunchTemplateGC1849B9F5", + "Arn", + ], + }, + ":*", + ], + ], + }, + ], + }, + { + "Action": [ + "logs:CreateLogDelivery", + "logs:GetLogDelivery", + "logs:UpdateLogDelivery", + "logs:DeleteLogDelivery", + "logs:ListLogDeliveries", + "logs:PutResourcePolicy", + "logs:DescribeResourcePolicies", + "logs:DescribeLogGroups", + ], + "Effect": "Allow", + "Resource": "*", + }, + { + "Action": [ + "xray:PutTraceSegments", + "xray:PutTelemetryRecords", + "xray:GetSamplingRules", + "xray:GetSamplingTargets", + ], + "Effect": "Allow", + "Resource": "*", + }, + ], + "Version": "2012-10-17", + }, + "PolicyName": "CnstrctInstanceToAmiStateMachineRoleDefaultPolicy5C292EAF", + "Roles": [ + { + "Ref": "CnstrctInstanceToAmiStateMachineRoleA9BBDE99", + }, + ], + }, + "Type": "AWS::IAM::Policy", + }, + "CnstrctLink3BE2E5AA": { + "Properties": { + "Name": "stk-cnstrct-link", + "SecurityGroupIds": [ + { + "Fn::GetAtt": [ + "CnstrctSgF5C70BA4", + "GroupId", + ], + }, + ], + "SubnetIds": [ + { + "Ref": "VpcPrivateSubnet1Subnet536B997A", + }, + { + "Ref": "VpcPrivateSubnet2Subnet3788AAA1", + }, + ], + }, + "Type": "AWS::ApiGatewayV2::VpcLink", + }, + "CnstrctMyCiHost001A9DE30324": { + "Properties": { + "HostedZoneId": "test-hostedZoneId", + "Name": "my-ci-host-001.test-zoneName.", + "ResourceRecords": [ + { + "Fn::GetAtt": [ + "CnstrctMyCiHost001Instance0F6CFAF181cec7ef931262b3", + "PrivateIp", + ], + }, + ], + "TTL": "60", + "Type": "A", + }, + "Type": "AWS::Route53::RecordSet", + }, + "CnstrctMyCiHost001ApiCCC990DF": { + "Properties": { + "Name": "stk-cnstrct-mycihost001", + "ProtocolType": "HTTP", + }, + "Type": "AWS::ApiGatewayV2::Api", + }, + "CnstrctMyCiHost001ApiDefaultStage4EC87F30": { + "Properties": { + "ApiId": { + "Ref": "CnstrctMyCiHost001ApiCCC990DF", + }, + "AutoDeploy": true, + "StageName": "$default", + }, + "Type": "AWS::ApiGatewayV2::Stage", + }, + "CnstrctMyCiHost001ApiPOSTproxy7FABAABA": { + "Properties": { + "ApiId": { + "Ref": "CnstrctMyCiHost001ApiCCC990DF", + }, + "AuthorizationType": "NONE", + "RouteKey": "POST /{proxy+}", + "Target": { + "Fn::Join": [ + "", + [ + "integrations/", + { + "Ref": "CnstrctMyCiHost001ApiPOSTproxyIntegration881CB7BF", + }, + ], + ], + }, + }, + "Type": "AWS::ApiGatewayV2::Route", + }, + "CnstrctMyCiHost001ApiPOSTproxyIntegration881CB7BF": { + "Properties": { + "ApiId": { + "Ref": "CnstrctMyCiHost001ApiCCC990DF", + }, + "ConnectionId": { + "Ref": "CnstrctLink3BE2E5AA", + }, + "ConnectionType": "VPC_LINK", + "IntegrationMethod": "ANY", + "IntegrationType": "HTTP_PROXY", + "IntegrationUri": { + "Ref": "CnstrctMyCiHost001NlbListener85589A96", + }, + "PayloadFormatVersion": "1.0", + }, + "Type": "AWS::ApiGatewayV2::Integration", + }, + "CnstrctMyCiHost001Instance0F6CFAF181cec7ef931262b3": { + "CreationPolicy": { + "ResourceSignal": { + "Count": 1, + "Timeout": "PT15M", + }, + }, + "Properties": { + "AvailabilityZone": { + "Fn::Select": [ + 0, + { + "Fn::GetAZs": "", + }, + ], + }, + "InstanceType": "t3.large", + "LaunchTemplate": { + "LaunchTemplateId": { + "Ref": "CnstrctMyCiHost001Lt509F8571", + }, + "Version": { + "Fn::GetAtt": [ + "CnstrctMyCiHost001Lt509F8571", + "LatestVersionNumber", + ], + }, + }, + "SubnetId": { + "Ref": "VpcPrivateSubnet1Subnet536B997A", + }, + "Tags": [ + { + "Key": "Name", + "Value": "my-ci-host-001.test-zoneName", + }, + ], + }, + "Type": "AWS::EC2::Instance", + }, + "CnstrctMyCiHost001Lt509F8571": { + "DependsOn": [ + "CnstrctHostRoleDefaultPolicy8426BDC1", + "CnstrctHostRole5DD9F366", + ], + "Properties": { + "LaunchTemplateData": { + "BlockDeviceMappings": [ + { + "DeviceName": "/dev/sda1", + "Ebs": { + "DeleteOnTermination": true, + "Encrypted": true, + "VolumeSize": 20, + "VolumeType": "gp2", + }, + }, + ], + "IamInstanceProfile": { + "Arn": { + "Fn::GetAtt": [ + "CnstrctMyCiHost001LtProfileF9C5FA56", + "Arn", + ], + }, + }, + "ImageId": { + "Ref": "SsmParameterValuetestimagessmnameC96584B6F00A464EAD1953AFF4B05118Parameter", + }, + "KeyName": { + "Fn::GetAtt": [ + "CnstrctSshIdRsaEC2KeyPairstkcnstrctsshidrsaF4F38F69", + "KeyPairName", + ], + }, + "MetadataOptions": { + "HttpPutResponseHopLimit": 2, + "HttpTokens": "required", + }, + "Monitoring": { + "Enabled": true, + }, + "SecurityGroupIds": [ + { + "Fn::GetAtt": [ + "CnstrctSgF5C70BA4", + "GroupId", + ], + }, + ], + "TagSpecifications": [ + { + "ResourceType": "instance", + "Tags": [ + { + "Key": "Name", + "Value": "Stk/Cnstrct/MyCiHost001Lt", + }, + ], + }, + { + "ResourceType": "volume", + "Tags": [ + { + "Key": "Name", + "Value": "Stk/Cnstrct/MyCiHost001Lt", + }, + ], + }, + ], + "UserData": { + "Fn::Base64": { + "Fn::Join": [ + "", + [ + "#cloud-config +timezone: America/Los_Angeles +fqdn: my-ci-host-001.test-zoneName +hostname: my-ci-host-001.test-zoneName +bootcmd: + - | + #!/bin/sh + cat <<'EOT' > /etc/profile.ssm-user + #!/bin/sh + [ "$0$@" = "sh" ] && ENV= sudo -u "ubuntu" -i + EOT + dir=/etc/systemd/system/snap.amazon-ssm-agent.amazon-ssm-agent.service.d + mkdir -p $dir + cat <<'EOT' > $dir/override.conf + [Service] + Environment="ENV=/etc/profile.ssm-user" + EOT + - | + #!/bin/sh + set -e -x + mkdir -p /var/lib/docker + mount -t tmpfs -o 'defaults,noatime,exec,nr_inodes=0,mode=710,size=4G' tmpfs /var/lib/docker + - mkdir -p /var/log/atop +apt: + sources: + github-cli.list: + source: deb https://cli.github.com/packages stable main + keyid: 23F3D4EA75716059 + docker.list: + source: deb https://download.docker.com/linux/ubuntu $RELEASE stable + keyid: 9DC858229FC7DD38854AE2D88D81803C0EBFCD88 +packages: + - awscli + - gh + - docker-ce + - docker-ce-cli + - containerd.io + - docker-compose-plugin + - qemu + - qemu-user-static + - binfmt-support + - git + - gosu + - mc + - curl + - apt-transport-https + - ca-certificates + - tzdata + - atop + - iotop + - htop + - bwm-ng + - jq + - expect +write_files: + - path: /etc/sysctl.d/enable-ipv4-forwarding.conf + content: | + net.ipv4.conf.all.forwarding=1 + - path: /etc/default/atop + content: | + LOGOPTS="-R" + LOGINTERVAL=15 + LOGGENERATIONS=4 + - path: /etc/docker/daemon.json + permissions: "0644" + content: | + { + "log-driver": "syslog", + "log-opts": { + "tag": "docker/{{.Name}}" + }, + "runtimes": { + "sysbox-runc": { + "path": "/usr/bin/sysbox-runc" + } + }, + "default-runtime": "sysbox-runc", + "userns-remap": "sysbox" + } + - path: /etc/systemd/system/docker.service.d/override.conf + permissions: "0644" + content: | + [Service] + TimeoutStopSec=3600 + ExecStartPost=/home/ubuntu/run-docker-compose.sh + - path: /etc/cron.d/git-pull-and-rerun-docker-compose-periodically + permissions: "0644" + content: | + */1 * * * * ubuntu /home/ubuntu/run-docker-compose.sh --no-print-compose-logs 2>&1 | logger -t docker/run-docker-compose + - path: /etc/rsyslog.d/01-docker-tag-to-serial-console.conf + permissions: "0644" + content: | + if $syslogtag startswith 'docker/' then -/dev/console + # It will also write to /var/log/syslog as usual. + - path: /var/lib/cloud/scripts/per-once/00-append-etc-environment.sh + permissions: "0755" + content: | + #!/bin/bash + set -e -o pipefail && echo "================= $(date) | Up $(awk '{print $1}' /proc/uptime) seconds =================" && echo "Running $BASH_SOURCE as $(whoami)" && set -x && export AWS_DEFAULT_REGION=$(ec2metadata --availability-zone | sed "s/[a-z]$//") + echo "TZ=America/Los_Angeles" >> /etc/environment + echo "LESS=RS" >> /etc/environment + - path: /var/lib/cloud/scripts/per-once/make-apt-get-install-not-run-after-restoring-from-snapshot-to-speedup-boot.sh + permissions: "0755" + content: | + #!/bin/bash + set -e -o pipefail && echo "================= $(date) | Up $(awk '{print $1}' /proc/uptime) seconds =================" && echo "Running $BASH_SOURCE as $(whoami)" && set -x && export AWS_DEFAULT_REGION=$(ec2metadata --availability-zone | sed "s/[a-z]$//") + sed -i -E 's/(- (package-update-upgrade-install|apt-configure|apt-pipelining))/# \\1/g' /etc/cloud/cloud.cfg + - path: /var/lib/cloud/scripts/per-once/apply-services-configs.sh + permissions: "0755" + content: | + #!/bin/bash + set -e -o pipefail && echo "================= $(date) | Up $(awk '{print $1}' /proc/uptime) seconds =================" && echo "Running $BASH_SOURCE as $(whoami)" && set -x && export AWS_DEFAULT_REGION=$(ec2metadata --availability-zone | sed "s/[a-z]$//") + systemctl daemon-reload + service atop restart || true + sysctl --system + - path: /var/lib/cloud/scripts/per-once/add-ubuntu-user-to-docker-group-to-access-socket.sh + permissions: "0755" + content: | + #!/bin/bash + set -e -o pipefail && echo "================= $(date) | Up $(awk '{print $1}' /proc/uptime) seconds =================" && echo "Running $BASH_SOURCE as $(whoami)" && set -x && export AWS_DEFAULT_REGION=$(ec2metadata --availability-zone | sed "s/[a-z]$//") + usermod -aG docker ubuntu + - path: /var/lib/cloud/scripts/per-once/install-sysbox-for-docker-in-docker.sh + permissions: "0755" + content: | + #!/bin/bash + set -e -o pipefail && echo "================= $(date) | Up $(awk '{print $1}' /proc/uptime) seconds =================" && echo "Running $BASH_SOURCE as $(whoami)" && set -x && export AWS_DEFAULT_REGION=$(ec2metadata --availability-zone | sed "s/[a-z]$//") + systemctl stop docker docker.socket || true + for svc in sysbox-fs sysbox-mgr; do + dir="/etc/systemd/system/$svc.service.d" + mkdir -p "$dir" + { + echo "[Service]" + echo "Restart=always" + echo "RestartSec=5" + if [[ "$svc" == "sysbox-fs" ]]; then + echo "ExecStartPost=/bin/systemctl start sysbox-mgr.service" + fi + } > "$dir/override.conf" + done + wget -nv -O /tmp/sysbox-ce.deb "https://downloads.nestybox.com/sysbox/releases/v0.6.4/sysbox-ce_0.6.4-0.linux_$(dpkg --print-architecture).deb" + dpkg -i /tmp/sysbox-ce.deb + rm -f /tmp/sysbox-ce.deb + - path: /var/lib/cloud/scripts/per-once/rsync-tmpfs-volume-from-old-instance.sh + permissions: "0755" + content: | + #!/bin/bash + set -e -o pipefail && echo "================= $(date) | Up $(awk '{print $1}' /proc/uptime) seconds =================" && echo "Running $BASH_SOURCE as $(whoami)" && set -x && export AWS_DEFAULT_REGION=$(ec2metadata --availability-zone | sed "s/[a-z]$//") + + instance_id=$(ec2metadata --instance-id) + stack_name=$( + aws ec2 describe-tags \\ + --filters "Name=resource-id,Values=$instance_id" "Name=key,Values=aws:cloudformation:stack-name" \\ + --query "Tags[0].Value" --output text + ) + logical_id=$( + aws ec2 describe-tags \\ + --filters "Name=resource-id,Values=$instance_id" "Name=key,Values=aws:cloudformation:logical-id" \\ + --query "Tags[0].Value" --output text + ) + old_instance_ip_addr=$( + aws ec2 describe-instances \\ + --filters "Name=tag:Name,Values=my-ci-host-001.test-zoneName" "Name=instance-state-name,Values=running" \\ + --query "Reservations[*].Instances[*].[InstanceId,PrivateIpAddress]" --output text \\ + | grep -v "$instance_id" | awk '{print $2}' | head -n1 || true + ) + + if [[ "$old_instance_ip_addr" != "" ]]; then + mkdir -p ~/.ssh && chmod 700 ~/.ssh + aws secretsmanager get-secret-value \\ + --secret-id "ec2-ssh-key/", + { + "Fn::GetAtt": [ + "CnstrctSshIdRsaEC2KeyPairstkcnstrctsshidrsaF4F38F69", + "KeyPairName", + ], + }, + "/private" \\ + --query SecretString --output text \\ + > ~/.ssh/id_rsa + chmod 600 ~/.ssh/id_rsa + + systemctl stop docker docker.socket || true + + ssh -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null \\ + "ubuntu@$old_instance_ip_addr" "sudo systemctl stop docker docker.socket || true" + + time rsync \\ + -aHXS --one-file-system --numeric-ids --delete $@ \\ + --rsh="ssh -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null" \\ + --rsync-path="sudo rsync" \\ + "ubuntu@$old_instance_ip_addr:/var/lib/docker/" "/var/lib/docker/" + + fi + + aws cloudformation signal-resource \\ + --stack-name "$stack_name" --logical-resource-id "$logical_id" \\ + --unique-id "$instance_id" --status SUCCESS + - path: /var/lib/cloud/scripts/per-once/allow-rsyslog-write-to-serial-console.sh + permissions: "0755" + content: | + #!/bin/bash + set -e -o pipefail && echo "================= $(date) | Up $(awk '{print $1}' /proc/uptime) seconds =================" && echo "Running $BASH_SOURCE as $(whoami)" && set -x && export AWS_DEFAULT_REGION=$(ec2metadata --availability-zone | sed "s/[a-z]$//") + usermod -a -G tty syslog + systemctl restart rsyslog + - path: /home/ubuntu/docker-pull-if-changed-rate-limit-friendly.sh + owner: ubuntu:ubuntu + permissions: "0755" + defer: true + content: | + #!/bin/bash + set -e -o pipefail + repo="$1" + name="$2" + tag="$3" + + if [[ "$repo" == "" || "$name" == "" || "$tag" == "" ]]; then + echo "Usage: $0 repo name tag" + exit 1 + fi + + bearer=$(jq -r ".auths[\\"$repo\\"].auth" ~/.docker/config.json || true) + if [[ "$bearer" == "" ]]; then + echo "There is no auth token for $repo in ~/.docker/config.json; did you run \\"docker login\\"?" + exit 2 + fi + + file=~/.docker/$(echo "$repo-$name-$tag" | sed -E 's/[^-_a-zA-Z0-9]/_/g').digest + old_digest=$(cat "$file" 2>/dev/null || true) + cur_digest=$( + curl -sS --fail --head \\ + -H "Authorization: Bearer $bearer" \\ + -H "Accept: application/vnd.oci.image.index.v1+json" \\ + https://$repo/v2/$name/manifests/$tag \\ + | grep -i 'Docker-Content-Digest' | awk '{print $2}' | sed -E 's/\\s+//' + ) + + if [[ "$old_digest" != "$cur_digest" ]]; then + if docker pull $repo/$name:$tag; then + echo "$cur_digest" > $file + else + code="$?" + echo "Failed to run \\"docker pull $repo/$name:$tag\\"" + exit "$code" + fi + else + echo "Not pulling $repo/$name:$tag - no changes in manifest since previous pull (digest: $cur_digest)" + fi + - path: /home/ubuntu/run-docker-compose.sh + owner: ubuntu:ubuntu + permissions: "0755" + defer: true + content: | + #!/bin/bash + + if [[ $(whoami) != "ubuntu" ]]; then + exec gosu ubuntu "$BASH_SOURCE" "$@" + fi + + exec {FD}<$BASH_SOURCE + flock -n "$FD" || { echo "Already running."; exit 0; } + set -e -o pipefail && echo "================= $(date) | Up $(awk '{print $1}' /proc/uptime) seconds =================" && echo "Running $BASH_SOURCE as $(whoami)" && set -x && export AWS_DEFAULT_REGION=$(ec2metadata --availability-zone | sed "s/[a-z]$//") + + source /etc/environment + export TZ + + mkdir -p ~/.ssh && chmod 700 ~/.ssh + aws secretsmanager get-secret-value \\ + --secret-id "ec2-ssh-key/", + { + "Fn::GetAtt": [ + "CnstrctSshIdRsaEC2KeyPairstkcnstrctsshidrsaF4F38F69", + "KeyPairName", + ], + }, + "/private" \\ + --query SecretString --output text \\ + > ~/.ssh/ci-storage + chmod 600 ~/.ssh/ci-storage + ssh-keygen -f ~/.ssh/ci-storage -y > ~/.ssh/ci-storage.pub + + aws secretsmanager get-secret-value \\ + --secret-id "ci-storage/gh-token" \\ + --query SecretString --output text \\ + | gh auth login --with-token + gh auth setup-git + + config=~/.docker/config.json + if [[ ! -f $config ]] || find "$config" -type f -mmin +60 | grep -q .; then + gh auth token | docker login ghcr.io -u "$(gh api user -q .login)" --password-stdin + fi + + mkdir -p ~/git && cd ~/git + if [[ ! -d .git ]]; then + git clone -n --depth=1 --filter=tree:0 "https://github.com/dimikot/ci-storage" . + if [[ "docker" != "." ]]; then + git sparse-checkout set --no-cone "docker" + fi + git checkout + else + git pull --rebase + fi + + { set +x; } &> /dev/null + export GH_TOKEN=$(gh auth token) + { set -x; } &> /dev/null + + export BTIME=$(cat /proc/stat | grep btime | awk '{print $2}') + export DOMAIN="", + { + "Fn::GetAtt": [ + "CnstrctMyCiHost001ApiCCC990DF", + "ApiEndpoint", + ], + }, + "" + export ASGS="time-loop/slapdash:ci-small:stk-cnstrct-runnercismall time-loop/slapdash:ci-large:stk-cnstrct-runnercilarge" + + file=~/.docker-started-after-first-git-clone + if [[ ! -f $file ]]; then + sudo systemctl start docker docker.socket + touch $file + fi + + cd "docker" + ~/docker-pull-if-changed-rate-limit-friendly.sh "ghcr.io" "dimikot/ci-storage" "main" + ~/docker-pull-if-changed-rate-limit-friendly.sh "ghcr.io" "dimikot/ci-storage" "latest" + ~/docker-pull-if-changed-rate-limit-friendly.sh "ghcr.io" "dimikot/ci-scaler" "main" + ~/docker-pull-if-changed-rate-limit-friendly.sh "ghcr.io" "dimikot/ci-scaler" "latest" + docker compose --profile=ci up --build --remove-orphans -d + + if [[ "$1" != "--no-print-compose-logs" ]]; then + sleep 5 + docker compose logs -n 10 + fi + + docker system prune --volumes -f + - path: /home/ubuntu/.bash_profile + owner: ubuntu:ubuntu + permissions: "0644" + defer: true + content: | + #!/bin/bash + C_CMD="\\033[0;36m" + C_NO="\\033[0m" + if [[ -d ~/git/"docker" ]]; then + cd ~/git/"docker" + echo "Hint: want to know, how did this instance boot and how did" + echo "the containers initialize? Run on the instance:" + echo + echo '$ less /var/log/cloud-init-output.log' + echo '$ less /var/log/cloud-init.log' + echo '$ less /var/log/syslog' + echo + echo -e "$C_CMD\\$ docker compose ps$C_NO" + COLUMNS=500 docker --log-level=ERROR compose ps --format="table {{.Service}}\\t{{.Status}}\\t{{.Ports}}" + echo + services=$(docker compose ps --format '{{.Service}}' 2>/dev/null) + if [[ "$services" != "" && $(echo "$services" | wc -l) -eq 1 ]]; then + echo "Hint: only one service is running on this instance, $services." + echo + echo "For your convenience, we are automatically logging you in" + echo "its container. Feel free to use any regular Linux commands" + echo "(ls, pwd, mc etc.) or any dev tools." + echo + echo "You can alway exit back to the instance by pressing ^D." + echo + cmd="docker compose exec $services bash -l" + echo -e "$C_CMD\\$ $cmd$C_NO" + eval "$cmd" + fi + fi +", + ], + ], + }, + }, + }, + "LaunchTemplateName": "stk-cnstrct-mycihost001", + "TagSpecifications": [ + { + "ResourceType": "launch-template", + "Tags": [ + { + "Key": "Name", + "Value": "Stk/Cnstrct/MyCiHost001Lt", + }, + ], + }, + ], + }, + "Type": "AWS::EC2::LaunchTemplate", + }, + "CnstrctMyCiHost001LtProfileF9C5FA56": { + "Properties": { + "Roles": [ + { + "Ref": "CnstrctHostRole5DD9F366", + }, + ], + }, + "Type": "AWS::IAM::InstanceProfile", + }, + "CnstrctMyCiHost001Nlb3D4B3C4E": { + "Properties": { + "LoadBalancerAttributes": [ + { + "Key": "deletion_protection.enabled", + "Value": "false", + }, + ], + "Name": "stk-cnstrct-mycihost001", + "Scheme": "internal", + "SecurityGroups": [ + { + "Fn::GetAtt": [ + "CnstrctSgF5C70BA4", + "GroupId", + ], + }, + ], + "Subnets": [ + { + "Ref": "VpcPrivateSubnet1Subnet536B997A", + }, + { + "Ref": "VpcPrivateSubnet2Subnet3788AAA1", + }, + ], + "Type": "network", + }, + "Type": "AWS::ElasticLoadBalancingV2::LoadBalancer", + }, + "CnstrctMyCiHost001NlbListener85589A96": { + "Properties": { + "DefaultActions": [ + { + "TargetGroupArn": { + "Ref": "CnstrctMyCiHost001TgA01EA825", + }, + "Type": "forward", + }, + ], + "LoadBalancerArn": { + "Ref": "CnstrctMyCiHost001Nlb3D4B3C4E", + }, + "Port": 28088, + "Protocol": "TCP", + }, + "Type": "AWS::ElasticLoadBalancingV2::Listener", + }, + "CnstrctMyCiHost001TgA01EA825": { + "Properties": { + "HealthCheckIntervalSeconds": 5, + "HealthCheckTimeoutSeconds": 2, + "HealthyThresholdCount": 2, + "Name": "stk-cnstrct-mycihost001", + "Port": 28088, + "Protocol": "TCP", + "TargetType": "instance", + "Targets": [ + { + "Id": { + "Ref": "CnstrctMyCiHost001Instance0F6CFAF181cec7ef931262b3", + }, + }, + ], + "UnhealthyThresholdCount": 10, + "VpcId": { + "Ref": "Vpc8378EB38", + }, + }, + "Type": "AWS::ElasticLoadBalancingV2::TargetGroup", + }, + "CnstrctRunnerCiLargeAsgASGA90353E3": { + "Properties": { + "AutoScalingGroupName": "stk-cnstrct-runnercilarge", + "CapacityRebalance": true, + "Cooldown": "30", + "DefaultInstanceWarmup": 60, + "MaxInstanceLifetime": 86400, + "MaxSize": "20", + "MetricsCollection": [ + { + "Granularity": "1Minute", + }, + ], + "MinSize": "5", + "MixedInstancesPolicy": { + "InstancesDistribution": { + "OnDemandAllocationStrategy": "lowest-price", + "OnDemandPercentageAboveBaseCapacity": 10, + "SpotAllocationStrategy": "price-capacity-optimized", + }, + "LaunchTemplate": { + "LaunchTemplateSpecification": { + "LaunchTemplateId": { + "Ref": "CnstrctRunnerCiLargeLt8A1A8B86", + }, + "Version": { + "Fn::GetAtt": [ + "CnstrctRunnerCiLargeLt8A1A8B86", + "LatestVersionNumber", + ], + }, + }, + "Overrides": [ + { + "InstanceRequirements": { + "MemoryMiB": { + "Max": 16384, + "Min": 8192, + }, + "VCpuCount": { + "Max": 8, + "Min": 4, + }, + }, + }, + ], + }, + }, + "Tags": [ + { + "Key": "Name", + "PropagateAtLaunch": true, + "Value": "my-ci-runner-ci-large", + }, + ], + "VPCZoneIdentifier": [ + { + "Ref": "VpcPrivateSubnet1Subnet536B997A", + }, + ], + }, + "Type": "AWS::AutoScaling::AutoScalingGroup", + "UpdatePolicy": { + "AutoScalingRollingUpdate": { + "SuspendProcesses": [ + "HealthCheck", + "ReplaceUnhealthy", + "AZRebalance", + "AlarmNotification", + "ScheduledActions", + "InstanceRefresh", + ], + }, + "AutoScalingScheduledAction": { + "IgnoreUnmodifiedGroupSizeProperties": true, + }, + }, + }, + "CnstrctRunnerCiLargeAsgScheduledActionCaWorkDayEnds7274DFDF": { + "Properties": { + "AutoScalingGroupName": { + "Ref": "CnstrctRunnerCiLargeAsgASGA90353E3", + }, + "MinSize": 5, + "Recurrence": "0 18 * * *", + "TimeZone": "America/Los_Angeles", + }, + "Type": "AWS::AutoScaling::ScheduledAction", + }, + "CnstrctRunnerCiLargeAsgScheduledActionCaWorkDayStartsAE0D0F6C": { + "Properties": { + "AutoScalingGroupName": { + "Ref": "CnstrctRunnerCiLargeAsgASGA90353E3", + }, + "MinSize": 10, + "Recurrence": "0 8 * * *", + "TimeZone": "America/Los_Angeles", + }, + "Type": "AWS::AutoScaling::ScheduledAction", + }, + "CnstrctRunnerCiLargeLt8A1A8B86": { + "DependsOn": [ + "CnstrctRunnerRoleDefaultPolicyCA04641E", + "CnstrctRunnerRole341E54FA", + ], + "Properties": { + "LaunchTemplateData": { + "BlockDeviceMappings": [ + { + "DeviceName": "/dev/sda1", + "Ebs": { + "DeleteOnTermination": true, + "Encrypted": true, + "VolumeSize": 40, + "VolumeType": "gp3", + }, + }, + { + "DeviceName": "/dev/sdb", + "Ebs": { + "DeleteOnTermination": true, + "Encrypted": true, + "VolumeSize": 13, + "VolumeType": "gp3", + }, + }, + ], + "IamInstanceProfile": { + "Arn": { + "Fn::GetAtt": [ + "CnstrctRunnerCiLargeLtProfile04AD45AD", + "Arn", + ], + }, + }, + "ImageId": { + "Fn::Select": [ + 1, + { + "Fn::Split": [ + """, + { + "Fn::GetAtt": [ + "CnstrctRunnerCiLargeLtImageCustomResourceC3e9d7e17B38AEE3", + "Payload", + ], + }, + ], + }, + ], + }, + "KeyName": { + "Fn::GetAtt": [ + "CnstrctSshIdRsaEC2KeyPairstkcnstrctsshidrsaF4F38F69", + "KeyPairName", + ], + }, + "MetadataOptions": { + "HttpPutResponseHopLimit": 2, + "HttpTokens": "required", + }, + "Monitoring": { + "Enabled": true, + }, + "SecurityGroupIds": [ + { + "Fn::GetAtt": [ + "CnstrctSgF5C70BA4", + "GroupId", + ], + }, + ], + "TagSpecifications": [ + { + "ResourceType": "instance", + "Tags": [ + { + "Key": "Name", + "Value": "Stk/Cnstrct/RunnerCiLargeLt", + }, + ], + }, + { + "ResourceType": "volume", + "Tags": [ + { + "Key": "Name", + "Value": "Stk/Cnstrct/RunnerCiLargeLt", + }, + ], + }, + { + "ResourceType": "instance", + "Tags": [ + { + "Key": "InstanceToAmi:UserDataHash", + "Value": "46cf1a6641f4d946", + }, + ], + }, + { + "ResourceType": "volume", + "Tags": [ + { + "Key": "InstanceToAmi:UserDataHash", + "Value": "46cf1a6641f4d946", + }, + ], + }, + ], + "UserData": { + "Fn::Base64": { + "Fn::Join": [ + "", + [ + "#cloud-config +timezone: America/Los_Angeles +bootcmd: + - | + #!/bin/sh + cat <<'EOT' > /etc/profile.ssm-user + #!/bin/sh + [ "$0$@" = "sh" ] && ENV= sudo -u "ubuntu" -i + EOT + dir=/etc/systemd/system/snap.amazon-ssm-agent.amazon-ssm-agent.service.d + mkdir -p $dir + cat <<'EOT' > $dir/override.conf + [Service] + Environment="ENV=/etc/profile.ssm-user" + EOT + - | + #!/bin/sh + set -e -x + mkdir -p /var/log + empty_device="" + if ! blkid -L ephemeral; then + for path in $(find /dev/disk/by-id -name "*Amazon_Elastic_Block_Store*" -a ! -name "*-part*"); do + find /dev/disk/by-id -wholename "$path-part*" | grep . || empty_device="$path" + done + if [ "$empty_device" = "" ]; then + echo "Cannot find a disk without partitions. Devices:" + find /dev/disk/by-id + exit 1 + fi + mkfs.ext4 -L ephemeral -F "$empty_device" + fi + mount -o 'defaults,noatime,exec' -L ephemeral /var/log + chown 0:syslog /var/log + chmod 775 /var/log + mount --bind -o ro / /mnt + (cd /mnt; if [ "$empty_device" != "" ]; then cp -af "./var/log/." "/var/log"; fi) + umount /mnt + - mkdir -p /var/log/atop + - | + #!/bin/sh + set -e -x + fallocate -l 8G /var/log/swapfile + chmod 0600 /var/log/swapfile + mkswap /var/log/swapfile + swapon /var/log/swapfile +apt: + sources: + github-cli.list: + source: deb https://cli.github.com/packages stable main + keyid: 23F3D4EA75716059 + docker.list: + source: deb https://download.docker.com/linux/ubuntu $RELEASE stable + keyid: 9DC858229FC7DD38854AE2D88D81803C0EBFCD88 +packages: + - awscli + - gh + - docker-ce + - docker-ce-cli + - containerd.io + - docker-compose-plugin + - qemu + - qemu-user-static + - binfmt-support + - git + - gosu + - mc + - curl + - apt-transport-https + - ca-certificates + - tzdata + - atop + - iotop + - htop + - bwm-ng + - jq + - expect +write_files: + - path: /etc/sysctl.d/enable-ipv4-forwarding.conf + content: | + net.ipv4.conf.all.forwarding=1 + - path: /etc/default/atop + content: | + LOGOPTS="-R" + LOGINTERVAL=15 + LOGGENERATIONS=4 + - path: /etc/docker/daemon.json + permissions: "0644" + content: | + { + "log-driver": "syslog", + "log-opts": { + "tag": "docker/{{.Name}}" + }, + "runtimes": { + "sysbox-runc": { + "path": "/usr/bin/sysbox-runc" + } + }, + "default-runtime": "sysbox-runc", + "userns-remap": "sysbox" + } + - path: /etc/systemd/system/docker.service.d/override.conf + permissions: "0644" + content: | + [Service] + TimeoutStopSec=3600 + ExecStartPost=/home/ubuntu/run-docker-compose.sh + - path: /etc/cron.d/git-pull-and-rerun-docker-compose-periodically + permissions: "0644" + content: | + */1 * * * * ubuntu /home/ubuntu/run-docker-compose.sh --no-print-compose-logs 2>&1 | logger -t docker/run-docker-compose + - path: /etc/rsyslog.d/01-docker-tag-to-serial-console.conf + permissions: "0644" + content: | + if $syslogtag startswith 'docker/' then -/dev/console + # It will also write to /var/log/syslog as usual. + - path: /var/lib/cloud/scripts/per-once/00-append-etc-environment.sh + permissions: "0755" + content: | + #!/bin/bash + set -e -o pipefail && echo "================= $(date) | Up $(awk '{print $1}' /proc/uptime) seconds =================" && echo "Running $BASH_SOURCE as $(whoami)" && set -x && export AWS_DEFAULT_REGION=$(ec2metadata --availability-zone | sed "s/[a-z]$//") + echo "TZ=America/Los_Angeles" >> /etc/environment + echo "LESS=RS" >> /etc/environment + - path: /var/lib/cloud/scripts/per-once/make-apt-get-install-not-run-after-restoring-from-snapshot-to-speedup-boot.sh + permissions: "0755" + content: | + #!/bin/bash + set -e -o pipefail && echo "================= $(date) | Up $(awk '{print $1}' /proc/uptime) seconds =================" && echo "Running $BASH_SOURCE as $(whoami)" && set -x && export AWS_DEFAULT_REGION=$(ec2metadata --availability-zone | sed "s/[a-z]$//") + sed -i -E 's/(- (package-update-upgrade-install|apt-configure|apt-pipelining))/# \\1/g' /etc/cloud/cloud.cfg + - path: /var/lib/cloud/scripts/per-once/apply-services-configs.sh + permissions: "0755" + content: | + #!/bin/bash + set -e -o pipefail && echo "================= $(date) | Up $(awk '{print $1}' /proc/uptime) seconds =================" && echo "Running $BASH_SOURCE as $(whoami)" && set -x && export AWS_DEFAULT_REGION=$(ec2metadata --availability-zone | sed "s/[a-z]$//") + systemctl daemon-reload + service atop restart || true + sysctl --system + - path: /var/lib/cloud/scripts/per-once/add-ubuntu-user-to-docker-group-to-access-socket.sh + permissions: "0755" + content: | + #!/bin/bash + set -e -o pipefail && echo "================= $(date) | Up $(awk '{print $1}' /proc/uptime) seconds =================" && echo "Running $BASH_SOURCE as $(whoami)" && set -x && export AWS_DEFAULT_REGION=$(ec2metadata --availability-zone | sed "s/[a-z]$//") + usermod -aG docker ubuntu + - path: /var/lib/cloud/scripts/per-once/install-sysbox-for-docker-in-docker.sh + permissions: "0755" + content: | + #!/bin/bash + set -e -o pipefail && echo "================= $(date) | Up $(awk '{print $1}' /proc/uptime) seconds =================" && echo "Running $BASH_SOURCE as $(whoami)" && set -x && export AWS_DEFAULT_REGION=$(ec2metadata --availability-zone | sed "s/[a-z]$//") + systemctl stop docker docker.socket || true + for svc in sysbox-fs sysbox-mgr; do + dir="/etc/systemd/system/$svc.service.d" + mkdir -p "$dir" + { + echo "[Service]" + echo "Restart=always" + echo "RestartSec=5" + if [[ "$svc" == "sysbox-fs" ]]; then + echo "ExecStartPost=/bin/systemctl start sysbox-mgr.service" + fi + } > "$dir/override.conf" + done + wget -nv -O /tmp/sysbox-ce.deb "https://downloads.nestybox.com/sysbox/releases/v0.6.4/sysbox-ce_0.6.4-0.linux_$(dpkg --print-architecture).deb" + dpkg -i /tmp/sysbox-ce.deb + rm -f /tmp/sysbox-ce.deb + - path: /var/lib/cloud/scripts/per-once/allow-rsyslog-write-to-serial-console.sh + permissions: "0755" + content: | + #!/bin/bash + set -e -o pipefail && echo "================= $(date) | Up $(awk '{print $1}' /proc/uptime) seconds =================" && echo "Running $BASH_SOURCE as $(whoami)" && set -x && export AWS_DEFAULT_REGION=$(ec2metadata --availability-zone | sed "s/[a-z]$//") + usermod -a -G tty syslog + systemctl restart rsyslog + - path: /home/ubuntu/docker-pull-if-changed-rate-limit-friendly.sh + owner: ubuntu:ubuntu + permissions: "0755" + defer: true + content: | + #!/bin/bash + set -e -o pipefail + repo="$1" + name="$2" + tag="$3" + + if [[ "$repo" == "" || "$name" == "" || "$tag" == "" ]]; then + echo "Usage: $0 repo name tag" + exit 1 + fi + + bearer=$(jq -r ".auths[\\"$repo\\"].auth" ~/.docker/config.json || true) + if [[ "$bearer" == "" ]]; then + echo "There is no auth token for $repo in ~/.docker/config.json; did you run \\"docker login\\"?" + exit 2 + fi + + file=~/.docker/$(echo "$repo-$name-$tag" | sed -E 's/[^-_a-zA-Z0-9]/_/g').digest + old_digest=$(cat "$file" 2>/dev/null || true) + cur_digest=$( + curl -sS --fail --head \\ + -H "Authorization: Bearer $bearer" \\ + -H "Accept: application/vnd.oci.image.index.v1+json" \\ + https://$repo/v2/$name/manifests/$tag \\ + | grep -i 'Docker-Content-Digest' | awk '{print $2}' | sed -E 's/\\s+//' + ) + + if [[ "$old_digest" != "$cur_digest" ]]; then + if docker pull $repo/$name:$tag; then + echo "$cur_digest" > $file + else + code="$?" + echo "Failed to run \\"docker pull $repo/$name:$tag\\"" + exit "$code" + fi + else + echo "Not pulling $repo/$name:$tag - no changes in manifest since previous pull (digest: $cur_digest)" + fi + - path: /home/ubuntu/run-docker-compose.sh + owner: ubuntu:ubuntu + permissions: "0755" + defer: true + content: | + #!/bin/bash + + if [[ $(whoami) != "ubuntu" ]]; then + exec gosu ubuntu "$BASH_SOURCE" "$@" + fi + + exec {FD}<$BASH_SOURCE + flock -n "$FD" || { echo "Already running."; exit 0; } + set -e -o pipefail && echo "================= $(date) | Up $(awk '{print $1}' /proc/uptime) seconds =================" && echo "Running $BASH_SOURCE as $(whoami)" && set -x && export AWS_DEFAULT_REGION=$(ec2metadata --availability-zone | sed "s/[a-z]$//") + + source /etc/environment + export TZ + + mkdir -p ~/.ssh && chmod 700 ~/.ssh + aws secretsmanager get-secret-value \\ + --secret-id "ec2-ssh-key/", + { + "Fn::GetAtt": [ + "CnstrctSshIdRsaEC2KeyPairstkcnstrctsshidrsaF4F38F69", + "KeyPairName", + ], + }, + "/private" \\ + --query SecretString --output text \\ + > ~/.ssh/ci-storage + chmod 600 ~/.ssh/ci-storage + ssh-keygen -f ~/.ssh/ci-storage -y > ~/.ssh/ci-storage.pub + + aws secretsmanager get-secret-value \\ + --secret-id "ci-storage/gh-token" \\ + --query SecretString --output text \\ + | gh auth login --with-token + gh auth setup-git + + config=~/.docker/config.json + if [[ ! -f $config ]] || find "$config" -type f -mmin +60 | grep -q .; then + gh auth token | docker login ghcr.io -u "$(gh api user -q .login)" --password-stdin + fi + + mkdir -p ~/git && cd ~/git + if [[ ! -d .git ]]; then + git clone -n --depth=1 --filter=tree:0 "https://github.com/dimikot/ci-storage" . + if [[ "docker" != "." ]]; then + git sparse-checkout set --no-cone "docker" + fi + git checkout + else + git pull --rebase + fi + + { set +x; } &> /dev/null + export GH_TOKEN=$(gh auth token) + { set -x; } &> /dev/null + + export BTIME=$(cat /proc/stat | grep btime | awk '{print $2}') + export GH_REPOSITORY="time-loop/slapdash" + export GH_LABELS="my-ci,ci-large" + export FORWARD_HOST="my-ci-host-001.test-zoneName" + + file=~/.docker-started-after-first-git-clone + if [[ ! -f $file ]]; then + sudo systemctl start docker docker.socket + touch $file + fi + + cd "docker" + ~/docker-pull-if-changed-rate-limit-friendly.sh "ghcr.io" "dimikot/ci-runner" "main" + ~/docker-pull-if-changed-rate-limit-friendly.sh "ghcr.io" "dimikot/ci-runner" "latest" + docker compose up --build --remove-orphans -d + + if [[ "$1" != "--no-print-compose-logs" ]]; then + sleep 5 + docker compose logs -n 10 + fi + + docker system prune --volumes -f + export deps=$(docker image ls --format "{{.Repository}}:{{.ID}}:{{.Tag}}" | grep dimikot/ci-runner) + export instanceId=$(cloud-init query ds.meta_data.instance_id) + aws lambda invoke --function-name "stk-cnstrct-instancetoami" \\ + --payload "$(jq -nc '{"instanceId":$ENV.instanceId,"deps":$ENV.deps}')" \\ + /dev/stdout | jq -s '.[0]' + - path: /home/ubuntu/.bash_profile + owner: ubuntu:ubuntu + permissions: "0644" + defer: true + content: | + #!/bin/bash + C_CMD="\\033[0;36m" + C_NO="\\033[0m" + if [[ -d ~/git/"docker" ]]; then + cd ~/git/"docker" + echo "Hint: want to know, how did this instance boot and how did" + echo "the containers initialize? Run on the instance:" + echo + echo '$ less /var/log/cloud-init-output.log' + echo '$ less /var/log/cloud-init.log' + echo '$ less /var/log/syslog' + echo + echo -e "$C_CMD\\$ docker compose ps$C_NO" + COLUMNS=500 docker --log-level=ERROR compose ps --format="table {{.Service}}\\t{{.Status}}\\t{{.Ports}}" + echo + services=$(docker compose ps --format '{{.Service}}' 2>/dev/null) + if [[ "$services" != "" && $(echo "$services" | wc -l) -eq 1 ]]; then + echo "Hint: only one service is running on this instance, $services." + echo + echo "For your convenience, we are automatically logging you in" + echo "its container. Feel free to use any regular Linux commands" + echo "(ls, pwd, mc etc.) or any dev tools." + echo + echo "You can alway exit back to the instance by pressing ^D." + echo + cmd="docker compose exec $services bash -l" + echo -e "$C_CMD\\$ $cmd$C_NO" + eval "$cmd" + fi + fi +", + ], + ], + }, + }, + }, + "LaunchTemplateName": "stk-cnstrct-runnercilarge", + "TagSpecifications": [ + { + "ResourceType": "launch-template", + "Tags": [ + { + "Key": "Name", + "Value": "Stk/Cnstrct/RunnerCiLargeLt", + }, + ], + }, + ], + }, + "Type": "AWS::EC2::LaunchTemplate", + }, + "CnstrctRunnerCiLargeLtImageCustomResourceC3e9d7e17B38AEE3": { + "DeletionPolicy": "Delete", + "DependsOn": [ + "CnstrctRunnerCiLargeLtImageCustomResourceC3e9d7e1CustomResourcePolicyF65A30BE", + "CnstrctRunnerRoleDefaultPolicyCA04641E", + "CnstrctRunnerRole341E54FA", + ], + "Properties": { + "Create": { + "Fn::Join": [ + "", + [ + "{"service":"Lambda","action":"invoke","parameters":{"FunctionName":"", + { + "Ref": "CnstrctRunnerCiLargeLtImageFunc67F3A45E", + }, + "","InvocationType":"RequestResponse","Payload":"{\\"launchTemplateName\\":\\"stk-cnstrct-runnercilarge\\",\\"defaultImageId\\":\\"", + { + "Ref": "SsmParameterValuetestimagessmnameC96584B6F00A464EAD1953AFF4B05118Parameter", + }, + "\\",\\"userDataHash\\":\\"46cf1a6641f4d946\\"}"},"physicalResourceId":{"id":"StkCnstrctRunnerCiLargeLtResourceImageCustomResourceC3e9d7e1"}}", + ], + ], + }, + "InstallLatestAwsSdk": true, + "ServiceToken": { + "Fn::GetAtt": [ + "AWS679f53fac002430cb0da5b7982bd22872D164C4C", + "Arn", + ], + }, + "Update": { + "Fn::Join": [ + "", + [ + "{"service":"Lambda","action":"invoke","parameters":{"FunctionName":"", + { + "Ref": "CnstrctRunnerCiLargeLtImageFunc67F3A45E", + }, + "","InvocationType":"RequestResponse","Payload":"{\\"launchTemplateName\\":\\"stk-cnstrct-runnercilarge\\",\\"defaultImageId\\":\\"", + { + "Ref": "SsmParameterValuetestimagessmnameC96584B6F00A464EAD1953AFF4B05118Parameter", + }, + "\\",\\"userDataHash\\":\\"46cf1a6641f4d946\\"}"},"physicalResourceId":{"id":"StkCnstrctRunnerCiLargeLtResourceImageCustomResourceC3e9d7e1"}}", + ], + ], + }, + }, + "Type": "Custom::AWS", + "UpdateReplacePolicy": "Delete", + }, + "CnstrctRunnerCiLargeLtImageCustomResourceC3e9d7e1CustomResourcePolicyF65A30BE": { + "DependsOn": [ + "CnstrctRunnerRoleDefaultPolicyCA04641E", + "CnstrctRunnerRole341E54FA", + ], + "Properties": { + "PolicyDocument": { + "Statement": [ + { + "Action": "lambda:InvokeFunction", + "Effect": "Allow", + "Resource": { + "Fn::GetAtt": [ + "CnstrctRunnerCiLargeLtImageFunc67F3A45E", + "Arn", + ], + }, + }, + ], + "Version": "2012-10-17", + }, + "PolicyName": "CnstrctRunnerCiLargeLtImageCustomResourceC3e9d7e1CustomResourcePolicyF65A30BE", + "Roles": [ + { + "Ref": "AWS679f53fac002430cb0da5b7982bd2287ServiceRoleC1EA0FF2", + }, + ], + }, + "Type": "AWS::IAM::Policy", + }, + "CnstrctRunnerCiLargeLtImageFunc67F3A45E": { + "DependsOn": [ + "CnstrctRunnerCiLargeLtImageFuncServiceRoleDefaultPolicyD4F8D404", + "CnstrctRunnerCiLargeLtImageFuncServiceRoleA8442DD5", + "CnstrctRunnerRoleDefaultPolicyCA04641E", + "CnstrctRunnerRole341E54FA", + ], + "Properties": { + "Code": { + "ZipFile": "from boto3 import client +from os import environ +def handler(event, context): + launchTemplateName=event["launchTemplateName"] + defaultImageId=event["defaultImageId"] + userDataHash=event["userDataHash"] + res_data = None + try: + res = client("ec2").describe_launch_template_versions( + LaunchTemplateName=launchTemplateName, + Versions=["$Latest"] + ) + res_data = res["LaunchTemplateVersions"][0]["LaunchTemplateData"] + image_id = res_data["ImageId"] + user_data_hash = next( + iter( + tag["Value"] + for spec in res_data.get("TagSpecifications", []) + if spec["ResourceType"] == "instance" + for tag in spec["Tags"] + if tag["Key"] == "InstanceToAmi:UserDataHash" + ), + None + ) + print(f"describe_launch_template_versions() returned ImageId={image_id} InstanceToAmi:UserDataHash={user_data_hash}") + if user_data_hash and userDataHash and user_data_hash != userDataHash: + print(f"returning defaultImageId={defaultImageId} to build the instance from scratch, since UserData has changed") + return defaultImageId + else: + print(f"returning previous Launch Template Version's ImageId={image_id}") + return image_id + except Exception as e: + e_str = f"{e.__class__.__name__}: {str(e)}" + if "NotFoundException" in e_str: + print(f"returning defaultImageId={defaultImageId} due to: {e_str}") + return defaultImageId + else: + if res_data: + print("describe_launch_template_versions() returned LaunchTemplateData: {res_data}") + print(f"got fatal exception: {e_str}") + # Hack: return the error instead of ImageId (not throw), since it + # will be shown right in the CDK deployment error output (because + # it is not of a valid ImageId format, so CDK will print that and + # fail anyways, but with a descriptive message). + return e_str +", + }, + "Handler": "index.handler", + "Role": { + "Fn::GetAtt": [ + "CnstrctRunnerCiLargeLtImageFuncServiceRoleA8442DD5", + "Arn", + ], + }, + "Runtime": "python3.9", + "Tags": [ + { + "Key": "Name", + "Value": "Stk/Cnstrct/RunnerCiLargeLt", + }, + ], + "Timeout": 20, + }, + "Type": "AWS::Lambda::Function", + }, + "CnstrctRunnerCiLargeLtImageFuncServiceRoleA8442DD5": { + "DependsOn": [ + "CnstrctRunnerRoleDefaultPolicyCA04641E", + "CnstrctRunnerRole341E54FA", + ], + "Properties": { + "AssumeRolePolicyDocument": { + "Statement": [ + { + "Action": "sts:AssumeRole", + "Effect": "Allow", + "Principal": { + "Service": "lambda.amazonaws.com", + }, + }, + ], + "Version": "2012-10-17", + }, + "ManagedPolicyArns": [ + { + "Fn::Join": [ + "", + [ + "arn:", + { + "Ref": "AWS::Partition", + }, + ":iam::aws:policy/service-role/AWSLambdaBasicExecutionRole", + ], + ], + }, + ], + "Tags": [ + { + "Key": "Name", + "Value": "Stk/Cnstrct/RunnerCiLargeLt", + }, + ], + }, + "Type": "AWS::IAM::Role", + }, + "CnstrctRunnerCiLargeLtImageFuncServiceRoleDefaultPolicyD4F8D404": { + "DependsOn": [ + "CnstrctRunnerRoleDefaultPolicyCA04641E", + "CnstrctRunnerRole341E54FA", + ], + "Properties": { + "PolicyDocument": { + "Statement": [ + { + "Action": [ + "ec2:DescribeLaunchTemplateVersions", + "logs:CreateLogGroup", + "logs:CreateLogStream", + "logs:PutLogEvents", + ], + "Effect": "Allow", + "Resource": "*", + }, + ], + "Version": "2012-10-17", + }, + "PolicyName": "CnstrctRunnerCiLargeLtImageFuncServiceRoleDefaultPolicyD4F8D404", + "Roles": [ + { + "Ref": "CnstrctRunnerCiLargeLtImageFuncServiceRoleA8442DD5", + }, + ], + }, + "Type": "AWS::IAM::Policy", + }, + "CnstrctRunnerCiLargeLtProfile04AD45AD": { + "Properties": { + "Roles": [ + { + "Ref": "CnstrctRunnerRole341E54FA", + }, + ], + }, + "Type": "AWS::IAM::InstanceProfile", + }, + "CnstrctRunnerCiSmallAsgASG02E51BB1": { + "Properties": { + "AutoScalingGroupName": "stk-cnstrct-runnercismall", + "CapacityRebalance": true, + "Cooldown": "30", + "DefaultInstanceWarmup": 60, + "MaxInstanceLifetime": 86400, + "MaxSize": "10", + "MetricsCollection": [ + { + "Granularity": "1Minute", + }, + ], + "MinSize": "5", + "MixedInstancesPolicy": { + "InstancesDistribution": { + "OnDemandAllocationStrategy": "lowest-price", + "OnDemandPercentageAboveBaseCapacity": 10, + "SpotAllocationStrategy": "price-capacity-optimized", + }, + "LaunchTemplate": { + "LaunchTemplateSpecification": { + "LaunchTemplateId": { + "Ref": "CnstrctRunnerCiSmallLtC3B1880A", + }, + "Version": { + "Fn::GetAtt": [ + "CnstrctRunnerCiSmallLtC3B1880A", + "LatestVersionNumber", + ], + }, + }, + "Overrides": [ + { + "InstanceRequirements": { + "MemoryMiB": { + "Max": 8192, + "Min": 4096, + }, + "VCpuCount": { + "Max": 4, + "Min": 2, + }, + }, + }, + ], + }, + }, + "Tags": [ + { + "Key": "Name", + "PropagateAtLaunch": true, + "Value": "my-ci-runner-ci-small", + }, + ], + "VPCZoneIdentifier": [ + { + "Ref": "VpcPrivateSubnet1Subnet536B997A", + }, + ], + }, + "Type": "AWS::AutoScaling::AutoScalingGroup", + "UpdatePolicy": { + "AutoScalingRollingUpdate": { + "SuspendProcesses": [ + "HealthCheck", + "ReplaceUnhealthy", + "AZRebalance", + "AlarmNotification", + "ScheduledActions", + "InstanceRefresh", + ], + }, + "AutoScalingScheduledAction": { + "IgnoreUnmodifiedGroupSizeProperties": true, + }, + }, + }, + "CnstrctRunnerCiSmallAsgScheduledActionCaWorkDayEnds460D21FE": { + "Properties": { + "AutoScalingGroupName": { + "Ref": "CnstrctRunnerCiSmallAsgASG02E51BB1", + }, + "MinSize": 5, + "Recurrence": "0 18 * * *", + "TimeZone": "America/Los_Angeles", + }, + "Type": "AWS::AutoScaling::ScheduledAction", + }, + "CnstrctRunnerCiSmallAsgScheduledActionCaWorkDayStarts9BD73C76": { + "Properties": { + "AutoScalingGroupName": { + "Ref": "CnstrctRunnerCiSmallAsgASG02E51BB1", + }, + "MinSize": 10, + "Recurrence": "0 8 * * *", + "TimeZone": "America/Los_Angeles", + }, + "Type": "AWS::AutoScaling::ScheduledAction", + }, + "CnstrctRunnerCiSmallLtC3B1880A": { + "DependsOn": [ + "CnstrctRunnerRoleDefaultPolicyCA04641E", + "CnstrctRunnerRole341E54FA", + ], + "Properties": { + "LaunchTemplateData": { + "BlockDeviceMappings": [ + { + "DeviceName": "/dev/sda1", + "Ebs": { + "DeleteOnTermination": true, + "Encrypted": true, + "VolumeSize": 20, + "VolumeType": "gp3", + }, + }, + { + "DeviceName": "/dev/sdb", + "Ebs": { + "DeleteOnTermination": true, + "Encrypted": true, + "VolumeSize": 9, + "VolumeType": "gp3", + }, + }, + ], + "IamInstanceProfile": { + "Arn": { + "Fn::GetAtt": [ + "CnstrctRunnerCiSmallLtProfile0EB069BA", + "Arn", + ], + }, + }, + "ImageId": { + "Fn::Select": [ + 1, + { + "Fn::Split": [ + """, + { + "Fn::GetAtt": [ + "CnstrctRunnerCiSmallLtImageCustomResource2db51ef0C46B9232", + "Payload", + ], + }, + ], + }, + ], + }, + "KeyName": { + "Fn::GetAtt": [ + "CnstrctSshIdRsaEC2KeyPairstkcnstrctsshidrsaF4F38F69", + "KeyPairName", + ], + }, + "MetadataOptions": { + "HttpPutResponseHopLimit": 2, + "HttpTokens": "required", + }, + "Monitoring": { + "Enabled": true, + }, + "SecurityGroupIds": [ + { + "Fn::GetAtt": [ + "CnstrctSgF5C70BA4", + "GroupId", + ], + }, + ], + "TagSpecifications": [ + { + "ResourceType": "instance", + "Tags": [ + { + "Key": "Name", + "Value": "Stk/Cnstrct/RunnerCiSmallLt", + }, + ], + }, + { + "ResourceType": "volume", + "Tags": [ + { + "Key": "Name", + "Value": "Stk/Cnstrct/RunnerCiSmallLt", + }, + ], + }, + { + "ResourceType": "instance", + "Tags": [ + { + "Key": "InstanceToAmi:UserDataHash", + "Value": "ac1bd295908bee9f", + }, + ], + }, + { + "ResourceType": "volume", + "Tags": [ + { + "Key": "InstanceToAmi:UserDataHash", + "Value": "ac1bd295908bee9f", + }, + ], + }, + ], + "UserData": { + "Fn::Base64": { + "Fn::Join": [ + "", + [ + "#cloud-config +timezone: America/Los_Angeles +bootcmd: + - | + #!/bin/sh + cat <<'EOT' > /etc/profile.ssm-user + #!/bin/sh + [ "$0$@" = "sh" ] && ENV= sudo -u "ubuntu" -i + EOT + dir=/etc/systemd/system/snap.amazon-ssm-agent.amazon-ssm-agent.service.d + mkdir -p $dir + cat <<'EOT' > $dir/override.conf + [Service] + Environment="ENV=/etc/profile.ssm-user" + EOT + - | + #!/bin/sh + set -e -x + mkdir -p /var/log + empty_device="" + if ! blkid -L ephemeral; then + for path in $(find /dev/disk/by-id -name "*Amazon_Elastic_Block_Store*" -a ! -name "*-part*"); do + find /dev/disk/by-id -wholename "$path-part*" | grep . || empty_device="$path" + done + if [ "$empty_device" = "" ]; then + echo "Cannot find a disk without partitions. Devices:" + find /dev/disk/by-id + exit 1 + fi + mkfs.ext4 -L ephemeral -F "$empty_device" + fi + mount -o 'defaults,noatime,exec' -L ephemeral /var/log + chown 0:syslog /var/log + chmod 775 /var/log + mount --bind -o ro / /mnt + (cd /mnt; if [ "$empty_device" != "" ]; then cp -af "./var/log/." "/var/log"; fi) + umount /mnt + - mkdir -p /var/log/atop + - | + #!/bin/sh + set -e -x + fallocate -l 4G /var/log/swapfile + chmod 0600 /var/log/swapfile + mkswap /var/log/swapfile + swapon /var/log/swapfile +apt: + sources: + github-cli.list: + source: deb https://cli.github.com/packages stable main + keyid: 23F3D4EA75716059 + docker.list: + source: deb https://download.docker.com/linux/ubuntu $RELEASE stable + keyid: 9DC858229FC7DD38854AE2D88D81803C0EBFCD88 +packages: + - awscli + - gh + - docker-ce + - docker-ce-cli + - containerd.io + - docker-compose-plugin + - qemu + - qemu-user-static + - binfmt-support + - git + - gosu + - mc + - curl + - apt-transport-https + - ca-certificates + - tzdata + - atop + - iotop + - htop + - bwm-ng + - jq + - expect +write_files: + - path: /etc/sysctl.d/enable-ipv4-forwarding.conf + content: | + net.ipv4.conf.all.forwarding=1 + - path: /etc/default/atop + content: | + LOGOPTS="-R" + LOGINTERVAL=15 + LOGGENERATIONS=4 + - path: /etc/docker/daemon.json + permissions: "0644" + content: | + { + "log-driver": "syslog", + "log-opts": { + "tag": "docker/{{.Name}}" + }, + "runtimes": { + "sysbox-runc": { + "path": "/usr/bin/sysbox-runc" + } + }, + "default-runtime": "sysbox-runc", + "userns-remap": "sysbox" + } + - path: /etc/systemd/system/docker.service.d/override.conf + permissions: "0644" + content: | + [Service] + TimeoutStopSec=3600 + ExecStartPost=/home/ubuntu/run-docker-compose.sh + - path: /etc/cron.d/git-pull-and-rerun-docker-compose-periodically + permissions: "0644" + content: | + */1 * * * * ubuntu /home/ubuntu/run-docker-compose.sh --no-print-compose-logs 2>&1 | logger -t docker/run-docker-compose + - path: /etc/rsyslog.d/01-docker-tag-to-serial-console.conf + permissions: "0644" + content: | + if $syslogtag startswith 'docker/' then -/dev/console + # It will also write to /var/log/syslog as usual. + - path: /var/lib/cloud/scripts/per-once/00-append-etc-environment.sh + permissions: "0755" + content: | + #!/bin/bash + set -e -o pipefail && echo "================= $(date) | Up $(awk '{print $1}' /proc/uptime) seconds =================" && echo "Running $BASH_SOURCE as $(whoami)" && set -x && export AWS_DEFAULT_REGION=$(ec2metadata --availability-zone | sed "s/[a-z]$//") + echo "TZ=America/Los_Angeles" >> /etc/environment + echo "LESS=RS" >> /etc/environment + - path: /var/lib/cloud/scripts/per-once/make-apt-get-install-not-run-after-restoring-from-snapshot-to-speedup-boot.sh + permissions: "0755" + content: | + #!/bin/bash + set -e -o pipefail && echo "================= $(date) | Up $(awk '{print $1}' /proc/uptime) seconds =================" && echo "Running $BASH_SOURCE as $(whoami)" && set -x && export AWS_DEFAULT_REGION=$(ec2metadata --availability-zone | sed "s/[a-z]$//") + sed -i -E 's/(- (package-update-upgrade-install|apt-configure|apt-pipelining))/# \\1/g' /etc/cloud/cloud.cfg + - path: /var/lib/cloud/scripts/per-once/apply-services-configs.sh + permissions: "0755" + content: | + #!/bin/bash + set -e -o pipefail && echo "================= $(date) | Up $(awk '{print $1}' /proc/uptime) seconds =================" && echo "Running $BASH_SOURCE as $(whoami)" && set -x && export AWS_DEFAULT_REGION=$(ec2metadata --availability-zone | sed "s/[a-z]$//") + systemctl daemon-reload + service atop restart || true + sysctl --system + - path: /var/lib/cloud/scripts/per-once/add-ubuntu-user-to-docker-group-to-access-socket.sh + permissions: "0755" + content: | + #!/bin/bash + set -e -o pipefail && echo "================= $(date) | Up $(awk '{print $1}' /proc/uptime) seconds =================" && echo "Running $BASH_SOURCE as $(whoami)" && set -x && export AWS_DEFAULT_REGION=$(ec2metadata --availability-zone | sed "s/[a-z]$//") + usermod -aG docker ubuntu + - path: /var/lib/cloud/scripts/per-once/install-sysbox-for-docker-in-docker.sh + permissions: "0755" + content: | + #!/bin/bash + set -e -o pipefail && echo "================= $(date) | Up $(awk '{print $1}' /proc/uptime) seconds =================" && echo "Running $BASH_SOURCE as $(whoami)" && set -x && export AWS_DEFAULT_REGION=$(ec2metadata --availability-zone | sed "s/[a-z]$//") + systemctl stop docker docker.socket || true + for svc in sysbox-fs sysbox-mgr; do + dir="/etc/systemd/system/$svc.service.d" + mkdir -p "$dir" + { + echo "[Service]" + echo "Restart=always" + echo "RestartSec=5" + if [[ "$svc" == "sysbox-fs" ]]; then + echo "ExecStartPost=/bin/systemctl start sysbox-mgr.service" + fi + } > "$dir/override.conf" + done + wget -nv -O /tmp/sysbox-ce.deb "https://downloads.nestybox.com/sysbox/releases/v0.6.4/sysbox-ce_0.6.4-0.linux_$(dpkg --print-architecture).deb" + dpkg -i /tmp/sysbox-ce.deb + rm -f /tmp/sysbox-ce.deb + - path: /var/lib/cloud/scripts/per-once/allow-rsyslog-write-to-serial-console.sh + permissions: "0755" + content: | + #!/bin/bash + set -e -o pipefail && echo "================= $(date) | Up $(awk '{print $1}' /proc/uptime) seconds =================" && echo "Running $BASH_SOURCE as $(whoami)" && set -x && export AWS_DEFAULT_REGION=$(ec2metadata --availability-zone | sed "s/[a-z]$//") + usermod -a -G tty syslog + systemctl restart rsyslog + - path: /home/ubuntu/docker-pull-if-changed-rate-limit-friendly.sh + owner: ubuntu:ubuntu + permissions: "0755" + defer: true + content: | + #!/bin/bash + set -e -o pipefail + repo="$1" + name="$2" + tag="$3" + + if [[ "$repo" == "" || "$name" == "" || "$tag" == "" ]]; then + echo "Usage: $0 repo name tag" + exit 1 + fi + + bearer=$(jq -r ".auths[\\"$repo\\"].auth" ~/.docker/config.json || true) + if [[ "$bearer" == "" ]]; then + echo "There is no auth token for $repo in ~/.docker/config.json; did you run \\"docker login\\"?" + exit 2 + fi + + file=~/.docker/$(echo "$repo-$name-$tag" | sed -E 's/[^-_a-zA-Z0-9]/_/g').digest + old_digest=$(cat "$file" 2>/dev/null || true) + cur_digest=$( + curl -sS --fail --head \\ + -H "Authorization: Bearer $bearer" \\ + -H "Accept: application/vnd.oci.image.index.v1+json" \\ + https://$repo/v2/$name/manifests/$tag \\ + | grep -i 'Docker-Content-Digest' | awk '{print $2}' | sed -E 's/\\s+//' + ) + + if [[ "$old_digest" != "$cur_digest" ]]; then + if docker pull $repo/$name:$tag; then + echo "$cur_digest" > $file + else + code="$?" + echo "Failed to run \\"docker pull $repo/$name:$tag\\"" + exit "$code" + fi + else + echo "Not pulling $repo/$name:$tag - no changes in manifest since previous pull (digest: $cur_digest)" + fi + - path: /home/ubuntu/run-docker-compose.sh + owner: ubuntu:ubuntu + permissions: "0755" + defer: true + content: | + #!/bin/bash + + if [[ $(whoami) != "ubuntu" ]]; then + exec gosu ubuntu "$BASH_SOURCE" "$@" + fi + + exec {FD}<$BASH_SOURCE + flock -n "$FD" || { echo "Already running."; exit 0; } + set -e -o pipefail && echo "================= $(date) | Up $(awk '{print $1}' /proc/uptime) seconds =================" && echo "Running $BASH_SOURCE as $(whoami)" && set -x && export AWS_DEFAULT_REGION=$(ec2metadata --availability-zone | sed "s/[a-z]$//") + + source /etc/environment + export TZ + + mkdir -p ~/.ssh && chmod 700 ~/.ssh + aws secretsmanager get-secret-value \\ + --secret-id "ec2-ssh-key/", + { + "Fn::GetAtt": [ + "CnstrctSshIdRsaEC2KeyPairstkcnstrctsshidrsaF4F38F69", + "KeyPairName", + ], + }, + "/private" \\ + --query SecretString --output text \\ + > ~/.ssh/ci-storage + chmod 600 ~/.ssh/ci-storage + ssh-keygen -f ~/.ssh/ci-storage -y > ~/.ssh/ci-storage.pub + + aws secretsmanager get-secret-value \\ + --secret-id "ci-storage/gh-token" \\ + --query SecretString --output text \\ + | gh auth login --with-token + gh auth setup-git + + config=~/.docker/config.json + if [[ ! -f $config ]] || find "$config" -type f -mmin +60 | grep -q .; then + gh auth token | docker login ghcr.io -u "$(gh api user -q .login)" --password-stdin + fi + + mkdir -p ~/git && cd ~/git + if [[ ! -d .git ]]; then + git clone -n --depth=1 --filter=tree:0 "https://github.com/dimikot/ci-storage" . + if [[ "docker" != "." ]]; then + git sparse-checkout set --no-cone "docker" + fi + git checkout + else + git pull --rebase + fi + + { set +x; } &> /dev/null + export GH_TOKEN=$(gh auth token) + { set -x; } &> /dev/null + + export BTIME=$(cat /proc/stat | grep btime | awk '{print $2}') + export GH_REPOSITORY="time-loop/slapdash" + export GH_LABELS="my-ci,ci-small" + export FORWARD_HOST="my-ci-host-001.test-zoneName" + + file=~/.docker-started-after-first-git-clone + if [[ ! -f $file ]]; then + sudo systemctl start docker docker.socket + touch $file + fi + + cd "docker" + ~/docker-pull-if-changed-rate-limit-friendly.sh "ghcr.io" "dimikot/ci-runner" "main" + ~/docker-pull-if-changed-rate-limit-friendly.sh "ghcr.io" "dimikot/ci-runner" "latest" + docker compose up --build --remove-orphans -d + + if [[ "$1" != "--no-print-compose-logs" ]]; then + sleep 5 + docker compose logs -n 10 + fi + + docker system prune --volumes -f + export deps=$(docker image ls --format "{{.Repository}}:{{.ID}}:{{.Tag}}" | grep dimikot/ci-runner) + export instanceId=$(cloud-init query ds.meta_data.instance_id) + aws lambda invoke --function-name "stk-cnstrct-instancetoami" \\ + --payload "$(jq -nc '{"instanceId":$ENV.instanceId,"deps":$ENV.deps}')" \\ + /dev/stdout | jq -s '.[0]' + - path: /home/ubuntu/.bash_profile + owner: ubuntu:ubuntu + permissions: "0644" + defer: true + content: | + #!/bin/bash + C_CMD="\\033[0;36m" + C_NO="\\033[0m" + if [[ -d ~/git/"docker" ]]; then + cd ~/git/"docker" + echo "Hint: want to know, how did this instance boot and how did" + echo "the containers initialize? Run on the instance:" + echo + echo '$ less /var/log/cloud-init-output.log' + echo '$ less /var/log/cloud-init.log' + echo '$ less /var/log/syslog' + echo + echo -e "$C_CMD\\$ docker compose ps$C_NO" + COLUMNS=500 docker --log-level=ERROR compose ps --format="table {{.Service}}\\t{{.Status}}\\t{{.Ports}}" + echo + services=$(docker compose ps --format '{{.Service}}' 2>/dev/null) + if [[ "$services" != "" && $(echo "$services" | wc -l) -eq 1 ]]; then + echo "Hint: only one service is running on this instance, $services." + echo + echo "For your convenience, we are automatically logging you in" + echo "its container. Feel free to use any regular Linux commands" + echo "(ls, pwd, mc etc.) or any dev tools." + echo + echo "You can alway exit back to the instance by pressing ^D." + echo + cmd="docker compose exec $services bash -l" + echo -e "$C_CMD\\$ $cmd$C_NO" + eval "$cmd" + fi + fi +", + ], + ], + }, + }, + }, + "LaunchTemplateName": "stk-cnstrct-runnercismall", + "TagSpecifications": [ + { + "ResourceType": "launch-template", + "Tags": [ + { + "Key": "Name", + "Value": "Stk/Cnstrct/RunnerCiSmallLt", + }, + ], + }, + ], + }, + "Type": "AWS::EC2::LaunchTemplate", + }, + "CnstrctRunnerCiSmallLtImageCustomResource2db51ef0C46B9232": { + "DeletionPolicy": "Delete", + "DependsOn": [ + "CnstrctRunnerCiSmallLtImageCustomResource2db51ef0CustomResourcePolicy51CC6984", + "CnstrctRunnerRoleDefaultPolicyCA04641E", + "CnstrctRunnerRole341E54FA", + ], + "Properties": { + "Create": { + "Fn::Join": [ + "", + [ + "{"service":"Lambda","action":"invoke","parameters":{"FunctionName":"", + { + "Ref": "CnstrctRunnerCiSmallLtImageFuncF5364B8D", + }, + "","InvocationType":"RequestResponse","Payload":"{\\"launchTemplateName\\":\\"stk-cnstrct-runnercismall\\",\\"defaultImageId\\":\\"", + { + "Ref": "SsmParameterValuetestimagessmnameC96584B6F00A464EAD1953AFF4B05118Parameter", + }, + "\\",\\"userDataHash\\":\\"ac1bd295908bee9f\\"}"},"physicalResourceId":{"id":"StkCnstrctRunnerCiSmallLtResourceImageCustomResource2db51ef0"}}", + ], + ], + }, + "InstallLatestAwsSdk": true, + "ServiceToken": { + "Fn::GetAtt": [ + "AWS679f53fac002430cb0da5b7982bd22872D164C4C", + "Arn", + ], + }, + "Update": { + "Fn::Join": [ + "", + [ + "{"service":"Lambda","action":"invoke","parameters":{"FunctionName":"", + { + "Ref": "CnstrctRunnerCiSmallLtImageFuncF5364B8D", + }, + "","InvocationType":"RequestResponse","Payload":"{\\"launchTemplateName\\":\\"stk-cnstrct-runnercismall\\",\\"defaultImageId\\":\\"", + { + "Ref": "SsmParameterValuetestimagessmnameC96584B6F00A464EAD1953AFF4B05118Parameter", + }, + "\\",\\"userDataHash\\":\\"ac1bd295908bee9f\\"}"},"physicalResourceId":{"id":"StkCnstrctRunnerCiSmallLtResourceImageCustomResource2db51ef0"}}", + ], + ], + }, + }, + "Type": "Custom::AWS", + "UpdateReplacePolicy": "Delete", + }, + "CnstrctRunnerCiSmallLtImageCustomResource2db51ef0CustomResourcePolicy51CC6984": { + "DependsOn": [ + "CnstrctRunnerRoleDefaultPolicyCA04641E", + "CnstrctRunnerRole341E54FA", + ], + "Properties": { + "PolicyDocument": { + "Statement": [ + { + "Action": "lambda:InvokeFunction", + "Effect": "Allow", + "Resource": { + "Fn::GetAtt": [ + "CnstrctRunnerCiSmallLtImageFuncF5364B8D", + "Arn", + ], + }, + }, + ], + "Version": "2012-10-17", + }, + "PolicyName": "CnstrctRunnerCiSmallLtImageCustomResource2db51ef0CustomResourcePolicy51CC6984", + "Roles": [ + { + "Ref": "AWS679f53fac002430cb0da5b7982bd2287ServiceRoleC1EA0FF2", + }, + ], + }, + "Type": "AWS::IAM::Policy", + }, + "CnstrctRunnerCiSmallLtImageFuncF5364B8D": { + "DependsOn": [ + "CnstrctRunnerCiSmallLtImageFuncServiceRoleDefaultPolicyC2BBC915", + "CnstrctRunnerCiSmallLtImageFuncServiceRole12072151", + "CnstrctRunnerRoleDefaultPolicyCA04641E", + "CnstrctRunnerRole341E54FA", + ], + "Properties": { + "Code": { + "ZipFile": "from boto3 import client +from os import environ +def handler(event, context): + launchTemplateName=event["launchTemplateName"] + defaultImageId=event["defaultImageId"] + userDataHash=event["userDataHash"] + res_data = None + try: + res = client("ec2").describe_launch_template_versions( + LaunchTemplateName=launchTemplateName, + Versions=["$Latest"] + ) + res_data = res["LaunchTemplateVersions"][0]["LaunchTemplateData"] + image_id = res_data["ImageId"] + user_data_hash = next( + iter( + tag["Value"] + for spec in res_data.get("TagSpecifications", []) + if spec["ResourceType"] == "instance" + for tag in spec["Tags"] + if tag["Key"] == "InstanceToAmi:UserDataHash" + ), + None + ) + print(f"describe_launch_template_versions() returned ImageId={image_id} InstanceToAmi:UserDataHash={user_data_hash}") + if user_data_hash and userDataHash and user_data_hash != userDataHash: + print(f"returning defaultImageId={defaultImageId} to build the instance from scratch, since UserData has changed") + return defaultImageId + else: + print(f"returning previous Launch Template Version's ImageId={image_id}") + return image_id + except Exception as e: + e_str = f"{e.__class__.__name__}: {str(e)}" + if "NotFoundException" in e_str: + print(f"returning defaultImageId={defaultImageId} due to: {e_str}") + return defaultImageId + else: + if res_data: + print("describe_launch_template_versions() returned LaunchTemplateData: {res_data}") + print(f"got fatal exception: {e_str}") + # Hack: return the error instead of ImageId (not throw), since it + # will be shown right in the CDK deployment error output (because + # it is not of a valid ImageId format, so CDK will print that and + # fail anyways, but with a descriptive message). + return e_str +", + }, + "Handler": "index.handler", + "Role": { + "Fn::GetAtt": [ + "CnstrctRunnerCiSmallLtImageFuncServiceRole12072151", + "Arn", + ], + }, + "Runtime": "python3.9", + "Tags": [ + { + "Key": "Name", + "Value": "Stk/Cnstrct/RunnerCiSmallLt", + }, + ], + "Timeout": 20, + }, + "Type": "AWS::Lambda::Function", + }, + "CnstrctRunnerCiSmallLtImageFuncServiceRole12072151": { + "DependsOn": [ + "CnstrctRunnerRoleDefaultPolicyCA04641E", + "CnstrctRunnerRole341E54FA", + ], + "Properties": { + "AssumeRolePolicyDocument": { + "Statement": [ + { + "Action": "sts:AssumeRole", + "Effect": "Allow", + "Principal": { + "Service": "lambda.amazonaws.com", + }, + }, + ], + "Version": "2012-10-17", + }, + "ManagedPolicyArns": [ + { + "Fn::Join": [ + "", + [ + "arn:", + { + "Ref": "AWS::Partition", + }, + ":iam::aws:policy/service-role/AWSLambdaBasicExecutionRole", + ], + ], + }, + ], + "Tags": [ + { + "Key": "Name", + "Value": "Stk/Cnstrct/RunnerCiSmallLt", + }, + ], + }, + "Type": "AWS::IAM::Role", + }, + "CnstrctRunnerCiSmallLtImageFuncServiceRoleDefaultPolicyC2BBC915": { + "DependsOn": [ + "CnstrctRunnerRoleDefaultPolicyCA04641E", + "CnstrctRunnerRole341E54FA", + ], + "Properties": { + "PolicyDocument": { + "Statement": [ + { + "Action": [ + "ec2:DescribeLaunchTemplateVersions", + "logs:CreateLogGroup", + "logs:CreateLogStream", + "logs:PutLogEvents", + ], + "Effect": "Allow", + "Resource": "*", + }, + ], + "Version": "2012-10-17", + }, + "PolicyName": "CnstrctRunnerCiSmallLtImageFuncServiceRoleDefaultPolicyC2BBC915", + "Roles": [ + { + "Ref": "CnstrctRunnerCiSmallLtImageFuncServiceRole12072151", + }, + ], + }, + "Type": "AWS::IAM::Policy", + }, + "CnstrctRunnerCiSmallLtProfile0EB069BA": { + "Properties": { + "Roles": [ + { + "Ref": "CnstrctRunnerRole341E54FA", + }, + ], + }, + "Type": "AWS::IAM::InstanceProfile", + }, + "CnstrctRunnerRole341E54FA": { + "Properties": { + "AssumeRolePolicyDocument": { + "Statement": [ + { + "Action": "sts:AssumeRole", + "Effect": "Allow", + "Principal": { + "Service": "ec2.amazonaws.com", + }, + }, + ], + "Version": "2012-10-17", + }, + "ManagedPolicyArns": [ + { + "Fn::Join": [ + "", + [ + "arn:", + { + "Ref": "AWS::Partition", + }, + ":iam::aws:policy/service-role/AmazonEC2RoleforSSM", + ], + ], + }, + { + "Fn::Join": [ + "", + [ + "arn:", + { + "Ref": "AWS::Partition", + }, + ":iam::aws:policy/CloudWatchAgentServerPolicy", + ], + ], + }, + ], + "Policies": [ + { + "PolicyDocument": { + "Statement": [ + { + "Action": "secretsmanager:GetSecretValue", + "Effect": "Allow", + "Resource": { + "Fn::Join": [ + "", + [ + "arn:", + { + "Ref": "AWS::Partition", + }, + ":secretsmanager:", + { + "Ref": "AWS::Region", + }, + ":", + { + "Ref": "AWS::AccountId", + }, + ":secret:ec2-ssh-key/", + { + "Fn::GetAtt": [ + "CnstrctSshIdRsaEC2KeyPairstkcnstrctsshidrsaF4F38F69", + "KeyPairName", + ], + }, + "/private*", + ], + ], + }, + }, + ], + "Version": "2012-10-17", + }, + "PolicyName": "KeyPairPolicy", + }, + { + "PolicyDocument": { + "Statement": [ + { + "Action": "secretsmanager:GetSecretValue", + "Effect": "Allow", + "Resource": { + "Fn::Join": [ + "", + [ + "arn:", + { + "Ref": "AWS::Partition", + }, + ":secretsmanager:", + { + "Ref": "AWS::Region", + }, + ":", + { + "Ref": "AWS::AccountId", + }, + ":secret:ci-storage/gh-token*", + ], + ], + }, + }, + ], + "Version": "2012-10-17", + }, + "PolicyName": "GhTokenPolicy", + }, + { + "PolicyDocument": { + "Statement": [ + { + "Action": "ec2:DescribeInstances", + "Effect": "Allow", + "Resource": "*", + }, + { + "Action": "cloudformation:SignalResource", + "Effect": "Allow", + "Resource": { + "Fn::Join": [ + "", + [ + "arn:", + { + "Ref": "AWS::Partition", + }, + ":cloudformation:", + { + "Ref": "AWS::Region", + }, + ":", + { + "Ref": "AWS::AccountId", + }, + ":stack/Stk/*", + ], + ], + }, + }, + ], + "Version": "2012-10-17", + }, + "PolicyName": "SignalResourcePolicy", + }, + { + "PolicyDocument": { + "Statement": [ { - "Fn::GetAtt": [ - "CnstrctSshIdRsaEC2KeyPairstkcnstrctsshidrsaF4F38F69", - "KeyPairName", - ], + "Action": "ec2:DescribeTags", + "Effect": "Allow", + "Resource": "*", + }, + { + "Action": "ec2:CreateTags", + "Condition": { + "StringEquals": { + "ec2:ResourceTag/aws:autoscaling:groupName": [ + "stk-cnstrct-runnercismall", + "stk-cnstrct-runnercilarge", + ], + }, + }, + "Effect": "Allow", + "Resource": { + "Fn::Join": [ + "", + [ + "arn:", + { + "Ref": "AWS::Partition", + }, + ":ec2:", + { + "Ref": "AWS::Region", + }, + ":", + { + "Ref": "AWS::AccountId", + }, + ":instance/*", + ], + ], + }, }, - "/private" \\ - --query SecretString --output text \\ - > ~/.ssh/ci-storage - chmod 600 ~/.ssh/ci-storage - ssh-keygen -f ~/.ssh/ci-storage -y > ~/.ssh/ci-storage.pub - - # Load GitHub PAT from Secrets Manager and log in to GitHub. - aws secretsmanager get-secret-value \\ - --secret-id "ci-storage/gh-token" \\ - --query SecretString --output text \\ - | gh auth login --with-token - gh auth setup-git - - # Log in to ghcr.io every hour. - config=~/.docker/config.json - if [[ ! -f $config ]] || find "$config" -type f -mmin +60 | grep -q .; then - gh auth token | docker login ghcr.io -u "$(gh api user -q .login)" --password-stdin - fi - - # Pull the repository. - mkdir -p ~/git && cd ~/git - if [[ ! -d .git ]]; then - git clone -n --depth=1 --filter=tree:0 "https://github.com/dimikot/ci-storage" . - if [[ "docker" != "." ]]; then - git sparse-checkout set --no-cone "docker" - fi - git checkout - else - git pull --rebase - fi - - # Process some tokens and print rate limits without xtrace. - set +o xtrace - GH_TOKEN=$(gh auth token) - echo "Docker Hub Rate Limits:" - docker_hub_token=$(curl -s "https://auth.docker.io/token?service=registry.docker.io&scope=repository:ratelimitpreview/test:pull" | jq -r .token || true) - curl -s --head -H "Authorization: Bearer $docker_hub_token" https://registry-1.docker.io/v2/ratelimitpreview/test/manifests/latest | grep ratelimit || true - echo "GitHub Core Rate Limits:" - gh api -i -X HEAD /rate_limit | grep Ratelimit - set -o xtrace - - # Export env vars for docker compose. - export GH_TOKEN - - # It it's the very 1st run, start Docker service. We do not start it every run, - # because otherwise we wouldn't be able to "systemctl stop docker docker.socket" - # manually or while copying files from the old host. - file=~/.docker-started-after-first-git-clone - if [[ ! -f $file ]]; then - sudo systemctl start docker docker.socket - touch $file - fi - - # Run docker compose. - cd "docker" - docker pull ghcr.io/dimikot/ci-storage:main || true - docker pull ghcr.io/dimikot/ci-runner:main || true - docker compose --profile=ci up --build --remove-orphans -d - sleep 5 - if [[ "$1" != "--no-logs" ]]; then - docker compose logs -n 10 - fi - docker system prune --volumes -f - - path: /home/ubuntu/.bash_profile - owner: ubuntu:ubuntu - permissions: "0644" - defer: true - content: | - #!/bin/bash - C_CMD="\\033[0;36m" - C_NO="\\033[0m" - if [[ -d ~/git/"docker" ]]; then - cd ~/git/"docker" - echo -e "$C_CMD\\$ docker compose ps$C_NO" - docker --log-level=ERROR compose ps --format="table {{.Service}}\\t{{.Status}}\\t{{.Ports}}" - services=$(docker compose ps --format '{{.Service}}' 2>/dev/null) - if [[ "$services" != "" && $(echo "$services" | wc -l) -eq 1 ]]; then - cmd="docker compose exec $services bash -l" - echo -e "$C_CMD\\$ $cmd$C_NO" - eval "$cmd" - fi - fi -", ], - ], + "Version": "2012-10-17", + }, + "PolicyName": "TagsPolicy", }, - }, + ], + "RoleName": "StkCnstrctRunnerRole", }, - "Type": "AWS::EC2::Instance", + "Type": "AWS::IAM::Role", }, - "CnstrctStkCnstrctMyCiHost001InstanceInstanceProfile2045E5AE": { + "CnstrctRunnerRoleDefaultPolicyCA04641E": { "Properties": { + "PolicyDocument": { + "Statement": [ + { + "Action": "lambda:InvokeFunction", + "Effect": "Allow", + "Resource": { + "Fn::GetAtt": [ + "CnstrctInstanceToAmiStartExecution3968560D", + "Arn", + ], + }, + }, + ], + "Version": "2012-10-17", + }, + "PolicyName": "CnstrctRunnerRoleDefaultPolicyCA04641E", "Roles": [ { - "Ref": "CnstrctHostRole5DD9F366", + "Ref": "CnstrctRunnerRole341E54FA", }, ], }, - "Type": "AWS::IAM::InstanceProfile", + "Type": "AWS::IAM::Policy", }, - "CnstrctStkCnstrctMyCiHost001InstanceLaunchTemplateD3F24170": { + "CnstrctSgF5C70BA4": { "Properties": { - "LaunchTemplateData": { - "MetadataOptions": { - "HttpTokens": "required", + "GroupDescription": "stk-cnstrct-sg", + "GroupName": "stk-cnstrct-sg", + "SecurityGroupEgress": [ + { + "CidrIp": "0.0.0.0/0", + "Description": "Allow all outbound traffic by default", + "IpProtocol": "-1", }, + ], + "Tags": [ + { + "Key": "Name", + "Value": "stk-cnstrct-sg", + }, + ], + "VpcId": { + "Ref": "Vpc8378EB38", }, - "LaunchTemplateName": "StkCnstrctMyCiHost001InstanceLaunchTemplate", }, - "Type": "AWS::EC2::LaunchTemplate", + "Type": "AWS::EC2::SecurityGroup", + }, + "CnstrctSgfromStkCnstrctSgCE9C2C74226B452137": { + "Properties": { + "Description": "from stk-cnstrct to SSH", + "FromPort": 22, + "GroupId": { + "Fn::GetAtt": [ + "CnstrctSgF5C70BA4", + "GroupId", + ], + }, + "IpProtocol": "tcp", + "SourceSecurityGroupId": { + "Fn::GetAtt": [ + "CnstrctSgF5C70BA4", + "GroupId", + ], + }, + "ToPort": 22, + }, + "Type": "AWS::EC2::SecurityGroupIngress", + }, + "CnstrctSgfromStkCnstrctSgCE9C2C74260223B2126BC": { + "Properties": { + "Description": "from stk-cnstrct to ci-storage", + "FromPort": 26022, + "GroupId": { + "Fn::GetAtt": [ + "CnstrctSgF5C70BA4", + "GroupId", + ], + }, + "IpProtocol": "tcp", + "SourceSecurityGroupId": { + "Fn::GetAtt": [ + "CnstrctSgF5C70BA4", + "GroupId", + ], + }, + "ToPort": 26022, + }, + "Type": "AWS::EC2::SecurityGroupIngress", + }, + "CnstrctSgfromStkCnstrctSgCE9C2C74280885707034B": { + "Properties": { + "Description": "from stk-cnstrct to ci-scaler", + "FromPort": 28088, + "GroupId": { + "Fn::GetAtt": [ + "CnstrctSgF5C70BA4", + "GroupId", + ], + }, + "IpProtocol": "tcp", + "SourceSecurityGroupId": { + "Fn::GetAtt": [ + "CnstrctSgF5C70BA4", + "GroupId", + ], + }, + "ToPort": 28088, + }, + "Type": "AWS::EC2::SecurityGroupIngress", + }, + "CnstrctSgfromStkCnstrctSgCE9C2C744200042042B0BB20A9": { + "Properties": { + "Description": "from stk-cnstrct to test ports", + "FromPort": 42000, + "GroupId": { + "Fn::GetAtt": [ + "CnstrctSgF5C70BA4", + "GroupId", + ], + }, + "IpProtocol": "tcp", + "SourceSecurityGroupId": { + "Fn::GetAtt": [ + "CnstrctSgF5C70BA4", + "GroupId", + ], + }, + "ToPort": 42042, + }, + "Type": "AWS::EC2::SecurityGroupIngress", + }, + "CnstrctSshIdRsaEC2KeyPairstkcnstrctsshidrsaF4F38F69": { + "DeletionPolicy": "Delete", + "Properties": { + "Description": "Used to access ci-storage host from self-hosted runner nodes.", + "ExposePublicKey": false, + "KmsPrivate": "alias/aws/secretsmanager", + "KmsPublic": "alias/aws/secretsmanager", + "Name": "stk-cnstrct-sshidrsa", + "PublicKey": "", + "PublicKeyFormat": "OPENSSH", + "RemoveKeySecretsAfterDays": 0, + "SecretPrefix": "ec2-ssh-key/", + "ServiceToken": { + "Fn::GetAtt": [ + "EC2KeyNameManagerLambdaBE629145", + "Arn", + ], + }, + "StackName": "Stk", + "StorePublicKey": false, + "Tags": { + "CreatedByCfnCustomResource": "CFN::Resource::Custom::EC2-Key-Pair", + }, + }, + "Type": "Custom::EC2-Key-Pair", + "UpdateReplacePolicy": "Delete", }, "EC2KeyNameManagerLambdaBE629145": { "DependsOn": [ diff --git a/src/internal/__tests__/namer.test.ts b/src/internal/__tests__/namer.test.ts index ace483d..366d641 100644 --- a/src/internal/__tests__/namer.test.ts +++ b/src/internal/__tests__/namer.test.ts @@ -4,4 +4,5 @@ test("namer", () => { expect(namer("OneTwo" as any).kebab).toBe("one-two"); expect(namer("one-two" as any).pascal).toBe("OneTwo"); expect(namer("one", "two").pascal).toBe("OneTwo"); + expect(namer("*").pathKebabFrom({ node: { path: "abc" } })).toBe("abc-*"); }); diff --git a/src/internal/cloudConfigBuild.ts b/src/internal/cloudConfigBuild.ts index e96e437..cb1f5a9 100644 --- a/src/internal/cloudConfigBuild.ts +++ b/src/internal/cloudConfigBuild.ts @@ -1,3 +1,9 @@ +import { + bootCmdMountDisk, + bootCmdMountSwap, + bootCmdMountTmpfs, + bootCmdSwitchSsmUserOnLogin, +} from "@clickup/instance-to-ami-cdk"; import compact from "lodash/compact"; import { dedent } from "./dedent"; @@ -12,9 +18,12 @@ export function cloudConfigBuild({ ghTokenSecretName, dockerComposeEnv, dockerComposeProfiles, + dockerComposePrePullImages, + dockerComposeCmdAfter, ghDockerComposeDirectoryUrl, keyPairPrivateKeySecretName, timeZone, + ephemeral, tmpfs, swapSizeGb, }: { @@ -23,9 +32,16 @@ export function cloudConfigBuild({ ghDockerComposeDirectoryUrl: string; dockerComposeEnv: Record; dockerComposeProfiles: string[]; + dockerComposePrePullImages: Array<{ + repo: string; + image: string; + tags: string[]; + }>; + dockerComposeCmdAfter: string | null; keyPairPrivateKeySecretName: string; timeZone: string | undefined; - tmpfs: { path: string; maxSizeGb?: number } | undefined; + ephemeral: { path: string; chown: string; chmod: string } | undefined; + tmpfs: { path: string; chmod: string; maxSizeGb?: number } | undefined; swapSizeGb: number | undefined; }) { if (!ghDockerComposeDirectoryUrl.match(/^([^#]+)(?:#([^:]*):(.*))?$/s)) { @@ -40,9 +56,9 @@ export function cloudConfigBuild({ const path = (RegExp.$3 || ".").replace(/^\/+|\/+$/gs, ""); const preamble = [ "set -e -o pipefail", - "echo ================", + "echo \"================= $(date) | Up $(awk '{print $1}' /proc/uptime) seconds =================\"", 'echo "Running $BASH_SOURCE as $(whoami)"', - "set -o xtrace", + "set -x", 'export AWS_DEFAULT_REGION=$(ec2metadata --availability-zone | sed "s/[a-z]$//")', ].join(" && "); @@ -50,38 +66,50 @@ export function cloudConfigBuild({ timezone: timeZone, fqdn: fqdn || undefined, hostname: fqdn || undefined, - swap: swapSizeGb - ? { - filename: "/var/swapfile", - size: "auto", - maxsize: 1024 * 1024 * 1024 * swapSizeGb, - } - : undefined, - mounts: tmpfs - ? [ - [ - "tmpfs", - tmpfs.path, - "tmpfs", - "defaults,noatime,exec,mode=0710,nr_inodes=0" + - (tmpfs.maxSizeGb ? `,size=${tmpfs.maxSizeGb}G` : ""), - "0", - "0", - ], - ] - : undefined, - apt_sources: [ - { - source: "deb https://cli.github.com/packages stable main", - keyid: "23F3D4EA75716059", - filename: "github-cli.list", - }, - { - source: "deb https://download.docker.com/linux/ubuntu $RELEASE stable", - keyid: "9DC858229FC7DD38854AE2D88D81803C0EBFCD88", - filename: "docker.list", + // Debugging when booting from a snapshot: + // + // - https://github.com/canonical/cloud-init/blob/main/cloudinit/config/cc_disk_setup.py + // - issues with EBS: `lsblk` + // - some service deps: `systemd-analyze critical-chain docker.service` + // - boot logs: `journalctl -b -o short-iso` + // - cloud-init output logs: `less /var/log/cloud-init-output.log` + // - cloud-init steps logs: `less /var/log/cloud-init.log` + // + // Notice that bootcmd runs very early, even before disk_setup and + // write_files directives. + bootcmd: compact([ + bootCmdSwitchSsmUserOnLogin(), + tmpfs && bootCmdMountTmpfs(tmpfs), + ephemeral && + bootCmdMountDisk({ + label: "ephemeral", + path: ephemeral.path, + chown: ephemeral.chown, + chmod: ephemeral.chmod, + initOnceCmd: `cp -af ".${ephemeral.path}/." "${ephemeral.path}"`, + }), + "mkdir -p /var/log/atop", + swapSizeGb && + bootCmdMountSwap({ + path: `${ephemeral?.path ?? "/var"}/swapfile`, + sizeGb: swapSizeGb, + }), + ]), + // Packages are installed after bootcmd and after write_files, but before + // /var/lib/cloud/scripts/per-* scripts. + apt: { + sources: { + "github-cli.list": { + source: "deb https://cli.github.com/packages stable main", + keyid: "23F3D4EA75716059", + }, + "docker.list": { + source: + "deb https://download.docker.com/linux/ubuntu $RELEASE stable", + keyid: "9DC858229FC7DD38854AE2D88D81803C0EBFCD88", + }, }, - ], + }, packages: [ "awscli", "gh", @@ -104,8 +132,13 @@ export function cloudConfigBuild({ "htop", "bwm-ng", "jq", + "expect", ], + // Files are written after bootcmd, but before packages are installed. write_files: compact([ + // + // Regular system files and configs. + // { path: "/etc/sysctl.d/enable-ipv4-forwarding.conf", content: dedent(` @@ -120,20 +153,6 @@ export function cloudConfigBuild({ LOGGENERATIONS=4 `), }, - timeZone && { - path: "/etc/environment", - append: true, - content: dedent(` - TZ="${timeZone}" - `), - }, - { - path: "/etc/environment", - append: true, - content: dedent(` - LESS="RS" - `), - }, { path: "/etc/docker/daemon.json", permissions: "0644", @@ -154,23 +173,63 @@ export function cloudConfigBuild({ `), }, { - path: "/var/lib/cloud/scripts/per-once/apply-services-configs.sh", + path: "/etc/systemd/system/docker.service.d/override.conf", + permissions: "0644", + content: dedent(` + [Service] + # Increase grace period for stopping containers. + TimeoutStopSec=3600 + # Run docker compose ASAP after Docker starts. + ExecStartPost=/home/ubuntu/run-docker-compose.sh + `), + }, + { + path: "/etc/cron.d/git-pull-and-rerun-docker-compose-periodically", + permissions: "0644", + content: dedent(` + */1 * * * * ubuntu /home/ubuntu/run-docker-compose.sh --no-print-compose-logs 2>&1 | logger -t docker/run-docker-compose + `), + }, + { + path: "/etc/rsyslog.d/01-docker-tag-to-serial-console.conf", + permissions: "0644", + content: dedent(` + if $syslogtag startswith 'docker/' then -/dev/console + # It will also write to /var/log/syslog as usual. + `), + }, + // + // Per-once scripts (change FS permanently before snapshot+image are + // taken). Notice that they run in ~20 seconds after bootcmd, i.e. late. + // + { + path: "/var/lib/cloud/scripts/per-once/00-append-etc-environment.sh", permissions: "0755", content: dedent(` #!/bin/bash ${preamble} - service atop restart || true - sysctl --system + ${timeZone ? `echo "TZ=${timeZone}" >> /etc/environment` : ""} + echo "LESS=RS" >> /etc/environment + `), + }, + { + path: "/var/lib/cloud/scripts/per-once/make-apt-get-install-not-run-after-restoring-from-snapshot-to-speedup-boot.sh", + permissions: "0755", + content: dedent(` + #!/bin/bash + ${preamble} + sed -i -E 's/(- (package-update-upgrade-install|apt-configure|apt-pipelining))/# \\1/g' /etc/cloud/cloud.cfg `), }, { - path: "/var/lib/cloud/scripts/per-once/increase-docker-shutdown-timeout.sh", + path: "/var/lib/cloud/scripts/per-once/apply-services-configs.sh", permissions: "0755", content: dedent(` #!/bin/bash ${preamble} - sed -i -E '/TimeoutStartSec=.*/a TimeoutStopSec=3600' /usr/lib/systemd/system/docker.service systemctl daemon-reload + service atop restart || true + sysctl --system `), }, { @@ -185,31 +244,35 @@ export function cloudConfigBuild({ { path: "/var/lib/cloud/scripts/per-once/install-sysbox-for-docker-in-docker.sh", permissions: "0755", + // To debug crashes (like "program exceeds 10000-thread limit"): + // + // - sudo journalctl -u sysbox-fs.service + // - sudo journalctl -u sysbox-mgr.service + // + // Also, if sysbox-fs crashes and gets restarted, it also tries to + // restart sysbox-mgr (because otherwise, sysbox-mgr remains stopped: + // something stops it gracefully on a sysbox-fs crash). content: dedent(` #!/bin/bash ${preamble} systemctl stop docker docker.socket || true + for svc in sysbox-fs sysbox-mgr; do + dir="/etc/systemd/system/$svc.service.d" + mkdir -p "$dir" + { + echo "[Service]" + echo "Restart=always" + echo "RestartSec=5" + if [[ "$svc" == "sysbox-fs" ]]; then + echo "ExecStartPost=/bin/systemctl start sysbox-mgr.service" + fi + } > "$dir/override.conf" + done wget -nv -O /tmp/sysbox-ce.deb "https://downloads.nestybox.com/sysbox/releases/v0.6.4/sysbox-ce_0.6.4-0.linux_$(dpkg --print-architecture).deb" dpkg -i /tmp/sysbox-ce.deb rm -f /tmp/sysbox-ce.deb `), }, - { - path: "/var/lib/cloud/scripts/per-once/switch-ssm-user-to-ubuntu-on-login.sh", - permissions: "0755", - content: dedent(` - #!/bin/bash - ${preamble} - echo '[ "$0$@" = "sh" ] && ENV= sudo -u ubuntu -i' > /etc/profile.ssm-user - mkdir -p /etc/systemd/system/snap.amazon-ssm-agent.amazon-ssm-agent.service.d/ - ( - echo '[Service]' - echo 'Environment="ENV=/etc/profile.ssm-user"' - ) > /etc/systemd/system/snap.amazon-ssm-agent.amazon-ssm-agent.service.d/sh-env.conf - systemctl daemon-reload - systemctl restart snap.amazon-ssm-agent.amazon-ssm-agent.service || true - `), - }, tmpfs && fqdn && { path: "/var/lib/cloud/scripts/per-once/rsync-tmpfs-volume-from-old-instance.sh", @@ -276,14 +339,6 @@ export function cloudConfigBuild({ --unique-id "$instance_id" --status SUCCESS `), }, - { - path: "/etc/rsyslog.d/01-docker-tag-to-serial-console.conf", - permissions: "0644", - content: dedent(` - if $syslogtag startswith 'docker/' then -/dev/console - # It will also write to /var/log/syslog as usual. - `), - }, { path: "/var/lib/cloud/scripts/per-once/allow-rsyslog-write-to-serial-console.sh", permissions: "0755", @@ -294,14 +349,55 @@ export function cloudConfigBuild({ systemctl restart rsyslog `), }, + // + // User scripts and tools (must have "defer=true" to be run after ubuntu + // user is created). + // { - path: "/var/lib/cloud/scripts/per-boot/run-docker-compose-on-boot.sh", + path: "/home/ubuntu/docker-pull-if-changed-rate-limit-friendly.sh", + owner: "ubuntu:ubuntu", permissions: "0755", - content: dedent(` + defer: true, + content: dedent(String.raw` #!/bin/bash - ${preamble} - echo "*/2 * * * * ubuntu /home/ubuntu/run-docker-compose.sh --no-logs 2>&1 | logger -t docker/run-docker-compose" > /etc/cron.d/run-docker-compose - exec /home/ubuntu/run-docker-compose.sh + set -e -o pipefail + repo="$1" + name="$2" + tag="$3" + + if [[ "$repo" == "" || "$name" == "" || "$tag" == "" ]]; then + echo "Usage: $0 repo name tag" + exit 1 + fi + + # In case of e.g. ghcr.io, it's actually base64("used:pat"). + bearer=$(jq -r ".auths[\"$repo\"].auth" ~/.docker/config.json || true) + if [[ "$bearer" == "" ]]; then + echo "There is no auth token for $repo in ~/.docker/config.json; did you run \"docker login\"?" + exit 2 + fi + + file=~/.docker/$(echo "$repo-$name-$tag" | sed -E 's/[^-_a-zA-Z0-9]/_/g').digest + old_digest=$(cat "$file" 2>/dev/null || true) + cur_digest=$( + curl -sS --fail --head \ + -H "Authorization: Bearer $bearer" \ + -H "Accept: application/vnd.oci.image.index.v1+json" \ + https://$repo/v2/$name/manifests/$tag \ + | grep -i 'Docker-Content-Digest' | awk '{print $2}' | sed -E 's/\s+//' + ) + + if [[ "$old_digest" != "$cur_digest" ]]; then + if docker pull $repo/$name:$tag; then + echo "$cur_digest" > $file + else + code="$?" + echo "Failed to run \"docker pull $repo/$name:$tag\"" + exit "$code" + fi + else + echo "Not pulling $repo/$name:$tag - no changes in manifest since previous pull (digest: $cur_digest)" + fi `), }, { @@ -362,18 +458,13 @@ export function cloudConfigBuild({ git pull --rebase fi - # Process some tokens and print rate limits without xtrace. - set +o xtrace - GH_TOKEN=$(gh auth token) - echo "Docker Hub Rate Limits:" - docker_hub_token=$(curl -s "https://auth.docker.io/token?service=registry.docker.io&scope=repository:ratelimitpreview/test:pull" | jq -r .token || true) - curl -s --head -H "Authorization: Bearer $docker_hub_token" https://registry-1.docker.io/v2/ratelimitpreview/test/manifests/latest | grep ratelimit || true - echo "GitHub Core Rate Limits:" - gh api -i -X HEAD /rate_limit | grep Ratelimit - set -o xtrace + # Export token without xtrace. + { set +x; } &> /dev/null + export GH_TOKEN=$(gh auth token) + { set -x; } &> /dev/null # Export env vars for docker compose. - export GH_TOKEN + export BTIME=$(cat /proc/stat | grep btime | awk '{print $2}') ${Object.entries(dockerComposeEnv) .map(([k, v]) => `export ${k}="${v}"`) .join("\n")} @@ -389,14 +480,26 @@ export function cloudConfigBuild({ # Run docker compose. cd "${path}" - docker pull ghcr.io/dimikot/ci-storage:main || true - docker pull ghcr.io/dimikot/ci-runner:main || true + ${dockerComposePrePullImages + .flatMap(({ repo, image, tags }) => + tags.map( + (tag) => + `~/docker-pull-if-changed-rate-limit-friendly.sh "${repo}" "${image}" "${tag}"`, + ), + ) + .join("\n")} docker compose ${dockerComposeProfiles.map((profile) => `--profile=${profile} `).join("")}up --build --remove-orphans -d - sleep 5 - if [[ "$1" != "--no-logs" ]]; then + + if [[ "$1" != "--no-print-compose-logs" ]]; then + # Print logs before "docker system prune", otherwise they may be + # empty in case the container failed to start. We can always look + # at /var/log/syslog though. + sleep 5 docker compose logs -n 10 fi + docker system prune --volumes -f + ${dockerComposeCmdAfter ?? ""} `), }, { @@ -410,10 +513,26 @@ export function cloudConfigBuild({ C_NO="\\033[0m" if [[ -d ~/git/"${path}" ]]; then cd ~/git/"${path}" + echo "Hint: want to know, how did this instance boot and how did" + echo "the containers initialize? Run on the instance:" + echo + echo '$ less /var/log/cloud-init-output.log' + echo '$ less /var/log/cloud-init.log' + echo '$ less /var/log/syslog' + echo echo -e "$C_CMD\\$ docker compose ps$C_NO" - docker --log-level=ERROR compose ps --format="table {{.Service}}\\t{{.Status}}\\t{{.Ports}}" + COLUMNS=500 docker --log-level=ERROR compose ps --format="table {{.Service}}\\t{{.Status}}\\t{{.Ports}}" + echo services=$(docker compose ps --format '{{.Service}}' 2>/dev/null) if [[ "$services" != "" && $(echo "$services" | wc -l) -eq 1 ]]; then + echo "Hint: only one service is running on this instance, $services." + echo + echo "For your convenience, we are automatically logging you in" + echo "its container. Feel free to use any regular Linux commands" + echo "(ls, pwd, mc etc.) or any dev tools." + echo + echo "You can alway exit back to the instance by pressing ^D." + echo cmd="docker compose exec $services bash -l" echo -e "$C_CMD\\$ $cmd$C_NO" eval "$cmd" diff --git a/src/internal/dedent.ts b/src/internal/dedent.ts index 405cee7..2068e09 100644 --- a/src/internal/dedent.ts +++ b/src/internal/dedent.ts @@ -1,10 +1,12 @@ /** - * Removes leading indentation from each line of the text. Also, it some line - * contains only the indentation spaces immediately followed by \n, the line is - * removed entirely. + * Removes leading indentation from each line of the text. Removes lines which + * only include #-comments (to save some space; UserData is limited by 16K). + * Also, it some line contains only the indentation spaces immediately followed + * by \n, the line is removed entirely. */ export function dedent(text: string): string { text = text.replace(/^([ \t\r]*\n)+/s, "").trimEnd(); + text = text.replace(/^[ \t]*#(?![!])[^\n]*\n/gm, ""); const spacePrefix = text.match(/^([ \t]+)/s) ? RegExp.$1 : null; return ( (spacePrefix diff --git a/src/internal/namer.ts b/src/internal/namer.ts index 0686e0f..263748c 100644 --- a/src/internal/namer.ts +++ b/src/internal/namer.ts @@ -36,7 +36,7 @@ export function namer( namer.pathKebabFrom = (scope) => (scope.node.path + "/" + namer.pascal).replace(/\//g, "-").toLowerCase(); namer.pathPascalFrom = (scope) => - new NamerOrig([...compact(scope.node.path.split(/\W+/)), ...namer.parts]) + new NamerOrig(compact([...scope.node.path.split(/\W+/), ...namer.parts])) .pascal; return namer; } diff --git a/src/internal/userDataCausesReplacement.ts b/src/internal/userDataCausesReplacement.ts new file mode 100644 index 0000000..f352e29 --- /dev/null +++ b/src/internal/userDataCausesReplacement.ts @@ -0,0 +1,40 @@ +import { createHash } from "crypto"; +import type { UserData } from "aws-cdk-lib/aws-ec2"; +import { Lazy, type CfnElement, Stack } from "aws-cdk-lib/core"; + +/** + * Triggers replacement (via new logical ID) on user data change. We need it to + * recreate the instance, since its cloud-init script defines how the instance + * is created. + * + * The logic is copied from ec2.Instance construct which we can't use since it + * doesn't support httpPutResponseHopLimit attribute: + * https://github.com/aws/aws-cdk/blob/f470271864ee5/packages/aws-cdk-lib/aws-ec2/lib/instance.ts#L527 + */ +export function userDataCausesReplacement( + element: CfnElement, + userData: UserData, +) { + const originalLogicalId = Stack.of(element).getLogicalId(element); + let recursing = false; + element.overrideLogicalId( + Lazy.uncachedString({ + produce: (context) => { + const fragments: string[] = []; + recursing = true; + try { + fragments.push(JSON.stringify(context.resolve(userData.render()))); + } finally { + recursing = false; + } + + const digest = createHash("sha256") + .update(fragments.join("")) + .digest("hex") + .slice(0, 16); + return `${originalLogicalId}${digest}`; + }, + }), + ); + recursing; // TS quirk +}