-
Notifications
You must be signed in to change notification settings - Fork 121
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Fixes varios issues with rebuilding CI Docker images (#2077)
* Update Intel SDE from `9.21.1` to `9.44.0` as the older version is no longer available on at the mirror location. * Update the AL2023 SSM Agent file to pull the changed path location for the manifest for GitHub. * Update the CentOS 7 docker image's mirrors to use the vault location as the previous mirror is no longer serving the repository files. * Since cryptofuzz is no longer available on GitHub from the original upstream author, update it to pull an archived file from a CloudFront distribution from our team AWS account. I have also put a copy of the code in AWS CodeCommit for a complete git history. * Botan repository tip isn't building for us on AL2023 anymore (looks like C++ 20 related / issues with cryptofuzz maybe?), anyways pinned to the current cryptofuzz we've been using and botan version that was released around that time. * Update CDK dependencies for good hygiene * Turns out our Windows CDK image build was never truly automated....so fix that by automating the long-standing issue of not installing the Docker engine. By submitting this pull request, I confirm that my contribution is made under the terms of the Apache 2.0 license and the ISC license.
- Loading branch information
Showing
9 changed files
with
134 additions
and
70 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -1,66 +1,124 @@ | ||
| # Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. | ||
| # SPDX-License-Identifier: Apache-2.0 OR ISC | ||
|
|
||
| from aws_cdk import Stack, Tags, aws_ec2 as ec2, aws_s3 as s3, aws_iam as iam, aws_ssm as ssm | ||
| from aws_cdk import ( | ||
| Stack, | ||
| Tags, | ||
| aws_ec2 as ec2, | ||
| aws_s3 as s3, | ||
| aws_iam as iam, | ||
| aws_ssm as ssm, | ||
| ) | ||
| from constructs import Construct | ||
| from util.iam_policies import ecr_power_user_policy_in_json, s3_read_write_policy_in_json | ||
| from util.metadata import AWS_ACCOUNT, AWS_REGION, WINDOWS_X86_ECR_REPO, S3_BUCKET_NAME, GITHUB_REPO_OWNER, WIN_EC2_TAG_KEY, \ | ||
| WIN_EC2_TAG_VALUE, SSM_DOCUMENT_NAME, GITHUB_SOURCE_VERSION | ||
| from util.iam_policies import ( | ||
| ecr_power_user_policy_in_json, | ||
| s3_read_write_policy_in_json, | ||
| ) | ||
| from util.metadata import ( | ||
| AWS_ACCOUNT, | ||
| AWS_REGION, | ||
| WINDOWS_X86_ECR_REPO, | ||
| S3_BUCKET_NAME, | ||
| GITHUB_REPO_OWNER, | ||
| WIN_EC2_TAG_KEY, | ||
| WIN_EC2_TAG_VALUE, | ||
| SSM_DOCUMENT_NAME, | ||
| GITHUB_SOURCE_VERSION, | ||
| ) | ||
| from util.yml_loader import YmlLoader | ||
|
|
||
|
|
||
| class WindowsDockerImageBuildStack(Stack): | ||
| """Define a temporary stack used to build Windows Docker images. After build, this stack will be destroyed.""" | ||
|
|
||
| def __init__(self, | ||
| scope: Construct, | ||
| id: str, | ||
| **kwargs) -> None: | ||
| def __init__(self, scope: Construct, id: str, **kwargs) -> None: | ||
| super().__init__(scope, id, **kwargs) | ||
|
|
||
| # Define SSM command document. | ||
| ecr_repo = "{}.dkr.ecr.{}.amazonaws.com/{}".format(AWS_ACCOUNT, AWS_REGION, WINDOWS_X86_ECR_REPO) | ||
| placeholder_map = {"ECR_PLACEHOLDER": ecr_repo, "GITHUB_OWNER_PLACEHOLDER": GITHUB_REPO_OWNER, | ||
| "REGION_PLACEHOLDER": AWS_REGION, "GITHUB_SOURCE_VERSION_PLACEHOLDER": GITHUB_SOURCE_VERSION} | ||
| content = YmlLoader.load("./cdk/ssm/windows_docker_build_ssm_document.yaml", placeholder_map) | ||
| ssm.CfnDocument(scope=self, | ||
| id="{}-ssm-document".format(id), | ||
| name=SSM_DOCUMENT_NAME, | ||
| content=content, | ||
| document_type="Command") | ||
| ecr_repo = "{}.dkr.ecr.{}.amazonaws.com/{}".format( | ||
| AWS_ACCOUNT, AWS_REGION, WINDOWS_X86_ECR_REPO | ||
| ) | ||
| placeholder_map = { | ||
| "ECR_PLACEHOLDER": ecr_repo, | ||
| "GITHUB_OWNER_PLACEHOLDER": GITHUB_REPO_OWNER, | ||
| "REGION_PLACEHOLDER": AWS_REGION, | ||
| "GITHUB_SOURCE_VERSION_PLACEHOLDER": GITHUB_SOURCE_VERSION, | ||
| } | ||
| content = YmlLoader.load( | ||
| "./cdk/ssm/windows_docker_build_ssm_document.yaml", placeholder_map | ||
| ) | ||
| ssm.CfnDocument( | ||
| scope=self, | ||
| id="{}-ssm-document".format(id), | ||
| name=SSM_DOCUMENT_NAME, | ||
| content=content, | ||
| document_type="Command", | ||
| ) | ||
|
|
||
| # Define a S3 bucket to store windows docker files and build scripts. | ||
| s3.Bucket(scope=self, | ||
| id="{}-s3".format(id), | ||
| bucket_name=S3_BUCKET_NAME, | ||
| block_public_access=s3.BlockPublicAccess.BLOCK_ALL) | ||
| s3.Bucket( | ||
| scope=self, | ||
| id="{}-s3".format(id), | ||
| bucket_name=S3_BUCKET_NAME, | ||
| block_public_access=s3.BlockPublicAccess.BLOCK_ALL, | ||
| ) | ||
|
|
||
| # Define a role for EC2. | ||
| ecr_power_user_policy = iam.PolicyDocument.from_json(ecr_power_user_policy_in_json([WINDOWS_X86_ECR_REPO])) | ||
| s3_read_write_policy = iam.PolicyDocument.from_json(s3_read_write_policy_in_json(S3_BUCKET_NAME)) | ||
| inline_policies = {"ecr_power_user_policy": ecr_power_user_policy, "s3_read_write_policy": s3_read_write_policy} | ||
| role = iam.Role(scope=self, id="{}-role".format(id), | ||
| assumed_by=iam.ServicePrincipal("ec2.amazonaws.com"), | ||
| inline_policies=inline_policies, | ||
| managed_policies=[ | ||
| iam.ManagedPolicy.from_aws_managed_policy_name("AmazonSSMManagedInstanceCore") | ||
| ]) | ||
| ecr_power_user_policy = iam.PolicyDocument.from_json( | ||
| ecr_power_user_policy_in_json([WINDOWS_X86_ECR_REPO]) | ||
| ) | ||
| s3_read_write_policy = iam.PolicyDocument.from_json( | ||
| s3_read_write_policy_in_json(S3_BUCKET_NAME) | ||
| ) | ||
| inline_policies = { | ||
| "ecr_power_user_policy": ecr_power_user_policy, | ||
| "s3_read_write_policy": s3_read_write_policy, | ||
| } | ||
| role = iam.Role( | ||
| scope=self, | ||
| id="{}-role".format(id), | ||
| assumed_by=iam.ServicePrincipal("ec2.amazonaws.com"), | ||
| inline_policies=inline_policies, | ||
| managed_policies=[ | ||
| iam.ManagedPolicy.from_aws_managed_policy_name( | ||
| "AmazonSSMManagedInstanceCore" | ||
| ) | ||
| ], | ||
| ) | ||
|
|
||
| # Define Windows EC2 instance, where the SSM document will be executed. | ||
| # TODO: This AMI does not have docker installed by default anymore. Find another Windows machine | ||
| # that has docker by default or update the ssm document to properly install docker. | ||
| machine_image = ec2.MachineImage.latest_windows( | ||
| ec2.WindowsVersion.WINDOWS_SERVER_2019_ENGLISH_FULL_BASE) | ||
| ec2.WindowsVersion.WINDOWS_SERVER_2019_ENGLISH_FULL_BASE | ||
| ) | ||
| vpc = ec2.Vpc(scope=self, id="{}-vpc".format(id)) | ||
| block_device_volume = ec2.BlockDeviceVolume.ebs(volume_size=200, delete_on_termination=True) | ||
| block_device = ec2.BlockDevice(device_name="/dev/sda1", volume=block_device_volume) | ||
| instance = ec2.Instance(scope=self, | ||
| id="{}-instance".format(id), | ||
| instance_type=ec2.InstanceType(instance_type_identifier="m5d.xlarge"), | ||
| vpc=vpc, | ||
| role=role, | ||
| block_devices=[block_device], | ||
| vpc_subnets=ec2.SubnetSelection(subnet_type=ec2.SubnetType.PUBLIC), | ||
| machine_image=machine_image) | ||
| block_device_volume = ec2.BlockDeviceVolume.ebs( | ||
| volume_size=200, delete_on_termination=True | ||
| ) | ||
| block_device = ec2.BlockDevice( | ||
| device_name="/dev/sda1", volume=block_device_volume | ||
| ) | ||
|
|
||
| setup_user_data = ec2.UserData.for_windows() | ||
| setup_user_data.add_commands( | ||
| "Install-WindowsFeature -Name Containers -IncludeAllSubFeature -IncludeManagementTools", | ||
| "Set-ExecutionPolicy Bypass -Scope Process -Force; [Net.ServicePointManager]::SecurityProtocol = [Net.ServicePointManager]::SecurityProtocol -bor [Net.SecurityProtocolType]::Tls12; $env:chocolateyUseWindowsCompression = 'true'; [System.Net.ServicePointManager]::SecurityProtocol = [System.Net.ServicePointManager]::SecurityProtocol -bor 3072; iex ((New-Object System.Net.WebClient).DownloadString('https://chocolatey.org/install.ps1')) | Out-Null", | ||
| "choco install docker-cli -y", | ||
| "choco install docker-engine -y", | ||
| "choco install git --version 2.23.0 -y", | ||
| "Set-Service -Name docker -StartupType Automatic", | ||
| "Restart-Computer -Force", | ||
| ) | ||
|
|
||
| instance = ec2.Instance( | ||
| scope=self, | ||
| id="{}-instance".format(id), | ||
| instance_type=ec2.InstanceType(instance_type_identifier="m5d.xlarge"), | ||
| vpc=vpc, | ||
| role=role, | ||
| block_devices=[block_device], | ||
| vpc_subnets=ec2.SubnetSelection(subnet_type=ec2.SubnetType.PUBLIC), | ||
| machine_image=machine_image, | ||
| user_data=setup_user_data, | ||
| ) | ||
|
|
||
| Tags.of(instance).add(WIN_EC2_TAG_KEY, WIN_EC2_TAG_VALUE) |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters