diff --git a/.github/workflows/tf-apply.yml b/.github/workflows/tf-apply.yml index 325f58dd..ce2433d9 100644 --- a/.github/workflows/tf-apply.yml +++ b/.github/workflows/tf-apply.yml @@ -59,6 +59,11 @@ jobs: account: 659087519042 role: cds-aws-lz-apply + - account_folder: org_account + module: billing_extract_tags + account: 659087519042 + role: cds-aws-lz-apply + - account_folder: log_archive module: main account: 274536870005 diff --git a/.github/workflows/tf-plan.yml b/.github/workflows/tf-plan.yml index 17a504aa..0c58e246 100644 --- a/.github/workflows/tf-plan.yml +++ b/.github/workflows/tf-plan.yml @@ -50,7 +50,11 @@ jobs: module: sentinel_oidc account: 659087519042 role: cds-aws-lz-plan - admin_sso_role_arn: ADMIN_SSO_ROLE_ARN + + - account_folder: org_account + module: billing_extract_tags + account: 659087519042 + role: cds-aws-lz-plan - account_folder: log_archive module: main diff --git a/.gitignore b/.gitignore index 2dbb8a49..dc7595f8 100644 --- a/.gitignore +++ b/.gitignore @@ -12,4 +12,6 @@ debug.log creds.sh .tool-versions -.DS_Store \ No newline at end of file +.DS_Store + +__pycache__ \ No newline at end of file diff --git a/terragrunt/org_account/billing_extract_tags/.terraform.lock.hcl b/terragrunt/org_account/billing_extract_tags/.terraform.lock.hcl new file mode 100644 index 00000000..7094c1d1 --- /dev/null +++ b/terragrunt/org_account/billing_extract_tags/.terraform.lock.hcl @@ -0,0 +1,44 @@ +# This file is maintained automatically by "terraform init". +# Manual edits may be lost in future updates. + +provider "registry.terraform.io/hashicorp/archive" { + version = "2.4.2" + hashes = [ + "h1:G4v6F6Lhqlo3EKGBKEK/kJRhNcQiRrhEdUiVpBHKHOA=", + "zh:08faed7c9f42d82bc3d406d0d9d4971e2d1c2d34eae268ad211b8aca57b7f758", + "zh:3564112ed2d097d7e0672378044a69b06642c326f6f1584d81c7cdd32ebf3a08", + "zh:53cd9afd223c15828c1916e68cb728d2be1cbccb9545568d6c2b122d0bac5102", + "zh:5ae4e41e3a1ce9d40b6458218a85bbde44f21723943982bca4a3b8bb7c103670", + "zh:5b65499218b315b96e95c5d3463ea6d7c66245b59461217c99eaa1611891cd2c", + "zh:78d5eefdd9e494defcb3c68d282b8f96630502cac21d1ea161f53cfe9bb483b3", + "zh:7f45b35a8330bebd184c2545a41782ff58240ed6ba947274d9881dd5da44b02e", + "zh:87e67891033214e55cfead1391d68e6a3bf37993b7607753237e82aa3250bb71", + "zh:de3590d14037ad81fc5cedf7cfa44614a92452d7b39676289b704a962050bc5e", + "zh:e7e6f2ea567f2dbb3baa81c6203be69f9cd6aeeb01204fd93e3cf181e099b610", + "zh:fd24d03c89a7702628c2e5a3c732c0dede56fa75a08da4a1efe17b5f881c88e2", + "zh:febf4b7b5f3ff2adff0573ef6361f09b6638105111644bdebc0e4f575373935f", + ] +} + +provider "registry.terraform.io/hashicorp/aws" { + version = "5.11.0" + constraints = ">= 4.9.0, >= 4.40.0, <= 5.11.0" + hashes = [ + "h1:OyEBhYcTPChBb0gooSlLIcrxakh72qAN+Sd8Oo12uoc=", + "zh:2913af44f9b584f756e5548d5ddc5a251c6d68a7fcd7c41d1418a800a94ef113", + "zh:31d2bfa84608b74ff5896f41b09e5927d7c37d18875277a51dcd75a1fea3f909", + "zh:8538ff18e3b4822178e793f06764efdbb84c62227c1051af7d2409ab7be37bfc", + "zh:8a9295e623327613fc02a6994e73c61b9d0d195bf6fabdb31ee9fd0e6778f62b", + "zh:9b12af85486a96aedd8d7984b0ff811a4b42e3d88dad1a3fb4c0b580d04fa425", + "zh:a65877248951eadf0d16a3260e85f6b178645da7f1897bc7bda6f12fdbec8e47", + "zh:a70772851e2c87cc1e10c35389718a544746adc4acbbed129243c0972c367fc6", + "zh:b10ca631318f8d1d9a2baa318139bc9e545e51efaf677afece173badce75b44c", + "zh:ca2a5698c33158549fa084ad601610eae94498cba445458391b507da22355402", + "zh:cdbfc4d64161561bfbcaee5d9b078077ed986131a1eab32ff30e71be09037eec", + "zh:ce499f93835bf3d28c13ba98a0a220ff541a827fb400fa931601a375b907b56d", + "zh:da6af610e66e96280a299071a698568b505c2456bb15c906304d6f39578c72e3", + "zh:e42714e085126c10d8f29664143f97d771b6cc6887d27cdf6c4007ab12af4646", + "zh:e86dd0c561c73512acba69f55041adfc04d0467f592f52337a7ac600fbc93680", + "zh:f5da95bbd44809534c6678e9b1ae0b390331a5619f2ae353c6b88e96ae855cc0", + ] +} diff --git a/terragrunt/org_account/billing_extract_tags/eventbridge.tf b/terragrunt/org_account/billing_extract_tags/eventbridge.tf new file mode 100644 index 00000000..672a4319 --- /dev/null +++ b/terragrunt/org_account/billing_extract_tags/eventbridge.tf @@ -0,0 +1,11 @@ +resource "aws_cloudwatch_event_rule" "billing_extract_tags" { + name = "billing_extract_tags_daily" + schedule_expression = "cron(0 5 * * ? *)" + + tags = local.common_tags +} + +resource "aws_cloudwatch_event_target" "billing_extract_tags" { + rule = aws_cloudwatch_event_rule.billing_extract_tags.name + arn = aws_lambda_function.billing_extract_tags.arn +} diff --git a/terragrunt/org_account/billing_extract_tags/iam.tf b/terragrunt/org_account/billing_extract_tags/iam.tf new file mode 100644 index 00000000..b7864323 --- /dev/null +++ b/terragrunt/org_account/billing_extract_tags/iam.tf @@ -0,0 +1,80 @@ +resource "aws_iam_role" "billing_extract_tags" { + name = "BillingExtractTags" + assume_role_policy = data.aws_iam_policy_document.billing_extract_tags_assume.json + tags = local.common_tags +} + +data "aws_iam_policy_document" "billing_extract_tags_assume" { + statement { + effect = "Allow" + actions = ["sts:AssumeRole"] + principals { + type = "Service" + identifiers = [ + "lambda.amazonaws.com"] + } + } +} + +data "aws_iam_policy_document" "billing_extract_tags" { + statement { + effect = "Allow" + actions = ["logs:CreateLogGroup"] + resources = ["arn:aws:logs:${var.region}:${var.account_id}:*"] + } + + statement { + effect = "Allow" + actions = [ + "logs:CreateLogStream", + "logs:PutLogEvents" + ] + resources = [ + "arn:aws:logs:${var.region}:${var.account_id}:log-group:/aws/lambda/billing_extract_tags:*" + ] + } + + statement { + effect = "Allow" + actions = [ + "s3:PutObject*", + "s3:ListBucket", + "s3:GetObject*", + "s3:DeleteObject*", + "s3:GetBucketLocation" + ] + resources = [ + module.billing_extract_tags.s3_bucket_arn, + "${module.billing_extract_tags.s3_bucket_arn}/*", + ] + } +} + +resource "aws_iam_policy" "billing_extract_tags" { + name = "BillingExtractTags" + policy = data.aws_iam_policy_document.billing_extract_tags.json + tags = local.common_tags +} + +resource "aws_iam_role_policy_attachment" "billing_extract_tags" { + role = aws_iam_role.billing_extract_tags.name + policy_arn = aws_iam_policy.billing_extract_tags.arn +} + +data "aws_iam_policy" "org_read_only" { + arn = "arn:aws:iam::aws:policy/AWSOrganizationsReadOnlyAccess" +} + +resource "aws_iam_role_policy_attachment" "org_read_only" { + role = aws_iam_role.billing_extract_tags.name + policy_arn = data.aws_iam_policy.org_read_only.arn +} + +data "aws_iam_policy" "lambda_insights" { + name = "CloudWatchLambdaInsightsExecutionRolePolicy" +} + +resource "aws_iam_role_policy_attachment" "lambda_insights" { + role = aws_iam_role.billing_extract_tags.name + policy_arn = data.aws_iam_policy.lambda_insights.arn +} diff --git a/terragrunt/org_account/billing_extract_tags/lambda.tf b/terragrunt/org_account/billing_extract_tags/lambda.tf new file mode 100644 index 00000000..16812146 --- /dev/null +++ b/terragrunt/org_account/billing_extract_tags/lambda.tf @@ -0,0 +1,44 @@ +data "archive_file" "billing_extract_tags" { + type = "zip" + source_file = "${path.module}/lambdas/billing_extract_tags/main.py" + output_path = "/tmp/main.py.zip" +} + +resource "aws_lambda_function" "billing_extract_tags" { + function_name = "billing_extract_tags" + role = aws_iam_role.billing_extract_tags.arn + runtime = "python3.11" + handler = "main.handler" + memory_size = 1024 + timeout = 30 + + filename = data.archive_file.billing_extract_tags.output_path + source_code_hash = filebase64sha256(data.archive_file.billing_extract_tags.output_path) + + environment { + variables = { + TARGET_BUCKET = module.billing_extract_tags.s3_bucket_id + } + } + + tracing_config { + mode = "PassThrough" + } + + tags = local.common_tags +} + +resource "aws_lambda_permission" "billing_extract_tags" { + statement_id = "AllowBillingExtractTagsDaily" + action = "lambda:InvokeFunction" + function_name = aws_lambda_function.billing_extract_tags.function_name + principal = "events.amazonaws.com" + source_arn = aws_cloudwatch_event_rule.billing_extract_tags.arn +} + +resource "aws_cloudwatch_log_group" "billing_extract_tags" { + #checkov:skip=CKV_AWS_158:We trust the AWS provided keys + name = "/aws/lambda/${aws_lambda_function.billing_extract_tags.function_name}" + retention_in_days = "14" + tags = local.common_tags +} diff --git a/terragrunt/org_account/billing_extract_tags/lambdas/billing_extract_tags/Makefile b/terragrunt/org_account/billing_extract_tags/lambdas/billing_extract_tags/Makefile new file mode 100644 index 00000000..6a0cb861 --- /dev/null +++ b/terragrunt/org_account/billing_extract_tags/lambdas/billing_extract_tags/Makefile @@ -0,0 +1,20 @@ +default: + python3 main.py + +fmt: + black . $(ARGS) + +install: + pip3 install --user -r requirements_dev.txt + +lint: + flake8 main.py + +test: + python -m pytest -s -vv . + +.PHONY: \ + fmt \ + install \ + lint \ + test \ No newline at end of file diff --git a/terragrunt/org_account/billing_extract_tags/lambdas/billing_extract_tags/main.py b/terragrunt/org_account/billing_extract_tags/lambdas/billing_extract_tags/main.py new file mode 100644 index 00000000..e5c1f1a5 --- /dev/null +++ b/terragrunt/org_account/billing_extract_tags/lambdas/billing_extract_tags/main.py @@ -0,0 +1,73 @@ +""" +Get the tags for all accounts in the organization and save them to an s3 bucket. +This is then used to enrich the billing data with the account tags to allow +for business unit filtering. +""" +import json +import logging +import os + +import boto3 + +orgs = boto3.client("organizations") +s3 = boto3.client("s3") + +TARGET_BUCKET = os.getenv("TARGET_BUCKET") + + +def lambda_handler(event, context): + """ + Get the tags for all accounts in the organization and save them to an s3 bucket + """ + logging.info("Getting account tags") + accounts = [] + accounts_result = orgs.list_accounts() + accounts += accounts_result["Accounts"] + while "NextToken" in accounts_result: + logging.info("Paginating accounts...") + accounts_result = orgs.list_accounts(NextToken=accounts_result["NextToken"]) + accounts += accounts_result["Accounts"] + + # Iterate over the accounts and get the tags and then add them to the account in the list + logging.info("Getting account tags") + for account in accounts: + account_tags = orgs.list_tags_for_resource(ResourceId=account["Id"]) + + # Convert the tags from {'Key': 'Name', 'Value': 'Dev'} to {'Name': 'Dev'} + account_tags["Tags"] = { + tag["Key"]: tag["Value"] for tag in account_tags["Tags"] + } + account["Tags"] = account_tags["Tags"] + + # Get a set of all possible tag keys + tag_keys = set() + for account in accounts: + tag_keys.update(account["Tags"].keys()) + logging.info(f"Found tag keys: {tag_keys}") + + # Add empty strings for all the tags that are not present in the account + logging.info("Adding empty strings for missing tags") + for account in accounts: + for tag_key in tag_keys: + if tag_key not in account["Tags"]: + account["Tags"][tag_key] = "" + + # Convert the tags into the format tag_key_name: tag_value and add them to the base object + logging.info("Converting tags to tag_key_name: tag_value") + for account in accounts: + for tag_key, tag_value in account["Tags"].items(): + account[f"tag_{tag_key}"] = tag_value + del account["Tags"] + + # .write json to string and add a newline between each record + logging.info("Writing account tags to json") + accounts = json.dumps(accounts, default=str) + # accounts = accounts.replace('},', '},\n') + # accounts = accounts.replace('[{', '[\n{') + logging.info(f"Accounts: {accounts}") + + # save accounts to an s3 bucket + logging.info("Saving account tags to s3") + s3.put_object(Bucket=TARGET_BUCKET, Key="account_tags.json", Body=accounts) + + return {"statusCode": 200} diff --git a/terragrunt/org_account/billing_extract_tags/lambdas/billing_extract_tags/requirements_dev.txt b/terragrunt/org_account/billing_extract_tags/lambdas/billing_extract_tags/requirements_dev.txt new file mode 100644 index 00000000..bae51571 --- /dev/null +++ b/terragrunt/org_account/billing_extract_tags/lambdas/billing_extract_tags/requirements_dev.txt @@ -0,0 +1,4 @@ +boto3==1.34.54 +black==23.12.1 +flake8==7.0.0 +pytest==7.4.4 \ No newline at end of file diff --git a/terragrunt/org_account/billing_extract_tags/lambdas/billing_extract_tags/test_main.py b/terragrunt/org_account/billing_extract_tags/lambdas/billing_extract_tags/test_main.py new file mode 100644 index 00000000..bd6aa60e --- /dev/null +++ b/terragrunt/org_account/billing_extract_tags/lambdas/billing_extract_tags/test_main.py @@ -0,0 +1,64 @@ +import unittest +import os +from unittest.mock import call, patch, MagicMock +from main import lambda_handler + +TARGET_BUCKET = "TARGET_BUCKET" +ACCOUNT_TAGS_KEY = "account_tags.json" + + +class TestLambdaHandler(unittest.TestCase): + def setUp(self): + self.event = {} + self.context = MagicMock() + + @patch("main.orgs.list_accounts") + @patch("main.orgs.list_tags_for_resource") + @patch("main.s3.put_object") + @patch("main.TARGET_BUCKET", TARGET_BUCKET) + def test_lambda_handler( + self, mock_s3_put, mock_orgs_list_tags, mock_orgs_list_accounts + ): + mock_orgs_list_accounts.return_value = { + "Accounts": [{"Id": "123"}], + } + mock_orgs_list_tags.return_value = {"Tags": [{"Key": "Name", "Value": "Dev"}]} + + response = lambda_handler(self.event, self.context) + + mock_orgs_list_accounts.assert_called() + mock_orgs_list_tags.assert_called_with(ResourceId="123") + mock_s3_put.assert_called_with( + Bucket=TARGET_BUCKET, + Key=ACCOUNT_TAGS_KEY, + Body='[{"Id": "123", "tag_Name": "Dev"}]', + ) + + self.assertEqual(response, {"statusCode": 200}) + + @patch("main.orgs.list_accounts") + @patch("main.orgs.list_tags_for_resource") + @patch("main.s3.put_object") + @patch("main.TARGET_BUCKET", TARGET_BUCKET) + def test_lambda_handler_pagination( + self, mock_s3_put, mock_orgs_list_tags, mock_orgs_list_accounts + ): + mock_orgs_list_accounts.side_effect = [ + {"Accounts": [{"Id": "123"}], "NextToken": "token"}, + {"Accounts": [{"Id": "456"}]}, + ] + mock_orgs_list_tags.side_effect = [ + {"Tags": [{"Key": "Name", "Value": "Dev"}]}, + {"Tags": [{"Key": "Name", "Value": "Prod"}]}, + ] + + lambda_handler(self.event, self.context) + + mock_orgs_list_accounts.assert_any_call(NextToken="token") + mock_orgs_list_tags.assert_any_call(ResourceId="123") + mock_orgs_list_tags.assert_any_call(ResourceId="456") + mock_s3_put.assert_called_with( + Bucket=TARGET_BUCKET, + Key=ACCOUNT_TAGS_KEY, + Body='[{"Id": "123", "tag_Name": "Dev"}, {"Id": "456", "tag_Name": "Prod"}]', + ) diff --git a/terragrunt/org_account/billing_extract_tags/locals.tf b/terragrunt/org_account/billing_extract_tags/locals.tf new file mode 100644 index 00000000..9c562e80 --- /dev/null +++ b/terragrunt/org_account/billing_extract_tags/locals.tf @@ -0,0 +1,6 @@ +locals { + common_tags = { + CostCentre = var.billing_code + Terraform = "true" + } +} \ No newline at end of file diff --git a/terragrunt/org_account/billing_extract_tags/s3.tf b/terragrunt/org_account/billing_extract_tags/s3.tf new file mode 100644 index 00000000..5078c676 --- /dev/null +++ b/terragrunt/org_account/billing_extract_tags/s3.tf @@ -0,0 +1,71 @@ +import { + to = module.billing_extract_tags.aws_s3_bucket.this + id = "5bf89a78-1503-4e02-9621-3ac658f558fb" +} + +import { + to = module.billing_extract_tags.aws_s3_bucket_public_access_block.this + id = "5bf89a78-1503-4e02-9621-3ac658f558fb" +} + +import { + to = aws_s3_bucket_policy.billing_extract_tags + id = "5bf89a78-1503-4e02-9621-3ac658f558fb" +} + +module "billing_extract_tags" { + source = "github.com/cds-snc/terraform-modules//S3?ref=v9.2.4" + bucket_name = "5bf89a78-1503-4e02-9621-3ac658f558fb" + + versioning = { + enabled = true + } + + billing_tag_value = var.billing_code +} + +resource "aws_s3_bucket_policy" "billing_extract_tags" { + bucket = module.billing_extract_tags.s3_bucket_id + policy = data.aws_iam_policy_document.billing_extract_tags_bucket.json +} + +data "aws_iam_policy_document" "billing_extract_tags_bucket" { + statement { + effect = "Allow" + principals { + type = "AWS" + identifiers = [aws_iam_role.billing_extract_tags.arn] + } + actions = [ + "s3:PutObject*", + "s3:ListBucket", + "s3:GetObject*", + "s3:DeleteObject*", + "s3:GetBucketLocation" + ] + resources = [ + module.billing_extract_tags.s3_bucket_arn, + "${module.billing_extract_tags.s3_bucket_arn}/*", + ] + } + + statement { + effect = "Allow" + principals { + type = "AWS" + identifiers = ["arn:aws:iam::066023111852:root"] + } + actions = [ + "s3:GetBucketLocation", + "s3:GetObject", + "s3:ListBucket", + "s3:ListBucketMultipartUploads", + "s3:ListMultipartUploadParts", + "s3:AbortMultipartUpload" + ] + resources = [ + module.billing_extract_tags.s3_bucket_arn, + "${module.billing_extract_tags.s3_bucket_arn}/*", + ] + } +} diff --git a/terragrunt/org_account/billing_extract_tags/terragrunt.hcl b/terragrunt/org_account/billing_extract_tags/terragrunt.hcl new file mode 100644 index 00000000..e68b7cf1 --- /dev/null +++ b/terragrunt/org_account/billing_extract_tags/terragrunt.hcl @@ -0,0 +1,3 @@ +include { + path = find_in_parent_folders() +} \ No newline at end of file