Updating DLAC to use CDKv2.

Author: EC2 Default User

DeployOrUpdateDataLake.sh

@@ -1,9 +1,5 @@
 #!/bin/sh
 npm run build
-currentPrincipalArn=$(aws sts get-caller-identity --query Arn --output text)
-#Just in case you are using an IAM role, we will switch the identity from your STS arn to the underlying role ARN.
-currentPrincipalArn=$(sed 's/\(sts\)\(.*\)\(assumed-role\)\(.*\)\(\/.*\)/iam\2role\4/' <<< $currentPrincipalArn)
-jq '.context.starterLakeFormationAdmin = $currentPrincipalArn' --arg currentPrincipalArn $currentPrincipalArn cdk.json > tmp.$$.json && mv tmp.$$.json cdk.json
 cdk deploy CoreDataLake --require-approval never
 #cdk deploy ExampleS3DataSet --require-approval never
 #cdk deploy ExamplePgRdsDataSet --require-approval never

InstallCdkDependencies.sh

@@ -1,7 +1,4 @@
 npm install -g aws-cdk
 npm install "@types/node" --save-dev
 npm update
-sudo apt install jq -y
-sudo yum install jq -y
-npm install
 cdk bootstrap

bin/aws.ts

@@ -14,8 +14,7 @@ import { BaselineStack } from '../lib/Baseline-stack';
 const app = new App();
 
 const coreDataLake = new DataLakeStack(app, 'CoreDataLake', {
-    description: "AWS Data Lake as Code core data lake template. (ib-87ce095eDf)",
-    starterLakeFormationAdminPrincipalArn: app.node.tryGetContext("starterLakeFormationAdmin")
+    description: "AWS Data Lake as Code core data lake template. (ib-87ce095eDf)"
 });
 
 

cdk.json

@@ -3,7 +3,6 @@
   "context": {
     "@aws-cdk/aws-rds:lowercaseDbIdentifier": false,
     "@aws-cdk/aws-apigateway:usagePlanKeyOrderInsensitiveId": false,
-    "@aws-cdk/core:stackRelativeExports": false,
-    "starterLakeFormationAdmin": "arn:aws:iam::XXXXX:role/Cloud9-Roda-IAM-Role"
+    "@aws-cdk/core:stackRelativeExports": false
   }
 }

lib/constructs/data-lake-enrollment.ts

@@ -20,7 +20,8 @@ export class DataLakeEnrollment extends Construct {
     private CoarseResourceAccessPolicy: iam.ManagedPolicy;
     private CoarseIamPolciesApplied: boolean;
     private WorkflowCronScheduleExpression?: string;
-    
+    public SourceCfnResource?: lakeformation.CfnResource;
+    public DatalakeCfnResource?: lakeformation.CfnResource;
 
     constructor(scope: Construct, id: string, props: DataLakeEnrollment.DataLakeEnrollmentProps) {
         super(scope, id);
@@ -32,19 +33,19 @@ export class DataLakeEnrollment extends Construct {
 
     }
 
-    grantGlueRoleLakeFormationPermissions(DataSetGlueRole: iam.Role, DataSetName: string) {
+    grantGlueRoleLakeFormationPermissions(DataSetGlueRole: iam.Role, DataSetName: string, description: string, resource?: lakeformation.CfnResource) {
 
         this.grantDataLocationPermissions(this.DataEnrollment.DataSetGlueRole, {
             Grantable: true,
-            GrantResourcePrefix: `${DataSetName}locationGrant`,
+            GrantResourcePrefix: `${DataSetName}${description}locationGrant`,
             Location: this.DataEnrollment.DataLakeBucketName,
             LocationPrefix: this.DataEnrollment.DataLakePrefix
-        });
+        }, resource);
 
         this.grantDatabasePermission(this.DataEnrollment.DataSetGlueRole,  {		     
 		     DatabasePermissions: [DataLakeEnrollment.DatabasePermission.All],
              GrantableDatabasePermissions: [DataLakeEnrollment.DatabasePermission.All],
-             GrantResourcePrefix: `${DataSetName}RoleGrant`
+             GrantResourcePrefix: `${DataSetName}${description}RoleGrant`
 		}, true);
     }
 
@@ -75,8 +76,8 @@ export class DataLakeEnrollment extends Construct {
             "Resource": [
                 `arn:aws:glue:${Stack.of(this).region}:${Stack.of(this).account}:catalog`,
                 `arn:aws:glue:${Stack.of(this).region}:${Stack.of(this).account}:database/default`,
-                this.DataEnrollment.Dataset_Datalake.ref,
-                `arn:aws:glue:${Stack.of(this).region}:${Stack.of(this).account}:table/${this.DataEnrollment.Dataset_Datalake.getAtt('DatabaseInput.Name').toString()}/*`
+                `arn:aws:glue:${Stack.of(this).region}:${Stack.of(this).account}:database/${this.DataEnrollment.Dataset_DatalakeDatabaseName}`,
+                `arn:aws:glue:${Stack.of(this).region}:${Stack.of(this).account}:table/${this.DataEnrollment.Dataset_DatalakeDatabaseName}/*`
             ],
             "Effect": "Allow"
 		};
@@ -121,7 +122,12 @@ export class DataLakeEnrollment extends Construct {
 
     	const policyParams = {
     	  policyName: `${this.DataSetName}-coarseIamDataLakeAccessPolicy`,
-    	  statements: [s3PolicyStatement,gluePolicyStatement, athenaPolicyStatement, coarseLakeFormationPolicy]
+    	  statements: [
+    	      s3PolicyStatement,
+    	      gluePolicyStatement, 
+    	      athenaPolicyStatement, 
+    	      coarseLakeFormationPolicy
+    	      ]
         }
 
 	    this.CoarseResourceAccessPolicy = new iam.ManagedPolicy(this, `${this.DataSetName}-coarseIamDataLakeAccessPolicy`, policyParams );
@@ -356,7 +362,7 @@ export class DataLakeEnrollment extends Construct {
 
     public grantDatabasePermission(principal: iam.IPrincipal, permissionGrant: DataLakeEnrollment.DatabasePermissionGrant, includeSourceDb: boolean = false){
 
-        const databaseName = this.DataEnrollment.Dataset_Datalake.getAtt('DatabaseInput.Name').toString();
+        const databaseName = this.DataEnrollment.Dataset_DatalakeDatabaseName;
         var grantIdPrefix = ""
         var dataLakePrincipal : lakeformation.CfnPermissions.DataLakePrincipalProperty = {
             dataLakePrincipalIdentifier: ""
@@ -404,7 +410,7 @@ export class DataLakeEnrollment extends Construct {
     public grantTablePermissions(principal: iam.IPrincipal, permissionGrant: DataLakeEnrollment.TablePermissionGrant){
 
         const coreGrant = this.setupIamAndLakeFormationDatabasePermissionForPrincipal(principal, permissionGrant.DatabasePermissions, permissionGrant.GrantableDatabasePermissions);
-        const databaseName = this.DataEnrollment.Dataset_Datalake.getAtt('DatabaseInput.Name').toString();
+        const databaseName = this.DataEnrollment.Dataset_DatalakeDatabaseName;   
         permissionGrant.tables.forEach(table => {
             var tableResourceProperty : lakeformation.CfnPermissions.ResourceProperty = {
                 tableResource:{
@@ -487,7 +493,7 @@ export class DataLakeEnrollment extends Construct {
 	private setupIamAndLakeFormationDatabasePermissionForPrincipal(principal: iam.IPrincipal, databasePermissions: Array<DataLakeEnrollment.DatabasePermission>, grantableDatabasePermissions: Array<DataLakeEnrollment.DatabasePermission> ){
 
         this.grantCoarseIamRead(principal);
-        const databaseName = this.DataEnrollment.Dataset_Datalake.getAtt('DatabaseInput.Name').toString();
+        const databaseName = this.DataEnrollment.Dataset_DatalakeDatabaseName;
 
         var grantIdPrefix = ""
         var dataLakePrincipal : lakeformation.CfnPermissions.DataLakePrincipalProperty = {

lib/constructs/data-set-enrollment.ts

@@ -1,5 +1,5 @@
 import { Construct,  } from 'constructs';
-import { Aws, App, Stack, Resource, StackProps, CustomResource,  Duration } from 'aws-cdk-lib';
+import { Aws, App, Stack, Resource, StackProps, CustomResource,  Duration, Fn, Token } from 'aws-cdk-lib';
 
 import * as ec2 from 'aws-cdk-lib/aws-ec2';
 import * as iam from 'aws-cdk-lib/aws-iam';
@@ -186,20 +186,24 @@ export class DataSetEnrollment extends Construct {
     public readonly Dataset_Source: glue.CfnDatabase;
     public readonly Dataset_Datalake: glue.CfnDatabase;
 
+	public readonly Dataset_SourceDatabaseName: string;
+    public readonly Dataset_DatalakeDatabaseName: string;
+    
     public readonly DataLakeBucketName: string;
     public readonly DataLakePrefix: string;
 	public readonly DataLakeTargets: glue.CfnCrawler.TargetsProperty;
 
 
-	private setupCrawler(targetGlueDatabase: glue.CfnDatabase, targets: glue.CfnCrawler.TargetsProperty, isSourceCrawler: boolean){
+	private setupCrawler(targetGlueDatabase: glue.CfnDatabase, targets: glue.CfnCrawler.TargetsProperty, isSourceCrawler: boolean, databaseName: string){
 
 		var sourceCrawler = isSourceCrawler ? "src" : "dl";
 
+		
 		return new glue.CfnCrawler(this,  `${this.DataSetName}-${sourceCrawler}-crawler`,{
 			name: `${this.DataSetName}_${sourceCrawler}_crawler`, 
 			targets: targets, 
 			role: this.DataSetGlueRole.roleName,
-			databaseName: targetGlueDatabase.getAtt('DatabaseInput.Name').toString(), 
+			databaseName: databaseName,
 			schemaChangePolicy: {
 				deleteBehavior: "DEPRECATE_IN_DATABASE", 
 				updateBehavior: "UPDATE_IN_DATABASE",
@@ -219,17 +223,20 @@ export class DataSetEnrollment extends Construct {
 
 		this.DataSetName = props.dataSetName;
 
+		this.Dataset_SourceDatabaseName = `${props.dataSetName}_src`;
+		this.Dataset_DatalakeDatabaseName = `${props.dataSetName}_dl`;
+		
 		this.Dataset_Source = new glue.CfnDatabase(this, `${props.dataSetName}_src`, {
 			catalogId: Aws.ACCOUNT_ID,
 			databaseInput: {
-				name: `${props.dataSetName}_src`,
+				name: this.Dataset_SourceDatabaseName,
 				locationUri: `s3://${props.dataLakeBucket.bucketName}/${props.dataSetName}/`
 			}
 		});
 		this.Dataset_Datalake = new glue.CfnDatabase(this, `${props.dataSetName}_dl`, {
 			catalogId: Aws.ACCOUNT_ID,
 			databaseInput: {
-				name:  `${props.dataSetName}_dl`,
+				name:  this.Dataset_DatalakeDatabaseName,
 				locationUri: `s3://${props.dataLakeBucket.bucketName}/${props.dataSetName}/`
 			}
 		});
@@ -261,9 +268,8 @@ export class DataSetEnrollment extends Construct {
 		if(typeof props.SourceAccessPolicy !== 'undefined'){
 			props.SourceAccessPolicy.attachToRole(this.DataSetGlueRole);	
 		}
-		 
-		 
-		const sourceCrawler = this.setupCrawler(this.Dataset_Source, props.SourceTargets, true);
+
+		const sourceCrawler = this.setupCrawler(this.Dataset_Source, props.SourceTargets, true, this.Dataset_SourceDatabaseName);
 
 
 
@@ -300,16 +306,7 @@ export class DataSetEnrollment extends Construct {
 		const etl_job = new glue.CfnJob(this, `${props.dataSetName}-EtlJob`, jobParams );
 
 
-		const datalake_crawler = this.setupCrawler(this.Dataset_Datalake, this.DataLakeTargets, false);
-		
-		// const datalake_crawler = this.setupCrawler(this.Dataset_Datalake, {
-		// 		s3Targets: [
-		// 			{
-		// 				path: `s3://${props.dataLakeBucket.bucketName}/${props.dataSetName}/`
-		// 			}
-		// 		]
-		// }, false);
-
+		const datalake_crawler = this.setupCrawler(this.Dataset_Datalake, this.DataLakeTargets, false, this.Dataset_DatalakeDatabaseName);
 
 		const datalakeEnrollmentWorkflow = new DataLakeEnrollmentWorkflow(this,`${props.dataSetName}DataLakeWorkflow`,{
 			workfowName: `${props.dataSetName}_DataLakeEnrollmentWorkflow`,

lib/constructs/rds-data-set-enrollment.ts

@@ -80,7 +80,7 @@ export class RDSDataSetEnrollment extends DataLakeEnrollment {
 
 
 		this.createCoarseIamPolicy();
-		this.grantGlueRoleLakeFormationPermissions(this.DataEnrollment.DataSetGlueRole, props.DataSetName); 
+		this.grantGlueRoleLakeFormationPermissions(this.DataEnrollment.DataSetGlueRole, props.DataSetName, 'src'); 
 		this.grantCoarseIamRead(this.DataEnrollment.DataSetGlueRole);
 	}
 }

lib/constructs/s3-data-set-enrollment.ts

@@ -23,24 +23,24 @@ export class S3dataSetEnrollment extends DataLakeEnrollment{
 
     private readonly sourceBucket: s3.IBucket;
 
-    setupGlueRoleLakeFormationPermissions(DataSetGlueRole: iam.Role, DataSetName: string, sourceDataBucket: s3.IBucket) {
+    setupGlueRoleLakeFormationPermissions(DataSetGlueRole: iam.Role, DataSetName: string, sourceDataBucket: s3.IBucket, locationDescription: string) {
 
-        const sourceLakeFormationLocation = new lakeformation.CfnResource(  this, "sourceLakeFormationLocation",
+        const sourceLakeFormationLocation = new lakeformation.CfnResource(  this, `sourceLakeFormationLocation-${sourceDataBucket}`,
           {
             resourceArn: sourceDataBucket.bucketArn,
             roleArn: this.DataEnrollment.DataSetGlueRole.roleArn,
             useServiceLinkedRole: true,
           }
         );
 
-        super.grantGlueRoleLakeFormationPermissions(DataSetGlueRole, DataSetName);
-
         this.grantDataLocationPermissions(this.DataEnrollment.DataSetGlueRole, {
             Grantable: true,
-            GrantResourcePrefix: `${DataSetName}SourcelocationGrant`,
+            GrantResourcePrefix: `${DataSetName}-${locationDescription}-locationGrant`,
             Location: sourceDataBucket.bucketName,
             LocationPrefix: "/"
         }, sourceLakeFormationLocation);
+        
+        return sourceLakeFormationLocation;
 
     }
 
@@ -114,8 +114,11 @@ export class S3dataSetEnrollment extends DataLakeEnrollment{
 
         this.createCoarseIamPolicy();
 
+        this.SourceCfnResource = this.setupGlueRoleLakeFormationPermissions(this.DataEnrollment.DataSetGlueRole, props.DataSetName, props.sourceBucket, "src"); 
+        this.DatalakeCfnResource = this.setupGlueRoleLakeFormationPermissions(this.DataEnrollment.DataSetGlueRole, props.DataSetName, props.dataLakeBucket, "dl"); 
 
-        this.setupGlueRoleLakeFormationPermissions(this.DataEnrollment.DataSetGlueRole, props.DataSetName, props.sourceBucket); 
+        super.grantGlueRoleLakeFormationPermissions(this.DataEnrollment.DataSetGlueRole, props.DataSetName, 'src', this.SourceCfnResource);
+        super.grantGlueRoleLakeFormationPermissions(this.DataEnrollment.DataSetGlueRole, props.DataSetName, 'dl', this.DatalakeCfnResource);
 
         this.grantCoarseIamRead(this.DataEnrollment.DataSetGlueRole);
 

lib/stacks/datalake-stack.ts

@@ -1,5 +1,5 @@
 import { Construct } from 'constructs';
-import { App, Stack, Resource, StackProps, CustomResource,  Duration } from 'aws-cdk-lib';
+import { App, Stack, Resource, StackProps, CustomResource,  Duration, DefaultStackSynthesizer, Fn } from 'aws-cdk-lib';
 
 import * as ec2 from 'aws-cdk-lib/aws-ec2';
 import * as iam from 'aws-cdk-lib/aws-iam';
@@ -19,7 +19,7 @@ import {
 } from "../constructs/data-set-enrollment";
 
 export interface DataLakeStackProps extends StackProps {
-  starterLakeFormationAdminPrincipalArn: string;
+
 }
 
 export class DataLakeStack extends Stack {
@@ -29,7 +29,6 @@ export class DataLakeStack extends Stack {
   public readonly LakeFormationResource: lakeformation.CfnResource;
   public readonly PrimaryAthenaWorkgroup: athena.CfnWorkGroup;
   private readonly bucketRole: iam.Role;
-  private readonly starterAdminArn: string;
 
   public grantAthenaResultsBucketPermission(principal: iam.IPrincipal) {
     if (principal instanceof iam.Role) {
@@ -63,15 +62,17 @@ export class DataLakeStack extends Stack {
   constructor(scope: Construct, id: string, props: DataLakeStackProps) {
     super(scope, id, props);
 
+
     this.DataLakeBucket = new s3.Bucket(this, 'dataLakeBucket',{
+      bucketName: 'aws-roda-fintech-datalake'
     });
     this.AthenaResultsBucket = new s3.Bucket(this, "athenaResultsBucket");
 
-    new lakeformation.CfnDataLakeSettings(this, "starterAdminPermission", {
+
+    new lakeformation.CfnDataLakeSettings(this, "cdkCfnExecRoleAdminPermission", {
       admins: [
         {
-          dataLakePrincipalIdentifier:
-            props.starterLakeFormationAdminPrincipalArn,
+          dataLakePrincipalIdentifier: Fn.sub((this.synthesizer as DefaultStackSynthesizer).cloudFormationExecutionRoleArn)
         },
       ],
     });
@@ -94,9 +95,7 @@ export class DataLakeStack extends Stack {
       coarseAthenaResultBucketAccess
     );
 
-    this.AthenaResultsBucketAccessPolicy = new iam.ManagedPolicy(
-      this,
-      `athenaResultBucketAccessPolicy`,
+    this.AthenaResultsBucketAccessPolicy = new iam.ManagedPolicy(this, `athenaResultBucketAccessPolicy`,
       {
         document: coarseAthenaResultBucketAccessPolicyDoc,
         description: `AthenaResultBucketAccessPolicy`,
@@ -111,19 +110,15 @@ export class DataLakeStack extends Stack {
 
     this.DataLakeBucket.grantReadWrite(this.bucketRole);
 
-    this.LakeFormationResource = new lakeformation.CfnResource(
-      this,
-      "dataLakeBucketLakeFormationResource",
+    this.LakeFormationResource = new lakeformation.CfnResource(this,"dataLakeBucketLakeFormationResource",
       {
         resourceArn: this.DataLakeBucket.bucketArn,
         roleArn: this.bucketRole.roleArn,
         useServiceLinkedRole: true,
       }
     );
 
-    const workGroupConfigCustResourceRole = new iam.Role(
-      this,
-      "workGroupConfigCustResourceRole",
+    const workGroupConfigCustResourceRole = new iam.Role(this,"workGroupConfigCustResourceRole",
       {
         assumedBy: new iam.ServicePrincipal("lambda.amazonaws.com"),
       }