From 44f5cc3cbb4adf1071a5505d8b121a5b4422cab5 Mon Sep 17 00:00:00 2001 From: Kazuho CryerShinozuka Date: Thu, 14 Nov 2024 09:31:37 +0900 Subject: [PATCH 1/2] add validation for ray job --- packages/@aws-cdk/aws-glue-alpha/lib/job.ts | 11 +++++++ .../@aws-cdk/aws-glue-alpha/test/job.test.ts | 29 +++++++++++++++++++ 2 files changed, 40 insertions(+) diff --git a/packages/@aws-cdk/aws-glue-alpha/lib/job.ts b/packages/@aws-cdk/aws-glue-alpha/lib/job.ts index 6c6a1d5168f5d..d6f9a954861cd 100644 --- a/packages/@aws-cdk/aws-glue-alpha/lib/job.ts +++ b/packages/@aws-cdk/aws-glue-alpha/lib/job.ts @@ -738,6 +738,17 @@ export class Job extends JobBase { } } + // Validate Ray job properties + // See https://github.com/aws/aws-cdk/issues/29612 + if (executable.type.name === JobType.RAY.name) { + if (props.workerType !== WorkerType.Z_2X) { + throw new Error(`WorkerType must be Z_2X for Ray jobs, got: ${props.workerType}`); + } + if (props.timeout !== undefined) { + throw new Error('Timeout cannot be set for Ray jobs'); + } + } + let maxCapacity = props.maxCapacity; if (maxCapacity !== undefined && (props.workerType && props.workerCount !== undefined)) { throw new Error('maxCapacity cannot be used when setting workerType and workerCount'); diff --git a/packages/@aws-cdk/aws-glue-alpha/test/job.test.ts b/packages/@aws-cdk/aws-glue-alpha/test/job.test.ts index c8df118da127c..fbe8104abe5ce 100644 --- a/packages/@aws-cdk/aws-glue-alpha/test/job.test.ts +++ b/packages/@aws-cdk/aws-glue-alpha/test/job.test.ts @@ -896,6 +896,35 @@ describe('Job', () => { workerCount: 2, })).toThrow('Runtime is required for Ray jobs'); }); + + test.each([ + glue.WorkerType.G_025X, + glue.WorkerType.G_1X, + glue.WorkerType.G_2X, + glue.WorkerType.G_4X, + glue.WorkerType.G_8X, + ])('throw error for unsupported worker type', (workerType) => { + expect(() => new glue.Job(stack, 'Job', { + executable: glue.JobExecutable.pythonRay({ + glueVersion: glue.GlueVersion.V4_0, + pythonVersion: glue.PythonVersion.THREE_NINE, + script, + }), + workerType, + workerCount: 2, + })).toThrow(`WorkerType must be Z_2X for Ray jobs, got: ${workerType}`); + }); + }); + + test('throw error for specifying timeout', () => { + expect(() => new glue.Job(stack, 'Job', { + executable: glue.JobExecutable.pythonEtl({ + glueVersion: glue.GlueVersion.V2_0, + pythonVersion: glue.PythonVersion.THREE, + script, + }), + timeout: cdk.Duration.minutes(5), + })).toThrow('Timeout cannot be set for Ray jobs'); }); test('etl job with all props should synthesize correctly', () => { From fa9de80a5c5399f3b6ac9d352bf226a7fd2007c8 Mon Sep 17 00:00:00 2001 From: Kazuho CryerShinozuka Date: Thu, 14 Nov 2024 13:00:42 +0900 Subject: [PATCH 2/2] fix unit test --- packages/@aws-cdk/aws-glue-alpha/test/job.test.ts | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/packages/@aws-cdk/aws-glue-alpha/test/job.test.ts b/packages/@aws-cdk/aws-glue-alpha/test/job.test.ts index fbe8104abe5ce..dd5d92fd7c3bf 100644 --- a/packages/@aws-cdk/aws-glue-alpha/test/job.test.ts +++ b/packages/@aws-cdk/aws-glue-alpha/test/job.test.ts @@ -908,6 +908,7 @@ describe('Job', () => { executable: glue.JobExecutable.pythonRay({ glueVersion: glue.GlueVersion.V4_0, pythonVersion: glue.PythonVersion.THREE_NINE, + runtime: glue.Runtime.RAY_TWO_FOUR, script, }), workerType, @@ -918,11 +919,13 @@ describe('Job', () => { test('throw error for specifying timeout', () => { expect(() => new glue.Job(stack, 'Job', { - executable: glue.JobExecutable.pythonEtl({ - glueVersion: glue.GlueVersion.V2_0, - pythonVersion: glue.PythonVersion.THREE, + executable: glue.JobExecutable.pythonRay({ + glueVersion: glue.GlueVersion.V4_0, + pythonVersion: glue.PythonVersion.THREE_NINE, + runtime: glue.Runtime.RAY_TWO_FOUR, script, }), + workerType: glue.WorkerType.Z_2X, timeout: cdk.Duration.minutes(5), })).toThrow('Timeout cannot be set for Ray jobs'); });