Unexpected throttling during deployment from UpdateRolesWithIDPFunction

[NickDario]

Amplify CLI Version

12.10.1

Question

amplify push fails as we create new environments.
We examine our cloud formation logs see the UpdateRolesWithIDPFunctionOutputs lambda function for amplify environment is the source of the error. We visit the cloudwatch logs and find this error.

Previously we used amplify v9.2.1 and deployed environments without issue, however after upgrading to v12.10.1 we are seeing this issue. This appears to occur intermittently as we have deployed some environments successfully.

Error in cloud watch:

{
    "Status": "FAILED",
    "Reason": "See the details in CloudWatch Log Stream: 2024/04/01/[$LATEST]80d0d2816e064b43824d00206af9b1b7",
    "PhysicalResourceId": "2024/04/01/[$LATEST]80d0d2816e064b43824d00206af9b1b7",
    "StackId": "arn:aws:cloudformation:us-west-2:370837355406:stack/amplify-www-lemona-140512/88476010-f06b-11ee-b2a3-02b61020254d",
    "RequestId": "c5c4f4fe-99b3-45eb-9d8f-d88fe8582fd0",
    "LogicalResourceId": "UpdateRolesWithIDPFunctionOutputs",
    "NoEcho": false,
    "Data": {
        "Error": {
            "name": "Throttling",
            "$fault": "client",
            "$metadata": {
                "httpStatusCode": 400,
                "requestId": "63175dcc-b2a1-4a43-85e3-806b74e7a9cd",
                "attempts": 3,
                "totalRetryDelay": 134
            },
            "Type": "Sender",
            "Code": "Throttling",
            "message": "Rate exceeded"
        }
    }
}

error.stack

2024-04-01T21:23:29.839Z	101336b9-30e1-45d2-84f6-68ca61681f54	INFO	Throttling: Rate exceeded
    at throwDefaultError (/var/runtime/node_modules/@aws-sdk/node_modules/@smithy/smithy-client/dist-cjs/index.js:838:20)
    at /var/runtime/node_modules/@aws-sdk/node_modules/@smithy/smithy-client/dist-cjs/index.js:847:5
    at de_CommandError (/var/runtime/node_modules/@aws-sdk/client-iam/dist-cjs/index.js:4635:14)
    at process.processTicksAndRejections (node:internal/process/task_queues:95:5)
    at async /var/runtime/node_modules/@aws-sdk/node_modules/@smithy/middleware-serde/dist-cjs/index.js:35:20
    at async /var/runtime/node_modules/@aws-sdk/node_modules/@smithy/core/dist-cjs/index.js:165:18
    at async /var/runtime/node_modules/@aws-sdk/node_modules/@smithy/middleware-retry/dist-cjs/index.js:320:38
    at async /var/runtime/node_modules/@aws-sdk/middleware-logger/dist-cjs/index.js:33:22
    at async Promise.all (index 0)
    at async handleEvent (/var/task/index.js:62:25)

Lambda source

const response = require('cfn-response');
const { IAMClient, GetRoleCommand, UpdateAssumeRolePolicyCommand } = require('@aws-sdk/client-iam');
exports.handler = function(event, context) {
    // Don't return promise, response.send() marks context as done internally
    const ignoredPromise = handleEvent(event, context)
};
async function handleEvent(event, context) {
    try {
        let authRoleName = event.ResourceProperties.authRoleName;
        let unauthRoleName = event.ResourceProperties.unauthRoleName;
        let idpId = event.ResourceProperties.idpId;
        let authParamsJson = {
            'Version': '2012-10-17',
            'Statement': [{
                'Effect': 'Allow',
                'Principal': {'Federated': 'cognito-identity.amazonaws.com'},
                'Action': 'sts:AssumeRoleWithWebIdentity',
                'Condition': {
                    'StringEquals': {'cognito-identity.amazonaws.com:aud': idpId},
                    'ForAnyValue:StringLike': {'cognito-identity.amazonaws.com:amr': 'authenticated'}
                }
            }]
        };
        let unauthParamsJson = {
            'Version': '2012-10-17',
            'Statement': [{
                'Effect': 'Allow',
                'Principal': {'Federated': 'cognito-identity.amazonaws.com'},
                'Action': 'sts:AssumeRoleWithWebIdentity',
                'Condition': {
                    'StringEquals': {'cognito-identity.amazonaws.com:aud': idpId},
                    'ForAnyValue:StringLike': {'cognito-identity.amazonaws.com:amr': 'unauthenticated'}
                }
            }]
        };
        if (event.RequestType === 'Delete') {
            try {
                delete authParamsJson.Statement[0].Condition;
                delete unauthParamsJson.Statement[0].Condition;
                authParamsJson.Statement[0].Effect = 'Deny'
                unauthParamsJson.Statement[0].Effect = 'Deny'
                let authParams = {PolicyDocument: JSON.stringify(authParamsJson), RoleName: authRoleName};
                let unauthParams = {PolicyDocument: JSON.stringify(unauthParamsJson), RoleName: unauthRoleName};
                const iam = new IAMClient({region: event.ResourceProperties.region});
                let res = await Promise.all([
                    iam.send(new GetRoleCommand({RoleName: authParams.RoleName})),
                    iam.send(new GetRoleCommand({RoleName: unauthParams.RoleName}))
                ]);
                res = await Promise.all([
                    iam.send(new UpdateAssumeRolePolicyCommand(authParams)),
                    iam.send(new UpdateAssumeRolePolicyCommand(unauthParams))
                ]);
                response.send(event, context, response.SUCCESS, {});
            } catch (err) {
                console.log(err.stack);
                response.send(event, context, response.SUCCESS, {Error: err});
            }
        } else if (event.RequestType === 'Update' || event.RequestType === 'Create') {
            const iam = new IAMClient({region: event.ResourceProperties.region});
            let authParams = {PolicyDocument: JSON.stringify(authParamsJson), RoleName: authRoleName};
            let unauthParams = {PolicyDocument: JSON.stringify(unauthParamsJson), RoleName: unauthRoleName};
            const res = await Promise.all([
                iam.send(new UpdateAssumeRolePolicyCommand(authParams)),
                iam.send(new UpdateAssumeRolePolicyCommand(unauthParams))
            ]);
            response.send(event, context, response.SUCCESS, {});
        }
    } catch (err) {
        console.log(err.stack);
        response.send(event, context, response.FAILED, {Error: err});
    }
};

[ykethan]

Hey @NickDario, thank you for reaching out. Could you let us know the number of resource the new environment is currently creating. Does waiting for a couple of minutes and running push mitigate the issue?

[NickDario]

Currently we are creating about 1-200 IAM roles per environment. During our amplify cli upgrade we needed to create about half a dozen new environments, We deployed 2 before encountering this issue.

After the two were deployed we were unable to create any more for about 24 hours, and then we were only able to deploy one before encountering this issue again.

We verified we are not hitting any IAM quota limits as well.

What is unclear to me why this lambda is throwing. it seems to only update two policies, the auth and unauth roles. it consistently was this lamdba that threw the throttling error.

[ykethan]

@NickDario apologies on the delay and thank you for the information. The issues seems to be occurring on the AWS SDK call UpdateAssumeRolePolicyCommand .
Marking this as bug for improvements to implement a backoff and retry strategy on the calls.

[NickDario]

thanks @ykethan we'll be happy to see those fixes get in!