Skip to content

Tfcloud migrate #638

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 7 commits into from
Jun 18, 2024
Merged
Show file tree
Hide file tree
Changes from 4 commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
14 changes: 0 additions & 14 deletions .aws/src/backfillLambda.ts
Original file line number Diff line number Diff line change
Expand Up @@ -80,20 +80,6 @@ export class BackfillLambda extends Resource {
],
},
],
alarms: {
errors: {
// The backfill lambda is throttled to concurrency of 10.
evaluationPeriods: 1,
comparisonOperator: 'GreaterThanOrEqualToThreshold',
period: 1800, // 30 minutes
// approx. 5% failure rate (taken from test runs on EN_INTL,
// which is the shortest backfill run)
threshold: 150,
actions: config.isDev
? []
: [pagerDuty!.snsNonCriticalAlarmTopic.arn],
},
},
},
tags: config.tags,
});
Expand Down
6 changes: 2 additions & 4 deletions .aws/src/datasyncLambda.ts
Original file line number Diff line number Diff line change
Expand Up @@ -260,10 +260,8 @@ export class DatasyncLambda extends Resource {
period: period,
threshold: threshold,
statistic: 'Sum',
alarmActions: config.isDev
? []
: [this.pagerDuty.snsCriticalAlarmTopic.arn],
okActions: config.isDev ? [] : [this.pagerDuty.snsCriticalAlarmTopic.arn],
alarmActions: [],
okActions: [],
});
}

Expand Down
55 changes: 8 additions & 47 deletions .aws/src/main.ts
Original file line number Diff line number Diff line change
@@ -1,15 +1,13 @@
import { Construct } from 'constructs';
import {

Check failure on line 2 in .aws/src/main.ts

View workflow job for this annotation

GitHub Actions / Lint Code Base

Replace `⏎··App,⏎··S3Backend,⏎··TerraformStack,⏎` with `·App,·S3Backend,·TerraformStack·`
App,
DataTerraformRemoteState,
RemoteBackend,
S3Backend,
TerraformStack,
} from 'cdktf';
import { AwsProvider } from '@cdktf/provider-aws/lib/provider';
import { S3Bucket } from '@cdktf/provider-aws/lib/s3-bucket';
import { config } from './config';
import { PocketPagerDuty, PocketVPC } from '@pocket-tools/terraform-modules';
import { PagerdutyProvider } from '@cdktf/provider-pagerduty/lib/provider';
import { PocketVPC } from '@pocket-tools/terraform-modules';
import { LocalProvider } from '@cdktf/provider-local/lib/provider';
import { NullProvider } from '@cdktf/provider-null/lib/provider';
import { ArchiveProvider } from '@cdktf/provider-archive/lib/provider';
Expand All @@ -23,35 +21,33 @@
super(scope, name);

new AwsProvider(this, 'aws', { region: 'us-east-1' });
new PagerdutyProvider(this, 'pagerduty_provider', { token: undefined });
new LocalProvider(this, 'local_provider');
new NullProvider(this, 'null_provider');
new ArchiveProvider(this, 'archive_provider');

new RemoteBackend(this, {
hostname: 'app.terraform.io',
organization: 'Pocket',
workspaces: [{ prefix: `${config.name}-` }],
new S3Backend(this, {
bucket: `mozilla-content-team-${config.environment.toLowerCase()}-terraform-state`,
dynamodbTable: `mozilla-content-team-${config.environment.toLowerCase()}-terraform-state`,
key: config.name,
region: 'us-east-1',
});

// ** shared infrastructure between backfill and datasync
const vpc = new PocketVPC(this, 'pocket-shared-vpc');
const pagerDuty = this.createPagerDuty();
//dynamo db to map curatedRecId - scheduledItem's externalId and store approvedItem's externalId
const idMapperDynamoDb = new DynamoDB(this, 'curation-migration-id-mapper');

// ** infrastructure for backfill process **
//bucket for storing all the required csv files
this.createMigrationBucket();

new BackfillAuthorsLambda(this, 'backfill-author-lambda', vpc, pagerDuty);
new BackfillAuthorsLambda(this, 'backfill-author-lambda', vpc);

new BackfillLambda(
this,
'backfill-lambda',
vpc,
idMapperDynamoDb.curationMigrationTable,
pagerDuty,
);

// ** infrastructure for datasync process **
Expand All @@ -60,44 +56,9 @@
'datasync-lambda',
vpc,
idMapperDynamoDb.curationMigrationTable,
pagerDuty,
);
}

/**
* Create PagerDuty service for alerts
* @private
*/
private createPagerDuty() {
// don't create any pagerduty resources if in dev
if (config.isDev) {
return undefined;
}

const incidentManagement = new DataTerraformRemoteState(
this,
'incident_management',
{
organization: 'Pocket',
workspaces: {
name: 'incident-management',
},
},
);

return new PocketPagerDuty(this, 'pagerduty', {
prefix: config.prefix,
service: {
criticalEscalationPolicyId: incidentManagement
.get('policy_default_critical_id')
.toString(),
nonCriticalEscalationPolicyId: incidentManagement
.get('policy_default_non_critical_id')
.toString(),
},
});
}

/**
* Create the migration S3 bucket
* This bucket is used to store all the required csv files
Expand Down
6 changes: 0 additions & 6 deletions buildspec.yml
Original file line number Diff line number Diff line change
Expand Up @@ -23,7 +23,6 @@ env:
USERNAME: 'codebuild'
secrets-manager:
#Pull in the default terraform cloud token
TERRAFORM_TOKEN: 'CodeBuild/Default:terraform_token'
PAGERDUTY_TOKEN: 'CodeBuild/Default:mozilla_pagerduty_token'
GITHUB_ACCESS_TOKEN: 'CodeBuild/Default:github_access_token'

Expand All @@ -35,11 +34,6 @@ phases:
- |
. /home/circleci/.codebuild_shims_wrapper.sh
echo $CODEBUILD_WEBHOOK_HEAD_REF
echo Setting Up Terraform Token
rc="credentials \"app.terraform.io\" { "
rc="${rc} token=\"$TERRAFORM_TOKEN\" "
rc="${rc}}"
echo "$rc" > ~/.terraformrc
echo Setting Github Access Token
echo "//npm.pkg.github.com/:_authToken=${GITHUB_ACCESS_TOKEN}" > ~/.npmrc
echo Setting environment variables
Expand Down
Loading