diff --git a/.aws/src/backfillLambda.ts b/.aws/src/backfillLambda.ts index 69786f7..c4dca80 100644 --- a/.aws/src/backfillLambda.ts +++ b/.aws/src/backfillLambda.ts @@ -80,20 +80,6 @@ export class BackfillLambda extends Resource { ], }, ], - alarms: { - errors: { - // The backfill lambda is throttled to concurrency of 10. - evaluationPeriods: 1, - comparisonOperator: 'GreaterThanOrEqualToThreshold', - period: 1800, // 30 minutes - // approx. 5% failure rate (taken from test runs on EN_INTL, - // which is the shortest backfill run) - threshold: 150, - actions: config.isDev - ? [] - : [pagerDuty!.snsNonCriticalAlarmTopic.arn], - }, - }, }, tags: config.tags, }); diff --git a/.aws/src/datasyncLambda.ts b/.aws/src/datasyncLambda.ts index b9a7aa7..0bd40ed 100644 --- a/.aws/src/datasyncLambda.ts +++ b/.aws/src/datasyncLambda.ts @@ -260,10 +260,8 @@ export class DatasyncLambda extends Resource { period: period, threshold: threshold, statistic: 'Sum', - alarmActions: config.isDev - ? [] - : [this.pagerDuty.snsCriticalAlarmTopic.arn], - okActions: config.isDev ? [] : [this.pagerDuty.snsCriticalAlarmTopic.arn], + alarmActions: [], + okActions: [], }); } diff --git a/.aws/src/main.ts b/.aws/src/main.ts index eab784a..3b2b285 100644 --- a/.aws/src/main.ts +++ b/.aws/src/main.ts @@ -1,15 +1,9 @@ import { Construct } from 'constructs'; -import { - App, - DataTerraformRemoteState, - RemoteBackend, - TerraformStack, -} from 'cdktf'; +import { App, S3Backend, TerraformStack } from 'cdktf'; import { AwsProvider } from '@cdktf/provider-aws/lib/provider'; import { S3Bucket } from '@cdktf/provider-aws/lib/s3-bucket'; import { config } from './config'; -import { PocketPagerDuty, PocketVPC } from '@pocket-tools/terraform-modules'; -import { PagerdutyProvider } from '@cdktf/provider-pagerduty/lib/provider'; +import { PocketVPC } from '@pocket-tools/terraform-modules'; import { LocalProvider } from '@cdktf/provider-local/lib/provider'; import { NullProvider } from '@cdktf/provider-null/lib/provider'; import { ArchiveProvider } from '@cdktf/provider-archive/lib/provider'; @@ -23,20 +17,19 @@ class CurationToolsDataSync extends TerraformStack { super(scope, name); new AwsProvider(this, 'aws', { region: 'us-east-1' }); - new PagerdutyProvider(this, 'pagerduty_provider', { token: undefined }); new LocalProvider(this, 'local_provider'); new NullProvider(this, 'null_provider'); new ArchiveProvider(this, 'archive_provider'); - new RemoteBackend(this, { - hostname: 'app.terraform.io', - organization: 'Pocket', - workspaces: [{ prefix: `${config.name}-` }], + new S3Backend(this, { + bucket: `mozilla-content-team-${config.environment.toLowerCase()}-terraform-state`, + dynamodbTable: `mozilla-content-team-${config.environment.toLowerCase()}-terraform-state`, + key: config.name, + region: 'us-east-1', }); // ** shared infrastructure between backfill and datasync const vpc = new PocketVPC(this, 'pocket-shared-vpc'); - const pagerDuty = this.createPagerDuty(); //dynamo db to map curatedRecId - scheduledItem's externalId and store approvedItem's externalId const idMapperDynamoDb = new DynamoDB(this, 'curation-migration-id-mapper'); @@ -44,14 +37,13 @@ class CurationToolsDataSync extends TerraformStack { //bucket for storing all the required csv files this.createMigrationBucket(); - new BackfillAuthorsLambda(this, 'backfill-author-lambda', vpc, pagerDuty); + new BackfillAuthorsLambda(this, 'backfill-author-lambda', vpc); new BackfillLambda( this, 'backfill-lambda', vpc, idMapperDynamoDb.curationMigrationTable, - pagerDuty, ); // ** infrastructure for datasync process ** @@ -60,44 +52,9 @@ class CurationToolsDataSync extends TerraformStack { 'datasync-lambda', vpc, idMapperDynamoDb.curationMigrationTable, - pagerDuty, ); } - /** - * Create PagerDuty service for alerts - * @private - */ - private createPagerDuty() { - // don't create any pagerduty resources if in dev - if (config.isDev) { - return undefined; - } - - const incidentManagement = new DataTerraformRemoteState( - this, - 'incident_management', - { - organization: 'Pocket', - workspaces: { - name: 'incident-management', - }, - }, - ); - - return new PocketPagerDuty(this, 'pagerduty', { - prefix: config.prefix, - service: { - criticalEscalationPolicyId: incidentManagement - .get('policy_default_critical_id') - .toString(), - nonCriticalEscalationPolicyId: incidentManagement - .get('policy_default_non_critical_id') - .toString(), - }, - }); - } - /** * Create the migration S3 bucket * This bucket is used to store all the required csv files diff --git a/bin/load-queued-items.ts b/bin/load-queued-items.ts index 7662486..838abb4 100755 --- a/bin/load-queued-items.ts +++ b/bin/load-queued-items.ts @@ -53,7 +53,7 @@ function getRandomInt(min, max) { async function insertIntoCuratedFeedItem( queuedItem, timeLive, - trx: Knex.Transaction + trx: Knex.Transaction, ) { const now = Math.floor(new Date().getTime() / 1000); @@ -107,7 +107,7 @@ async function moveQueuedItems(feedId, baseTime) { const curatedRecId = await insertIntoCuratedFeedItem( queuedItem, i * 3600 + baseTime, - trx + trx, ); await updateQueuedItem(queuedItem['queued_id'], trx); await insertTileSource(curatedRecId, trx); @@ -116,7 +116,9 @@ async function moveQueuedItems(feedId, baseTime) { } catch (e) { await trx.rollback(); console.log( - `Number of items emptied from the queue before error occurred: ${i - 1}` + `Number of items emptied from the queue before error occurred: ${ + i - 1 + }`, ); console.log(`Failed at queued item ID: ${queuedItem['queued_id']}`); console.log(`Failed for the time_live: ${i * 3600 + baseTime}`); @@ -137,7 +139,7 @@ async function load() { const count = await moveQueuedItems( feedId, // Starting from the next hour - startTime + startTime, ); return { count, startTime }; diff --git a/bin/rollback-queued-items.ts b/bin/rollback-queued-items.ts index 05bd827..21665ef 100755 --- a/bin/rollback-queued-items.ts +++ b/bin/rollback-queued-items.ts @@ -28,7 +28,7 @@ async function rollbackQueuedItems(queuedIds: number[], trx: Knex.Transaction) { async function rollbackTileSource( curatedRecIds: number[], - trx: Knex.Transaction + trx: Knex.Transaction, ) { await dbClient('tile_source') .delete() @@ -38,7 +38,7 @@ async function rollbackTileSource( async function rollbackCuratedItems( curatedRecIds: number[], - trx: Knex.Transaction + trx: Knex.Transaction, ) { return trx('curated_feed_items') .delete() @@ -70,7 +70,7 @@ async function rollback() { rollback() .then((res) => { console.log( - `Successfully rollback feed ID: ${res.feedId} after time_live: ${res.timeLive}` + `Successfully rollback feed ID: ${res.feedId} after time_live: ${res.timeLive}`, ); console.log(`Number of items affected back: ${res.curatedRecIds.length}`); console.log('Curated Rec IDs: ', JSON.stringify(res.curatedRecIds)); diff --git a/buildspec.yml b/buildspec.yml index d7f622e..443948b 100644 --- a/buildspec.yml +++ b/buildspec.yml @@ -23,7 +23,6 @@ env: USERNAME: 'codebuild' secrets-manager: #Pull in the default terraform cloud token - TERRAFORM_TOKEN: 'CodeBuild/Default:terraform_token' PAGERDUTY_TOKEN: 'CodeBuild/Default:mozilla_pagerduty_token' GITHUB_ACCESS_TOKEN: 'CodeBuild/Default:github_access_token' @@ -35,11 +34,6 @@ phases: - | . /home/circleci/.codebuild_shims_wrapper.sh echo $CODEBUILD_WEBHOOK_HEAD_REF - echo Setting Up Terraform Token - rc="credentials \"app.terraform.io\" { " - rc="${rc} token=\"$TERRAFORM_TOKEN\" " - rc="${rc}}" - echo "$rc" > ~/.terraformrc echo Setting Github Access Token echo "//npm.pkg.github.com/:_authToken=${GITHUB_ACCESS_TOKEN}" > ~/.npmrc echo Setting environment variables diff --git a/package.json b/package.json index 38bf985..13ad3d8 100644 --- a/package.json +++ b/package.json @@ -12,8 +12,8 @@ "test-integration": "jest \"\\.integration\\.ts\" --runInBand", "test-all": "npm run test-spec && npm run test-integration", "postinstall": "(cd bin && npm install); (cd curation-authors-backfill && npm install); (cd curation-migration-backfill && npm install); (cd curation-migration-datasync && npm install)", - "lint-check": "eslint --fix-dry-run \"curation-*/**/*.ts\"", - "lint-fix": "eslint --fix \"curation*/**/*.ts\"" + "lint-check": "eslint \"**/*.ts\"", + "lint-fix": "eslint --fix \"**/*.ts\"" }, "repository": { "type": "git",