feat: new lambda function to check and retry failed webhook deliveries

Author: jackie-linz

.eslintrc.json

@@ -19,6 +19,7 @@ const project = new awscdk.AwsCdkConstructLibrary({
     '@octokit/auth-app',
     '@octokit/request-error',
     '@octokit/rest',
+    '@octokit/types',
     '@aws-sdk/client-cloudformation',
     '@aws-sdk/client-codebuild',
     '@aws-sdk/client-ec2',

.gitattributes

@@ -1,5 +1,6 @@
 import { createAppAuth } from '@octokit/auth-app';
 import { Octokit } from '@octokit/rest';
+import { Endpoints } from '@octokit/types';
 import { getSecretValue, getSecretJsonValue } from './lambda-helpers';
 
 export function baseUrlFromDomain(domain: string): string {
@@ -132,3 +133,79 @@ export async function deleteRunner(octokit: Octokit, runnerLevel: RunnerLevel, o
     });
   }
 }
+
+
+export type WebhookDeliveries = Endpoints['GET /app/hook/deliveries']['response']['data'];
+export type WebhookDelivery = WebhookDeliveries[number];
+export type WebhookDeliveryDetail = Endpoints['GET /app/hook/deliveries/{delivery_id}']['response']['data'];
+export type WorkflowJob = Endpoints['GET /repos/{owner}/{repo}/actions/jobs/{job_id}']['response']['data'];
+
+export async function getFailedDeliveries(
+  octokit: Octokit,
+  sinceDeliveryId: number,
+): Promise<{
+  failedDeliveries: WebhookDeliveries;
+  latestDeliveryId: number;
+}> {
+  const failedDeliveries: WebhookDeliveries = [];
+  if (sinceDeliveryId === 0) {
+    // If no last delivery ID was set, just fetch the latest delivery to get the latest ID
+    const deliveriesResponse = await octokit.rest.apps.listWebhookDeliveries({ per_page: 1 });
+    if (deliveriesResponse.status !== 200) {
+      throw new Error(`Failed to fetch webhook deliveries`);
+    }
+    return {
+      failedDeliveries,
+      latestDeliveryId: deliveriesResponse.data[0]?.id || 0,
+    };
+  }
+
+  let latestDeliveryId = 0;
+  let deliveryCountSinceLastCheck = 0;
+  for await (const response of octokit.paginate.iterator('GET /app/hook/deliveries')) {
+    if (response.status !== 200) {
+      throw new Error(`Failed to fetch webhook deliveries`);
+    }
+    latestDeliveryId = Math.max(latestDeliveryId, ...response.data.map((delivery) => delivery.id));
+
+    const deliveriesSinceLastCheck = response.data.filter((delivery) => delivery.id > sinceDeliveryId);
+    deliveryCountSinceLastCheck += deliveriesSinceLastCheck.length;
+    failedDeliveries.push(...deliveriesSinceLastCheck.filter((delivery) => delivery.status !== 'OK'));
+
+    if (deliveriesSinceLastCheck.length < response.data.length) {
+      break;
+    }
+  }
+  console.debug(
+    `Searched through ${deliveryCountSinceLastCheck} deliveries since last check, found ${failedDeliveries.length} failed`,
+  );
+
+  return {
+    failedDeliveries,
+    latestDeliveryId,
+  };
+}
+
+export async function getDeliveryDetail(
+  octokit: Octokit,
+  deliveryId: number,
+): Promise<WebhookDeliveryDetail> {
+  const response = await octokit.rest.apps.getWebhookDelivery({
+    delivery_id: deliveryId,
+  });
+  if (response.status !== 200) {
+    throw new Error(`Failed to fetch webhook delivery with ID ${deliveryId}`);
+  }
+  return response.data;
+}
+
+export async function redeliver(octokit: Octokit, deliveryId: number): Promise<void> {
+  const response = await octokit.rest.apps.redeliverWebhookDelivery({
+    delivery_id: deliveryId,
+  });
+
+  if (response.status !== 202) {
+    throw new Error(`Failed to redeliver webhook delivery with ID ${deliveryId}`);
+  }
+  console.log(`Successfully redelivered webhook delivery with ID ${deliveryId}`);
+}

.gitignore

@@ -1,4 +1,5 @@
 import { SecretsManagerClient, GetSecretValueCommand, UpdateSecretCommand } from '@aws-sdk/client-secrets-manager';
+import { GetParameterCommand, PutParameterCommand, SSMClient } from '@aws-sdk/client-ssm';
 
 export interface StepFunctionLambdaInput {
   readonly owner: string;
@@ -13,6 +14,7 @@ export interface StepFunctionLambdaInput {
 }
 
 const sm = new SecretsManagerClient();
+const ssm = new SSMClient();
 
 export async function getSecretValue(arn: string | undefined) {
   if (!arn) {
@@ -79,3 +81,28 @@ export async function customResourceRespond(event: AWSLambda.CloudFormationCusto
     }
   });
 }
+
+export async function getLastDeliveryId(): Promise<number> {
+  if (!process.env.LAST_DELIVERY_ID_PARAM) {
+    throw new Error('Missing LAST_DELIVERY_ID_PARAM environment variable');
+  }
+
+  const paramName = process.env.LAST_DELIVERY_ID_PARAM;
+
+  const response = await ssm.send(new GetParameterCommand({ Name: paramName }));
+  if (!response.Parameter?.Value) {
+    throw new Error(`No Parameter.Value in ${paramName}`);
+  }
+
+  return parseInt(response.Parameter.Value, 10);
+}
+
+export async function setLastDeliveryId(id: number): Promise<void> {
+  if (!process.env.LAST_DELIVERY_ID_PARAM) {
+    throw new Error('Missing LAST_DELIVERY_ID_PARAM environment variable');
+  }
+
+  const paramName = process.env.LAST_DELIVERY_ID_PARAM;
+
+  await ssm.send(new PutParameterCommand({ Name: paramName, Value: id.toString(), Overwrite: true }));
+}