load_mentors.js: Only delete mentors that no longer exist in database

scripts/load_mentors.js

@@ -2,11 +2,13 @@ const Airtable = require("airtable");
 const { Client } = require("@elastic/elasticsearch");
 const AWS = require("aws-sdk");
 
+const AIRTABLE_BASE = "appDfSlmYKDyuAj51";
 const AWS_REGION = "ap-southeast-1";
 const AWS_SECRET_AIRTABLE_NAME = "airtable_api_key";
 const AWS_SECRET_ELASTIC_NAME = "elasticsearch_api_key";
 const ELASTIC_ENDPOINT =
   "https://advisorysg.es.ap-southeast-1.aws.found.io:9243";
+const ELASTIC_INDEX = ".ent-search-engine-documents-mentorship-page";
 
 const PLACEHOLDER_THUMBNAIL_URL = "/mentor-thumbnail.png";
 const WAVE_INFO = [
@@ -51,9 +53,7 @@ exports.handler = async (event) => {
     .promise()
     .then((data) => JSON.parse(data.SecretString)["APIKey"]);
 
-  const base = new Airtable({ apiKey: airtableApiKey }).base(
-    "appDfSlmYKDyuAj51"
-  );
+  const base = new Airtable({ apiKey: airtableApiKey }).base(AIRTABLE_BASE);
 
   const elasticApiKey = await awsClient
     .getSecretValue({ SecretId: AWS_SECRET_ELASTIC_NAME })
@@ -77,18 +77,28 @@ exports.handler = async (event) => {
     )
   );
 
-  await elasticClient.deleteByQuery({
-    index: ".ent-search-engine-documents-mentorship-page",
-    body: {
-      query: {
-        match_all: {},
-      },
-    },
+  const mentorIds = new Set(mentors.map(({ id }) => id));
+  const oldMentorIds = [];
+
+  const scrollSearch = await elasticClient.helpers.scrollDocuments({
+    index: ELASTIC_INDEX,
+    query: { match_all: {} },
   });
+  for await (const result of scrollSearch) {
+    if (!mentorIds.has(result.id)) {
+      oldMentorIds.push(result.id);
+    }
+  }
 
-  const body = mentors.flatMap((mentor) => [
-    { index: { _index: ".ent-search-engine-documents-mentorship-page" } },
+  const deleteBody = [...oldMentorIds].map((mentorId) => ({
+    delete: { _index: ELASTIC_INDEX, _id: mentorId },
+  }));
+  const indexBody = mentors.flatMap((mentor) => [
+    { index: { _index: ELASTIC_INDEX, _id: mentor.id } },
     mentor,
   ]);
-  await elasticClient.bulk({ refresh: true, body });
+  await elasticClient.bulk({
+    refresh: true,
+    body: deleteBody.concat(indexBody),
+  });
 };