Lambda: Update script to pull data from Airtable into Elasticsearch

.github/workflows/deploy-lambda.yml

@@ -15,16 +15,17 @@ jobs:
         uses: actions/cache@v1
         with:
           path: ~/.npm
-          key: ${{ runner.os }}-node-airtable-${{ hashFiles('**/package-lock.json') }}
+          key: ${{ runner.os }}-node-lambda-${{ hashFiles('**/package-lock.json') }}
           restore-keys: |
-            ${{ runner.os }}-node-airtable-
+            ${{ runner.os }}-node-lambda-
       - name: Setup Node
         uses: actions/setup-node@v1
         with:
           node-version: ${{ matrix.node-version }}
       - run: export VERSION_AIRTABLE=`node -p -e "require('./package-lock.json').packages['node_modules/airtable'].version"`
+      - run: export VERSION_ELASTICSEARCH=`node -p -e "require('./package-lock.json').packages['node_modules/@elastic/elasticsearch'].version"`
       - run: rm package.json package-lock.json
-      - run: npm install --no-save airtable@"$VERSION_AIRTABLE"
+      - run: npm install --no-save airtable@"$VERSION_AIRTABLE" @elastic/elasticsearch@"$VERSION_ELASTICSEARCH"
 
       - run: mkdir lambda
       - run: cp scripts/load_mentors.js lambda/index.js

package.json

@@ -4,6 +4,7 @@
   "homepage": "https://mentorship.advisory.sg/",
   "private": true,
   "dependencies": {
+    "@elastic/elasticsearch": "^7.16.0",
     "@material-ui/core": "^4.12.3",
     "@material-ui/lab": "^4.0.0-alpha.60",
     "body-scroll-lock": "^3.1.5",

scripts/load_mentors.js

@@ -1,8 +1,13 @@
 const Airtable = require("airtable");
+const { Client } = require("@elastic/elasticsearch");
 const AWS = require("aws-sdk");
 
 const AWS_REGION = "ap-southeast-1";
-const AWS_SECRET_NAME = "airtable_api_key";
+const AWS_SECRET_AIRTABLE_NAME = "airtable_api_key";
+const AWS_SECRET_ELASTIC_NAME = "elasticsearch_api_key";
+const ELASTIC_ENDPOINT =
+  "https://advisorysg.es.ap-southeast-1.aws.found.io:9243";
+
 const PLACEHOLDER_THUMBNAIL_URL = "/mentor-thumbnail.png";
 const WAVE_INFO = [
   { tableId: "4 Tech", name: "2021 Wave 1" },
@@ -39,12 +44,22 @@ const formatMentor = (id, waveId, fields) => ({
 exports.handler = async (event) => {
   const mentors = [];
 
-  const client = new AWS.SecretsManager({ region: AWS_REGION });
-  const apiKey = await client
-    .getSecretValue({ SecretId: AWS_SECRET_NAME })
+  const awsClient = new AWS.SecretsManager({ region: AWS_REGION });
+  const airtableApiKey = await awsClient
+    .getSecretValue({ SecretId: AWS_SECRET_AIRTABLE_NAME })
+    .promise()
+    .then((data) => JSON.parse(data.SecretString)["APIKey"]);
+
+  const base = new Airtable({ airtableApiKey }).base("appDfSlmYKDyuAj51");
+
+  const elasticApiKey = await awsClient
+    .getSecretValue({ SecretId: AWS_SECRET_ELASTIC_NAME })
     .promise()
     .then((data) => JSON.parse(data.SecretString)["APIKey"]);
-  const base = new Airtable({ apiKey }).base("appDfSlmYKDyuAj51");
+  const elasticClient = new Client({
+    node: ELASTIC_ENDPOINT,
+    auth: { apiKey: elasticApiKey },
+  });
 
   await Promise.all(
     WAVE_INFO.map(async ({ tableId }, i) =>
@@ -59,5 +74,19 @@ exports.handler = async (event) => {
     )
   );
 
-  return JSON.stringify({ mentors });
+  await elasticClient.deleteByQuery({
+    index: ".ent-search-engine-documents-mentorship-page",
+    type: "",
+    body: {
+      query: {
+        match_all: {},
+      },
+    },
+  });
+
+  const body = mentors.flatMap((mentor) => [
+    { index: { _index: ".ent-search-engine-documents-mentorship-page" } },
+    mentor,
+  ]);
+  await elasticClient.bulk({ refresh: true, body });
 };