Merge pull request #975 from techmatters/CHI-3568-conversation_cleanup

CHI-3568: Conversation cleanup job

Author: stephenhand

hrm-domain/hrm-core/case/caseService.ts

@@ -47,7 +47,6 @@ import {
   hrmSearchConfiguration,
 } from '@tech-matters/hrm-search-config';
 import { publishCaseChangeNotification } from '../notifications/entityChangeNotify';
-import { enablePublishHrmSearchIndex } from '../featureFlags';
 import { getClient } from '@tech-matters/elasticsearch-client';
 import {
   CaseListCondition,
@@ -123,10 +122,6 @@ const doCaseChangeNotification =
     caseRecord?: CaseRecord;
   }) => {
     try {
-      if (!enablePublishHrmSearchIndex) {
-        return;
-      }
-
       const caseObj =
         caseRecord ?? (await getById(parseInt(caseId), accountSid, maxPermissions.user));
 

hrm-domain/hrm-core/contact/contactService.ts

@@ -59,7 +59,6 @@ import { actionsMaps } from '../permissions';
 import type { TwilioUser } from '@tech-matters/twilio-worker-auth';
 import { createReferral } from '../referral/referralService';
 import { createContactJob } from '../contact-job/contact-job';
-import { enablePublishHrmSearchIndex } from '../featureFlags';
 import {
   type ConversationMedia,
   type NewConversationMedia,
@@ -207,10 +206,6 @@ const doContactChangeNotification =
     contactId: Contact['id'];
   }) => {
     try {
-      if (!enablePublishHrmSearchIndex) {
-        return;
-      }
-
       const contact = await getById(accountSid, parseInt(contactId));
 
       if (contact) {

hrm-domain/hrm-core/featureFlags.ts

@@ -13,15 +13,8 @@
  * You should have received a copy of the GNU Affero General Public License
  * along with this program.  If not, see https://www.gnu.org/licenses/.
  */
-export const enableCleanupJobs = /^true$/i.test(process.env.ENABLE_CLEANUP_JOBS);
-export const enableProfileFlagsCleanup = /^true$/i.test(
-  process.env.ENABLE_PROFILE_FLAGS_CLEANUP,
-);
-export const enablePublishHrmSearchIndex = /^true$/i.test(
-  process.env.ENABLE_PUBLISH_HRM_SEARCH_INDEX,
-);
-export const enableSnsHrmSearchIndex = /^true$/i.test(
-  process.env.ENABLE_SNS_HRM_SEARCH_INDEX,
+export const enableConversationsCleanup = /^true$/i.test(
+  process.env.ENABLE_CONVERSATIONS_CLEANUP,
 );
 export const enableDbUserPerAccount = /^true$/i.test(
   process.env.ENABLE_DB_USER_PER_ACCOUNT,

hrm-domain/hrm-service/package.json

@@ -6,6 +6,7 @@
     "start": "cd dist && node ./www",
     "start:job-processor": "cd dist && node ./jobProcessor",
     "start:job-cleanup": "cd dist && node ./jobCleanup",
+    "start:conversation-cleanup": "cd dist && node ./conversationCleanup",
     "start:job-case-status-transition": "cd dist && node ./jobCaseStatusTransition",
     "start:job-cleanup:local": "AWS_REGION=us-east-1 SSM_ENDPOINT=http://localhost:4566 SQS_ENDPOINT=http://localhost:4566 run-s start:job-cleanup",
     "start:generate-ai-training-set": "cd dist && node ./generateAiTrainingSet",
@@ -46,6 +47,7 @@
   "dependencies": {
     "@tech-matters/case-status-transition": "^1.0.0",
     "@tech-matters/contact-job-cleanup": "^1.0.0",
+    "@tech-matters/conversation-cleanup": "^1.0.0",
     "@tech-matters/generate-ai-training-set": "^1.0.0",
     "@tech-matters/hrm-core": "^1.0.0",
     "@tech-matters/hrm-types": "^1.0.0",

hrm-domain/hrm-service/service-tests/contact-job/contactJobCleanup.test.ts

@@ -70,7 +70,7 @@ beforeAll(async () => {
   process.env.TWILIO_AUTH_TOKEN = 'mockAuthToken';
   process.env.TWILIO_CLIENT_USE_ENV_AUTH_TOKEN = 'true';
   const client = await getClient({ accountSid });
-  twilioSpy = jest.spyOn(client.chat.v2, 'services');
+  twilioSpy = jest.spyOn(client.conversations.v1.conversations, 'get');
 
   await mockingProxy.start();
   const mockttp = await mockingProxy.mockttpServer();

hrm-domain/hrm-service/src/conversationCleanup.ts

@@ -0,0 +1,31 @@
+/**
+ * Copyright (C) 2021-2023 Technology Matters
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as published
+ * by the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see https://www.gnu.org/licenses/.
+ */
+
+import { handleSignals } from './handleSignals';
+import { cleanupConversations } from '@tech-matters/conversation-cleanup';
+import { enableConversationsCleanup } from '@tech-matters/hrm-core/featureFlags';
+
+const gracefulExit = async () => {
+  //Only a Twilio and AWS client is used, so nothing to do here.
+};
+
+if (enableConversationsCleanup) {
+  cleanupConversations();
+
+  handleSignals(gracefulExit);
+} else {
+  console.debug('enableCleanupJobs not set, skipping conversation cleanup.');
+}

hrm-domain/hrm-service/src/jobCleanup.ts

@@ -16,14 +16,11 @@
 
 import { handleSignals } from './handleSignals';
 import { cleanupContactJobs } from '@tech-matters/contact-job-cleanup';
-import { enableCleanupJobs } from '@tech-matters/hrm-core/featureFlags';
 
 const gracefulExit = async () => {
   //TODO: this should probably handle closing any running processes and open db connections
 };
 
-if (enableCleanupJobs) {
-  cleanupContactJobs();
+cleanupContactJobs();
 
-  handleSignals(gracefulExit);
-}
+handleSignals(gracefulExit);

hrm-domain/hrm-service/src/profileFlagsCleanup.ts

@@ -15,15 +15,12 @@
  */
 
 import { handleSignals } from './handleSignals';
-import { enableProfileFlagsCleanup } from '@tech-matters/hrm-core/featureFlags';
 import { cleanupProfileFlags } from '@tech-matters/profile-flags-cleanup';
 
 const gracefulExit = async () => {
   //TODO: this should probably handle closing any running processes and open db connections
 };
 
-if (enableProfileFlagsCleanup) {
-  cleanupProfileFlags();
+cleanupProfileFlags();
 
-  handleSignals(gracefulExit);
-}
+handleSignals(gracefulExit);

hrm-domain/hrm-service/tsconfig.build.json

@@ -20,6 +20,7 @@
     { "path": "hrm-domain/packages/hrm-search-config" },
     { "path": "hrm-domain/hrm-core" },
     { "path": "hrm-domain/scheduled-tasks/contact-job-cleanup" },
+    { "path": "hrm-domain/scheduled-tasks/conversation-cleanup" },
     { "path": "hrm-domain/scheduled-tasks/case-status-transition" },
     { "path": "hrm-domain/scheduled-tasks/profile-flags-cleanup" },
     { "path": "hrm-domain/scheduled-tasks/generate-ai-training-set" },

hrm-domain/scheduled-tasks/contact-job-cleanup/index.ts

@@ -16,7 +16,7 @@
 
 import { ContactJobType } from '@tech-matters/types';
 import { getClient } from '@tech-matters/twilio-client';
-import { getSsmParameter } from '@tech-matters/ssm-cache';
+import { getSsmParameter, SsmParameterNotFound } from '@tech-matters/ssm-cache';
 import {
   ContactJob,
   deleteContactJob,
@@ -82,17 +82,27 @@ const deleteTranscript = async (job: RetrieveContactTranscriptJob): Promise<bool
   }
 
   await setContactJobCleanupActive(id);
+  const client = await getClient({ accountSid });
   try {
-    const client = await getClient({ accountSid });
-    await client.chat.v2
-      .services(job.resource.serviceSid)
-      .channels.get(job.resource.channelSid)
-      .remove();
+    await client.conversations.v1.conversations.get(job.resource.channelSid).remove();
   } catch (err) {
     if (err.status === 404) {
-      console.log(
-        `Channel ${channelSid} not found, assuming it has already been deleted`,
+      console.info(
+        `Conversation ${channelSid} not found, checking legacy chat API just in case.`,
       );
+      try {
+        await client.chat.v2
+          .services(job.resource.serviceSid)
+          .channels.get(job.resource.channelSid)
+          .remove();
+      } catch (chatError) {
+        if (err.status === 404) {
+          console.info(
+            `${channelSid} not found as a chat channel or conversation, assuming it has already been deleted`,
+          );
+          return true;
+        }
+      }
     } else {
       console.error(
         new ContactJobCleanupError(
@@ -118,9 +128,11 @@ const cleanupContactJob = async (job: ContactJob): Promise<void> => {
     if (!(await deleteTranscript(job))) return;
   }
 
-  const { accountSid, id } = job;
+  const { accountSid, id, jobType, contactId } = job;
   await deleteContactJob(accountSid, id);
-  console.log(`Successfully cleaned up contact job ${id}`);
+  console.info(
+    `Successfully cleaned up contact job ${id} (${jobType}) for contact ${contactId}`,
+  );
 };
 
 /**
@@ -137,10 +149,20 @@ const getCleanupRetentionDays = async (accountSid): Promise<number | undefined>
           `/${process.env.NODE_ENV}/hrm/${accountSid}/transcript_retention_days`,
         ),
       ) || MAX_CLEANUP_JOB_RETENTION_DAYS;
-  } catch (err) {
-    console.error(
-      `Error trying to fetch /${process.env.NODE_ENV}/hrm/${accountSid}/transcript_retention_days ${err}, using default`,
+    console.debug(
+      `SSM parameter for transcript retention days set to ${ssmRetentionDays} for account ${accountSid}, so using that`,
     );
+  } catch (err) {
+    if ((err as any) instanceof SsmParameterNotFound) {
+      console.debug(
+        `SSM parameter for transcript retention days not set for account ${accountSid}, so using default ${MAX_CLEANUP_JOB_RETENTION_DAYS}`,
+      );
+    } else {
+      console.error(
+        `Error trying to fetch /${process.env.NODE_ENV}/hrm/${accountSid}/transcript_retention_days ${err}, using default`,
+        err,
+      );
+    }
     ssmRetentionDays = MAX_CLEANUP_JOB_RETENTION_DAYS;
   }
 
@@ -159,10 +181,10 @@ export const cleanupContactJobs = async (): Promise<void> => {
   try {
     const accountSids = await getPendingCleanupJobAccountSids();
 
-    console.log(`Cleaning up contact jobs for accounts:`, accountSids);
+    console.info(`Cleaning up contact jobs for accounts:`, accountSids);
 
     for (const accountSid of accountSids) {
-      console.log(`Cleaning up contact jobs for account:`, accountSid);
+      console.info(`Cleaning up contact jobs for account:`, accountSid);
       const cleanupRetentionDays = await getCleanupRetentionDays(accountSid);
       const pendingJobs = await getPendingCleanupJobs(accountSid, cleanupRetentionDays);
 
@@ -171,7 +193,9 @@ export const cleanupContactJobs = async (): Promise<void> => {
           await cleanupContactJob(job);
         } catch (err) {
           console.error(
-            new ContactJobCleanupError(`Error processing job ${job.id}: ${err}`),
+            new ContactJobCleanupError(
+              `Error processing job ${job.id} (contact: ${job.contactId}): ${err}`,
+            ),
           );
         }
       }