[Lock Manager] Handle alias resolution when checking lock index mappings (#244559)

Relates to: elastic/sdh-kibana#5923

## Summary

- Handle Lock Manager's alias resolution during migration:
  - when checking lock index mappings
- in cases when concrete index creation is attempted, but the name is
already used as alias

## Testing
New unit tests:
```bash
yarn test:jest src/platform/packages/shared/kbn-lock-manager/src/setup_lock_manager_index.test.ts
```

Updated integration tests:

```bash
node scripts/functional_tests_server --config x-pack/solutions/observability/test/api_integration_deployment_agnostic/configs/stateful/oblt.ai_assistant.stateful.config.ts

# Then run in a separate terminal:
node scripts/functional_test_runner --config x-pack/solutions/observability/test/api_integration_deployment_agnostic/configs/stateful/oblt.ai_assistant.stateful.config.ts --grep="LockManager"
```
### Checklist

Check the PR satisfies following conditions. 

Reviewers should verify this PR satisfies this list as well.
- [x] [Unit or functional
tests](https://www.elastic.co/guide/en/kibana/master/development-tests.html)
were updated or added to match the most common scenarios


### Identify risks

- This PR aims to mitigate risks that occur during migration. The
implementation makes certain assumptions on the way migration tooling
functions. In the unlikely events of that changing, the logic may need
to be revisited


### Reviewer notes
- AI assisted coding with Cursor and Claude 4.5 Opus:
  - Plan generation (to help cover all relevant places
  - Unit test generation

---------

Co-authored-by: Søren Louv-Jansen <[email protected]>
Co-authored-by: kibanamachine <[email protected]>

Author: 3 people

src/platform/packages/shared/kbn-lock-manager/moon.yml

@@ -20,6 +20,8 @@ project:
 dependsOn:
   - '@kbn/logging'
   - '@kbn/core'
+  - '@kbn/core-elasticsearch-client-server-mocks'
+  - '@kbn/core-logging-server-mocks'
 tags:
   - shared-server
   - package

src/platform/packages/shared/kbn-lock-manager/src/setup_lock_manager_index.test.ts

@@ -0,0 +1,221 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the "Elastic License
+ * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side
+ * Public License v 1"; you may not use this file except in compliance with, at
+ * your election, the "Elastic License 2.0", the "GNU Affero General Public
+ * License v3.0 only", or the "Server Side Public License, v 1".
+ */
+
+import { errors } from '@elastic/elasticsearch';
+import type { MappingTypeMapping } from '@elastic/elasticsearch/lib/api/types';
+import { elasticsearchClientMock } from '@kbn/core-elasticsearch-client-server-mocks';
+import { loggingSystemMock } from '@kbn/core-logging-server-mocks';
+import {
+  removeLockIndexWithIncorrectMappings,
+  ensureTemplatesAndIndexCreated,
+  LOCKS_CONCRETE_INDEX_NAME,
+} from './setup_lock_manager_index';
+
+describe('removeLockIndexWithIncorrectMappings', () => {
+  let esClient: ReturnType<typeof elasticsearchClientMock.createInternalClient>;
+  const logger = loggingSystemMock.createLogger();
+
+  const correctMappings: MappingTypeMapping = {
+    dynamic: 'false',
+    properties: {
+      token: { type: 'keyword' },
+      expiresAt: { type: 'date' },
+      createdAt: { type: 'date' },
+      metadata: { enabled: false, type: 'object' },
+    },
+  };
+
+  const incorrectMappings: MappingTypeMapping = {
+    dynamic: 'false',
+    properties: {
+      token: { type: 'text' }, // incorrect - should be keyword
+      expiresAt: { type: 'text' }, // incorrect - should be date
+      createdAt: { type: 'date' },
+      metadata: { enabled: false, type: 'object' },
+    },
+  };
+
+  beforeEach(() => {
+    esClient = elasticsearchClientMock.createInternalClient();
+  });
+
+  describe('when index name matches LOCKS_CONCRETE_INDEX_NAME', () => {
+    it('does not delete index when mappings are correct', async () => {
+      esClient.indices.getMapping.mockResolvedValueOnce({
+        [LOCKS_CONCRETE_INDEX_NAME]: { mappings: correctMappings },
+      });
+
+      await removeLockIndexWithIncorrectMappings(esClient, logger);
+
+      expect(esClient.indices.delete).not.toHaveBeenCalled();
+    });
+
+    it('deletes index when mappings are incorrect', async () => {
+      esClient.indices.getMapping.mockResolvedValueOnce({
+        [LOCKS_CONCRETE_INDEX_NAME]: { mappings: incorrectMappings },
+      });
+
+      await removeLockIndexWithIncorrectMappings(esClient, logger);
+
+      expect(esClient.indices.delete).toHaveBeenCalledWith({ index: LOCKS_CONCRETE_INDEX_NAME });
+    });
+  });
+
+  describe('when index is accessed via alias (response key differs from requested index)', () => {
+    const reindexedIndexName = `${LOCKS_CONCRETE_INDEX_NAME}-reindexed-for-10`;
+
+    it('does not delete index when mappings are correct', async () => {
+      esClient.indices.getMapping.mockResolvedValueOnce({
+        [reindexedIndexName]: { mappings: correctMappings },
+      });
+
+      await removeLockIndexWithIncorrectMappings(esClient, logger);
+
+      expect(esClient.indices.delete).not.toHaveBeenCalled();
+    });
+
+    it('deletes index when mappings are incorrect', async () => {
+      esClient.indices.getMapping.mockResolvedValueOnce({
+        [reindexedIndexName]: { mappings: incorrectMappings },
+      });
+
+      await removeLockIndexWithIncorrectMappings(esClient, logger);
+
+      expect(esClient.indices.delete).toHaveBeenCalledWith({ index: LOCKS_CONCRETE_INDEX_NAME });
+    });
+  });
+
+  describe('when index does not exist (404 error)', () => {
+    it('returns early without error', async () => {
+      esClient.indices.getMapping.mockRejectedValueOnce(
+        new errors.ResponseError({
+          statusCode: 404,
+          body: { error: { type: 'index_not_found_exception' } },
+          headers: {},
+          meta: {} as any,
+          warnings: [],
+        })
+      );
+
+      await removeLockIndexWithIncorrectMappings(esClient, logger);
+
+      expect(esClient.indices.delete).not.toHaveBeenCalled();
+    });
+  });
+
+  describe('when getMapping returns empty response', () => {
+    it('returns early without error', async () => {
+      esClient.indices.getMapping.mockResolvedValueOnce({});
+
+      await removeLockIndexWithIncorrectMappings(esClient, logger);
+
+      expect(esClient.indices.delete).not.toHaveBeenCalled();
+    });
+  });
+
+  describe('when mappings object is empty', () => {
+    it('returns early without error', async () => {
+      esClient.indices.getMapping.mockResolvedValueOnce({
+        [LOCKS_CONCRETE_INDEX_NAME]: { mappings: undefined as any },
+      });
+
+      await removeLockIndexWithIncorrectMappings(esClient, logger);
+
+      expect(esClient.indices.delete).not.toHaveBeenCalled();
+    });
+  });
+
+  describe('when getMapping fails with non-404 error', () => {
+    it('returns without throwing', async () => {
+      esClient.indices.getMapping.mockRejectedValueOnce(
+        new errors.ResponseError({
+          statusCode: 500,
+          body: { error: { type: 'internal_server_error' } },
+          headers: {},
+          meta: {} as any,
+          warnings: [],
+        })
+      );
+
+      await removeLockIndexWithIncorrectMappings(esClient, logger);
+
+      expect(esClient.indices.delete).not.toHaveBeenCalled();
+    });
+  });
+
+  describe('when delete fails', () => {
+    it('does not throw', async () => {
+      esClient.indices.getMapping.mockResolvedValueOnce({
+        [LOCKS_CONCRETE_INDEX_NAME]: { mappings: incorrectMappings },
+      });
+      esClient.indices.delete.mockRejectedValueOnce(new Error('Delete failed'));
+
+      await expect(removeLockIndexWithIncorrectMappings(esClient, logger)).resolves.not.toThrow();
+    });
+  });
+});
+
+describe('ensureTemplatesAndIndexCreated', () => {
+  let esClient: ReturnType<typeof elasticsearchClientMock.createInternalClient>;
+  const logger = loggingSystemMock.createLogger();
+
+  beforeEach(() => {
+    esClient = elasticsearchClientMock.createInternalClient();
+  });
+
+  it('creates index successfully', async () => {
+    await ensureTemplatesAndIndexCreated(esClient, logger);
+
+    expect(esClient.cluster.putComponentTemplate).toHaveBeenCalled();
+    expect(esClient.indices.putIndexTemplate).toHaveBeenCalled();
+    expect(esClient.indices.create).toHaveBeenCalledWith({ index: LOCKS_CONCRETE_INDEX_NAME });
+  });
+
+  it('handles resource_already_exists_exception without throwing', async () => {
+    esClient.indices.create.mockRejectedValueOnce(
+      new errors.ResponseError({
+        statusCode: 400,
+        body: { error: { type: 'resource_already_exists_exception' } },
+        headers: {},
+        meta: {} as any,
+        warnings: [],
+      })
+    );
+
+    await expect(ensureTemplatesAndIndexCreated(esClient, logger)).resolves.not.toThrow();
+  });
+
+  it('handles invalid_index_name_exception (index name exists as alias) without throwing', async () => {
+    esClient.indices.create.mockRejectedValueOnce(
+      new errors.ResponseError({
+        statusCode: 400,
+        body: { error: { type: 'invalid_index_name_exception' } },
+        headers: {},
+        meta: {} as any,
+        warnings: [],
+      })
+    );
+
+    await expect(ensureTemplatesAndIndexCreated(esClient, logger)).resolves.not.toThrow();
+  });
+
+  it('throws on other errors', async () => {
+    esClient.indices.create.mockRejectedValueOnce(
+      new errors.ResponseError({
+        statusCode: 500,
+        body: { error: { type: 'internal_server_error' } },
+        headers: {},
+        meta: {} as any,
+        warnings: [],
+      })
+    );
+
+    await expect(ensureTemplatesAndIndexCreated(esClient, logger)).rejects.toThrow();
+  });
+});

src/platform/packages/shared/kbn-lock-manager/src/setup_lock_manager_index.ts

@@ -35,7 +35,16 @@ export async function removeLockIndexWithIncorrectMappings(
     return;
   }
 
-  const { mappings } = res[LOCKS_CONCRETE_INDEX_NAME];
+  const indexMappings = Object.values(res);
+  if (indexMappings.length === 0) {
+    logger.warn(`No mappings found for "${LOCKS_CONCRETE_INDEX_NAME}"`);
+    return;
+  }
+  const { mappings } = indexMappings[0];
+  if (!mappings) {
+    logger.debug(`Mappings object is empty for "${LOCKS_CONCRETE_INDEX_NAME}"`);
+    return;
+  }
   const hasIncorrectMappings =
     mappings.properties?.token?.type !== 'keyword' ||
     mappings.properties?.expiresAt?.type !== 'date';
@@ -96,7 +105,12 @@ export async function ensureTemplatesAndIndexCreated(
       error instanceof errors.ResponseError &&
       error.body.error.type === 'resource_already_exists_exception';
 
-    if (isIndexAlreadyExistsError) {
+    // Handle the case where the index name already exists as an alias (e.g., after a reindex operation during cluster upgrade)
+    const isIndexNameExistsAsAliasError =
+      error instanceof errors.ResponseError &&
+      error.body.error.type === 'invalid_index_name_exception';
+
+    if (isIndexAlreadyExistsError || isIndexNameExistsAsAliasError) {
       logger.debug(`Index ${LOCKS_CONCRETE_INDEX_NAME} already exists. Skipping creation.`);
       return;
     }

src/platform/packages/shared/kbn-lock-manager/tsconfig.json

@@ -16,5 +16,7 @@
   "kbn_references": [
     "@kbn/logging",
     "@kbn/core",
+    "@kbn/core-elasticsearch-client-server-mocks",
+    "@kbn/core-logging-server-mocks",
   ]
 }

x-pack/solutions/observability/test/api_integration_deployment_agnostic/apis/ai_assistant/distributed_lock_manager/distributed_lock_manager.spec.ts

@@ -804,6 +804,72 @@ export default function ApiTest({ getService }: DeploymentAgnosticFtrProviderCon
           expect(indexExists).to.be(true);
         });
       });
+
+      describe('when lock index is accessed via alias (simulating reindexed scenario)', () => {
+        const reindexedIndexName = `${LOCKS_CONCRETE_INDEX_NAME}-reindexed-for-10`;
+
+        before(async () => {
+          await deleteLockIndexAssets(es, log);
+
+          // Create the actual index with correct mappings but different name
+          await es.indices.create({
+            index: reindexedIndexName,
+            settings: {
+              number_of_shards: 1,
+              auto_expand_replicas: '0-1',
+              hidden: true,
+            },
+            mappings: {
+              dynamic: false,
+              properties: {
+                token: { type: 'keyword' },
+                metadata: { enabled: false },
+                createdAt: { type: 'date' },
+                expiresAt: { type: 'date' },
+              },
+            },
+          });
+
+          // Add alias pointing LOCKS_CONCRETE_INDEX_NAME to the reindexed index
+          await es.indices.putAlias({
+            index: reindexedIndexName,
+            name: LOCKS_CONCRETE_INDEX_NAME,
+          });
+        });
+
+        after(async () => {
+          // Clean up alias and reindexed index
+          await es.indices.deleteAlias(
+            { index: reindexedIndexName, name: LOCKS_CONCRETE_INDEX_NAME },
+            { ignore: [404] }
+          );
+          await es.indices.delete({ index: reindexedIndexName }, { ignore: [404] });
+        });
+
+        it('should not throw when getMapping returns a different index name than requested', async () => {
+          // Verify the alias setup
+          const aliasExists = await es.indices.existsAlias({ name: LOCKS_CONCRETE_INDEX_NAME });
+          expect(aliasExists).to.be(true);
+
+          // This should NOT throw "Cannot destructure property 'mappings' of 'res[LOCKS_CONCRETE_INDEX_NAME]' as it is undefined"
+          try {
+            await setupLockManagerIndex(es, logger);
+          } catch (error) {
+            expect().fail(
+              `setupLockManagerIndex should not throw when index is accessed via alias, but got: ${error.message}`
+            );
+          }
+        });
+
+        it('should successfully acquire a lock when index is accessed via alias', async () => {
+          const lockManager = new LockManager('alias_test_lock', es, logger);
+          const acquired = await lockManager.acquire();
+          expect(acquired).to.be(true);
+
+          // Cleanup
+          await lockManager.release();
+        });
+      });
     });
 
     describe('setup robustness', () => {