diff --git a/source/common/changes/@cdf/assetlibrary-client/feature-enhanced-search_2022-06-30-16-47.json b/source/common/changes/@cdf/assetlibrary-client/feature-enhanced-search_2022-06-30-16-47.json new file mode 100644 index 000000000..6b0cd5da4 --- /dev/null +++ b/source/common/changes/@cdf/assetlibrary-client/feature-enhanced-search_2022-06-30-16-47.json @@ -0,0 +1,11 @@ +{ + "changes": [ + { + "packageName": "@cdf/assetlibrary-client", + "comment": "New \"enhanced mode\" for Asset Library", + "type": "patch" + } + ], + "packageName": "@cdf/assetlibrary-client", + "email": "jonasneu@amazon.com" +} \ No newline at end of file diff --git a/source/common/changes/@cdf/assetlibrary/feature-enhanced-search_2022-06-30-16-47.json b/source/common/changes/@cdf/assetlibrary/feature-enhanced-search_2022-06-30-16-47.json new file mode 100644 index 000000000..e72483f9c --- /dev/null +++ b/source/common/changes/@cdf/assetlibrary/feature-enhanced-search_2022-06-30-16-47.json @@ -0,0 +1,11 @@ +{ + "changes": [ + { + "packageName": "@cdf/assetlibrary", + "comment": "New \"enhanced mode\" for Asset Library", + "type": "minor" + } + ], + "packageName": "@cdf/assetlibrary", + "email": "jonasneu@amazon.com" +} \ No newline at end of file diff --git a/source/common/changes/@cdf/installer/feature-enhanced-search_2022-06-30-16-47.json b/source/common/changes/@cdf/installer/feature-enhanced-search_2022-06-30-16-47.json new file mode 100644 index 000000000..c8aca0620 --- /dev/null +++ b/source/common/changes/@cdf/installer/feature-enhanced-search_2022-06-30-16-47.json @@ -0,0 +1,11 @@ +{ + "changes": [ + { + "packageName": "@cdf/installer", + "comment": "New \"enhanced mode\" for Asset Library", + "type": "patch" + } + ], + "packageName": "@cdf/installer", + "email": "jonasneu@amazon.com" +} \ No newline at end of file diff --git a/source/common/changes/@cdf/integration-tests/feature-enhanced-search_2022-06-30-16-47.json b/source/common/changes/@cdf/integration-tests/feature-enhanced-search_2022-06-30-16-47.json new file mode 100644 index 000000000..5013916e0 --- /dev/null +++ b/source/common/changes/@cdf/integration-tests/feature-enhanced-search_2022-06-30-16-47.json @@ -0,0 +1,11 @@ +{ + "changes": [ + { + "packageName": "@cdf/integration-tests", + "comment": "New \"enhanced mode\" for Asset Library", + "type": "patch" + } + ], + "packageName": "@cdf/integration-tests", + "email": "jonasneu@amazon.com" +} \ No newline at end of file diff --git a/source/docs/migration.md b/source/docs/migration.md index 8934bdddb..e8aecb795 100644 --- a/source/docs/migration.md +++ b/source/docs/migration.md @@ -2,6 +2,10 @@ While we endeavor to always make backward compatible changes, there may be times when we need to make changes that are not backward compatible. If these changes are made at the API level then the affected modules REST API vendor mime types will be versioned supporting both new and old versions, as well as the modules minor version bumped. But if the change affect something else such as how configuration is handled, or how applications are deployed, then the major versions of the modules will be bumped with migration notes added here. +## Migrating an existing Asset Library deployment to Enhanced Search + +Starting with CDF Asset Library version 6.0.10 (part of CDF version 1.0.13), a new "enhanced" mode is available for CDF Asset Library. See the section [Migrating from full mode to enhanced mode](../packages/services/assetlibrary/docs/enhanced-search.md#migrating-from-full-mode-to-enhanced-mode) for guidance on migrating to enanced mode. + ## Migrating from Release <=1.0.10 to 1.0.11 ### Asset Library is now optional modules diff --git a/source/infrastructure/install-policy-3.json b/source/infrastructure/install-policy-3.json index d6e6645cd..0332df6e9 100644 --- a/source/infrastructure/install-policy-3.json +++ b/source/infrastructure/install-policy-3.json @@ -108,12 +108,37 @@ } } }, + { + "Sid": "OpenSearchServiceLinkedRoleCreate", + "Action": "iam:CreateServiceLinkedRole", + "Effect": "Allow", + "Resource": "arn:aws:iam::*:role/aws-service-role/es.amazonaws.com/AWSServiceRoleForAmazonOpenSearchService", + "Condition": { + "StringLike": { + "iam:AWSServiceName": [ + "opensearchservice.amazonaws.com", + "es.amazonaws.com" + ] + } + } + }, + { + "Sid": "OpenSearchServiceLinkedRoleGet", + "Action": "iam:GetRole", + "Effect": "Allow", + "Resource": "arn:aws:iam::*:role/aws-service-role/es.amazonaws.com/AWSServiceRoleForAmazonOpenSearchService" + }, + { + "Sid": "OpenSearchLegacyServiceLinkedRoleGet", + "Action": "iam:GetRole", + "Effect": "Allow", + "Resource": "arn:aws:iam::*:role/aws-service-role/es.amazonaws.com/AWSServiceRoleForAmazonElasticsearchService" + }, { "Sid": "ECS", "Action": "ecs:*", "Effect": "Allow", "Resource": "*" - } - + } ] -} \ No newline at end of file +} diff --git a/source/packages/integration-tests/features/assetlibrary/README.md b/source/packages/integration-tests/features/assetlibrary/README.md index 1005c7c0d..d6b8ef3ce 100644 --- a/source/packages/integration-tests/features/assetlibrary/README.md +++ b/source/packages/integration-tests/features/assetlibrary/README.md @@ -1,38 +1,47 @@ -# Asset Library integration tests +# Asset Library Integration Tests -Note: only the _full_ mode is tested as part of the CI/CD pipeline. We need to add both the _full (witht FGAC)_ and _lite_ modes to it. But for now, these need testing manually... +After following the [general steps for setting up an environment for integration testing](../README.md), the commands below can be used to run integration tests for Asset Library in various configurations. +The tests are executed locally on your development environment and send HTTP requests to an Asset Library API deployed in an AWS account. -### Testing full-with-authz mode +The subset of tests must match the mode and configuration of the Asset Library deployment at the URL configured in `ASSETLIBRARY_BASE_URL` in `path/to/local/.env`. +For example, integration tests in the `full-with-authz` folder only pass with an Asset Library deployment that is configured with `AUTHORIZATION_ENABLED=true`. -- Run asset library with FGAC enabled (only supported in _full_ mode): -```sh -$ assetlibrary> CONFIG_LOCATION="path/to/local/.env" ASSETLIBRARY_AUTHORIZATION_ENABLED=true npm run start -``` -- Run FGAC specific integration tests: -```sh -$ integration-tests> CONFIG_LOCATION="path/to/integrationtest/.env" pnpm run integration-test -- features/assetlibrary/full-with-authz/* -``` +Note: Only the _full_ mode is currently tested as part of the CI/CD pipeline. -### Testing lite mode +## Testing full/enhanced mode -See the [special notes for running lite mode tests](./lite/README.md). +To run integration tests for Asset Library in _full_ or _enhanced_ mode, irrespective of `AUTHORIZATION_ENABLED` setting: -- Run asset library in _lite_ mode: ```sh -$ assetlibrary> CONFIG_LOCATION="path/to/local/.env" ASSETLIBRARY_MODE=lite npm run start +$ source/packages/integration-tests> CONFIG_LOCATION="path/to/integrationtest/.env" pnpm run integration-test -- features/assetlibrary/full/* ``` -- Run FGAC specific integration tests: + +## Testing full-with-authz mode + +To run integration tests for Asset Library in _full_ mode, when `AUTHORIZATION_ENABLED` is set to `true`: + ```sh -$ integration-tests> CONFIG_LOCATION="path/to/integrationtest/.env" pnpm run integration-test -- features/assetlibrary/lite/* +$ source/packages/integration-tests> CONFIG_LOCATION="path/to/integrationtest/.env" npm run integration-test -- features/assetlibrary/full-with-authz/* ``` -### Testing full mode +These tests are an addition to, not a superset of, "Testing full/enhanced mode". + +## Testing enhanced mode + +To run integration tests for Asset Library in _enhanced_ mode, irrespective of `AUTHORIZATION_ENABLED` setting: -- Run asset library in the default _full_ mode (requires a Neptune tunnel of running loc) ```sh -$ assetlibrary> CONFIG_LOCATION="path/to/local/.env" npm run start +$ source/packages/integration-tests> CONFIG_LOCATION="path/to/integrationtest/.env" npm run integration-test -- features/assetlibrary/full-with-enhancedsearch/* ``` -- Run FGAC specific integration tests: + +These tests are an addition to, not a superset of, "Testing full/enhanced mode". + +## Testing lite mode + +See the [special notes for running lite mode tests](lite/README.md). + +To run integration tests for Asset Library in _lite_ mode: + ```sh -$ integration-tests> CONFIG_LOCATION="path/to/integrationtest/.env" pnpm run integration-test -- features/assetlibrary/full/* +$ source/packages/integration-tests> CONFIG_LOCATION="path/to/integrationtest/.env" npm run integration-test -- features/assetlibrary/lite/* ``` diff --git a/source/packages/integration-tests/features/assetlibrary/full-with-enhancedsearch/deviceSearch.feature b/source/packages/integration-tests/features/assetlibrary/full-with-enhancedsearch/deviceSearch.feature new file mode 100644 index 000000000..87e17cd08 --- /dev/null +++ b/source/packages/integration-tests/features/assetlibrary/full-with-enhancedsearch/deviceSearch.feature @@ -0,0 +1,249 @@ +#----------------------------------------------------------------------------------------------------------------------- +# Copyright Amazon.com Inc. or its affiliates. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"). You may not use this file except in compliance +# with the License. A copy of the License is located at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# or in the 'license' file accompanying this file. This file is distributed on an 'AS IS' BASIS, WITHOUT WARRANTIES +# OR CONDITIONS OF ANY KIND, express or implied. See the License for the specific language governing permissions +# and limitations under the License. +#----------------------------------------------------------------------------------------------------------------------- + +Feature: Device enhanced search + + @setup_deviceSearch_enhanced_feature + Scenario: Setup + Given published assetlibrary device template "test-enhancedsearch-deviceTpl" exists + And group "/enhancedSearchGroup_all" exists + And group "/enhancedSearchGroup_xxyy" exists + And group "/enhancedSearchGroup_xyyx" exists + And device "test-enhancedsearch-aaaa" exists + And device "test-enhancedsearch-aaab" exists + + Scenario: Baseline search for all devices + When I search with summary with following attributes: + | ancestorPath | /enhancedSearchGroup_all | + Then search result contains 16 total + + Scenario: Startswith search using enhanced search 1 + When I search with following attributes: + | ancestorPath | /enhancedSearchGroup_all | + | startsWith | characters:a | + Then search result contains 8 total + And search result contains device "test-enhancedsearch-aaaa" + And search result contains device "test-enhancedsearch-aaab" + And search result contains device "test-enhancedsearch-aaba" + And search result contains device "test-enhancedsearch-aabb" + And search result contains device "test-enhancedsearch-abaa" + And search result contains device "test-enhancedsearch-abab" + And search result contains device "test-enhancedsearch-abba" + And search result contains device "test-enhancedsearch-abbb" + + Scenario: Startswith search using enhanced search 2 + When I search with following attributes: + | ancestorPath | /enhancedSearchGroup_all | + | startsWith | characters:aa | + Then search result contains 4 total + And search result contains device "test-enhancedsearch-aaaa" + And search result contains device "test-enhancedsearch-aaab" + And search result contains device "test-enhancedsearch-aaba" + And search result contains device "test-enhancedsearch-aabb" + + Scenario: Startswith search using enhanced search 3 + When I search with following attributes: + | ancestorPath | /enhancedSearchGroup_all | + | startsWith | characters:aaa | + Then search result contains 2 total + And search result contains device "test-enhancedsearch-aaaa" + And search result contains device "test-enhancedsearch-aaab" + + Scenario: Endswith search using enhanced search 1 + When I search with following attributes: + | ancestorPath | /enhancedSearchGroup_all | + | endsWith | characters:a | + Then search result contains 8 total + And search result contains device "test-enhancedsearch-aaaa" + And search result contains device "test-enhancedsearch-aaba" + And search result contains device "test-enhancedsearch-abaa" + And search result contains device "test-enhancedsearch-abba" + And search result contains device "test-enhancedsearch-baaa" + And search result contains device "test-enhancedsearch-baba" + And search result contains device "test-enhancedsearch-bbaa" + And search result contains device "test-enhancedsearch-bbba" + + Scenario: Endswith search using enhanced search 2 + When I search with following attributes: + | ancestorPath | /enhancedSearchGroup_all | + | endsWith | characters:aa | + Then search result contains 4 total + And search result contains device "test-enhancedsearch-aaaa" + And search result contains device "test-enhancedsearch-abaa" + And search result contains device "test-enhancedsearch-baaa" + And search result contains device "test-enhancedsearch-bbaa" + + Scenario: Endswith search using enhanced search 3 + When I search with following attributes: + | ancestorPath | /enhancedSearchGroup_all | + | endsWith | characters:aaa | + Then search result contains 2 total + And search result contains device "test-enhancedsearch-aaaa" + And search result contains device "test-enhancedsearch-baaa" + + Scenario: Contains search using enhanced search 1 + When I search with following attributes: + | ancestorPath | /enhancedSearchGroup_all | + | contains | characters:aaa | + Then search result contains 3 total + And search result contains device "test-enhancedsearch-aaaa" + And search result contains device "test-enhancedsearch-baaa" + And search result contains device "test-enhancedsearch-aaab" + + Scenario: Contains search using enhanced search 2 + When I search with following attributes: + | ancestorPath | /enhancedSearchGroup_all | + | contains | characters:aba | + Then search result contains 4 total + And search result contains device "test-enhancedsearch-aaba" + And search result contains device "test-enhancedsearch-abaa" + And search result contains device "test-enhancedsearch-baba" + And search result contains device "test-enhancedsearch-abab" + + Scenario: Regex search 1 + When I search with following attributes: + | ancestorPath | /enhancedSearchGroup_all | + | regex | characters:a[ab]aa | + Then search result contains 2 total + And search result contains device "test-enhancedsearch-aaaa" + And search result contains device "test-enhancedsearch-abaa" + + Scenario: Regex search 2 + When I search with following attributes: + | ancestorPath | /enhancedSearchGroup_all | + | regex | characters:a.*a | + Then search result contains 4 total + And search result contains device "test-enhancedsearch-aaaa" + And search result contains device "test-enhancedsearch-aaba" + And search result contains device "test-enhancedsearch-abaa" + And search result contains device "test-enhancedsearch-abba" + + Scenario: Fulltext search Apple + When I search with following attributes: + | ancestorPath | /enhancedSearchGroup_all | + | fulltext | words:apple | + Then search result contains 5 total + And search result contains device "test-enhancedsearch-aaaa" + And search result contains device "test-enhancedsearch-abaa" + And search result contains device "test-enhancedsearch-abab" + And search result contains device "test-enhancedsearch-baba" + And search result contains device "test-enhancedsearch-bbba" + + Scenario: Fulltext search wildcard match for Apple and Pineapple + When I search with following attributes: + | ancestorPath | /enhancedSearchGroup_all | + | fulltext | words:*apple | + Then search result contains 9 total + And search result contains device "test-enhancedsearch-aaaa" + And search result contains device "test-enhancedsearch-aaba" + And search result contains device "test-enhancedsearch-abaa" + And search result contains device "test-enhancedsearch-abab" + And search result contains device "test-enhancedsearch-baaa" + And search result contains device "test-enhancedsearch-baba" + And search result contains device "test-enhancedsearch-babb" + And search result contains device "test-enhancedsearch-bbaa" + And search result contains device "test-enhancedsearch-bbba" + + Scenario: Fulltext fuzzy search Cherry misspelling + When I search with following attributes: + | ancestorPath | /enhancedSearchGroup_all | + | fulltext | words:chery~ | + Then search result contains 4 total + And search result contains device "test-enhancedsearch-aaab" + And search result contains device "test-enhancedsearch-abaa" + And search result contains device "test-enhancedsearch-baba" + And search result contains device "test-enhancedsearch-bbbb" + + Scenario: Fulltext two words without operator + When I search with following attributes: + | ancestorPath | /enhancedSearchGroup_all | + | fulltext | words:kiwi banana | + Then search result contains 10 total + + Scenario: Fulltext two words with AND operator + When I search with following attributes: + | ancestorPath | /enhancedSearchGroup_all | + | fulltext | words:kiwi AND banana | + Then search result contains 1 total + And search result contains device "test-enhancedsearch-bbab" + + Scenario: Lucene query with simple equality + When I search with following attributes: + | ancestorPath | /enhancedSearchGroup_all | + # important: must URL-encode the filter value, otherwise the step definition will incorrectly parse it because of + # the ":" characters in the lucene query + | lucene | *:predicates.characters.value%3Abbba | + Then search result contains 1 total + And search result contains device "test-enhancedsearch-bbba" + + Scenario: Lucene query with boolean OR + When I search with following attributes: + | ancestorPath | /enhancedSearchGroup_all | + | lucene | *:predicates.characters.value%3Abbba%20OR%20predicates.characters.value%3Abbbb | + Then search result contains 2 total + And search result contains device "test-enhancedsearch-bbba" + And search result contains device "test-enhancedsearch-bbbb" + + Scenario: Lucene query with boolean AND + When I search with following attributes: + | ancestorPath | /enhancedSearchGroup_all | + | lucene | *:predicates.characters.value%3Abbba%20AND%20predicates.characters.value%3Abbbb | + Then search result contains 0 total + + Scenario: Lucene query with fulltext exact match + When I search with following attributes: + | ancestorPath | /enhancedSearchGroup_all | + | lucene | *:predicates.words.value%3Agrapefruit | + Then search result contains 3 total + + Scenario: Lucene query with fulltext fuzzy match + When I search with following attributes: + | ancestorPath | /enhancedSearchGroup_all | + | lucene | *:predicates.words.value%3Agripefruit~ | + Then search result contains 3 total + + Scenario: Lucene query with fulltext regex match + When I search with following attributes: + | ancestorPath | /enhancedSearchGroup_all | + | lucene | *:predicates.words.value%3A/g.+fruit/ | + Then search result contains 3 total + + Scenario: Lucene query with regex fuzzy and boolean operators + When I search with following attributes: + | ancestorPath | /enhancedSearchGroup_all | + | lucene | *:predicates.characters.value%3Ababa%20AND%20predicates.words.value%3Agrapefruit | + Then search result contains 1 total + And search result contains device "test-enhancedsearch-baba" + + Scenario: Regex search with traversal + When I search with following attributes: + | ancestorPath | /enhancedSearchGroup_all | + | regex | part_of_group:out:name:enhancedSearchGroup_xy.* | + Then search result contains 8 total + + Scenario: Regex search with traversal and non-traversal startsWith + When I search with following attributes: + | ancestorPath | /enhancedSearchGroup_all | + | regex | part_of_group:out:name:enhancedSearchGroup_x[xy]{2}x | + | startsWith | characters:a | + Then search result contains 0 total + + @teardown_deviceSearch_enhanced_feature + Scenario: Teardown + Given draft assetlibrary device template "test-enhancedsearch-deviceTpl" does not exist + And published assetlibrary device template "test-enhancedsearch-deviceTpl" does not exist + And group "/enhancedSearchGroup_all" does not exist + And group "/enhancedSearchGroup_xxyy" does not exist + And group "/enhancedSearchGroup_xyyx" does not exist + And device "test-enhancedsearch-aaaa" does not exist + And device "test-enhancedsearch-aaab" does not exist diff --git a/source/packages/integration-tests/src/step_definitions/assetlibrary/search.steps.ts b/source/packages/integration-tests/src/step_definitions/assetlibrary/search.steps.ts index b41980a5e..6cc9ed390 100644 --- a/source/packages/integration-tests/src/step_definitions/assetlibrary/search.steps.ts +++ b/source/packages/integration-tests/src/step_definitions/assetlibrary/search.steps.ts @@ -80,7 +80,9 @@ function buildSearchRequest(data: DataTable): SearchRequestModel { const filter: SearchRequestFilter = { field: attrs[attrs.length - 2], - value: attrs[attrs.length - 1], + // test cases can optionally URL-encode the filter value, for example for lucene search operator + // where the filter value contains ":" characters + value: decodeURIComponent(attrs[attrs.length - 1]), }; // do we have traversals defined? if (attrs.length > 2) { diff --git a/source/packages/integration-tests/src/support/assetLibrary_hooks.ts b/source/packages/integration-tests/src/support/assetLibrary_hooks.ts index 59486472e..43d7d0238 100644 --- a/source/packages/integration-tests/src/support/assetLibrary_hooks.ts +++ b/source/packages/integration-tests/src/support/assetLibrary_hooks.ts @@ -210,6 +210,27 @@ const templatesService: TemplatesService = container.get( ASSETLIBRARY_CLIENT_TYPES.TemplatesService ); const profilesService: ProfilesService = container.get(ASSETLIBRARY_CLIENT_TYPES.ProfilesService); +const ENHANCEDSEARCH_FEATURES_DEVICE_TEMPLATE_ID = 'TEST-enhancedSearch-deviceTpl'; +const ENHANCEDSEARCH_FEATURES_GROUP_ALL = 'enhancedSearchGroup_all'; +const ENHANCEDSEARCH_FEATURES_GROUP_SUFFIXES = ['xxyy', 'xyyx']; +const ENHANCEDSEARCH_FEATURES_DEVICES = [ + { characters: 'aaaa', groups: ['xxyy'], words: 'apple orange kiwi' }, + { characters: 'aaab', groups: ['xxyy'], words: 'cherry blackberry grapefruit' }, + { characters: 'aaba', groups: ['xxyy'], words: 'pineapple pear orange' }, + { characters: 'aabb', groups: ['xxyy'], words: 'pear kiwi orange' }, + { characters: 'abaa', groups: ['xxyy'], words: 'cherry apple blackberry' }, + { characters: 'abab', groups: ['xxyy'], words: 'orange pineapple apple' }, + { characters: 'abba', groups: ['xxyy'], words: 'kiwi orange blackberry' }, + { characters: 'abbb', groups: ['xxyy'], words: 'blackberry orange kiwi' }, + { characters: 'baaa', groups: ['xyyx'], words: 'pineapple kiwi peach' }, + { characters: 'baab', groups: ['xyyx'], words: 'banana pear grapefruit' }, + { characters: 'baba', groups: ['xyyx'], words: 'grapefruit cherry apple' }, + { characters: 'babb', groups: ['xyyx'], words: 'blackberry orange pineapple' }, + { characters: 'bbaa', groups: ['xyyx'], words: 'orange banana pineapple' }, + { characters: 'bbab', groups: ['xyyx'], words: 'kiwi strawberry banana' }, + { characters: 'bbba', groups: ['xyyx'], words: 'orange apple banana' }, + { characters: 'bbbb', groups: ['xyyx'], words: 'cherry banana potato' }, +]; /* Cucumber describes current scenario context as “World”. It can be used to store the state of the scenario @@ -1464,3 +1485,95 @@ Before({ tags: '@setup_groupSearch_lite_feature' }, async function () { Before({ tags: '@teardown_groupSearch_lite_feature' }, async function () { await teardown_groupSearch_lite_feature(); }); + +async function teardown_deviceSearch_enhanced_feature() { + await deleteAssetLibraryDevices( + ENHANCEDSEARCH_FEATURES_DEVICES.map( + ({ characters }) => `TEST-enhancedSearch-${characters}` + ) + ); + await deleteAssetLibraryGroups([ + `/${ENHANCEDSEARCH_FEATURES_GROUP_ALL}`, + ...ENHANCEDSEARCH_FEATURES_GROUP_SUFFIXES.map( + (suffix) => `/enhancedSearchGroup_${suffix}` + ), + ]); + await deleteAssetLibraryTemplates(CategoryEnum.device, [ + ENHANCEDSEARCH_FEATURES_DEVICE_TEMPLATE_ID, + ]); +} + +Before({ tags: '@setup_deviceSearch_enhanced_feature' }, async function () { + // teardown first just in case + await teardown_deviceSearch_enhanced_feature(); + + // device template + const deviceType: TypeResource = { + templateId: ENHANCEDSEARCH_FEATURES_DEVICE_TEMPLATE_ID, + category: 'device', + properties: { + words: { type: ['string'] }, + characters: { type: ['string'] }, + }, + relations: { + out: { + part_of_group: ['root'], + }, + }, + }; + await templatesService.createTemplate(deviceType, additionalHeaders); + await templatesService.publishTemplate( + CategoryEnum.device, + ENHANCEDSEARCH_FEATURES_DEVICE_TEMPLATE_ID, + additionalHeaders + ); + + // create group that all devices are part of + await groupsService.createGroup({ + templateId: 'root', + parentPath: '/', + name: ENHANCEDSEARCH_FEATURES_GROUP_ALL, + attributes: {}, + }); + + // create sub-groups that only a subset of devices is part of + await Promise.all( + ENHANCEDSEARCH_FEATURES_GROUP_SUFFIXES.map((suffix) => + groupsService.createGroup({ + templateId: 'root', + parentPath: '/', + name: `enhancedSearchGroup_${suffix}`, + attributes: {}, + }) + ) + ); + + // create devices + await Promise.all( + ENHANCEDSEARCH_FEATURES_DEVICES.map(({ characters, words, groups }) => + devicesService.createDevice({ + templateId: ENHANCEDSEARCH_FEATURES_DEVICE_TEMPLATE_ID, + deviceId: `TEST-enhancedSearch-${characters}`, + attributes: { + words: words, + characters: characters, + }, + groups: { + out: { + part_of_group: [ + `/${ENHANCEDSEARCH_FEATURES_GROUP_ALL}`, + ...groups.map((suffix) => `/enhancedSearchGroup_${suffix}`), + ], + }, + }, + }) + ) + ); + + // wait a short while for data to arrive in the OpenSearch index + return new Promise((resolve) => setTimeout(resolve, 10000)); +}); + +Before({ tags: '@teardown_deviceSearch_enhanced_feature' }, async function () { + await teardown_deviceSearch_enhanced_feature(); +}); diff --git a/source/packages/libraries/clients/assetlibrary-client/src/client/search.model.spec.ts b/source/packages/libraries/clients/assetlibrary-client/src/client/search.model.spec.ts index 4391f98f7..b73fb8d45 100644 --- a/source/packages/libraries/clients/assetlibrary-client/src/client/search.model.spec.ts +++ b/source/packages/libraries/clients/assetlibrary-client/src/client/search.model.spec.ts @@ -125,6 +125,9 @@ describe('SearchRequestModel', () => { 'startsWith', 'endsWith', 'contains', + 'fulltext', + 'regex', + 'lucene', 'exist', 'nexist', ]; diff --git a/source/packages/libraries/clients/assetlibrary-client/src/client/search.model.ts b/source/packages/libraries/clients/assetlibrary-client/src/client/search.model.ts index 670d25a47..45ceca910 100644 --- a/source/packages/libraries/clients/assetlibrary-client/src/client/search.model.ts +++ b/source/packages/libraries/clients/assetlibrary-client/src/client/search.model.ts @@ -48,6 +48,9 @@ export class SearchRequestModel { startsWith?: SearchRequestFilters; endsWith?: SearchRequestFilters; contains?: SearchRequestFilters; + fulltext?: SearchRequestFilters; + regex?: SearchRequestFilters; + lucene?: SearchRequestFilters; exist?: SearchRequestFilters; nexist?: SearchRequestFilters; @@ -72,6 +75,9 @@ export class SearchRequestModel { this.startsWith = other.startsWith; this.endsWith = other.endsWith; this.contains = other.contains; + this.fulltext = other.fulltext; + this.regex = other.regex; + this.lucene = other.lucene; this.exist = other.exist; this.nexist = other.nexist; this.facetField = other.facetField; @@ -169,6 +175,18 @@ export class SearchRequestModel { qs = qs.concat(this.buildQSValues('endsWith', this.endsWith, true)); } + if (this.fulltext) { + qs = qs.concat(this.buildQSValues('fulltext', this.fulltext, true)); + } + + if (this.regex) { + qs = qs.concat(this.buildQSValues('regex', this.regex, true)); + } + + if (this.lucene) { + qs = qs.concat(this.buildQSValues('lucene', this.lucene, true)); + } + if (this.contains) { qs = qs.concat(this.buildQSValues('contains', this.contains, true)); } @@ -239,6 +257,26 @@ export class SearchRequestModel { qs['lte'] = values.map((v) => v.split('=')[1]); } + if (this.fulltext) { + const values = this.buildQSValues('fulltext', this.fulltext); + qs['fulltext'] = values.map((v) => v.split('=')[1]); + } + + if (this.regex) { + const values = this.buildQSValues('regex', this.regex); + qs['regex'] = values.map((v) => v.split('=')[1]); + } + + if (this.lucene) { + const values = this.buildQSValues('lucene', this.lucene); + qs['lucene'] = values.map((v) => v.split('=')[1]); + } + + if (this.exist) { + const values = this.buildQSValues('exist', this.exist); + qs['exist'] = values.map((v) => v.split('=')[1]); + } + if (this.gt) { const values = this.buildQSValues('gt', this.gt); qs['gt'] = values.map((v) => v.split('=')[1]); @@ -263,10 +301,6 @@ export class SearchRequestModel { const values = this.buildQSValues('contains', this.contains); qs['contains'] = values.map((v) => v.split('=')[1]); } - if (this.exist) { - const values = this.buildQSValues('exist', this.exist); - qs['exist'] = values.map((v) => v.split('=')[1]); - } if (this.nexist) { const values = this.buildQSValues('nexist', this.nexist); diff --git a/source/packages/services/assetlibrary/README.md b/source/packages/services/assetlibrary/README.md index cce4c9732..4cec3e51a 100644 --- a/source/packages/services/assetlibrary/README.md +++ b/source/packages/services/assetlibrary/README.md @@ -192,8 +192,10 @@ In the example above, retrieving the list of policies for `device001`, `device00 - [Application configuration](docs/configuration.md) - [Events](docs/events.md) - [Fine-gained access controll](docs/fine-grained-access-control.md) -- [Full vs lite mode](docs/modes.md) +- [Asset Library modes: lite, full, enhanced](docs/modes.md) +- [Enhanced Search](docs/enhanced-search.md) - [Profiles](docs/profiles.md) - [Swagger](docs/swagger.yml) - [Templates user guide](docs/templates-user.md) - [Templates developer guide](docs/templates-developer.md) +- [Integration Testing Asset Library](../../../packages/integration-tests/features/assetlibrary/README.md) diff --git a/source/packages/services/assetlibrary/docs/configuration.md b/source/packages/services/assetlibrary/docs/configuration.md index 2ececfdc6..2fde9a459 100644 --- a/source/packages/services/assetlibrary/docs/configuration.md +++ b/source/packages/services/assetlibrary/docs/configuration.md @@ -52,9 +52,7 @@ CORS_EXPOSED_HEADERS=content-type,location # the base path from the request to allow the module to map the incoming request to the correct lambda handler CUSTOMDOMAIN_BASEPATH= -# The Asset Library mode. `full` (default) will enable the full feature set and -# use Neptune as its datastore, whereas `lite` will offer a reduced feature set -# (see documentation) and use the AWS IoT Device Registry as its datastore. +# The Asset Library mode: `full`, `enhanced`, or `lite`. See docs/modes.md for details. MODE=full # If true, fine-grained access control will be enabled. Refer to documentation diff --git a/source/packages/services/assetlibrary/docs/enhanced-search.md b/source/packages/services/assetlibrary/docs/enhanced-search.md new file mode 100644 index 000000000..4add7097d --- /dev/null +++ b/source/packages/services/assetlibrary/docs/enhanced-search.md @@ -0,0 +1,111 @@ +# ASSET LIBRARY ENHANCED SEARCH + +## Migrating from `full` mode to `enhanced` mode + +Creating a new CDF Asset Library deployment is readily achieved by running the [CDF Installer](../../installer/README.md) which creates a Neptune database, an OpenSearch cluster, and serverless components that synchronize changes from Neptune to OpenSearch. +The migration of an _existing_ CDF Asset Library from `full` mode to `enhanced` mode, requires additional steps to import the existing data first. + +There is no migration path from `lite` mode to `enhanced` mode. + +The following instructions use the [Export Neptune to ElasticSearch](https://github.com/awslabs/amazon-neptune-tools/tree/master/export-neptune-to-elasticsearch) solution in a way that should be sufficient for most CDF Asset Library deployments. +Users are encouraged to review the solution's documentation for additional customization options and considerations related to large databases. + +### Step 1: Update the CDF configuration file + +1. Ensure that the [CDF Installer](../../installer/README.md) configuration file for the CDF deployment you want to migrate is available at the location where the CDF Installer expects it. For example, on Linux and macOS the CDF Installer expects the file to be located at a path formatted like: `~/aws-connected-device-framework/config///.json`. Recommended best practice is to check the configuration file(s) into source control. +2. [Configure your terminal/shell](https://docs.aws.amazon.com/cli/latest/userguide/cli-configure-envvars.html) so that the AWS CLI has permissions to deploy CDF in the AWS account where the CDF deployment you wish to migrate is located. For example, you can set the `AWS_ACCESS_KEY_ID`, `AWS_SECRET_ACCESS_KEY`, and `AWS_DEFAULT_REGION` environment variables. +3. Run the CDF Installer's configuration wizard: `cdf-cli deploy --dryrun`. The `--dryrun` option ensures that the CDF configuration file gets modified but the deployment is _not_ performed yet. + +Confirm that the modified config file specifies `enhanced` mode and contains the configuration settings required to deploy enhanced search: + +```json +{ + "environment": "", + "region": "", + "accountId": "", + "assetLibrary": { + "mode": "enhanced", + "openSearchDataNodeInstanceType": "**.******.search", + "openSearchDataNodeInstanceCount": x, + "openSearchEBSVolumeSize": xx + } +} +``` + +Note: In the configuration file snippet above, only relevant fields are shown for illustration. The actual file is much longer. + +### Step 2: Pause Asset Library database changes + +No changes should be made to the Asset Library database until the initial data import is complete. +Take the appropriate steps to temporarily stop write traffic to the Asset Library database, for example by switching your CDF Facade to maintenance mode. + +### Step 3: Deploy the updated configuration file + +Deploy the updated CDF configuration with the command + +```sh +cdf-cli deploy -c ~/aws-connected-device-framework/config///.json +``` + +Confirm that this created a new CloudFormation stack named `cdf-assetlibrary-enhancedsearch-`. + +> :warning: Note: This solution includes adding a Neptune DB Cluster parameter group to enable Neptune streams. In order for those parameter group changes to take effect, all instances of the Neptune DB cluster need to be rebooted. +### Step 4: Deploy the "Export Neptune to ElasticSearch" solution + +1. Log into the AWS Console account that contains the CDF deployment you wish to migrate. +This ensures that clicking the link in the next step opens in the correct account. +2. Launch the Neptune-to-ElasticSearch CloudFormation stack. +You can do so by clicking the link for the AWS region that contains the CDF deployment you wish to migrate in the [installation instructions for the "Export Neptune to ElasticSearch" solution](https://github.com/awslabs/amazon-neptune-tools/blob/master/export-neptune-to-elasticsearch/readme.md#installation). The table below shows suggested values for the stack parameters. Keep the stack name as the default `neptune-index`. +3. Acknowledge the CloudFormation templates' use of IAM capabilities at the bottom of the form. + +Stack parameters for the Neptune-to-ElasticSearch CloudFormation stack: + +| Stack parameter | Value | +| --- | --- | +| _Network Configuration_ | +| VPC | The `VpcId` output of the stack `cdf-network-` | +| Subnet1 | The first comma-separated value in the `PrivateSubnetIds` of the stack `cdf-network-` | +| _Neptune Configuration_ | +| NeptuneEndpoint | The `DBClusterReadEndpoint` output of the stack `cdf-assetlibrary-neptune-` | +| NeptunePort | Keep default `8182` +| NeptuneEngine | Select `gremlin` | +| ExportScope | Select `all` | +| CloneCluster | Select `yes` | +| NeptuneClientSecurityGroup | The `NeptuneSecurityGroupID` output of the stack `cdf-assetlibrary-neptune-` | +| AdditionalParams | Leave empty | +| _ElasticSearch Configuration_ | +| ElasticSearchEndpoint | The `OpenSearchDomainEndpoint` output of the stack `cdf-assetlibrary-enhancedsearch-` _without the `https://` prefix_. | +| NumberOfShards | Keep default. Only change if you modified the `NumberOfReplica` parameter of the stack `cdf-assetlibrary-enhancedsearch-`. | +| NumberOfReplica | Keep default. Only change if you modified the `NumberOfShards` parameter of the stack `cdf-assetlibrary-enhancedsearch-`. | +| GeoLocationFields | Leave empty | +| ElasticSearchClientSecurityGroup | The `HTTPSAccessSG` output of the stack `cdf-assetlibrary-enhancedsearch-` | +| _Advanced_ | +| Concurrency | Keep default | +| KinesisShardCount | Keep default | +| BatchSize | Keep default | + +Confirm that the CloudFormation stack `neptune-index` exists and has status `CREATE_COMPLETE`. + +Note that the `neptune-index` CloudFormation stack contains a nested stack named `neptune-index-EbsVolumeSizeStack-xxxxxxxxxxxx`. +You will not interact with this nested stack directly. + +### Step 5: Use the "Export Neptune to ElasticSearch" solution + +Invoke the AWS Lambda function created as part of the `neptune-index` CloudFormation stack. + +You can do either by navigating to the AWS Lambda console or the AWS CLI. + +Using the AWS CLI, copy the invoke command from the `StartExportCommand` stack output of the `neptune-index` stack and run it from the terminal/shell configured in [Step 1](#step-1-update-the-cdf-configuration-file). + +Alternatively, in the console, navigate to the function named `export-neptune-to-kinesis-xxxx` and invoke it. + +### Step 6: Resume Asset Library database changes + +At this point, all existing Asset Library data has been synchronized into the OpenSearch cluster and is available for search queries that include enhanced search operators (see [swagger.yml](./swagger.yml) for examples). +Incremental changes to the Amazon Neptune database content can resume. + +Reverse the steps you have taken in [Step 2](#step-2-pause-asset-library-database-changes). + +### Step 7: Clean Up + +Delete the "Export Neptune to ElasticSearch" solution by removing the CloudFormation stack created in [Step 4](#step-4-deploy-the-export-neptune-to-elasticsearch-solution). diff --git a/source/packages/services/assetlibrary/docs/modes.md b/source/packages/services/assetlibrary/docs/modes.md index 214fabf65..7d41bc800 100644 --- a/source/packages/services/assetlibrary/docs/modes.md +++ b/source/packages/services/assetlibrary/docs/modes.md @@ -2,9 +2,10 @@ ## Introduction -The Asset Library is capable of running in one of two modes: `full` and `lite`. +The Asset Library is capable of running in one of three modes: `full`, `enhanced`, and `lite`. -The `lite` version uses The AWS IoT Device Registry to store all devices and groups data, whereas the `full` version utilizes [AWS Neptune](https://aws.amazon.com/neptune/) to provide more advanced data modelling features. +The `lite` version uses the AWS IoT Device Registry to store all devices and groups data, whereas the `full` version utilizes [Amazon Neptune](https://aws.amazon.com/neptune/) to provide more advanced data modeling features. +In `enhanced` mode, an OpenSearch cluster is deployed as secondary data store and provides enhanced search functionality. The mode is determined via a configuration property at the time of deployment. The following describes the differences in functionality between the two modes. @@ -14,7 +15,7 @@ The following table indicates which REST API's are available in which mode: ### Devices -| Endpoint | Description | `full` mode | `lite` mode | +| Endpoint | Description | `full` and `enhanced` modes | `lite` mode | | ------------------------------------------------------------------- | ------------------------------------------------------------------------- | ------------------------------------------------------ | ---------------------------------------------------------------------------------------------------------------------------------------------- | | `POST /devices` | Adds a new device to the Asset Library | ✅ (adding to a default parent group if none provided) | ✅ (creating components not supported, and no default parent group set if none provided) | | `POST /bulkdevices` | Adds a batch of devices to the Asset Library | ✅ | ✅ (see `POST /devices`) | @@ -32,75 +33,90 @@ The following table indicates which REST API's are available in which mode: ### Groups -| Endpoint | Description | `full` mode | `lite` mode | -| -------------------------------------------------------------------------- | -------------------------------------------------------------------------------------------------------------- | ----------- | ------------------------------------------------------------------------------------------------------------------------------ | -| `POST /groups` | Adds a new group to the device library as a child of the `parentPath` as specified in the request body | ✅ | ✅ (specifying a parent is optional, specifying a template is not supported, and linking groups to other groups not supported) | -| `POST /bulkgroups` | Adds a batch of new group to the asset library as a child of the `parentPath` as specified in the request body | ✅ | ✅ (see `POST /groups`) | -| `GET /groups/{groupPath}` | Find group by Group's path | ✅ | ✅ | -| `DELETE /groups/{groupPath}` | Delete group with supplied path | ✅ | ✅ | -| `PATCH /groups/{groupPath}` | Update an existing group's attributes, including changing its parent group | ✅ | ✅ (see `POST /groups`) | -| `GET /groups/{groupPath}/members/devices` | List device members of group for supplied Group name | ✅ | ✅ (filtering by template or state not supported) | -| `GET /groups/{groupPath}/members/groups` | List group members of group for supplied Group name | ✅ | ✅ (filtering by template not supported) | -| `GET /groups/{groupPath}/memberships` | List all ancestor groups of a specific group | ✅ | ✅ | -| `PUT /groups/{sourceGroupPath}/{relationship}/groups/{targetGroupPath}` | Associates a group with another group, giving context to its relationship | ✅ | ⛔ | -| `DELETE /groups/{sourceGroupPath}/{relationship}/groups/{targetGroupPath}` | Removes a group from an associated group | ✅ | ⛔ | +| Endpoint | Description | `full` and `enhanced` modes | `lite` mode | +| -------------------------------------------------------------------------- | -------------------------------------------------------------------------------------------------------------- | --------------------------- | ------------------------------------------------------------------------------------------------------------------------------ | +| `POST /groups` | Adds a new group to the device library as a child of the `parentPath` as specified in the request body | ✅ | ✅ (specifying a parent is optional, specifying a template is not supported, and linking groups to other groups not supported) | +| `POST /bulkgroups` | Adds a batch of new group to the asset library as a child of the `parentPath` as specified in the request body | ✅ | ✅ (see `POST /groups`) | +| `GET /groups/{groupPath}` | Find group by Group's path | ✅ | ✅ | +| `DELETE /groups/{groupPath}` | Delete group with supplied path | ✅ | ✅ | +| `PATCH /groups/{groupPath}` | Update an existing group's attributes, including changing its parent group | ✅ | ✅ (see `POST /groups`) | +| `GET /groups/{groupPath}/members/devices` | List device members of group for supplied Group name | ✅ | ✅ (filtering by template or state not supported) | +| `GET /groups/{groupPath}/members/groups` | List group members of group for supplied Group name | ✅ | ✅ (filtering by template not supported) | +| `GET /groups/{groupPath}/memberships` | List all ancestor groups of a specific group | ✅ | ✅ | +| `PUT /groups/{sourceGroupPath}/{relationship}/groups/{targetGroupPath}` | Associates a group with another group, giving context to its relationship | ✅ | ⛔ | +| `DELETE /groups/{sourceGroupPath}/{relationship}/groups/{targetGroupPath}` | Removes a group from an associated group | ✅ | ⛔ | ### Device Templates -| Endpoint | Description | `full` mode | `lite` mode | -| -------------------------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------ | ----------- | -------------------------------------------------------------------------------------------------------------------------------------- | -| `POST /templates/device/{templateId}` | Registers a new device template within the system, using the JSON Schema standard to define the device template attributes and constraints | ✅ | ✅ (string types supported only, defining allowed relations to other group types not supported, and required attributes not supported) | -| `GET /templates/device/{templateId}` | Find device template by ID | ✅ | ✅ | -| `PATCH /templates/device/{templateId}` | Update an existing device template | ✅ | ✅ (see `POST /templates/devices/{templateId}`) | -| `DELETE /templates/device/{templateId}` | Deletes an existing device template | ✅ | ✅ (deleting a template will deprecate the Thing Type, not delete it) | -| `PUT /templates/device/{templateId}/publish` | Publishes an existing device template | ✅ | ⛔ | +| Endpoint | Description | `full` and `enhanced` modes | `lite` mode | +| -------------------------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------ | --------------------------- | -------------------------------------------------------------------------------------------------------------------------------------- | +| `POST /templates/device/{templateId}` | Registers a new device template within the system, using the JSON Schema standard to define the device template attributes and constraints | ✅ | ✅ (string types supported only, defining allowed relations to other group types not supported, and required attributes not supported) | +| `GET /templates/device/{templateId}` | Find device template by ID | ✅ | ✅ | +| `PATCH /templates/device/{templateId}` | Update an existing device template | ✅ | ✅ (see `POST /templates/devices/{templateId}`) | +| `DELETE /templates/device/{templateId}` | Deletes an existing device template | ✅ | ✅ (deleting a template will deprecate the Thing Type, not delete it) | +| `PUT /templates/device/{templateId}/publish` | Publishes an existing device template | ✅ | ⛔ | ### Group Templates -| Endpoint | Description | `full` mode | `lite` mode | -| ------------------------------------------- | ---------------------------------------------------------------------------------------------------------------------------------------- | ----------- | ----------- | -| `POST /templates/group/{templateId}` | Registers a new group template within the system, using the JSON Schema standard to define the group template attributes and constraints | ✅ | ⛔ | -| `GET /templates/group/{templateId}` | Find group template by ID | ✅ | ⛔ | -| `PATCH /templates/group/{templateId}` | Update an existing group template | ✅ | ⛔ | -| `DELETE /templates/group/{templateId}` | Deletes an existing group template | ✅ | ⛔ | -| `PUT /templates/group/{templateId}/publish` | Publishes an existing group template | ✅ | ⛔ | +| Endpoint | Description | `full` and `enhanced` modes | `lite` mode | +| ------------------------------------------- | ---------------------------------------------------------------------------------------------------------------------------------------- | --------------------------- | ----------- | +| `POST /templates/group/{templateId}` | Registers a new group template within the system, using the JSON Schema standard to define the group template attributes and constraints | ✅ | ⛔ | +| `GET /templates/group/{templateId}` | Find group template by ID | ✅ | ⛔ | +| `PATCH /templates/group/{templateId}` | Update an existing group template | ✅ | ⛔ | +| `DELETE /templates/group/{templateId}` | Deletes an existing group template | ✅ | ⛔ | +| `PUT /templates/group/{templateId}/publish` | Publishes an existing group template | ✅ | ⛔ | ### Device Profiles -| Endpoint | Description | `full` mode | `lite` mode | -| -------------------------------------------------- | -------------------------------------------------- | ----------- | ----------- | -| `POST /profiles/device/{templateId}` | Adds a new device profile for a specific template | ✅ | ⛔ | -| `GET /profiles/device/{templateId}` | Return all device profiles for a specific template | ✅ | ⛔ | -| `GET /profiles/device/{templateId}/{profileId}` | Retrieve a device profile | ✅ | ⛔ | -| `DELETE /profiles/device/{templateId}/{profileId}` | Delete a specific device profile | ✅ | ⛔ | -| `PATCH /profiles/device/{templateId}/{profileId}` | Update an existing device profile | ✅ | ⛔ | +| Endpoint | Description | `full` and `enhanced` modes | `lite` mode | +| -------------------------------------------------- | -------------------------------------------------- | --------------------------- | ----------- | +| `POST /profiles/device/{templateId}` | Adds a new device profile for a specific template | ✅ | ⛔ | +| `GET /profiles/device/{templateId}` | Return all device profiles for a specific template | ✅ | ⛔ | +| `GET /profiles/device/{templateId}/{profileId}` | Retrieve a device profile | ✅ | ⛔ | +| `DELETE /profiles/device/{templateId}/{profileId}` | Delete a specific device profile | ✅ | ⛔ | +| `PATCH /profiles/device/{templateId}/{profileId}` | Update an existing device profile | ✅ | ⛔ | ### Group Profiles -| Endpoint | Description | `full` mode | `lite` mode | -| ------------------------------------------------- | ------------------------------------------------- | ----------- | ----------- | -| `POST /profiles/group/{templateId}` | Adds a new group profile for a specific template | ✅ | ⛔ | -| `GET /profiles/group/{templateId}` | Return all group profiles for a specific template | ✅ | ⛔ | -| `GET /profiles/group/{templateId}/{profileId}` | Retrieve a group profile | ✅ | ⛔ | -| `DELETE /profiles/group/{templateId}/{profileId}` | Delete a specific group profile | ✅ | ⛔ | -| `PATCH /profiles/group/{templateId}/{profileId}` | Update an existing group profile | ✅ | ⛔ | +| Endpoint | Description | `full` and `enhanced` modes | `lite` mode | +| ------------------------------------------------- | ------------------------------------------------- | --------------------------- | ----------- | +| `POST /profiles/group/{templateId}` | Adds a new group profile for a specific template | ✅ | ⛔ | +| `GET /profiles/group/{templateId}` | Return all group profiles for a specific template | ✅ | ⛔ | +| `GET /profiles/group/{templateId}/{profileId}` | Retrieve a group profile | ✅ | ⛔ | +| `DELETE /profiles/group/{templateId}/{profileId}` | Delete a specific group profile | ✅ | ⛔ | +| `PATCH /profiles/group/{templateId}/{profileId}` | Update an existing group profile | ✅ | ⛔ | ### Policies -| Endpoint | Description | `full` mode | `lite` mode | -| ----------------------------- | ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | ----------- | ----------- | -| `POST /policies` | Creates a new `Policy`, and applies it to the provided `Groups` | ✅ | ⛔ | -| `GET /policies` | List policies, optionally filtered by policy type | ✅ | ⛔ | -| `GET /policies/inherited` | Returns all inherited `Policies` for a `Device` or set of `Groups` where the `Device`/`Groups` are associated with all the hierarchies that the `Policy` applies to. Either `deviceId` or `groupPath` must be provided | ✅ | ⛔ | -| `PATCH /policies/{policyId}` | Update the attributes of an existing policy | ✅ | ⛔ | -| `DELETE /policies/{policyId}` | Delete an existing policy | ✅ | ⛔ | -| `GET /policies/{policyId}` | Retrieve a specific policy | ✅ | ⛔ | +| Endpoint | Description | `full` and `enhanced` modes | `lite` mode | +| ----------------------------- | ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | --------------------------- | ----------- | +| `POST /policies` | Creates a new `Policy`, and applies it to the provided `Groups` | ✅ | ⛔ | +| `GET /policies` | List policies, optionally filtered by policy type | ✅ | ⛔ | +| `GET /policies/inherited` | Returns all inherited `Policies` for a `Device` or set of `Groups` where the `Device`/`Groups` are associated with all the hierarchies that the `Policy` applies to. Either `deviceId` or `groupPath` must be provided | ✅ | ⛔ | +| `PATCH /policies/{policyId}` | Update the attributes of an existing policy | ✅ | ⛔ | +| `DELETE /policies/{policyId}` | Delete an existing policy | ✅ | ⛔ | +| `GET /policies/{policyId}` | Retrieve a specific policy | ✅ | ⛔ | ### Search -| Endpoint | Description | `full` mode | `lite` mode | -| ------------- | ----------------------------- | ----------- | ----------- | -| `GET /search` | Search for groups and devices | ✅ | ✅ | +| Endpoint/Parameter | `full` mode | `enhanced` mode | `lite` mode | +| ----------------------------------- | ------------------------ | --------------- | ----------- | +| `GET /search?type={filter}` | ✅ | ✅ | ✅ | +| `GET /search?ancestorPath={filter}` | ✅ | ✅ | ✅ | +| `GET /search?eq={filter}` | ✅ | ✅ | ✅ | +| `GET /search?neq={filter}` | ✅ | ✅ | ✅ | +| `GET /search?lt={filter}` | ✅ | ✅ | ✅ | +| `GET /search?lte={filter}` | ✅ | ✅ | ✅ | +| `GET /search?gt={filter}` | ✅ | ✅ | ✅ | +| `GET /search?gte={filter}` | ✅ | ✅ | ✅ | +| `GET /search?exists={filter}` | ✅ | ✅ | ✅ | +| `GET /search?nexists={filter}` | ✅ | ✅ | ✅ | +| `GET /search?startsWith={filter}` | ✅ | ✅ (faster) | ✅ | +| `GET /search?endsWith={filter}` | ✅ (since version 5.4.0) | ✅ (faster) | ⛔ | +| `GET /search?contains={filter}` | ✅ (since version 5.4.0) | ✅ (faster) | ⛔ | +| `GET /search?fulltext={filter}` | ⛔ | ✅ | ⛔ | +| `GET /search?regex={filter}` | ⛔ | ✅ | ⛔ | +| `GET /search?lucene={filter}` | ⛔ | ✅ | ⛔ | ## Supported Functionality by Area @@ -157,9 +173,11 @@ Not supported in `lite` mode. ### Search -| Description | `full` mode | `lite` mode | -| ---------------------------- | ----------------------- | ------------------------------------------------------------ | -| No. query terms | Maximum 2048 characters | Maximum 2048 characters, and maximum 5 query terms per query | -| No. results | Unlimited | Maximm 500 per query | -| Aggregation | Supported | Not supported | -| Searching by group ancestors | Supported | Supports filtering by directly linked groups only | +| Description | `full` mode | `enhanced` mode | `lite` mode | +| --------------------------------------- | -------------------------------------- | --------------------------- | ------------------------------------------------------------ | +| No. query terms | Maximum 2048 characters | Maximum 2048 characters | Maximum 2048 characters, and maximum 5 query terms per query | +| No. results | Unlimited | Unlimited | Maximm 500 per query | +| Aggregation | Supported | Supported | Not supported | +| Searching by group ancestors | Supported | Supported | Supports filtering by directly linked groups only | +| `endsWith` and `contains` operators | Supported, using Neptune string search | Supported, using OpenSearch | Not supported | +| `fulltext`, `regex`, `lucene` operators | Not supported | Supported | Not supported | diff --git a/source/packages/services/assetlibrary/docs/swagger.yml b/source/packages/services/assetlibrary/docs/swagger.yml index 85fe2c6c8..ebaca1345 100644 --- a/source/packages/services/assetlibrary/docs/swagger.yml +++ b/source/packages/services/assetlibrary/docs/swagger.yml @@ -11,18 +11,6 @@ # OR CONDITIONS OF ANY KIND, express or implied. See the License for the specific language governing permissions # and limitations under the License. #----------------------------------------------------------------------------------------------------------------------- -#----------------------------------------------------------------------------------------------------------------------- -# Copyright Amazon.com Inc. or its affiliates. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"). You may not use this file except in compliance -# with the License. A copy of the License is located at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# or in the 'license' file accompanying this file. This file is distributed on an 'AS IS' BASIS, WITHOUT WARRANTIES -# OR CONDITIONS OF ANY KIND, express or implied. See the License for the specific language governing permissions -# and limitations under the License. -#----------------------------------------------------------------------------------------------------------------------- openapi: 3.0.0 info: title: 'AWS Connected Device Framework: Asset Library' @@ -48,7 +36,7 @@ info: Likewise, `Device Templates` can be created to represent the different types of devices within your fleet, each with their own attributes. - `Profiles` can be created and applied to device and groups to populate with default attirbutes and/or relations. + `Profiles` can be created and applied to device and groups to populate with default attributes and/or relations. `Policies` represent a document that can be attached to one or more groups within a hierarchy, and are automatically inherited by the devices and groups. @@ -96,7 +84,7 @@ tags: each with their own attributes and constraints. - Devices are identified by a unique `deviceId`, each have the following built-in attributes: + Devices are identified by a unique `deviceId`. Each device has the following built-in attributes: - `templateId`: a specific device template that represents what custom attributes the device can have @@ -120,7 +108,7 @@ tags: When a Device is created as a component of another Device, it has all the same built-in attributes as described above with the exception of `groups`. - Groups are identified by a unique `path`, and each have the following built-on attributes: + Groups are identified by a unique `path`. Each group has the following built-on attributes: - `templateId`: a specific group template that represents what custom attributes the group can have @@ -195,7 +183,7 @@ tags: A good use for policies is to look up appropriate documents or authorization levels based on a device or groups associations to specific hierarchies. As an example, let's say you need to apply different AWS IoT security policies when registering devices as Things depending upon their location. This would be handled by assigning a policy representing a provisoning template to different groups within a hierarchy representing the location. The appropriate provisioning template will be returned for the device/group depending on which and where in a hierarchy they are attached to. - name: Search description: > - The search api allows you to search across both devices and groups + The search API allows you to search across both devices and groups applying a variety of different filters. @@ -1376,25 +1364,27 @@ paths: - Search summary: Search for groups and devices. description: > - Search results can be filtered by type, ancestorPath, and an arbitrary number of + Search results can be filtered by type, ancestorPath, and an arbitrary number of additional filter parameters. Each filter can reference a field of the search - result, for example `eq=fieldname:value` or traverse the asset library graph to + result, for example `eq=fieldname:value` or traverse the asset library graph to reference a related entry, for example `eq=traversal1:out:traversal2:in:fieldname:value`. All parameters are combined with a logical AND. - For all search parameters that include a search key and search value separated - by a colon (:) character, the HTTP parameter must be assembled using the following + For all search filters that include a search key and search value separated + by a colon (:) character, the HTTP parameter must be assembled using the following sequence of steps: 1. URL-encode the search value. - 2. Concatenate the search key (incl. any traversals), the colon character, and + 2. Concatenate the search key (incl. any traversals), the colon character, and result of step 1. 3. URL-encode the output of step 2. Failure to do so can yield incorrect search results for any search values that include the colon character. - For example, a search for entries with an outgoing "manufactured_by" relation whose + For example, a search for entries with an outgoing "manufactured_by" relation whose name starts with "Mfg+Asy Inc" should be expressed as: `/search?startsWith=manufactured_by%3Aout%3Aname%3AMfg%252BAsy%2520Inc` + + Search filters are case-sensitive unless otherwise noted for the URL parameter. operationId: search parameters: - name: type @@ -1517,7 +1507,7 @@ paths: - name: exist in: query description: - Return a match if the device/group in context has a matching relation/atrribute. E.g. + Return a match if the device/group in context has a matching relation/attribute. E.g. `?exists=installed_in:out:groupPath:/vehicle/001` explode: true schema: @@ -1534,6 +1524,87 @@ paths: type: array items: type: string + - name: fulltext + in: query + description: > + Filter by an attribute based on an OpenSearch query string. This filter is only available + in the "enhanced" mode of Asset Library. This filter is case-insensitive. + + The fulltext filter matches individual words in a string as opposed to the complete string. + If a field contains more than one word, only one of the words needs to match, for example + `fulltext=widgets` matches "Widgets Incorporated". If the filter query contains more than one + word, they are combined using logical `or``, for example `fulltext=Widgets Gadgets Incorporated` + matches "Widgets Incorporated" and "Gadgets Incorporated" and "Gadgets Ltd". Field and search + strings are split into words using the rules of the OpenSearch standard query analyzer: + https://opensearch.org/docs/latest/opensearch/query-dsl/full-text/ + + Append the `~` character to any word to allow for fuzzy matches. Specify the edit distance + allowed by fuzzy match in the format `~n`, for example `Masachussets~3` matches "Massachusetts". + explode: true + schema: + type: array + items: + type: string + - name: regex + in: query + description: > + Filter by an attribute based on a regex pattern. This filter is only available in the + "enhanced" mode of Asset Library. + + For example: `?regex=serialNo:XY[A-Z]\-[0-4][^9].*`. Regular expressions use the Lucene syntax, which + differs from more standardized implementations. Notably, the Lucene syntax does NOT support: + 1. anchor operators, such as ^ and $ for start and end of a string + 2. character class tokens such as \d (any digit) and \S (all non-whitespace character) + See also: https://opensearch.org/docs/latest/opensearch/query-dsl/term/#regex. + + Regex filters are performed against the original complete value of the field, not individual + words. For example, a device with attribute "mfg" set to "Widgets Incorporated" can be found with + the query `regex=mfg:[wW]idgets Inc.*` but not with the query `regex=mfg:[wW]idgets`. + + Regex filters are case-sensitive. Do not include slash delimiters at the beginning or + end of your regular expression, i.e. use `regex=abc[0-9]`, not `regex=/abc[0-9]/`. + + Correct URL-encoding is especially important with the regex parameter because regular expressions + can contain many special characters. Note that the aforementioned examples are shown before + URL-encoding for readability but may not work unless correctly URL-encoded. Examples of regex + parameter values before and after URL-encoding: + * `regex=mfg:[wW]idgets Inc.*` --> `regex=mfg%3A%255BwW%255Didgets%2520Inc.*` + * `regex=mpn:ABC\[[0-9]\]` --> `regex=mpn:ABC%255C%255B%255B0-9%255D%255C%255D` + * `regex=desc:back\\slash` (literal backslash) --> `regex=desc%3Aback%255C%255Cslash` + explode: true + schema: + type: array + items: + type: string + - name: lucene + in: query + description: > + Filter by one or more fields of the same node using Apache Lucene query syntax. This + filter is only available in the "enhanced" mode of Asset Library. + + This filter exposes low level access to Amazon Neptune full text search (FTS): + https://docs.aws.amazon.com/neptune/latest/userguide/full-text-search.html Parameter values + are passed to FTS using the `query_string` query type with minimal processing. + + Values can include any valid Lucene query syntax, for example: + `?lucene=fieldname:(fuzzyvalue~2 AND /regexval.*/) OR exactvalue` + + To reference more than one field in the same query, use `*` as the filter key and follow the + Amazon Neptune FTS documentation for how to reference attributes in the query. For example, + to query for a device/group by attributes `attr1` and `attr2`: + `?lucene=*:(predicates.attr1.value:foo AND predicates.attr2.value:bar)` + Traversals are support as follows: + `?lucene=relation:direction:*:(predicates.attr1.value:foo AND predicates.attr2.value:bar)` + It is not possible to reference fields from different nodes in a single lucene query parameter. + + Correct URL-encoding is especially important with the lucene parameter because Lucene queries + can contain many special characters. Note that the aforementioned examples are shown before + URL-encoding for readability but may not work unless correctly URL-encoded. + explode: true + schema: + type: array + items: + type: string - name: facetField in: query description: Perform a faceted query. Specify in the format of @@ -1572,23 +1643,23 @@ paths: - Search summary: Search for groups and devices, and delete the results. description: > - Search results can be filtered by type, ancestorPath, and an arbitrary number of + Search results can be filtered by type, ancestorPath, and an arbitrary number of additional filter parameters. Each filter can reference a field of the search - result, for example `eq=fieldname:value` or traverse the asset library graph to + result, for example `eq=fieldname:value` or traverse the asset library graph to reference a related entry, for example `eq=traversal1:out:traversal2:in:fieldname:value`. All parameters are combined with a logical AND. - For all search parameters that include a search key and search value separated - by a colon (:) character, the HTTP parameter must be assembled using the following + For all search parameters that include a search key and search value separated + by a colon (:) character, the HTTP parameter must be assembled using the following sequence of steps: 1. URL-encode the search value. - 2. Concatenate the search key (incl. any traversals), the colon character, and + 2. Concatenate the search key (incl. any traversals), the colon character, and result of step 1. 3. URL-encode the output of step 2. Failure to do so can yield incorrect search results for any search values that include the colon character. - For example, a search for entries with an outgoing "manufactured_by" relation whose + For example, a search for entries with an outgoing "manufactured_by" relation whose name starts with "Mfg+Asy Inc" should be expressed as: `/search?startsWith=manufactured_by%3Aout%3Aname%3AMfg%252BAsy%2520Inc` operationId: deleteSearch @@ -2158,6 +2229,14 @@ components: required: true schemas: + Pagination: + type: object + properties: + offset: + type: integer + count: + type: integer + Entity: type: object properties: @@ -2439,15 +2518,10 @@ components: - $ref: '#/components/schemas/Group_1_0' - $ref: '#/components/schemas/Group_2_0' pagination: - type: object - properties: - offset: - type: integer - count: - type: integer + $ref: '#/components/schemas/Pagination' total: type: number - description: Total number of search results. Only returned by the search API's + description: Total number of search results. Only returned by the search API when `summarize` is set to true. DeviceList: @@ -2460,15 +2534,10 @@ components: - $ref: '#/components/schemas/Device_1_0' - $ref: '#/components/schemas/Device_2_0' pagination: - type: object - properties: - offset: - type: integer - count: - type: integer + $ref: '#/components/schemas/Pagination' total: type: number - description: Total number of search results. Only returned by the search API's + description: Total number of search results. Only returned by the search API when `summarize` is set to true. DeviceProfileList: @@ -2481,12 +2550,7 @@ components: - $ref: '#/components/schemas/DeviceProfile_1_0' - $ref: '#/components/schemas/DeviceProfile_2_0' pagination: - type: object - properties: - offset: - type: integer - count: - type: integer + $ref: '#/components/schemas/Pagination' GroupProfileList: type: object @@ -2498,12 +2562,7 @@ components: - $ref: '#/components/schemas/GroupProfile_1_0' - $ref: '#/components/schemas/GroupProfile_2_0' pagination: - type: object - properties: - offset: - type: integer - count: - type: integer + $ref: '#/components/schemas/Pagination' SearchResults: type: object @@ -2518,6 +2577,8 @@ components: - anyOf: - $ref: '#/components/schemas/Device_2_0' - $ref: '#/components/schemas/Group_2_0' + pagination: + $ref: '#/components/schemas/Pagination' TemplateInfoProperties: type: object @@ -2626,12 +2687,7 @@ components: items: $ref: '#/components/schemas/TemplateInfo' pagination: - type: object - properties: - offset: - type: integer - count: - type: integer + $ref: '#/components/schemas/Pagination' TemplateDefinition: type: object @@ -2719,12 +2775,7 @@ components: items: $ref: '#/components/schemas/Policy' pagination: - type: object - properties: - offset: - type: integer - count: - type: integer + $ref: '#/components/schemas/Pagination' Error: type: object diff --git a/source/packages/services/assetlibrary/infrastructure/cfn-assetLibrary.yaml b/source/packages/services/assetlibrary/infrastructure/cfn-assetLibrary.yaml index 6e4a92b98..b3322afef 100644 --- a/source/packages/services/assetlibrary/infrastructure/cfn-assetLibrary.yaml +++ b/source/packages/services/assetlibrary/infrastructure/cfn-assetLibrary.yaml @@ -43,12 +43,13 @@ Parameters: Type: CommaDelimitedList Mode: - Description: Run in 'lite' mode which includes device registry only, or 'full' mode which augments the device registry with an additional datastore + Description: Run in 'lite' mode which includes device registry only, 'full' mode which augments the device registry with an additional datastore, or 'enhanced' mode which adds enhanced search to full mode Type: String Default: full AllowedValues: - full - lite + - enhanced MinLength: 1 PrivateApiGatewayVPCEndpoint: @@ -94,7 +95,7 @@ Parameters: Default: 'N/A' ProvisionedConcurrentExecutions: - Description: The no. of desired concurrent executions to provision. Set to 0 to disable. + Description: The no. of desired concurrent executions to provision. Set to 0 to disable. Type: Number Default: 0 @@ -131,7 +132,7 @@ Parameters: MinLength: 1 Conditions: - DeployFullMode: !Equals [!Ref Mode, 'full'] + DeployFullOrEnhancedMode: !Or [!Equals [!Ref Mode, 'full'], !Equals [!Ref Mode, 'enhanced']] DeployLiteMode: !Equals [!Ref Mode, 'lite'] DeployInVPC: !Not [!Equals [!Ref VpcId, 'N/A']] @@ -336,7 +337,7 @@ Resources: AssetLibraryInit: Type: Custom::AssetLibraryInit - Condition: DeployFullMode + Condition: DeployFullOrEnhancedMode Version: 1.0 Properties: ServiceToken: !Ref CustomResourceVPCLambdaArn diff --git a/source/packages/services/assetlibrary/infrastructure/cfn-enhancedsearch.yaml b/source/packages/services/assetlibrary/infrastructure/cfn-enhancedsearch.yaml new file mode 100644 index 000000000..f5c03dd69 --- /dev/null +++ b/source/packages/services/assetlibrary/infrastructure/cfn-enhancedsearch.yaml @@ -0,0 +1,504 @@ +#----------------------------------------------------------------------------------------------------------------------- +# Copyright Amazon.com Inc. or its affiliates. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"). You may not use this file except in compliance +# with the License. A copy of the License is located at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# or in the 'license' file accompanying this file. This file is distributed on an 'AS IS' BASIS, WITHOUT WARRANTIES +# OR CONDITIONS OF ANY KIND, express or implied. See the License for the specific language governing permissions +# and limitations under the License. +#----------------------------------------------------------------------------------------------------------------------- +AWSTemplateFormatVersion : '2010-09-09' +Transform: AWS::Serverless-2016-10-31 +Description: CDF Asset Library Service Neptune-to-OpenSearch connection for enhanced search + +Parameters: + Environment: + Description: Name of environment. Used to name the created resources. + Type: String + MinLength: 1 + MaxLength: 24 + ConstraintDescription: Must be ≤24 chars to fit within 28 char limit for OpenSearch domain name with "cdf-" prefix. + VpcId: + Description: ID of VPC to deploy the OpenSearch domain into + Type: AWS::EC2::VPC::Id + PrivateSubNetIds: + Description: > + Comma delimited list of private subnetIds to deploy OpenSearch into. Number of subnets must match the number of + availability zones deployed into, i.e. only pass one subnet if operating in a single AZ. + Type: List + NeptuneSecurityGroupId: + Description: ID of an existing security group that contains the Neptune nodes + Type: AWS::EC2::SecurityGroup::Id + KmsKeyId: + Description: The KMS key ID used to encrypt the OpenSearch database + Type: String + MinLength: 1 + OpenSearchAvailabilityZoneCount: + Description: > + If deploying more than one data instance (defined in OpenSearchInstanceCount), the instances can be distributed + across availability zones. Default is 1, i.e. single-AZ. Choose 2 or 3 for multi-AZ. The list of subnets given in + PrivateSubNetIDs must have a length equal to the number given here. The number of data instances in + OpenSearchInstanceCount must be an integer multiple of the number given here. + Type: Number + Default: 1 + MinValue: 1 + MaxValue: 3 + ConstraintDescription: OpenSearchAvailabilityZoneCount must equal 1, 2, or 3. + OpenSearchInstanceType: + Description: > + OpenSearch data instance type. Must be a supported OpenSearch instance type in the region and must support + encryption at rest. See also: + https://docs.aws.amazon.com/opensearch-service/latest/developerguide/supported-instance-types.html + Type: String + Default: t3.small.search + AllowedPattern: "^[a-z][a-z0-9]+\\.[a-z0-9]+\\.search$" + ConstraintDescription: Must be a valid OpenSearch instance type. + OpenSearchInstanceCount: + Type: Number + Default: 1 + Description: > + The number of data nodes (instances) to use in the OpenSearch domain. Must be an integer multiple of the number of + availability zones specified in OpenSearchAvailabilityZoneCount. + OpenSearchDedicatedMasterCount: + Type: Number + Default: 0 + Description: > + The number of dedicated master nodes (instances) to use in the OpenSearch domain. The number must be larger than 2 + and should never be an even number. See also: + https://docs.aws.amazon.com/opensearch-service/latest/developerguide/managedomains-dedicatedmasternodes.html + OpenSearchDedicatedMasterInstanceType: + Description: > + OpenSearch master instance type. Must be a supported OpenSearch instance type in the region and must support + encryption at rest. See also: + https://docs.aws.amazon.com/opensearch-service/latest/developerguide/supported-instance-types.html + Type: String + Default: t3.small.search + AllowedPattern: "^[a-z][a-z0-9]+\\.[a-z0-9]+\\.search$" + ConstraintDescription: Must be a valid OpenSearch instance type. + OpenSearchEBSVolumeSize: + Type: Number + Default: 10 + MinValue: 10 + Description: > + Size of EBS volumes attached to each OpenSearch node, in GiB. Allowed ranges depend on the + instance type chosen in OpenSearchInstanceType and are documented here: + https://docs.aws.amazon.com/opensearch-service/latest/developerguide/limits.html#ebsresource + OpenSearchEBSVolumeType: + Type: String + Default: gp2 + Description: Type of the EBS volume attached to each OpenSearch node. + AllowedValues: + - gp2 + - gp3 + - io1 + - io2 + - standard + ConstraintDescription: > + See list of valid EBS volume types at https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/ebs-volume-types.html. + Not every instance type is compatible with every volume type. + OpenSearchEBSProvisionedIOPS: + Type: Number + Default: 16000 + Description: > + The number of I/O operations per second (IOPS) that the volume supports. This property applies only to the + Provisioned IOPS EBS volume type (io1). + OpenSearchAuditLogsCloudWatchLogsLogGroupArn: + Type: String + Default: '' + Description: > + ARN of Cloudwatch Log Group where OpenSearch audit Logs are sent. If left empty, logs will not be generated. + OpenSearchApplicationLogsCloudWatchLogsLogGroupArn: + Type: String + Default: '' + Description: > + ARN of Cloudwatch Log Group where OpenSearch application Logs are sent. If left empty, logs will not be + generated. + OpenSearchIndexSlowLogsCloudWatchLogsLogGroupArn: + Type: String + Default: '' + Description: > + ARN of Cloudwatch Log Group where OpenSearch Index Slow Logs are sent. If left empty, logs will not be + generated. + OpenSearchSearchSlowLogsCloudWatchLogsLogGroupArn: + Type: String + Default: '' + Description: > + ARN of Cloudwatch Log Group where OpenSearch Search Slow Logs are sent. If left empty, logs will not be + generated. + NeptuneClusterEndpoint: + Description: 'Neptune cluster endpoint. Format: :' + Type: String + NeptunePollerLambdaMemorySize: + Type: Number + Default: 512 + Description: Neptune Poller Lambda memory size (in MB). + AllowedValues: + - 128 + - 256 + - 512 + - 1024 + - 2048 + - 3008 + NeptunePollerLambdaLoggingLevel: + Type: String + Default: INFO + Description: Poller Lambda logging level. + AllowedValues: + - DEBUG + - INFO + - WARN + - ERROR + - FATAL + NeptunePollerStreamRecordsBatchSize: + Type: Number + Default: 5000 + MaxValue: 50000 + MinValue: 1 + Description: "Number of records to be read from stream in each batch. Should be between 1 to 50000." + NeptunePollerMaxPollingWaitTime: + Type: Number + Default: 10 + MaxValue: 3600 + MinValue: 0 + Description: "Maximum wait time in seconds between two successive polling from stream. Set value to 0 sec for continuous polling. Maximum value can be 3600 sec (1 hour)." + NeptunePollerMaxPollingInterval: + Type: Number + Default: 600 + MaxValue: 900 + MinValue: 5 + Description: "Number of seconds for which we can continuously poll stream for records on one Lambda instance. Should be between 5 sec to 900 sec. This parameter is used to set Poller Lambda Timeout." + NeptunePollerStepFunctionFallbackPeriod: + Type: Number + Default: 5 + Description: "Period after which Step function is invoked using Cloud Watch Events to recover from failure. Unit for Step Function Fallback period is set separately." + NeptunePollerStepFunctionFallbackPeriodUnit: + Type: String + Default: minutes + AllowedValues: + - minutes + - minute + - hours + - hour + - days + - day + Description: "Step Function FallbackPeriod unit. Should be one of minutes, minute, hours, hour, days, day" + EnableNonStringIndexing: + Type: String + Default: 'true' + AllowedValues: + - 'true' + - 'false' + ConstraintDescription: "Must be either true or false" + Description: Flag to enable/disable indexing Non-String fields + NumberOfShards: + Type: Number + Default: 5 + Description: Number of Shards for Elastic Search Index. Default value is 5. + NumberOfReplica: + Type: Number + Default: 1 + Description: Number of replicas for Elastic Search Index. Default value is 1. + GeoLocationFields: + Type: String + Default: '' + Description: 'Comma Delimited list of Property Keys to be mapped to Geo Point Type in Elastic Search. For Example: location,area. Currently, for a field to be mapped to Geo Point type, value should be in the format "latitude,longitude" Ex: "41.33,-11.69"' + PropertiesToExclude: + Type: String + Default: '' + Description: Comma delimited list of Property Keys to exclude from being indexed into Elastic Search. Optional Parameter - If left blank, all property keys will be indexed." + DatatypesToExclude: + Type: String + Default: '' + Description: 'Comma delimited list of Property Value Data Types to exclude from being indexed into Elastic Search. Optional Parameter - If left blank, all valid property values will be indexed. Type inputs that are unsupported for the specified query language will be ignored. Valid inputs for Gremlin data: [string, date, bool, byte, short, int, long, float, double]' + IgnoreMissingDocument: + Type: String + Default: 'true' + AllowedValues: + - 'true' + - 'false' + ConstraintDescription: Must be a either true or false + Description: 'Flag to determine if missing document error in Elastic Search can be ignored. Missing document error can occur rarely but will need manual intervention if not ignored.' + CdfService: + Description: Service name to tag resources. + Type: String + Default: assetlibrary + CreateCloudWatchAlarm: + Type: String + Default: false + Description: Flag used to determine whether to create Cloud watch alarm or not. + AllowedValues: + - 'true' + - 'false' + ConstraintDescription: Must be a either true or false + NotificationEmail: + Type: String + Default: "" + Description: "Email Address for CloudWatch Alarm Notification. Optional Parameter - Only needed when selecting option to create CloudWatch Alarm." + + +Conditions: + KmsKeyIdProvided: !Not [ !Equals [ !Ref KmsKeyId, "" ] ] + EnableDedicatedMasterNodes: !Not [ !Equals [ !Ref OpenSearchDedicatedMasterCount, 0 ] ] + CreateCloudWatchAlarmCondition: !Equals [ !Ref CreateCloudWatchAlarm, 'true' ] + EnableNonStringIndexingCondition: !Equals [ !Ref EnableNonStringIndexing, 'true' ] + OpenSearchAuditLogsCloudWatchLogsEnabled: !Not [ !Equals [ !Ref OpenSearchAuditLogsCloudWatchLogsLogGroupArn, '' ] ] + OpenSearchApplicationLogsCloudWatchLogsEnabled: !Not [ !Equals [ !Ref OpenSearchApplicationLogsCloudWatchLogsLogGroupArn, '' ] ] + OpenSearchIndexSlowLogsCloudWatchLogsEnabled: !Not [ !Equals [ !Ref OpenSearchIndexSlowLogsCloudWatchLogsLogGroupArn, '' ] ] + OpenSearchSearchSlowLogsCloudWatchLogsEnabled: !Not [ !Equals [ !Ref OpenSearchSearchSlowLogsCloudWatchLogsLogGroupArn, '' ] ] + OpenSearchEBSVolumeTypeIsIo1: !Equals [ !Ref OpenSearchEBSVolumeType, 'io1' ] + EnableZoneAwareness: !Not [ !Equals [ !Ref OpenSearchAvailabilityZoneCount, 1 ] ] + + +Resources: + + OpenSearchSG: + Type: 'AWS::EC2::SecurityGroup' + Properties: + VpcId: !Ref VpcId + GroupDescription: !Sub 'CDF Asset Library (${Environment}) OpenSearch Security Group' + + NeptuneStreamPollerSG: + Type: 'AWS::EC2::SecurityGroup' + Properties: + VpcId: !Ref VpcId + GroupDescription: !Sub 'CDF Asset Library (${Environment}) Neptune Stream Poller Security Group' + + OpenSearchSGIngressRule1: + Type: 'AWS::EC2::SecurityGroupIngress' + Properties: + GroupId: !Ref OpenSearchSG + FromPort: 443 + ToPort: 443 + IpProtocol: tcp + SourceSecurityGroupId: !Ref NeptuneSecurityGroupId + Description: Access from Neptune to OpenSearch + + OpenSearchSGIngressRule2: + Type: 'AWS::EC2::SecurityGroupIngress' + Properties: + GroupId: !Ref OpenSearchSG + FromPort: 443 + ToPort: 443 + IpProtocol: tcp + SourceSecurityGroupId: !Ref NeptuneStreamPollerSG + Description: Access from Neptune Stream Poller to OpenSearch + + OpenSearchSGIngressRule3: + Type: 'AWS::EC2::SecurityGroupIngress' + Properties: + GroupId: !Ref OpenSearchSG + FromPort: 443 + ToPort: 443 + IpProtocol: tcp + SourceSecurityGroupId: !GetAtt NeptuneStreamPoller.Outputs.HTTPSAccessSG + Description: Access for the Kinesis-to-opensearch lambda + + NeptuneSGIngressRule: + Type: 'AWS::EC2::SecurityGroupIngress' + Properties: + GroupId: !Ref NeptuneSecurityGroupId + FromPort: 8182 + ToPort: 8182 + IpProtocol: tcp + SourceSecurityGroupId: !Ref NeptuneStreamPollerSG + Description: Access from Neptune Stream Poller to Neptune + + OpenSearchDomain: + Type: AWS::OpenSearchService::Domain + DeletionPolicy: Snapshot + UpdateReplacePolicy: Snapshot + UpdatePolicy: + EnableVersionUpgrade: true + Properties: + DomainName: !Sub 'cdf-${Environment}' + AccessPolicies: + Version: '2012-10-17' + Statement: + - Effect: Allow + Principal: + AWS: '*' + Action: 'es:*' + Resource: !Join + - ':' + - - 'arn:aws:es' + - !Ref 'AWS::Region' + - !Ref 'AWS::AccountId' + - !Sub 'domain/cdf-${Environment}/*' + ClusterConfig: + InstanceCount: !Ref OpenSearchInstanceCount + InstanceType: !Ref OpenSearchInstanceType + DedicatedMasterEnabled: + Fn::If: + - EnableDedicatedMasterNodes + - true + - false + DedicatedMasterCount: + Fn::If: + - EnableDedicatedMasterNodes + - !Ref OpenSearchDedicatedMasterCount + - !Ref AWS::NoValue + DedicatedMasterType: + Fn::If: + - EnableDedicatedMasterNodes + - !Ref OpenSearchDedicatedMasterInstanceType + - !Ref AWS::NoValue + ZoneAwarenessEnabled: + Fn::If: + - EnableZoneAwareness + - true + - false + ZoneAwarenessConfig: + Fn::If: + - EnableZoneAwareness + - AvailabilityZoneCount: !Ref OpenSearchAvailabilityZoneCount + - !Ref AWS::NoValue + DomainEndpointOptions: + EnforceHTTPS: true + TLSSecurityPolicy: 'Policy-Min-TLS-1-2-2019-07' + EBSOptions: + EBSEnabled: true + VolumeSize: !Ref OpenSearchEBSVolumeSize + VolumeType: !Ref OpenSearchEBSVolumeType + Iops: + Fn::If: + - OpenSearchEBSVolumeTypeIsIo1 + - !Ref OpenSearchEBSProvisionedIOPS + - !Ref AWS::NoValue + EncryptionAtRestOptions: + Enabled: true + KmsKeyId: + Fn::If: + - KmsKeyIdProvided + - !Ref KmsKeyId + - !Ref AWS::NoValue + LogPublishingOptions: + ES_APPLICATION_LOGS: + Fn::If: + - OpenSearchApplicationLogsCloudWatchLogsEnabled + - CloudWatchLogsLogGroupArn: !Ref OpenSearchApplicationLogsCloudWatchLogsLogGroupArn + Enabled: true + - !Ref AWS::NoValue + SEARCH_SLOW_LOGS: + Fn::If: + - OpenSearchSearchSlowLogsCloudWatchLogsEnabled + - CloudWatchLogsLogGroupArn: !Ref OpenSearchSearchSlowLogsCloudWatchLogsLogGroupArn + Enabled: true + - !Ref AWS::NoValue + INDEX_SLOW_LOGS: + Fn::If: + - OpenSearchIndexSlowLogsCloudWatchLogsEnabled + - CloudWatchLogsLogGroupArn: !Ref OpenSearchIndexSlowLogsCloudWatchLogsLogGroupArn + Enabled: true + - !Ref AWS::NoValue + AUDIT_LOGS: + Fn::If: + - OpenSearchAuditLogsCloudWatchLogsEnabled + - CloudWatchLogsLogGroupArn: !Ref OpenSearchAuditLogsCloudWatchLogsLogGroupArn + Enabled: true + - !Ref AWS::NoValue + NodeToNodeEncryptionOptions: + Enabled: true + VPCOptions: + SecurityGroupIds: + - !Ref OpenSearchSG + SubnetIds: !Ref PrivateSubNetIds + Tags: + - Key: cdf_environment + Value: !Ref Environment + - Key: cdf_service + Value: !Ref CdfService + + OpenSearchAccessPolicy: + Type: 'AWS::IAM::ManagedPolicy' + Properties: + Description: "Allows OpenSearch access for Neptune Lambda Poller" + PolicyDocument: + Version: '2012-10-17' + Statement: + - Sid: "opensearchaccess" + Effect: Allow + Action: + - 'es:ESHttpDelete' + - 'es:ESHttpGet' + - 'es:ESHttpHead' + - 'es:ESHttpPost' + - 'es:ESHttpPut' + Resource: !Sub '${OpenSearchDomain.Arn}/*' + + NeptuneStreamPoller: + Type: 'AWS::CloudFormation::Stack' + Properties: + TemplateURL: 'https://s3.amazonaws.com/aws-neptune-customer-samples/neptune-stream/neptune_stream_poller_nested_full_stack.json' + Parameters: + AdditionalParams: + Fn::Sub: + - "{ \"ElasticSearchEndpoint\": \"${ElasticSearchEndpoint}\", \"NumberOfShards\": \"${NumberOfShards}\", \"NumberOfReplica\": \"${NumberOfReplica}\", \"IgnoreMissingDocument\": \"${IgnoreMissingDocument}\", \"ReplicationScope\": \"${ReplicationScope}\", \"GeoLocationFields\": \"${GeoLocationFields}\", \"DatatypesToExclude\": \"${DatatypesToExclude}\", \"PropertiesToExclude\": \"${PropertiesToExclude}\", \"EnableNonStringIndexing\": \"${EnableNonStringIndexing}\"}" + - ElasticSearchEndpoint: !GetAtt OpenSearchDomain.DomainEndpoint + NumberOfShards: !Ref "NumberOfShards" + NumberOfReplica: !Ref "NumberOfReplica" + GeoLocationFields: !Ref "GeoLocationFields" + PropertiesToExclude: !Ref "PropertiesToExclude" + DatatypesToExclude: !Ref "DatatypesToExclude" + IgnoreMissingDocument: !Ref "IgnoreMissingDocument" + ReplicationScope: "All" + EnableNonStringIndexing: !Ref EnableNonStringIndexing + ApplicationName: !Sub 'cdf-${CdfService}-enhancedsearch-${Environment}' + LambdaMemorySize: !Ref NeptunePollerLambdaMemorySize + LambdaRuntime: python3.9 + LambdaS3Bucket: !Join ["-", ["aws-neptune-customer-samples", !Ref "AWS::Region"]] + LambdaS3Key: "neptune-stream/lambda/python39/release_2022_06_06/neptune-to-es.zip" + ManagedPolicies: !Ref OpenSearchAccessPolicy + LambdaLoggingLevel: !Ref NeptunePollerLambdaLoggingLevel + StreamRecordsHandler: + # The published template in the Neptune documentation uses a three-level mapping here. + # The mapping is flattened into a single "If" because Gremlin and Python are fixed. + Fn::If: + - EnableNonStringIndexingCondition + - "neptune_to_es.neptune_gremlin_es_handler.ElasticSearchGremlinHandler" + - "neptune_to_es.neptune_gremlin_es_handler.ElasticSearchStringOnlyGremlinHandler" + StreamRecordsBatchSize: !Ref NeptunePollerStreamRecordsBatchSize + StepFunctionFallbackPeriod: !Ref NeptunePollerStepFunctionFallbackPeriod + StepFunctionFallbackPeriodUnit: !Ref NeptunePollerStepFunctionFallbackPeriodUnit + MaxPollingWaitTime: !Ref NeptunePollerMaxPollingWaitTime + NeptuneStreamEndpoint: !Sub 'https://${NeptuneClusterEndpoint}:8182/gremlin/stream' + IAMAuthEnabledOnSourceStream: false + MaxPollingInterval: !Ref NeptunePollerMaxPollingInterval + VPC: !Ref VpcId + CreateDDBVPCEndPoint: false + CreateMonitoringEndPoint: true + CreateCloudWatchAlarm: !Ref CreateCloudWatchAlarm + NotificationEmail: !Ref NotificationEmail + SubnetIds: !Join [ ",", !Ref PrivateSubNetIds ] + SecurityGroupIds: !Ref NeptuneStreamPollerSG + +Outputs: + OpenSearchDomainEndpoint: + Description: HTTPS endpoint URL for OpenSearch cluster + Value: !Sub 'https://${OpenSearchDomain.DomainEndpoint}' + Export: + Name: !Sub "cdf-assetlibrary-enhancedsearch-${Environment}-OpenSearchDomainEndpoint" + HTTPSAccessSG: + Description: 'HTTPS Access Security Group Arn' + Value: !GetAtt NeptuneStreamPoller.Outputs.HTTPSAccessSG + LeaseDynamoDBTable: + Description: 'Neptune Stream Poller Lease Table' + Value: !GetAtt NeptuneStreamPoller.Outputs.LeaseDynamoDBTable + StateMachineArn: + Description: 'Neptune Stream Poller State Machine Arn' + Value: !GetAtt NeptuneStreamPoller.Outputs.StateMachineArn + CronArn: + Description: 'Neptune Stream Poller Scheduler Cron Arn' + Value: !GetAtt NeptuneStreamPoller.Outputs.CronArn + StateMachineAlarmArn: + Description: 'Neptune Stream Poller State Machine Alarm Arn' + Condition: CreateCloudWatchAlarmCondition + Value: !GetAtt NeptuneStreamPoller.Outputs.StateMachineAlarmArn + NeptuneStreamPollerLambdaArn: + Description: 'Neptune Stream Poller Lambda Arn' + Value: !GetAtt NeptuneStreamPoller.Outputs.NeptuneStreamPollerLambdaArn + CloudWatchMetricsDashboardURI: + Description: 'CloudWatch Metrics Dashboard URI' + Value: !GetAtt NeptuneStreamPoller.Outputs.CloudWatchMetricsDashboardURI diff --git a/source/packages/services/assetlibrary/infrastructure/cfn-neptune-stack-policy.json b/source/packages/services/assetlibrary/infrastructure/cfn-neptune-stack-policy.json deleted file mode 100644 index d9a67d079..000000000 --- a/source/packages/services/assetlibrary/infrastructure/cfn-neptune-stack-policy.json +++ /dev/null @@ -1,10 +0,0 @@ -{ - "Statement": [ - { - "Effect": "Allow", - "Action": "Update:*", - "Principal": "*", - "Resource": "*" - } - ] -} diff --git a/source/packages/services/assetlibrary/infrastructure/cfn-neptune.yaml b/source/packages/services/assetlibrary/infrastructure/cfn-neptune.yaml index e73443b58..842da8450 100644 --- a/source/packages/services/assetlibrary/infrastructure/cfn-neptune.yaml +++ b/source/packages/services/assetlibrary/infrastructure/cfn-neptune.yaml @@ -30,7 +30,7 @@ Parameters: Type: List DbInstanceType: Description: > - Neptune DB instance type. The list of available instance types for your region can be found here: + Neptune DB instance type. The list of available instance types for your region can be found here: https://aws.amazon.com/neptune/pricing/ Type: String AllowedPattern: "^db\\.[tr]\\d+[a-z0-9]*\\.[a-z0-9]*$" @@ -53,6 +53,19 @@ Parameters: - 0 - 1 Description: Enable Audit Log. 0 means disable and 1 means enable. + NeptuneEnableStreams: + Type: Number + Default: 0 + AllowedValues: + - 0 + - 1 + Description: Enable Neptune Streams. 0 means disable and 1 means enable. Must be enabled for Asset Library Enhanced Search. + BackupRetentionDays: + Type: Number + Default: 1 + MinValue: 1 + MaxValue: 35 + Description: Days automated snapshots will be retained IamAuthEnabled: Type: String Default: 'false' @@ -133,7 +146,7 @@ Resources: ToPort: 8182 IpProtocol: tcp SourceSecurityGroupId: !Ref CDFSecurityGroupId - Description: Allow access from default securty group + Description: Allow access from default security group NeptuneEC2InstanceProfile: Type: 'AWS::IAM::InstanceProfile' @@ -238,6 +251,7 @@ Resources: Description: CDF parameters Parameters: neptune_enable_audit_log: !Ref NeptuneEnableAuditLog + neptune_streams: !Ref NeptuneEnableStreams Tags: - Key: cdf_environment Value: !Ref Environment diff --git a/source/packages/services/assetlibrary/src/di/inversify.config.enhanced.ts b/source/packages/services/assetlibrary/src/di/inversify.config.enhanced.ts new file mode 100644 index 000000000..daf3bae81 --- /dev/null +++ b/source/packages/services/assetlibrary/src/di/inversify.config.enhanced.ts @@ -0,0 +1,27 @@ +/********************************************************************************************************************* + * Copyright Amazon.com Inc. or its affiliates. All Rights Reserved. * + * * + * Licensed under the Apache License, Version 2.0 (the "License"). You may not use this file except in compliance * + * with the License. A copy of the License is located at * + * * + * http://www.apache.org/licenses/LICENSE-2.0 * + * * + * or in the 'license' file accompanying this file. This file is distributed on an 'AS IS' BASIS, WITHOUT WARRANTIES * + * OR CONDITIONS OF ANY KIND, express or implied. See the License for the specific language governing permissions * + * and limitations under the License. * + *********************************************************************************************************************/ +import { ContainerModule, interfaces } from 'inversify'; +import { SearchDaoEnhanced } from '../search/search.enhanced.dao'; +import { TYPES } from './types'; + +export const EnhancedContainerModule = new ContainerModule( + ( + bind: interfaces.Bind, + _unbind: interfaces.Unbind, + _isBound: interfaces.IsBound, + rebind: interfaces.Rebind + ) => { + bind('openSearchEndpoint').toConstantValue(process.env.OPENSEARCH_ENDPOINT); + rebind(TYPES.SearchDao).to(SearchDaoEnhanced).inSingletonScope(); + } +); diff --git a/source/packages/services/assetlibrary/src/di/inversify.config.full.ts b/source/packages/services/assetlibrary/src/di/inversify.config.full.ts index 59d13ed54..b7350f544 100644 --- a/source/packages/services/assetlibrary/src/di/inversify.config.full.ts +++ b/source/packages/services/assetlibrary/src/di/inversify.config.full.ts @@ -1,4 +1,3 @@ -import { structure } from 'gremlin'; /********************************************************************************************************************* * Copyright Amazon.com Inc. or its affiliates. All Rights Reserved. * * * @@ -12,6 +11,7 @@ import { structure } from 'gremlin'; * and limitations under the License. * *********************************************************************************************************************/ import { ContainerModule, decorate, injectable, interfaces } from 'inversify'; +import { structure } from 'gremlin'; import { AuthzDaoFull } from '../authz/authz.full.dao'; import { AuthzServiceFull } from '../authz/authz.full.service'; diff --git a/source/packages/services/assetlibrary/src/di/inversify.config.ts b/source/packages/services/assetlibrary/src/di/inversify.config.ts index 3d5faf8eb..85cdae5b6 100644 --- a/source/packages/services/assetlibrary/src/di/inversify.config.ts +++ b/source/packages/services/assetlibrary/src/di/inversify.config.ts @@ -25,6 +25,7 @@ import { HttpHeaderUtils } from '../utils/httpHeaders'; import { TypeUtils } from '../utils/typeUtils'; import * as full from './inversify.config.full'; import * as lite from './inversify.config.lite'; +import * as enhanced from './inversify.config.enhanced'; import { TYPES } from './types'; import AWS from 'aws-sdk'; @@ -33,6 +34,10 @@ export const container = new Container(); if (process.env.MODE === 'lite') { container.load(lite.LiteContainerModule); +} else if (process.env.MODE === 'enhanced') { + // EnhancedContainerModule extends, not replaces, FullContainerModule + container.load(full.FullContainerModule); + container.load(enhanced.EnhancedContainerModule); } else { container.load(full.FullContainerModule); } diff --git a/source/packages/services/assetlibrary/src/search/search.assembler.spec.ts b/source/packages/services/assetlibrary/src/search/search.assembler.spec.ts index e0c8dba4e..6a0b0f6b0 100644 --- a/source/packages/services/assetlibrary/src/search/search.assembler.spec.ts +++ b/source/packages/services/assetlibrary/src/search/search.assembler.spec.ts @@ -36,10 +36,13 @@ describe('SearchServiceAssembler', () => { neqs: string | string[] | undefined; lts: string | string[] | undefined; gtes: string | string[] | undefined; - facetField: string | undefined; startsWiths: string | string[] | undefined; endsWiths: string | string[] | undefined; containses: string | string[] | undefined; + fulltexts: string | string[] | undefined; + regexes: string | string[] | undefined; + lucenes: string | string[] | undefined; + facetField: string | undefined; }; beforeEach(() => { mockedDeviceAssembler = createMockInstance(DevicesAssembler); @@ -66,6 +69,9 @@ describe('SearchServiceAssembler', () => { startsWiths: undefined, endsWiths: undefined, containses: undefined, + fulltexts: undefined, + regexes: undefined, + lucenes: undefined, exists: undefined, nexists: undefined, facetField: undefined, @@ -85,6 +91,9 @@ describe('SearchServiceAssembler', () => { mockedSearchRequest.startsWiths, mockedSearchRequest.endsWiths, mockedSearchRequest.containses, + mockedSearchRequest.fulltexts, + mockedSearchRequest.regexes, + mockedSearchRequest.lucenes, mockedSearchRequest.exists, mockedSearchRequest.nexists, mockedSearchRequest.facetField @@ -118,6 +127,9 @@ describe('SearchServiceAssembler', () => { startsWiths: 'swfield:abc', endsWiths: 'ewfield:xyz', containses: 'confield:opq', + fulltexts: 'ftfield:*abc*', + regexes: 'refield:AB[CD]12++', + lucenes: undefined, exists: 'exfield:exval', nexists: 'nexfield:nexval', facetField: undefined, @@ -137,6 +149,9 @@ describe('SearchServiceAssembler', () => { mockedSearchRequest.startsWiths, mockedSearchRequest.endsWiths, mockedSearchRequest.containses, + mockedSearchRequest.fulltexts, + mockedSearchRequest.regexes, + mockedSearchRequest.lucenes, mockedSearchRequest.exists, mockedSearchRequest.nexists, mockedSearchRequest.facetField @@ -174,6 +189,12 @@ describe('SearchServiceAssembler', () => { expect(searchRequestModel.contains).toHaveLength(1); expect(searchRequestModel.contains[0].field).toEqual('confield'); expect(searchRequestModel.contains[0].value).toEqual('opq'); + expect(searchRequestModel.fulltext).toHaveLength(1); + expect(searchRequestModel.fulltext[0].field).toEqual('ftfield'); + expect(searchRequestModel.fulltext[0].value).toEqual('*abc*'); + expect(searchRequestModel.regex).toHaveLength(1); + expect(searchRequestModel.regex[0].field).toEqual('refield'); + expect(searchRequestModel.regex[0].value).toEqual('AB[CD]12++'); expect(searchRequestModel.exists).toHaveLength(1); expect(searchRequestModel.exists[0].field).toEqual('exfield'); expect(searchRequestModel.exists[0].value).toEqual('exval'); @@ -197,6 +218,9 @@ describe('SearchServiceAssembler', () => { startsWiths: undefined, endsWiths: undefined, containses: undefined, + fulltexts: undefined, + regexes: undefined, + lucenes: undefined, exists: undefined, nexists: undefined, facetField: undefined, @@ -216,6 +240,9 @@ describe('SearchServiceAssembler', () => { mockedSearchRequest.startsWiths, mockedSearchRequest.endsWiths, mockedSearchRequest.containses, + mockedSearchRequest.fulltexts, + mockedSearchRequest.regexes, + mockedSearchRequest.lucenes, mockedSearchRequest.exists, mockedSearchRequest.nexists, mockedSearchRequest.facetField @@ -249,6 +276,9 @@ describe('SearchServiceAssembler', () => { startsWiths: undefined, endsWiths: undefined, containses: undefined, + fulltexts: undefined, + regexes: undefined, + lucenes: undefined, exists: undefined, nexists: undefined, facetField: undefined, @@ -268,6 +298,9 @@ describe('SearchServiceAssembler', () => { mockedSearchRequest.startsWiths, mockedSearchRequest.endsWiths, mockedSearchRequest.containses, + mockedSearchRequest.fulltexts, + mockedSearchRequest.regexes, + mockedSearchRequest.lucenes, mockedSearchRequest.exists, mockedSearchRequest.nexists, mockedSearchRequest.facetField diff --git a/source/packages/services/assetlibrary/src/search/search.assembler.ts b/source/packages/services/assetlibrary/src/search/search.assembler.ts index af3584806..73ca4e642 100644 --- a/source/packages/services/assetlibrary/src/search/search.assembler.ts +++ b/source/packages/services/assetlibrary/src/search/search.assembler.ts @@ -48,9 +48,12 @@ export class SearchAssembler { startsWiths: string | string[], endsWiths: string | string[], containses: string | string[], + fulltexts: string | string[], + regexes: string | string[], + lucenes: string | string[], exists: string | string[], nexists: string | string[], - facetField?: string, + facetField: string, offset?: number, count?: number, sort?: string @@ -82,7 +85,9 @@ export class SearchAssembler { req.startsWith = this.toSearchRequestFilters(startsWiths); req.endsWith = this.toSearchRequestFilters(endsWiths); req.contains = this.toSearchRequestFilters(containses); - + req.fulltext = this.toSearchRequestFilters(fulltexts); + req.regex = this.toSearchRequestFilters(regexes); + req.lucene = this.toSearchRequestFilters(lucenes); req.exists = this.toSearchRequestFilters(exists); req.nexists = this.toSearchRequestFilters(nexists); diff --git a/source/packages/services/assetlibrary/src/search/search.controller.ts b/source/packages/services/assetlibrary/src/search/search.controller.ts index 8f8fdae63..26d34bd04 100644 --- a/source/packages/services/assetlibrary/src/search/search.controller.ts +++ b/source/packages/services/assetlibrary/src/search/search.controller.ts @@ -50,6 +50,9 @@ export class SearchController implements interfaces.Controller { @queryParam('startsWith') startsWiths: string | string[], @queryParam('endsWith') endsWiths: string | string[], @queryParam('contains') containses: string | string[], + @queryParam('fulltext') fulltexts: string | string[], + @queryParam('regex') regexes: string | string[], + @queryParam('lucene') lucenes: string | string[], @queryParam('exist') exists: string | string[], @queryParam('nexist') nexists: string | string[], @queryParam('facetField') facetField: string, @@ -61,7 +64,7 @@ export class SearchController implements interfaces.Controller { @response() res: Response ): Promise { logger.debug( - `search.controller search: in: types:${types}, ancestorPath:${ancestorPath}, eqs:${eqs}, neqs:${neqs}, lts:${lts}, ltes:${ltes}, gts:${gts}, gtes:${gtes}, startsWiths:${startsWiths}, endsWiths:${endsWiths}, containses:${containses}, exists:${exists}, nexists:${nexists}, facetField:${facetField}, summarize:${summarize}, offset:${offset}, count:${count}, sort:${sort}` + `search.controller search: in: types:${types}, ancestorPath:${ancestorPath}, eqs:${eqs}, neqs:${neqs}, lts:${lts}, ltes:${ltes}, gts:${gts}, gtes:${gtes}, startsWiths:${startsWiths}, endsWiths:${endsWiths}, containses:${containses}, fulltexts:${fulltexts}, regexes:${regexes}, lucenes:${lucenes}, exists:${exists}, nexists:${nexists}, facetField:${facetField}, summarize:${summarize}, offset:${offset}, count:${count}, sort:${sort}` ); const r: SearchResultsResource = { results: [] }; @@ -81,6 +84,9 @@ export class SearchController implements interfaces.Controller { startsWiths, endsWiths, containses, + fulltexts, + regexes, + lucenes, exists, nexists, facetField, @@ -135,8 +141,12 @@ export class SearchController implements interfaces.Controller { @queryParam('startsWith') startsWiths: string | string[], @queryParam('endsWith') endsWiths: string | string[], @queryParam('contains') containses: string | string[], + @queryParam('fulltext') fulltexts: string | string[], + @queryParam('regex') regexes: string | string[], + @queryParam('lucene') lucenes: string | string[], @queryParam('exist') exists: string | string[], @queryParam('nexist') nexists: string | string[], + @queryParam('facetField') facetField: string, @response() res: Response ): Promise { logger.debug( @@ -157,8 +167,12 @@ export class SearchController implements interfaces.Controller { startsWiths, endsWiths, containses, + fulltexts, + regexes, + lucenes, exists, - nexists + nexists, + facetField ); try { diff --git a/source/packages/services/assetlibrary/src/search/search.enhanced.dao.ts b/source/packages/services/assetlibrary/src/search/search.enhanced.dao.ts new file mode 100644 index 000000000..e4776eaf7 --- /dev/null +++ b/source/packages/services/assetlibrary/src/search/search.enhanced.dao.ts @@ -0,0 +1,261 @@ +/********************************************************************************************************************* + * Copyright Amazon.com Inc. or its affiliates. All Rights Reserved. * + * * + * Licensed under the Apache License, Version 2.0 (the "License"). You may not use this file except in compliance * + * with the License. A copy of the License is located at * + * * + * http://www.apache.org/licenses/LICENSE-2.0 * + * * + * or in the 'license' file accompanying this file. This file is distributed on an 'AS IS' BASIS, WITHOUT WARRANTIES * + * OR CONDITIONS OF ANY KIND, express or implied. See the License for the specific language governing permissions * + * and limitations under the License. * + *********************************************************************************************************************/ +import { logger } from '@awssolutions/simple-cdf-logger'; +import { process, structure } from 'gremlin'; +import { inject, injectable } from 'inversify'; +import { NodeAssembler } from '../data/assembler'; +import { NeptuneConnection } from '../data/base.full.dao'; +import { TYPES } from '../di/types'; +import { TypeUtils } from '../utils/typeUtils'; +import { SearchDaoFull } from './search.full.dao'; +import { SearchRequestModel } from './search.models'; + +const __ = process.statics; + +@injectable() +export class SearchDaoEnhanced extends SearchDaoFull { + public constructor( + @inject('neptuneUrl') neptuneUrl: string, + @inject('enableDfeOptimization') enableDfeOptimization: boolean, + @inject('openSearchEndpoint') private openSearchEndpoint: boolean, + @inject(TYPES.TypeUtils) typeUtils: TypeUtils, + @inject(TYPES.NodeAssembler) assembler: NodeAssembler, + @inject(TYPES.GraphSourceFactory) graphSourceFactory: () => structure.Graph + ) { + super(neptuneUrl, enableDfeOptimization, typeUtils, assembler, graphSourceFactory); + } + + protected buildSearchTraverser( + conn: NeptuneConnection, + request: SearchRequestModel, + authorizedPaths: string[] + ): process.GraphTraversal { + logger.debug( + `search.enhanced.dao buildSearchTraverser: in: request: ${JSON.stringify( + request + )}, authorizedPaths:${authorizedPaths}` + ); + + let source: process.GraphTraversalSource = conn.traversal; + if (this.enableDfeOptimization) { + source = source.withSideEffect('Neptune#useDFE', true); + } + source = source.withSideEffect('Neptune#fts.endpoint', this.openSearchEndpoint); + source = source.withSideEffect('Neptune#fts.queryType', 'query_string'); + + // if a path is provided, that becomes the starting point + let traverser: process.GraphTraversal; + if (request.ancestorPath !== undefined) { + const ancestorId = `group___${request.ancestorPath}`; + traverser = source.V(ancestorId).repeat(__.in_().simplePath().dedup()).emit().as('a'); + } else { + traverser = source.V().as('a'); + } + + // construct Gremlin traverser from request parameters + + if (request.types !== undefined) { + request.types.forEach((t) => traverser.select('a').hasLabel(t)); + } + + if (request.eq !== undefined) { + request.eq.forEach((f) => { + traverser.select('a'); + this.buildSearchFilterVBase(f, traverser); + traverser.has(f.field, f.value); + }); + } + if (request.neq !== undefined) { + request.neq.forEach((f) => { + traverser.select('a'); + this.buildSearchFilterVBase(f, traverser); + traverser.not(__.has(f.field, f.value)); + }); + } + if (request.lt !== undefined) { + request.lt.forEach((f) => { + traverser.select('a'); + this.buildSearchFilterVBase(f, traverser); + traverser.has(f.field, process.P.lt(Number(f.value))); + }); + } + if (request.lte !== undefined) { + request.lte.forEach((f) => { + traverser.select('a'); + this.buildSearchFilterVBase(f, traverser); + traverser.has(f.field, process.P.lte(Number(f.value))); + }); + } + if (request.gt !== undefined) { + request.gt.forEach((f) => { + traverser.select('a'); + this.buildSearchFilterVBase(f, traverser); + traverser.has(f.field, process.P.gt(Number(f.value))); + }); + } + if (request.gte !== undefined) { + request.gte.forEach((f) => { + traverser.select('a'); + this.buildSearchFilterVBase(f, traverser); + traverser.has(f.field, process.P.gte(Number(f.value))); + }); + } + if (request.startsWith !== undefined) { + request.startsWith.forEach((f) => { + const luceneQueryKey = this.buildLuceneQueryKey(f.field, true); + let luceneQueryVal = f.value.toString(); + [':', '/', ' '].forEach((char) => { + luceneQueryVal = luceneQueryVal.replace(new RegExp(char, 'g'), `\\${char}`); + }); + ['[', ']'].forEach((char) => { + luceneQueryVal = luceneQueryVal.replace( + new RegExp(`\\${char}`, 'g'), + `\\${char}` + ); + }); + traverser.select('a'); + this.buildSearchFilterVBase(f, traverser); + traverser.has('*', `Neptune#fts ${luceneQueryKey}:${luceneQueryVal}*`); + }); + } + + if (request.endsWith !== undefined) { + request.endsWith.forEach((f) => { + const luceneQueryKey = this.buildLuceneQueryKey(f.field, true); + let luceneQueryVal = f.value.toString(); + [':', '/', ' '].forEach((char) => { + luceneQueryVal = luceneQueryVal.replace(new RegExp(char, 'g'), `\\${char}`); + }); + ['[', ']'].forEach((char) => { + luceneQueryVal = luceneQueryVal.replace( + new RegExp(`\\${char}`, 'g'), + `\\${char}` + ); + }); + traverser.select('a'); + this.buildSearchFilterVBase(f, traverser); + traverser.has('*', `Neptune#fts ${luceneQueryKey}:*${luceneQueryVal}`); + }); + } + + if (request.contains !== undefined) { + request.contains.forEach((f) => { + const luceneQueryKey = this.buildLuceneQueryKey(f.field, true); + let luceneQueryVal = f.value.toString(); + [':', '/', ' '].forEach((char) => { + luceneQueryVal = luceneQueryVal.replace(new RegExp(char, 'g'), `\\${char}`); + }); + ['[', ']'].forEach((char) => { + luceneQueryVal = luceneQueryVal.replace( + new RegExp(`\\${char}`, 'g'), + `\\${char}` + ); + }); + traverser.select('a'); + this.buildSearchFilterVBase(f, traverser); + traverser.has('*', `Neptune#fts ${luceneQueryKey}:*${luceneQueryVal}*`); + }); + } + + if (request.exists !== undefined) { + request.exists.forEach((f) => { + traverser.select('a'); + this.buildSearchFilterEBase(f, traverser); + traverser.has(f.field, f.value); + }); + } + + if (request.nexists !== undefined) { + request.nexists.forEach((f) => { + traverser.select('a'); + this.buildSearchFilterEBaseNegated(f, traverser, f.field, f.value); + }); + } + + if (request.fulltext !== undefined) { + request.fulltext.forEach((f) => { + // Remove any characters that would be recognized by Lucene as control characters. + // Alternatively, could escape them but the standard query string analyzer will + // replace them with spaces anyway. + let luceneQueryVal = f.value.toString(); + [':', '/', '\\[', '\\]'].forEach((char) => { + luceneQueryVal = luceneQueryVal.replace(new RegExp(char, 'g'), ' '); + }); + // RegExp('\\\\') = regex for single backslash, '\\\\' = string with two backslashes + luceneQueryVal = luceneQueryVal.replace(new RegExp('\\\\', 'g'), `\\\\`); + traverser.select('a'); + this.buildSearchFilterVBase(f, traverser); + traverser.has(f.field, `Neptune#fts ${luceneQueryVal}`); + }); + } + + if (request.regex !== undefined) { + request.regex.forEach((f) => { + const luceneQueryKey = this.buildLuceneQueryKey(f.field, true); + // Escape characters that can appear, escaped or unescaped, in regex but are also + // control characters for Lucene. For example, "abc/def" is a valid regex but the contained + // slash is interpreted by Lucene as the end of the regex. Square brackets [] must not + // be escaped even though they denote range queries in Lucene because Lucene ignores + // them inside of regexes. + let luceneQueryVal = f.value.toString(); + [':', '/', ' '].forEach((char) => { + luceneQueryVal = luceneQueryVal.replace(new RegExp(char, 'g'), `\\${char}`); + }); + traverser.select('a'); + this.buildSearchFilterVBase(f, traverser); + traverser.has('*', `Neptune#fts ${luceneQueryKey}:/${luceneQueryVal}/`); + }); + } + + if (request.lucene !== undefined) { + request.lucene.forEach((f) => { + traverser.select('a'); + this.buildSearchFilterVBase(f, traverser); + // no escaping for Opensearch, user can send control characters and is responsible for + // escaping them in the search request when necessary + traverser.has(f.field, `Neptune#fts ${f.value}`); + }); + } + + // must reset all traversals so far as we may meed to use simplePath if FGAC is enabled to prevent cyclic checks + traverser.select('a').dedup().fold().unfold().as('matched'); + + // if authz is enabled, only return results that the user is authorized to view + if (authorizedPaths !== undefined && authorizedPaths.length > 0) { + const authorizedPathIds = authorizedPaths.map((path) => `group___${path}`); + traverser + .local( + __.until(__.hasId(process.P.within(authorizedPathIds))).repeat( + __.out().simplePath().dedup() + ) + ) + .as('authorization'); + } + + logger.debug( + `search.enhanced.dao buildSearchTraverser: traverser: ${traverser.toString()}` + ); + + return traverser.select('matched').dedup(); + } + + private buildLuceneQueryKey(field: string, keyword?: boolean): string { + if (field === 'id') return 'entity_id'; + if (field === 'label') return 'entity_type'; + + let components: string[] = []; + components = ['predicates', field, 'value']; + if (keyword) components.push('keyword'); + return components.join('.'); + } +} diff --git a/source/packages/services/assetlibrary/src/search/search.full.dao.ts b/source/packages/services/assetlibrary/src/search/search.full.dao.ts index 1939266c9..4a3ac90e9 100644 --- a/source/packages/services/assetlibrary/src/search/search.full.dao.ts +++ b/source/packages/services/assetlibrary/src/search/search.full.dao.ts @@ -32,7 +32,7 @@ const __ = process.statics; export class SearchDaoFull extends BaseDaoFull { public constructor( @inject('neptuneUrl') neptuneUrl: string, - @inject('enableDfeOptimization') private enableDfeOptimization: boolean, + @inject('enableDfeOptimization') protected enableDfeOptimization: boolean, @inject(TYPES.TypeUtils) private typeUtils: TypeUtils, @inject(TYPES.NodeAssembler) private assembler: NodeAssembler, @inject(TYPES.GraphSourceFactory) graphSourceFactory: () => structure.Graph @@ -40,7 +40,7 @@ export class SearchDaoFull extends BaseDaoFull { super(neptuneUrl, graphSourceFactory); } - private buildSearchTraverser( + protected buildSearchTraverser( conn: NeptuneConnection, request: SearchRequestModel, authorizedPaths: string[] @@ -184,7 +184,7 @@ export class SearchDaoFull extends BaseDaoFull { return traverser.select('a').dedup(); } - private buildSearchFilterVBase( + protected buildSearchFilterVBase( filter: SearchRequestFilter | SearchRequestFacet, traverser: process.GraphTraversal ): void { @@ -199,7 +199,7 @@ export class SearchDaoFull extends BaseDaoFull { } } - private buildSearchFilterEBase( + protected buildSearchFilterEBase( filter: SearchRequestFilter | SearchRequestFacet, traverser: process.GraphTraversal ): void { @@ -215,7 +215,7 @@ export class SearchDaoFull extends BaseDaoFull { } } - private buildSearchFilterEBaseNegated( + protected buildSearchFilterEBaseNegated( filter: SearchRequestFilter | SearchRequestFacet, traverser: process.GraphTraversal, field: unknown, diff --git a/source/packages/services/assetlibrary/src/search/search.lite.service.ts b/source/packages/services/assetlibrary/src/search/search.lite.service.ts index c0e36f338..a86f0ac9a 100644 --- a/source/packages/services/assetlibrary/src/search/search.lite.service.ts +++ b/source/packages/services/assetlibrary/src/search/search.lite.service.ts @@ -22,9 +22,10 @@ import { TypeCategory } from '../types/constants'; import { NotSupportedError } from '../utils/errors'; import { SearchDaoLite } from './search.lite.dao'; import { FacetResults, SearchRequestModel } from './search.models'; +import { SearchService } from './search.service'; @injectable() -export class SearchServiceLite { +export class SearchServiceLite implements SearchService { private readonly DEFAULT_SEARCH_COUNT = 200; constructor( diff --git a/source/packages/services/assetlibrary/src/search/search.models.ts b/source/packages/services/assetlibrary/src/search/search.models.ts index 850999a7d..9ac9f2ca4 100644 --- a/source/packages/services/assetlibrary/src/search/search.models.ts +++ b/source/packages/services/assetlibrary/src/search/search.models.ts @@ -48,6 +48,9 @@ export class SearchRequestModel { startsWith?: SearchRequestFilters; endsWith?: SearchRequestFilters; contains?: SearchRequestFilters; + fulltext?: SearchRequestFilters; + regex?: SearchRequestFilters; + lucene?: SearchRequestFilters; exists?: SearchRequestFilters; nexists?: SearchRequestFilters; diff --git a/source/packages/services/command-and-control/docs/configuration.md b/source/packages/services/command-and-control/docs/configuration.md index af2ea5a7a..84bb8ed03 100644 --- a/source/packages/services/command-and-control/docs/configuration.md +++ b/source/packages/services/command-and-control/docs/configuration.md @@ -22,9 +22,8 @@ CORS_EXPOSED_HEADERS=content-type,location # the base path from the request to allow the module to map the incoming request to the correct lambda handler CUSTOMDOMAIN_BASEPATH= -# The Asset Library mode. `full` (default) will enable the full feature set and -# use Neptune as its datastore, whereas `lite` will offer a reduced feature set -# (see documentation) and use the AWS IoT Device Registry as its datastore. +# The Asset Library mode: `full`, `enhanced`, or `lite`. See source/packages/services/assetlibrary/docs/modes.md +# for details. MODE=full #Application logging level. Set to (in order) error, warn, info, verbose, debug or silly. diff --git a/source/packages/services/installer/src/commands/modules/service/assetLibrary.ts b/source/packages/services/installer/src/commands/modules/service/assetLibrary.ts index f0f2c04a2..9d664eb47 100644 --- a/source/packages/services/installer/src/commands/modules/service/assetLibrary.ts +++ b/source/packages/services/installer/src/commands/modules/service/assetLibrary.ts @@ -10,11 +10,12 @@ * OR CONDITIONS OF ANY KIND, express or implied. See the License for the specific language governing permissions * * and limitations under the License. * *********************************************************************************************************************/ +import { CreateServiceLinkedRoleCommand, GetRoleCommand, IAMClient } from '@aws-sdk/client-iam'; import inquirer from 'inquirer'; import { ListrTask } from 'listr2'; import ow from 'ow'; import path from 'path'; -import { Answers } from '../../../models/answers'; +import { Answers, AssetLibraryMode } from '../../../models/answers'; import { ModuleName, PostmanEnvironment, RestModule } from '../../../models/modules'; import { applicationConfigurationPrompt } from '../../../prompts/applicationConfiguration.prompt'; import { @@ -44,6 +45,13 @@ const ASSUMED_NEPTUNE_ENGINE_VERSION = '1.1.0.0'; // AWS.RDS.DescribeOrderableDBInstanceOptions API. const DEFAULT_NEPTUNE_INSTANCE_TYPE = 'db.r5.xlarge'; +export function modeRequiresNeptune(mode: string): boolean { + return mode === AssetLibraryMode.Full || mode === AssetLibraryMode.Enhanced; +} + +export function modeRequiresOpenSearch(mode: string): boolean { + return mode === AssetLibraryMode.Enhanced; +} export class AssetLibraryInstaller implements RestModule { public readonly friendlyName = 'Asset Library'; public readonly name = 'assetLibrary'; @@ -51,13 +59,15 @@ export class AssetLibraryInstaller implements RestModule { public readonly type = 'SERVICE'; public readonly dependsOnMandatory: ModuleName[] = ['apigw', 'deploymentHelper']; - public readonly dependsOnOptional: ModuleName[] = ['vpc']; + public readonly dependsOnOptional: ModuleName[] = ['vpc', 'kms']; public readonly stackName: string; private readonly neptuneStackName: string; + private readonly enhancedSearchStackName: string; constructor(environment: string) { this.neptuneStackName = `cdf-assetlibrary-neptune-${environment}`; + this.enhancedSearchStackName = `cdf-assetlibrary-enhancedsearch-${environment}`; this.stackName = `cdf-assetlibrary-${environment}`; } includeOptionalModules: (answers: Answers) => Answers; @@ -79,11 +89,15 @@ export class AssetLibraryInstaller implements RestModule { updatedAnswers = await inquirer.prompt( [ { - message: `Run in 'full' mode (with Amazon Neptune), or 'lite' mode (using AWS IoT Device Registry only). Note that 'lite' mode supports a reduced set of Asset Library features (see documentation for further info).`, + message: `Asset library mode: 'full' uses Amazon Neptune as data store. 'enhanced' adds Amazon OpenSearch for enhanced search features. 'lite' uses only AWS IoT Device Registry and supports a reduced feature set. See documentation for details.`, type: 'list', - choices: ['full', 'lite'], + choices: [ + AssetLibraryMode.Full, + AssetLibraryMode.Lite, + AssetLibraryMode.Enhanced, + ], name: 'assetLibrary.mode', - default: answers.assetLibrary?.mode ?? 'full', + default: answers.assetLibrary?.mode ?? AssetLibraryMode.Full, askAnswered: true, validate(answer: string) { if (answer?.length === 0) { @@ -108,7 +122,7 @@ export class AssetLibraryInstaller implements RestModule { loop: false, pageSize: 10, when(answers: Answers) { - return answers.assetLibrary?.mode === 'full'; + return modeRequiresNeptune(answers.assetLibrary?.mode); }, validate(answer: string) { if ( @@ -129,7 +143,7 @@ export class AssetLibraryInstaller implements RestModule { default: answers.assetLibrary?.createDbReplicaInstance ?? false, askAnswered: true, when(answers: Answers) { - return answers.assetLibrary?.mode === 'full'; + return modeRequiresNeptune(answers.assetLibrary?.mode); }, validate(answer: string) { if (answer?.length === 0) { @@ -145,7 +159,7 @@ export class AssetLibraryInstaller implements RestModule { default: answers.assetLibrary?.restoreFromSnapshot ?? false, askAnswered: true, when(answers: Answers) { - return answers.assetLibrary?.mode === 'full'; + return modeRequiresNeptune(answers.assetLibrary?.mode); }, validate(answer: string) { if (answer?.length === 0) { @@ -154,6 +168,53 @@ export class AssetLibraryInstaller implements RestModule { return true; }, }, + { + message: `Select the OpenSearch cluster data node instance type:`, + type: 'input', + name: 'assetLibrary.openSearchDataNodeInstanceType', + default: + answers.assetLibrary?.openSearchDataNodeInstanceType ?? + 't3.small.search', + askAnswered: true, + when: (answers: Answers) => + modeRequiresOpenSearch(answers.assetLibrary?.mode), + validate(answer: string) { + if (answer?.length === 0) { + return 'You must enter the OpenSearch data node instance type.'; + } + return true; + }, + }, + { + message: `Enter the number of OpenSearch cluster data node instances. This number must either be 1 or a multiple of the number of private subnets in the VPC:`, + type: 'number', + name: 'assetLibrary.openSearchDataNodeInstanceCount', + default: answers.assetLibrary?.openSearchDataNodeInstanceCount ?? 1, + askAnswered: true, + when: (answers: Answers) => + modeRequiresOpenSearch(answers.assetLibrary?.mode), + validate(answer: number) { + if (answer < 1) { + return 'The number of OpenSearch data node instances must be a non-zero multiple of the number of availability zones.'; + } + return true; + }, + }, + { + message: `Size of the EBS volume attached to OpenSearch data nodes in GiB`, + type: 'number', + name: 'assetLibrary.openSearchEBSVolumeSize', + default: answers.assetLibrary?.openSearchEBSVolumeSize ?? 10, + askAnswered: true, + when: (answers: Answers) => + modeRequiresOpenSearch(answers.assetLibrary?.mode), + validate(answer: number) { + if (answer < 10) { + return `You must specify at least 10 GiB for EBS volume size. For detailed documentation, see: https://docs.aws.amazon.com/opensearch-service/latest/developerguide/limits.html#ebsresource`; + } + return true; + }, + }, { message: `Enter the Neptune database snapshot identifier:`, type: 'input', @@ -162,7 +223,7 @@ export class AssetLibraryInstaller implements RestModule { askAnswered: true, when(answers: Answers) { return ( - answers.assetLibrary?.mode === 'full' && + modeRequiresNeptune(answers.assetLibrary?.mode) && answers.assetLibrary?.restoreFromSnapshot ); }, @@ -216,7 +277,12 @@ export class AssetLibraryInstaller implements RestModule { includeOptionalModule( 'vpc', updatedAnswers.modules, - updatedAnswers.assetLibrary.mode === 'full' + modeRequiresNeptune(updatedAnswers.assetLibrary.mode) + ); + includeOptionalModule( + 'kms', + updatedAnswers.modules, + modeRequiresOpenSearch(updatedAnswers.assetLibrary.mode) ); return updatedAnswers; } @@ -290,7 +356,7 @@ export class AssetLibraryInstaller implements RestModule { ); addIfSpecified( 'CustomResourceVPCLambdaArn', - answers.assetLibrary.mode === 'full' + modeRequiresNeptune(answers.assetLibrary?.mode) ? answers.deploymentHelper.vpcLambdaArn : answers.deploymentHelper.lambdaArn ); @@ -320,6 +386,10 @@ export class AssetLibraryInstaller implements RestModule { addIfSpecified('DbInstanceType', answers.assetLibrary.neptuneDbInstanceType); addIfSpecified('CreateDBReplicaInstance', answers.assetLibrary.createDbReplicaInstance); addIfSpecified('SnapshotIdentifier', answers.assetLibrary.neptuneSnapshotIdentifier); + // The Neptune-to-OpenSearch integration relies on Neptune Streams + if (modeRequiresOpenSearch(answers.assetLibrary.mode)) { + parameterOverrides.push('NeptuneEnableStreams=1'); + } return parameterOverrides; } @@ -335,7 +405,7 @@ export class AssetLibraryInstaller implements RestModule { return [answers, tasks]; } - if (answers.assetLibrary.mode === 'full') { + if (modeRequiresNeptune(answers.assetLibrary?.mode)) { tasks.push({ title: `Deploying stack '${this.neptuneStackName}'`, task: async () => { @@ -367,6 +437,130 @@ export class AssetLibraryInstaller implements RestModule { answers.assetLibrary.neptuneUrl = byOutputKey('GremlinEndpoint'); }, }); + + if (modeRequiresOpenSearch(answers.assetLibrary.mode)) { + tasks.push({ + title: `Ensure service-linked role 'AWSServiceRoleForAmazonElasticsearchService' exists`, + task: async () => { + const iamClient = new IAMClient({}); + + const getCommand1 = new GetRoleCommand({ + RoleName: 'AWSServiceRoleForAmazonOpenSearchService', + }); + try { + const data1 = await iamClient.send(getCommand1); + console.log( + `First attempt at finding SLR ${data1.$metadata.httpStatusCode}` + ); + if (data1.$metadata.httpStatusCode === 200) return; + } catch { + /* do nothing */ + } + // also probe for the legacy name of the role + const getCommand2 = new GetRoleCommand({ + RoleName: 'AWSServiceRoleForAmazonElasticsearchService', + }); + try { + const data2 = await iamClient.send(getCommand2); + console.log( + `Second attempt at finding SLR ${data2.$metadata.httpStatusCode}` + ); + if (data2.$metadata.httpStatusCode === 200) return; + } catch { + /* do nothing */ + } + + // if neither role exists, create it + const createCommand = new CreateServiceLinkedRoleCommand({ + AWSServiceName: 'es.amazonaws.com', + }); + await iamClient.send(createCommand); + + // An error occurred (InvalidInput) when calling the CreateServiceLinkedRole operation: Service role name + // AWSServiceRoleForAmazonElasticsearchService has been taken in this account, please try a different suffix. + }, + }); + + tasks.push({ + title: `Deploying stack '${this.enhancedSearchStackName}'`, + task: async () => { + const vpcSubnetIdsArr = answers.vpc.privateSubnetIds.split(','); + const instanceCount = answers.assetLibrary.openSearchDataNodeInstanceCount; + let subnetIds = answers.vpc.privateSubnetIds; + if (instanceCount < vpcSubnetIdsArr.length) { + subnetIds = vpcSubnetIdsArr.slice(0, instanceCount).join(','); + } else if (instanceCount % vpcSubnetIdsArr.length != 0) { + throw new Error( + `The chosen number of OpenSearch data nodes (${instanceCount}) is not an integer multiple of the number of VPC subnets given (${vpcSubnetIdsArr.length}: ${answers.vpc.privateSubnetIds}).` + ); + } + const availabilityZoneCount = subnetIds.split(',').length; + + const parameterOverrides = [ + `Environment=${answers.environment}`, + `VpcId=${answers.vpc.id}`, + `CDFSecurityGroupId=${answers.vpc.securityGroupId}`, + `PrivateSubNetIds=${subnetIds}`, + `CustomResourceVPCLambdaArn=${answers.deploymentHelper.vpcLambdaArn}`, + `KmsKeyId=${answers.kms.id}`, + `NeptuneSecurityGroupId=${answers.assetLibrary.neptuneSecurityGroup}`, + `NeptuneClusterEndpoint=${answers.assetLibrary.neptuneClusterReadEndpoint}`, + `OpenSearchInstanceType=${answers.assetLibrary.openSearchDataNodeInstanceType}`, + `OpenSearchInstanceCount=${answers.assetLibrary.openSearchDataNodeInstanceCount}`, + `OpenSearchEBSVolumeSize=${answers.assetLibrary.openSearchEBSVolumeSize}`, + `OpenSearchAvailabilityZoneCount=${availabilityZoneCount}`, + // # Parameters available in the Cloudformation template but not currently exposed in the installer are + // # listed as comments below. + // ## Unused Parameters for defining OpenSearch cluster setup: + // OpenSearchDedicatedMasterCount + // OpenSearchDedicatedMasterInstanceType + // OpenSearchEBSVolumeType + // OpenSearchEBSProvisionedIOPS + // NumberOfShards + // NumberOfReplica + // ## Unused Parameters for defining stream poller lambda behavior: + // NeptunePollerLambdaMemorySize + // NeptunePollerLambdaLoggingLevel + // NeptunePollerStreamRecordsBatchSize + // NeptunePollerMaxPollingWaitTime + // NeptunePollerMaxPollingInterval + // NeptunePollerStepFunctionFallbackPeriod + // NeptunePollerStepFunctionFallbackPeriodUnit + // ## Unused Parameters for defining data syncing behavior: + // GeoLocationFields + // PropertiesToExclude + // DatatypesToExclude + // IgnoreMissingDocument + // EnableNonStringIndexing + // ## Unused Parameters for defining monitoring and alerting: + // OpenSearchAuditLogsCloudWatchLogsLogGroupArn + // OpenSearchApplicationLogsCloudWatchLogsLogGroupArn + // OpenSearchIndexSlowLogsCloudWatchLogsLogGroupArn + // OpenSearchSearchSlowLogsCloudWatchLogsLogGroupArn + // CreateCloudWatchAlarm + // NotificationEmail + ]; + + await packageAndDeployStack({ + answers: answers, + stackName: this.enhancedSearchStackName, + serviceName: 'assetlibrary', + templateFile: 'infrastructure/cfn-enhancedsearch.yaml', + cwd: path.join( + monorepoRoot, + 'source', + 'packages', + 'services', + 'assetlibrary' + ), + parameterOverrides: parameterOverrides, + needsPackaging: false, + needsCapabilityNamedIAM: true, + needsCapabilityAutoExpand: false, + }); + }, + }); + } } tasks.push({ @@ -433,12 +627,14 @@ export class AssetLibraryInstaller implements RestModule { }, }); - tasks.push({ - title: `Deleting stack '${this.neptuneStackName}'`, - task: async () => { - await deleteStack(this.neptuneStackName, answers.region); - }, - }); + if (modeRequiresNeptune(answers.assetLibrary?.mode)) { + tasks.push({ + title: `Deleting stack '${this.neptuneStackName}'`, + task: async () => { + await deleteStack(this.neptuneStackName, answers.region); + }, + }); + } return tasks; } } diff --git a/source/packages/services/installer/src/models/answers.ts b/source/packages/services/installer/src/models/answers.ts index 18a9238f8..0f97126c7 100644 --- a/source/packages/services/installer/src/models/answers.ts +++ b/source/packages/services/installer/src/models/answers.ts @@ -103,6 +103,7 @@ export interface ProvisionedConcurrencyModuleAttribues extends ServiceModuleAttr provisionedConcurrentExecutions?: number; enableAutoScaling?: boolean; } + export interface RestServiceModuleAttribues extends ServiceModuleAttributes { enableCustomDomain?: boolean; customDomainBasePath?: string; @@ -121,23 +122,37 @@ export interface OrganizationManager extends RestServiceModuleAttribues { artifactBucketPrefix?: string; } +export const enum AssetLibraryMode { + Lite = 'lite', + Full = 'full', + Enhanced = 'enhanced', +} + export interface AssetLibrary extends RestServiceModuleAttribues, ProvisionedConcurrencyModuleAttribues { - mode?: 'full' | 'lite'; + mode?: AssetLibraryMode; + // Neptune Configuration neptuneDbInstanceType?: string; createDbReplicaInstance?: boolean; neptuneSnapshotIdentifier?: string; restoreFromSnapshot?: boolean; - neptuneUrl?: string; + // OpenSearch Configuration + openSearchDataNodeInstanceType?: string; + openSearchDataNodeInstanceCount?: number; + openSearchEBSVolumeSize?: number; + neptuneSecurityGroup?: string; + neptuneClusterReadEndpoint?: string; // Application Configuration defaultAnswer?: boolean; + neptuneUrl?: string; defaultDevicesParentRelationName?: string; defaultDevicesParentPath?: string; defaultDevicesState?: string; defaultGroupsValidateAllowedParentPath?: string; enableDfeOptimization?: boolean; authorizationEnabled?: boolean; + openSearchEndpoint?: string; } export interface AssetLibraryExport extends ServiceModuleAttributes {