Merge branch '8.x' into 8.0

Author: untergeek

README.rst

@@ -18,6 +18,12 @@ released, Curator 8.x will work with Elasticsearch 8.x.
 
 Watch this space for updates when that is coming.
 
+Additional support for Elasticsearch 7.14.0 - 7.17.x
+****************************************************
+
+Starting with Curator 8.0.18, Curator 8 can execute against Elasticsearch versions
+7.14.0 - 7.17.x in addition to all versions of Elasticsearch 8.x.
+
 New Client Configuration
 ------------------------
 

curator/_version.py

@@ -1,3 +1,3 @@
 """Curator Version"""
 
-__version__ = '8.0.18'
+__version__ = '8.0.19'

curator/cli.py

@@ -128,18 +128,23 @@ def process_action(client, action_def, dry_run=False):
     mykwargs.update(prune_nones(action_def.options))
 
     # Pop out the search_pattern option, if present.
-    search_pattern = mykwargs.pop('search_pattern', '*')
+    ptrn = mykwargs.pop('search_pattern', '*')
+    hidn = mykwargs.pop('include_hidden', False)
 
     logger.debug('Action kwargs: %s', mykwargs)
-    logger.debug('Post search_pattern Action kwargs: %s', mykwargs)
+    logger.debug('Post search_pattern & include_hidden Action kwargs: %s', mykwargs)
 
     # Set up the action
     logger.debug('Running "%s"', action_def.action.upper())
     if action_def.action == 'alias':
         # Special behavior for this action, as it has 2 index lists
         action_def.instantiate('action_cls', **mykwargs)
-        action_def.instantiate('alias_adds', client, search_pattern=search_pattern)
-        action_def.instantiate('alias_removes', client, search_pattern=search_pattern)
+        action_def.instantiate(
+            'alias_adds', client, search_pattern=ptrn, include_hidden=hidn
+        )
+        action_def.instantiate(
+            'alias_removes', client, search_pattern=ptrn, include_hidden=hidn
+        )
         if 'remove' in action_def.action_dict:
             logger.debug('Removing indices from alias "%s"', action_def.options['name'])
             action_def.alias_removes.iterate_filters(action_def.action_dict['remove'])
@@ -163,7 +168,9 @@ def process_action(client, action_def, dry_run=False):
                 'list_obj', client, repository=action_def.options['repository']
             )
         else:
-            action_def.instantiate('list_obj', client, search_pattern=search_pattern)
+            action_def.instantiate(
+                'list_obj', client, search_pattern=ptrn, include_hidden=hidn
+            )
         action_def.list_obj.iterate_filters({'filters': action_def.filters})
         logger.debug('Pre Instantiation Action kwargs: %s', mykwargs)
         action_def.instantiate('action_cls', action_def.list_obj, **mykwargs)

curator/cli_singletons/alias.py

@@ -36,9 +36,22 @@
     default=False,
     show_default=True,
 )
+@click.option(
+    '--include_hidden/--no-include_hidden',
+    help='Allow Curator to operate on hidden indices (and data_streams).',
+    default=False,
+    show_default=True,
+)
 @click.pass_context
 def alias(
-    ctx, name, add, remove, warn_if_no_indices, extra_settings, allow_ilm_indices
+    ctx,
+    name,
+    add,
+    remove,
+    warn_if_no_indices,
+    extra_settings,
+    allow_ilm_indices,
+    include_hidden,
 ):
     """
     Add/Remove Indices to/from Alias
@@ -48,6 +61,7 @@ def alias(
         'name': name,
         'extra_settings': extra_settings,
         'allow_ilm_indices': allow_ilm_indices,
+        'include_hidden': include_hidden,
     }
     logger.debug('manual_options %s', manual_options)
     # ctx.info_name is the name of the function or name specified in

curator/cli_singletons/allocation.py

@@ -43,6 +43,12 @@
     default=False,
     show_default=True,
 )
+@click.option(
+    '--include_hidden/--no-include_hidden',
+    help='Allow Curator to operate on hidden indices (and data_streams).',
+    default=False,
+    show_default=True,
+)
 @click.option(
     '--filter_list',
     callback=validate_filter_json,
@@ -61,6 +67,7 @@ def allocation(
     wait_interval,
     ignore_empty_list,
     allow_ilm_indices,
+    include_hidden,
     filter_list,
 ):
     """
@@ -75,6 +82,7 @@ def allocation(
         'max_wait': max_wait,
         'wait_interval': wait_interval,
         'allow_ilm_indices': allow_ilm_indices,
+        'include_hidden': include_hidden,
     }
     # ctx.info_name is the name of the function or name specified in
     # @click.command decorator

curator/cli_singletons/close.py

@@ -28,6 +28,12 @@
     default=False,
     show_default=True,
 )
+@click.option(
+    '--include_hidden/--no-include_hidden',
+    help='Allow Curator to operate on hidden indices (and data_streams).',
+    default=False,
+    show_default=True,
+)
 @click.option(
     '--filter_list',
     callback=validate_filter_json,
@@ -42,6 +48,7 @@ def close(
     skip_flush,
     ignore_empty_list,
     allow_ilm_indices,
+    include_hidden,
     filter_list,
 ):
     """
@@ -52,6 +59,7 @@ def close(
         'skip_flush': skip_flush,
         'delete_aliases': delete_aliases,
         'allow_ilm_indices': allow_ilm_indices,
+        'include_hidden': include_hidden,
     }
     # ctx.info_name is the name of the function or name specified in
     # @click.command decorator

curator/cli_singletons/delete.py

@@ -67,6 +67,12 @@ def delete_indices(
     default=False,
     show_default=True,
 )
+@click.option(
+    '--include_hidden/--no-include_hidden',
+    help='Allow Curator to operate on hidden indices (and data_streams).',
+    default=False,
+    show_default=True,
+)
 @click.option(
     '--filter_list',
     callback=validate_filter_json,
@@ -81,6 +87,7 @@ def delete_snapshots(
     retry_interval,
     ignore_empty_list,
     allow_ilm_indices,
+    include_hidden,
     filter_list,
 ):
     """
@@ -90,6 +97,7 @@ def delete_snapshots(
         'retry_count': retry_count,
         'retry_interval': retry_interval,
         'allow_ilm_indices': allow_ilm_indices,
+        'include_hidden': include_hidden,
     }
     # ctx.info_name is the name of the function or name specified in @click.command
     # decorator

curator/cli_singletons/forcemerge.py

@@ -31,6 +31,12 @@
     default=False,
     show_default=True,
 )
+@click.option(
+    '--include_hidden/--no-include_hidden',
+    help='Allow Curator to operate on hidden indices (and data_streams).',
+    default=False,
+    show_default=True,
+)
 @click.option(
     '--filter_list',
     callback=validate_filter_json,
@@ -45,6 +51,7 @@ def forcemerge(
     delay,
     ignore_empty_list,
     allow_ilm_indices,
+    include_hidden,
     filter_list,
 ):
     """
@@ -55,6 +62,7 @@ def forcemerge(
         'max_num_segments': max_num_segments,
         'delay': delay,
         'allow_ilm_indices': allow_ilm_indices,
+        'include_hidden': include_hidden,
     }
     # ctx.info_name is the name of the function or name specified in
     # @click.command decorator

curator/cli_singletons/object_class.py

@@ -100,6 +100,7 @@ def __init__(
             self.options = option_dict
 
         self.search_pattern = self.options.pop('search_pattern', '*')
+        self.include_hidden = self.options.pop('include_hidden', False)
 
         # Extract allow_ilm_indices so it can be handled separately.
         if 'allow_ilm_indices' in self.options:
@@ -214,7 +215,9 @@ def get_list_object(self):
             self.list_object = SnapshotList(self.client, repository=self.repository)
         else:
             self.list_object = IndexList(
-                self.client, search_pattern=self.search_pattern
+                self.client,
+                search_pattern=self.search_pattern,
+                include_hidden=self.include_hidden,
             )
 
     def get_alias_obj(self):
@@ -230,7 +233,9 @@ def get_alias_obj(self):
                 )
                 self.logger.debug(msg)
                 self.alias[k]['ilo'] = IndexList(
-                    self.client, search_pattern=self.search_pattern
+                    self.client,
+                    search_pattern=self.search_pattern,
+                    include_hidden=self.include_hidden,
                 )
                 self.alias[k]['ilo'].iterate_filters(
                     {'filters': self.alias[k]['filters']}

curator/cli_singletons/open_indices.py

@@ -20,6 +20,12 @@
     default=False,
     show_default=True,
 )
+@click.option(
+    '--include_hidden/--no-include_hidden',
+    help='Allow Curator to operate on hidden indices (and data_streams).',
+    default=False,
+    show_default=True,
+)
 @click.option(
     '--filter_list',
     callback=validate_filter_json,
@@ -28,7 +34,12 @@
 )
 @click.pass_context
 def open_indices(
-    ctx, search_pattern, ignore_empty_list, allow_ilm_indices, filter_list
+    ctx,
+    search_pattern,
+    ignore_empty_list,
+    allow_ilm_indices,
+    include_hidden,
+    filter_list,
 ):
     """
     Open Indices
@@ -38,7 +49,11 @@ def open_indices(
     action = CLIAction(
         ctx.info_name,
         ctx.obj['configdict'],
-        {'search_pattern': search_pattern, 'allow_ilm_indices': allow_ilm_indices},
+        {
+            'search_pattern': search_pattern,
+            'allow_ilm_indices': allow_ilm_indices,
+            'include_hidden': include_hidden,
+        },
         filter_list,
         ignore_empty_list,
     )