Merge pull request #439 from untergeek/fix/420-1

Prepare for release of 3.2.3

Changelog.rst

@@ -3,8 +3,23 @@
 Changelog
 =========
 
-3.3.0 (? ? ?)
--------------
+3.2.3 (16 July 2015)
+--------------------
+
+**Bug fix**
+
+  * In order to address customer and community issues with bulk deletes, the
+    ``master_timeout`` is now invoked for delete operations.  This should address
+    503s with 30s timeouts in the debug log, even when ``--timeout`` is set to
+    a much higher value.  The ``master_timeout`` is tied to the ``--timeout``
+    flag value, but will not exceed 300 seconds. #420 (untergeek)
+
+**General**
+
+  * Mixing it up a bit here by putting `General` second!  The only other changes
+    are that logging has been improved for deletes so you won't need to have the
+    ``--debug`` flag to see if you have error codes >= 400, and some code
+    documentation improvements.
 
 3.2.2 (13 July 2015)
 --------------------

curator/_version.py

@@ -1 +1 @@
-__version__ = '3.3.0.dev2'
+__version__ = '3.2.3'

curator/api/delete.py

@@ -4,45 +4,50 @@
 import logging
 logger = logging.getLogger(__name__)
 
-def delete_indices(client, indices):
+def delete_indices(client, indices, master_timeout=30000):
     """
     Delete the indicated indices, including closed indices.
 
     :arg client: The Elasticsearch client connection
     :arg indices: A list of indices to act on
+    :arg master_timeout: Number of milliseconds to wait for master node response
     :rtype bool:
     """
     indices = ensure_list(indices)
     try:
         logger.info("Deleting indices as a batch operation:")
         for i in indices:
             logger.info("---deleting index {0}".format(i))
-        client.indices.delete(index=to_csv(indices))
+        client.indices.delete(index=to_csv(indices), master_timeout=master_timeout)
         return True
-    except Exception:
+    except Exception as e:
         logger.error("Error deleting one or more indices.  Run with --debug flag and/or check Elasticsearch logs for more information.")
+        try:
+            logger.error('Got a {0} response from Elasticsearch.  Error message: {1}'.format(e.status_code, e.error))
+        except AttributeError:
+            logger.error('Error message: {0}'.format(e))
         return False
 
-def delete(client, indices):
+def delete(client, indices, master_timeout=30000):
     """
     Helper method called by the CLI.  Tries up to 3x to delete indices.
 
     :arg client: The Elasticsearch client connection
     :arg indices: A list of indices to act on
+    :arg master_timeout: Number of milliseconds to wait for master node response
+    :rtype bool:
     """
     for count in range(1, 4): # Try 3 times
-        success = delete_indices(client, indices)
-        if success:
-            result = [ i for i in indices if i in get_indices(client) ]
-            if len(result) > 0:
-                logger.error("Indices failed to delete:")
-                for idx in result:
-                    logger.error("---{0}".format(idx))
-            else:
-                # break
-                return True # We leave the loop here, if everything deleted.
-            indices = result
+        logger.debug("master_timeout value: {0}".format(master_timeout))
+        delete_indices(client, indices, master_timeout)
+        result = [ i for i in indices if i in get_indices(client) ]
+        if len(result) > 0:
+            logger.error("Indices failed to delete:")
+            for idx in result:
+                logger.error("---{0}".format(idx))
         else:
-            return False # Encountered an exception
+            # break
+            return True # We leave the loop here, if everything deleted.
+        indices = result
     logger.error("Unable to delete indices after 3 attempts: {0}".format(result))
     return False # If we make it here, indices didn't delete after 3 tries.

curator/cli/index_selection.py

@@ -51,13 +51,27 @@ def indices(ctx, newer_than, older_than, prefix, suffix, time_unit,
         sys.exit(1)
 
     logger.info("Job starting: {0} indices".format(ctx.parent.info_name))
-    logger.debug("Params: {0}".format(ctx.parent.parent.params))
+
     # Base and client args are in the grandparent tier of the context
+    logger.debug("Params: {0}".format(ctx.parent.parent.params))
+
+    # Set master_timeout to match 'timeout' if less than or equal to 300 seconds,
+    # otherwise set to 300.  Also, set this before overriding the timeout.
+    master_timeout = ctx.parent.parent.params['timeout'] if ctx.parent.parent.params['timeout'] <= 300 else 300
+    master_timeout = master_timeout * 1000 # This value is in milliseconds, at least in Python.
+    # If this is in error, it may be somewhere in the Python module or urllib3
+    # The Elasticsearch output was timing out deletes because they failed to
+    # complete within 30ms (until I multiplied by 1000)
     override_timeout(ctx)
+
+    # Get the client
     client = get_client(**ctx.parent.parent.params)
+
     # Get a master-list of indices
     indices = get_indices(client)
     logger.debug("Full list of indices: {0}".format(indices))
+
+    # Build index list
     if index and not ctx.obj['filters']:
         working_list = []
     else:
@@ -67,6 +81,7 @@ def indices(ctx, newer_than, older_than, prefix, suffix, time_unit,
             click.echo(click.style('ERROR. No indices found in Elasticsearch.', fg='red', bold=True))
             sys.exit(1)
 
+    # Override any other flags if --all_indices specified
     if all_indices:
         working_list = indices
         logger.info('Matching all indices. Ignoring flags other than --exclude.')
@@ -124,12 +139,13 @@ def indices(ctx, newer_than, older_than, prefix, suffix, time_unit,
                     index_lists = chunk_index_list(working_list)
                     success = True
                     for l in index_lists:
-                        retval = do_command(client, ctx.parent.info_name, l, ctx.parent.params)
+                        retval = do_command(client, ctx.parent.info_name, l, ctx.parent.params, master_timeout)
                         if not retval:
                             success = False
+                        time.sleep(2) # Pause in between iterations
                     exit_msg(success)
                 else:
-                    retval = do_command(client, ctx.parent.info_name, working_list, ctx.parent.params)
+                    retval = do_command(client, ctx.parent.info_name, working_list, ctx.parent.params, master_timeout)
                     exit_msg(retval)
 
     else:

curator/cli/utils.py

@@ -201,7 +201,7 @@ def in_list(values, source_list):
             logger.warn('{0} not found!'.format(v))
     return retval
 
-def do_command(client, command, indices, params=None):
+def do_command(client, command, indices, params=None, master_timeout=30000):
     """
     Do the command.
     """
@@ -216,7 +216,7 @@ def do_command(client, command, indices, params=None):
     if command == "close":
         return close(client, indices)
     if command == "delete":
-        return delete(client, indices)
+        return delete(client, indices, master_timeout)
     if command == "open":
         return opener(client, indices)
     if command == "optimize":