Merge branch 'master' into 8.x

Author: untergeek

Dockerfile

@@ -1,6 +1,6 @@
 # syntax=docker/dockerfile:1
-ARG PYVER=3.11.7
-ARG ALPTAG=3.18
+ARG PYVER=3.11.9
+ARG ALPTAG=3.19
 FROM python:${PYVER}-alpine${ALPTAG} as builder
 
 # Add the community repo for access to patchelf binary package

curator/_version.py

@@ -1,2 +1,2 @@
 """Curator Version"""
-__version__ = '8.0.14'
+__version__ = '8.0.15'

curator/actions/alias.py

@@ -98,7 +98,7 @@ def remove(self, ilo, warn_if_no_indices=False):
 
             # Re-raise the exceptions.NoIndices so it will behave as before
             raise NoIndices('No indices to remove from alias') from exc
-        aliases = self.client.indices.get_alias()
+        aliases = self.client.indices.get_alias(expand_wildcards=['open', 'closed'])
         for index in ilo.working_list():
             if index in aliases:
                 self.loggit.debug('Index %s in get_aliases output', index)

curator/helpers/date_ops.py

@@ -4,7 +4,7 @@
 import re
 import string
 import time
-from datetime import timedelta, datetime
+from datetime import timedelta, datetime, timezone
 from elasticsearch8.exceptions import NotFoundError
 from curator.exceptions import ConfigurationError
 from curator.defaults.settings import date_regex
@@ -43,6 +43,7 @@ def get_epoch(self, searchme):
                 timestamp = match.group("date")
                 return datetime_to_epoch(get_datetime(timestamp, self.timestring))
 
+
 def absolute_date_range(
         unit, date_from, date_to,
         date_from_format=None, date_to_format=None
@@ -77,7 +78,7 @@ def absolute_date_range(
         raise ConfigurationError('Must provide "date_from_format" and "date_to_format"')
     try:
         start_epoch = datetime_to_epoch(get_datetime(date_from, date_from_format))
-        logger.debug('Start ISO8601 = %s', datetime.utcfromtimestamp(start_epoch).isoformat())
+        logger.debug('Start ISO8601 = %s', epoch2iso(start_epoch))
     except Exception as err:
         raise ConfigurationError(
             f'Unable to parse "date_from" {date_from} and "date_from_format" {date_from_format}. '
@@ -116,7 +117,7 @@ def absolute_date_range(
         end_epoch = get_point_of_reference(
             unit, -1, epoch=datetime_to_epoch(end_date)) -1
 
-    logger.debug('End ISO8601 = %s', datetime.utcfromtimestamp(end_epoch).isoformat())
+    logger.debug('End ISO8601 = %s', epoch2iso(end_epoch))
     return (start_epoch, end_epoch)
 
 def date_range(unit, range_from, range_to, epoch=None, week_starts_on='sunday'):
@@ -152,7 +153,7 @@ def date_range(unit, range_from, range_to, epoch=None, week_starts_on='sunday'):
     if not epoch:
         epoch = time.time()
     epoch = fix_epoch(epoch)
-    raw_point_of_ref = datetime.utcfromtimestamp(epoch)
+    raw_point_of_ref = datetime.fromtimestamp(epoch, timezone.utc)
     logger.debug('Raw point of Reference = %s', raw_point_of_ref)
     # Reverse the polarity, because -1 as last week makes sense when read by
     # humans, but datetime timedelta math makes -1 in the future.
@@ -208,7 +209,7 @@ def date_range(unit, range_from, range_to, epoch=None, week_starts_on='sunday'):
         start_date = point_of_ref - start_delta
     # By this point, we know our start date and can convert it to epoch time
     start_epoch = datetime_to_epoch(start_date)
-    logger.debug('Start ISO8601 = %s', datetime.utcfromtimestamp(start_epoch).isoformat())
+    logger.debug('Start ISO8601 = %s', epoch2iso(start_epoch))
     # This is the number of units we need to consider.
     count = (range_to - range_from) + 1
     # We have to iterate to one more month, and then subtract a second to get
@@ -234,7 +235,7 @@ def date_range(unit, range_from, range_to, epoch=None, week_starts_on='sunday'):
         # to get hours, days, or weeks, as they don't change
         end_epoch = get_point_of_reference(
             unit, count * -1, epoch=start_epoch) -1
-    logger.debug('End ISO8601 = %s', datetime.utcfromtimestamp(end_epoch).isoformat())
+    logger.debug('End ISO8601 = %s', epoch2iso(end_epoch))
     return (start_epoch, end_epoch)
 
 def datetime_to_epoch(mydate):
@@ -247,9 +248,53 @@ def datetime_to_epoch(mydate):
     :returns: An epoch timestamp based on ``mydate``
     :rtype: int
     """
-    tdelta = (mydate - datetime(1970, 1, 1))
+    tdelta = mydate - datetime(1970, 1, 1)
     return tdelta.seconds + tdelta.days * 24 * 3600
 
+def epoch2iso(epoch: int) -> str:
+    """
+    Return an ISO8601 value for epoch
+
+    :param epoch: An epoch timestamp
+    :type epoch: int
+
+    :returns: An ISO8601 timestamp
+    :rtype: str
+    """
+    # Because Python 3.12 now requires non-naive timezone declarations, we must change.
+    #
+    ### Example:
+    ### epoch == 1491256800
+    ###
+    ### The old way:
+    ###datetime.utcfromtimestamp(epoch)
+    ###   datetime.datetime(2017, 4, 3, 22, 0).isoformat()
+    ###   Result: 2017-04-03T22:00:00
+    ###
+    ### The new way:
+    ###     datetime.fromtimestamp(epoch, timezone.utc)
+    ###     datetime.datetime(2017, 4, 3, 22, 0, tzinfo=datetime.timezone.utc).isoformat()
+    ###     Result: 2017-04-03T22:00:00+00:00
+    ###
+    ### End Example
+    #
+    # Note that the +00:00 is appended now where we affirmatively declare the UTC timezone
+    #
+    # As a result, we will use this function to prune away the timezone if it is +00:00 and replace
+    # it with Z, which is shorter Zulu notation for UTC (which Elasticsearch uses)
+    #
+    # We are MANUALLY, FORCEFULLY declaring timezone.utc, so it should ALWAYS be +00:00, but could
+    # in theory sometime show up as a Z, so we test for that.
+
+    parts = datetime.fromtimestamp(epoch, timezone.utc).isoformat().split('+')
+    if len(parts) == 1:
+        if parts[0][-1] == 'Z':
+            return parts[0] # Our ISO8601 already ends with a Z for Zulu/UTC time
+        return f'{parts[0]}Z' # It doesn't end with a Z so we put one there
+    if parts[1] == '00:00':
+        return f'{parts[0]}Z' # It doesn't end with a Z so we put one there
+    return f'{parts[0]}+{parts[1]}' # Fallback publishes the +TZ, whatever that was
+
 def fix_epoch(epoch):
     """
     Fix value of ``epoch`` to be the count since the epoch in seconds only, which should be 10 or
@@ -537,7 +582,7 @@ def parse_date_pattern(name):
         if char == '%':
             pass
         elif char in date_regex() and prev == '%':
-            rendered += str(datetime.utcnow().strftime(f'%{char}'))
+            rendered += str(datetime.now(timezone.utc).strftime(f'%{char}'))
         else:
             rendered += char
         logger.debug('Partially rendered name: %s', rendered)

curator/indexlist.py

@@ -927,7 +927,6 @@ def filter_by_alias(self, aliases=None, exclude=False):
     def filter_by_count(self, count=None, reverse=True, use_age=False, pattern=None,
             source='creation_date', timestring=None, field=None, stats_result='min_value',
             exclude=True):
-        # pylint: disable=anomalous-backslash-in-string
         """
         Remove indices from the actionable list beyond the number ``count``, sorted
         reverse-alphabetically by default.  If you set ``reverse=False``, it will be sorted
@@ -951,7 +950,7 @@ def filter_by_count(self, count=None, reverse=True, use_age=False, pattern=None,
         :param pattern: Select indices to count from a regular expression pattern. This pattern
             must have one and only one capture group. This can allow a single ``count`` filter
             instance to operate against any number of matching patterns, and keep ``count`` of each
-            index in that group.  For example, given a ``pattern`` of ``'^(.*)-\d{6}$'``, it will
+            index in that group.  For example, given a ``pattern`` of ``'^(.*)-\\d{6}$'``, it will
             match both ``rollover-000001`` and ``index-999990``, but not ``logstash-2017.10.12``.
             Following the same example, if my cluster also had ``rollover-000002`` through
             ``rollover-000010`` and ``index-888888`` through ``index-999999``, it will process both

docker_test/.env

@@ -0,0 +1 @@
+export REMOTE_ES_SERVER="http://172.19.2.14:9201"

docker_test/scripts/create.sh

@@ -127,13 +127,11 @@ echo "Please select one of these environment variables to prepend your 'pytest'
 echo
 
 for IP in $IPLIST; do
-  echo "REMOTE_ES_SERVER=\"http://$IP:${REMOTE_PORT}\""
-  if [ "$AUTO_EXPORT" == "y" ]; then
-    # This puts our curatortestenv file where it can be purged easily by destroy.sh
-    cd $SCRIPTPATH
-    cd ..
-    echo "export REMOTE_ES_SERVER=http://$IP:$REMOTE_PORT" > curatortestenv
-  fi
+  REMOTE="REMOTE_ES_SERVER=\"http://$IP:${REMOTE_PORT}\""
+  echo ${REMOTE}
+  cd $SCRIPTPATH
+  cd ..
+  echo "export ${REMOTE}" > .env
 done
 
 echo

docker_test/scripts/destroy.sh

@@ -26,7 +26,7 @@ UPONE=$(pwd | awk -F\/ '{print $NF}')
 
 if [[ "$UPONE" = "docker_test" ]]; then
   rm -rf $(pwd)/repo/*
-  rm -rf $(pwd)/curatortestenv
+  cp /dev/null $(pwd)/.env
 else
   echo "WARNING: Unable to automatically empty bind mounted repo path."
   echo "Please manually empty the contents of the repo directory!"

docs/Changelog.rst

@@ -3,6 +3,36 @@
 Changelog
 =========
 
+8.0.15 (10 April 2024)
+----------------------
+
+**Announcement**
+
+  * Python 3.12 support becomes official. A few changes were necessary to ``datetime`` calls
+    which were still using naive timestamps. Tests across all minor Python versions from 3.8 - 3.12
+    verify everything is working as expected with regards to those changes. Note that Docker builds
+    are still running Python 3.11 as cx_Freeze still does not officially support Python 3.12.
+  * Added infrastructure to test multiple versions of Python against the code base. This requires
+    you to run:
+        * ``pip install -U hatch hatchling`` -- Install prerequisites
+        * ``hatch run docker:create X.Y.Z`` -- where ``X.Y.Z`` is an ES version on Docker Hub
+        * ``hatch run test:pytest`` -- Run the test suite for each supported version of Python
+        * ``hatch run docker:destroy`` -- Cleanup the Docker containers created in ``docker:create``
+
+**Bugfix**
+
+  * A bug reported in ``es_client`` with Python versions 3.8 and 3.9 has been addressed. Going
+    forward, testing protocol will be to ensure that Curator works with all supported versions of
+    Python, or support will be removed (when 3.8 is EOL, for example).
+
+**Changes**
+
+  * Address deprecation warning in ``get_alias()`` call by limiting indices to only open and
+    closed indices via ``expand_wildcards=['open', 'closed']``.
+  * Address test warnings for an improperly escaped ``\d`` in a docstring in ``indexlist.py``
+  * Updated Python version in Docker build. See Dockerfile for more information.
+  * Docker test scripts updated to make Hatch matrix testing easier (.env file)
+
 8.0.14 (2 April 2024)
 ---------------------
 

docs/asciidoc/index.asciidoc

@@ -1,10 +1,10 @@
-:curator_version: 8.0.14
+:curator_version: 8.0.15
 :curator_major: 8
 :curator_doc_tree: 8.0
 :es_py_version: 8.13.0
 :es_doc_tree: 8.13
 :stack_doc_tree: 8.13
-:pybuild_ver: 3.11.7
+:pybuild_ver: 3.11.9
 :copyright_years: 2011-2024
 :ref:  http://www.elastic.co/guide/en/elasticsearch/reference/{es_doc_tree}
 :esref: http://www.elastic.co/guide/en/elasticsearch/reference/{stack_doc_tree}