Merge pull request #89 from stackhpc/upstream/2023.1-2026-01-16

Synchronise 2023.1 with upstream

Author: priteau

.gitreview

@@ -2,4 +2,4 @@
 host=review.opendev.org
 port=29418
 project=openstack/cinder.git
-defaultbranch=stable/2023.1
+defaultbranch=unmaintained/2023.1

.zuul.yaml

@@ -58,6 +58,9 @@
             irrelevant-files: *gate-irrelevant-files
         - cinder-tempest-plugin-lvm-lio-barbican:
             irrelevant-files: *gate-irrelevant-files
+        - cinder-tempest-plugin-lvm-lio-barbican-fips:
+            voting: false
+            irrelevant-files: *gate-irrelevant-files
         - cinder-grenade-mn-sub-volbak:
             irrelevant-files: *gate-irrelevant-files
         - cinder-tempest-lvm-multibackend:
@@ -68,6 +71,9 @@
             irrelevant-files: *gate-irrelevant-files
         - devstack-plugin-nfs-tempest-full:
             irrelevant-files: *gate-irrelevant-files
+        - devstack-plugin-nfs-tempest-full-fips:
+            voting: false
+            irrelevant-files: *gate-irrelevant-files
         - tempest-slow-py3:
             irrelevant-files: *gate-irrelevant-files
         - tempest-integrated-storage:
@@ -174,6 +180,17 @@
               volume_revert: True
               volume_image_dep_tests: False
 
+- job:
+    # this depends on some ceph admin setup which is not yet complete
+    # TODO(alee) enable this test when ceph admin work is complete.
+    name: cinder-plugin-ceph-tempest-fips
+    parent: cinder-plugin-ceph-tempest
+    nodeset: devstack-single-node-centos-9-stream
+    pre-run: playbooks/enable-fips.yaml
+    vars:
+      configure_swap_size: 4096
+      nslookup_target: 'opendev.org'
+
 - job:
     name: cinder-plugin-ceph-tempest-mn-aa
     parent: devstack-plugin-ceph-multinode-tempest-py3

bindep.txt

@@ -29,6 +29,7 @@ postgresql
 postgresql-client [platform:dpkg]
 postgresql-devel [platform:rpm]
 postgresql-server [platform:rpm]
+python3-devel [platform:rpm test]
 libpq-dev [platform:dpkg]
 thin-provisioning-tools [platform:debian]
 libxml2-dev [platform:dpkg test]

cinder/api/schemas/volume_image_metadata.py

@@ -27,7 +27,7 @@
         'os-set_image_metadata': {
             'type': 'object',
             'properties': {
-                'metadata': parameter_types.extra_specs,
+                'metadata': parameter_types.image_metadata,
             },
             'required': ['metadata'],
             'additionalProperties': False,

cinder/api/v3/router.py

@@ -210,22 +210,28 @@ def _setup_routes(self, mapper, ext_mgr):
                         member={'accept': 'POST'})
 
         self.resources['default_types'] = default_types.create_resource()
-        mapper.connect("default-types", "/default-types/{id}",
-                       controller=self.resources['default_types'],
-                       action='create_update',
-                       conditions={"method": ['PUT']})
-
-        mapper.connect("default-types", "/default-types",
-                       controller=self.resources['default_types'],
-                       action='index',
-                       conditions={"method": ['GET']})
-
-        mapper.connect("default-types", "/default-types/{id}",
-                       controller=self.resources['default_types'],
-                       action='detail',
-                       conditions={"method": ['GET']})
-
-        mapper.connect("default-types", "/default-types/{id}",
-                       controller=self.resources['default_types'],
-                       action='delete',
-                       conditions={"method": ['DELETE']})
+        for path_prefix in ['/{project_id}', '']:
+            # project_id is optional
+            mapper.connect(
+                "default-types", "%s/default-types/{id}" % path_prefix,
+                controller=self.resources['default_types'],
+                action='create_update',
+                conditions={"method": ['PUT']})
+
+            mapper.connect(
+                "default-types", "%s/default-types" % path_prefix,
+                controller=self.resources['default_types'],
+                action='index',
+                conditions={"method": ['GET']})
+
+            mapper.connect(
+                "default-types", "%s/default-types/{id}" % path_prefix,
+                controller=self.resources['default_types'],
+                action='detail',
+                conditions={"method": ['GET']})
+
+            mapper.connect(
+                "default-types", "%s/default-types/{id}" % path_prefix,
+                controller=self.resources['default_types'],
+                action='delete',
+                conditions={"method": ['DELETE']})

cinder/api/validation/parameter_types.py

@@ -157,6 +157,15 @@ def valid_char(char):
     'additionalProperties': False
 }
 
+image_metadata = {
+    'type': 'object',
+    'patternProperties': {
+        '^[a-zA-Z0-9-_:. /]{1,255}$': {
+            'type': 'string', 'format': 'mysql_text'
+        }
+    },
+    'additionalProperties': False
+}
 
 extra_specs_with_no_spaces_key = {
     'type': 'object',

cinder/api/validation/validators.py

@@ -398,6 +398,14 @@ def _validate_key_size(param_value):
     return True
 
 
+@jsonschema.FormatChecker.cls_checks('mysql_text')
+def _validate_mysql_text(param_value):
+    length = len(param_value.encode('utf8'))
+    if length > 65535:
+        return False
+    return True
+
+
 class FormatChecker(jsonschema.FormatChecker):
     """A FormatChecker can output the message from cause exception
 

cinder/backup/drivers/ceph.py

@@ -47,6 +47,7 @@
 import os
 import re
 import subprocess
+import tempfile
 import time
 from typing import Dict, List, Optional, Tuple  # noqa: H301
 
@@ -618,33 +619,40 @@ def _piped_execute(self, cmd1: list, cmd2: list) -> Tuple[int, bytes]:
         LOG.debug("Piping cmd1='%s' into...", ' '.join(cmd1))
         LOG.debug("cmd2='%s'", ' '.join(cmd2))
 
-        try:
-            p1 = subprocess.Popen(cmd1, stdout=subprocess.PIPE,
-                                  stderr=subprocess.PIPE,
-                                  close_fds=True)
-        except OSError as e:
-            LOG.error("Pipe1 failed - %s ", e)
-            raise
+        with tempfile.TemporaryFile() as errfile:
 
-        # NOTE(dosaboy): ensure that the pipe is blocking. This is to work
-        # around the case where evenlet.green.subprocess is used which seems to
-        # use a non-blocking pipe.
-        assert p1.stdout is not None
-        flags = fcntl.fcntl(p1.stdout, fcntl.F_GETFL) & (~os.O_NONBLOCK)
-        fcntl.fcntl(p1.stdout, fcntl.F_SETFL, flags)
-
-        try:
-            p2 = subprocess.Popen(cmd2, stdin=p1.stdout,
-                                  stdout=subprocess.PIPE,
-                                  stderr=subprocess.PIPE,
-                                  close_fds=True)
-        except OSError as e:
-            LOG.error("Pipe2 failed - %s ", e)
-            raise
+            try:
+                p1 = subprocess.Popen(cmd1, stdout=subprocess.PIPE,
+                                      stderr=errfile,
+                                      close_fds=True)
+            except OSError as e:
+                LOG.error("Pipe1 failed - %s ", e)
+                raise
+
+            # NOTE(dosaboy): ensure that the pipe is blocking. This is to work
+            # around the case where evenlet.green.subprocess is used which
+            # seems to use a non-blocking pipe.
+            assert p1.stdout is not None
+            flags = fcntl.fcntl(p1.stdout, fcntl.F_GETFL) & (~os.O_NONBLOCK)
+            fcntl.fcntl(p1.stdout, fcntl.F_SETFL, flags)
 
-        p1.stdout.close()
-        stdout, stderr = p2.communicate()
-        return p2.returncode, stderr
+            try:
+                p2 = subprocess.Popen(cmd2, stdin=p1.stdout,
+                                      stdout=subprocess.PIPE,
+                                      stderr=errfile,
+                                      close_fds=True)
+            except OSError as e:
+                LOG.error("Pipe2 failed - %s ", e)
+                raise
+
+            p1.stdout.close()
+            p2.communicate()
+            p1.wait()
+
+            errfile.seek(0)
+            px_stderr = errfile.read()
+
+        return p1.returncode or p2.returncode, px_stderr
 
     def _rbd_diff_transfer(self, src_name: str, src_pool: str,
                            dest_name: str, dest_pool: str,

cinder/backup/drivers/swift.py

@@ -192,7 +192,8 @@ def _headers(self, headers=None):
 
         sa_plugin = service_auth.get_service_auth_plugin()
         if sa_plugin is not None:
-            result['X-Service-Token'] = sa_plugin.get_token()
+            sa_session = service_auth.get_service_session()
+            result['X-Service-Token'] = sa_plugin.get_token(session=sa_session)
 
         return result
 

cinder/cmd/manage.py

@@ -645,6 +645,16 @@ def update_host(self, currenthost: str, newhost: str) -> None:
             db.volume_update(ctxt, v['id'],
                              {'host': newhost})
 
+    def update_service(self):
+        """Modify the service uuid associated with a volume.
+
+        In certain upgrade cases, we create new cinder services and delete the
+        records of old ones, however, the volumes created with old service
+        still contain the service uuid of the old services.
+        """
+        ctxt = context.get_admin_context()
+        db.volume_update_all_by_service(ctxt)
+
 
 class ConfigCommands(object):
     """Class for exposing the flags defined by flag_file(s)."""