Synchronise 2023.1 with upstream

.gitreview

@@ -2,4 +2,4 @@
 host=review.opendev.org
 port=29418
 project=openstack/cinder.git
-defaultbranch=stable/2023.1
+defaultbranch=unmaintained/2023.1

.zuul.yaml

@@ -58,6 +58,9 @@
             irrelevant-files: *gate-irrelevant-files
         - cinder-tempest-plugin-lvm-lio-barbican:
             irrelevant-files: *gate-irrelevant-files
+        - cinder-tempest-plugin-lvm-lio-barbican-fips:
+            voting: false
+            irrelevant-files: *gate-irrelevant-files
         - cinder-grenade-mn-sub-volbak:
             irrelevant-files: *gate-irrelevant-files
         - cinder-tempest-lvm-multibackend:
@@ -68,6 +71,9 @@
             irrelevant-files: *gate-irrelevant-files
         - devstack-plugin-nfs-tempest-full:
             irrelevant-files: *gate-irrelevant-files
+        - devstack-plugin-nfs-tempest-full-fips:
+            voting: false
+            irrelevant-files: *gate-irrelevant-files
         - tempest-slow-py3:
             irrelevant-files: *gate-irrelevant-files
         - tempest-integrated-storage:
@@ -174,6 +180,17 @@
               volume_revert: True
               volume_image_dep_tests: False
 
+- job:
+    # this depends on some ceph admin setup which is not yet complete
+    # TODO(alee) enable this test when ceph admin work is complete.
+    name: cinder-plugin-ceph-tempest-fips
+    parent: cinder-plugin-ceph-tempest
+    nodeset: devstack-single-node-centos-9-stream
+    pre-run: playbooks/enable-fips.yaml
+    vars:
+      configure_swap_size: 4096
+      nslookup_target: 'opendev.org'
+
 - job:
     name: cinder-plugin-ceph-tempest-mn-aa
     parent: devstack-plugin-ceph-multinode-tempest-py3

bindep.txt

@@ -29,6 +29,7 @@ postgresql
 postgresql-client [platform:dpkg]
 postgresql-devel [platform:rpm]
 postgresql-server [platform:rpm]
+python3-devel [platform:rpm test]
 libpq-dev [platform:dpkg]
 thin-provisioning-tools [platform:debian]
 libxml2-dev [platform:dpkg test]

cinder/api/schemas/volume_image_metadata.py

@@ -27,7 +27,7 @@
         'os-set_image_metadata': {
             'type': 'object',
             'properties': {
-                'metadata': parameter_types.extra_specs,
+                'metadata': parameter_types.image_metadata,
             },
             'required': ['metadata'],
             'additionalProperties': False,

cinder/api/v3/router.py

@@ -210,22 +210,28 @@ def _setup_routes(self, mapper, ext_mgr):
                         member={'accept': 'POST'})
 
         self.resources['default_types'] = default_types.create_resource()
-        mapper.connect("default-types", "/default-types/{id}",
-                       controller=self.resources['default_types'],
-                       action='create_update',
-                       conditions={"method": ['PUT']})
-
-        mapper.connect("default-types", "/default-types",
-                       controller=self.resources['default_types'],
-                       action='index',
-                       conditions={"method": ['GET']})
-
-        mapper.connect("default-types", "/default-types/{id}",
-                       controller=self.resources['default_types'],
-                       action='detail',
-                       conditions={"method": ['GET']})
-
-        mapper.connect("default-types", "/default-types/{id}",
-                       controller=self.resources['default_types'],
-                       action='delete',
-                       conditions={"method": ['DELETE']})
+        for path_prefix in ['/{project_id}', '']:
+            # project_id is optional
+            mapper.connect(
+                "default-types", "%s/default-types/{id}" % path_prefix,
+                controller=self.resources['default_types'],
+                action='create_update',
+                conditions={"method": ['PUT']})
+
+            mapper.connect(
+                "default-types", "%s/default-types" % path_prefix,
+                controller=self.resources['default_types'],
+                action='index',
+                conditions={"method": ['GET']})
+
+            mapper.connect(
+                "default-types", "%s/default-types/{id}" % path_prefix,
+                controller=self.resources['default_types'],
+                action='detail',
+                conditions={"method": ['GET']})
+
+            mapper.connect(
+                "default-types", "%s/default-types/{id}" % path_prefix,
+                controller=self.resources['default_types'],
+                action='delete',
+                conditions={"method": ['DELETE']})

cinder/api/validation/parameter_types.py

@@ -157,6 +157,15 @@ def valid_char(char):
     'additionalProperties': False
 }
 
+image_metadata = {
+    'type': 'object',
+    'patternProperties': {
+        '^[a-zA-Z0-9-_:. /]{1,255}$': {
+            'type': 'string', 'format': 'mysql_text'
+        }
+    },
+    'additionalProperties': False
+}
 
 extra_specs_with_no_spaces_key = {
     'type': 'object',

cinder/api/validation/validators.py

@@ -398,6 +398,14 @@ def _validate_key_size(param_value):
     return True
 
 
+@jsonschema.FormatChecker.cls_checks('mysql_text')
+def _validate_mysql_text(param_value):
+    length = len(param_value.encode('utf8'))
+    if length > 65535:
+        return False
+    return True
+
+
 class FormatChecker(jsonschema.FormatChecker):
     """A FormatChecker can output the message from cause exception
 

cinder/backup/drivers/ceph.py

@@ -47,6 +47,7 @@
 import os
 import re
 import subprocess
+import tempfile
 import time
 from typing import Dict, List, Optional, Tuple  # noqa: H301
 
@@ -618,33 +619,40 @@ def _piped_execute(self, cmd1: list, cmd2: list) -> Tuple[int, bytes]:
         LOG.debug("Piping cmd1='%s' into...", ' '.join(cmd1))
         LOG.debug("cmd2='%s'", ' '.join(cmd2))
 
-        try:
-            p1 = subprocess.Popen(cmd1, stdout=subprocess.PIPE,
-                                  stderr=subprocess.PIPE,
-                                  close_fds=True)
-        except OSError as e:
-            LOG.error("Pipe1 failed - %s ", e)
-            raise
+        with tempfile.TemporaryFile() as errfile:
 
-        # NOTE(dosaboy): ensure that the pipe is blocking. This is to work
-        # around the case where evenlet.green.subprocess is used which seems to
-        # use a non-blocking pipe.
-        assert p1.stdout is not None
-        flags = fcntl.fcntl(p1.stdout, fcntl.F_GETFL) & (~os.O_NONBLOCK)
-        fcntl.fcntl(p1.stdout, fcntl.F_SETFL, flags)
-
-        try:
-            p2 = subprocess.Popen(cmd2, stdin=p1.stdout,
-                                  stdout=subprocess.PIPE,
-                                  stderr=subprocess.PIPE,
-                                  close_fds=True)
-        except OSError as e:
-            LOG.error("Pipe2 failed - %s ", e)
-            raise
+            try:
+                p1 = subprocess.Popen(cmd1, stdout=subprocess.PIPE,
+                                      stderr=errfile,
+                                      close_fds=True)
+            except OSError as e:
+                LOG.error("Pipe1 failed - %s ", e)
+                raise
+
+            # NOTE(dosaboy): ensure that the pipe is blocking. This is to work
+            # around the case where evenlet.green.subprocess is used which
+            # seems to use a non-blocking pipe.
+            assert p1.stdout is not None
+            flags = fcntl.fcntl(p1.stdout, fcntl.F_GETFL) & (~os.O_NONBLOCK)
+            fcntl.fcntl(p1.stdout, fcntl.F_SETFL, flags)
 
-        p1.stdout.close()
-        stdout, stderr = p2.communicate()
-        return p2.returncode, stderr
+            try:
+                p2 = subprocess.Popen(cmd2, stdin=p1.stdout,
+                                      stdout=subprocess.PIPE,
+                                      stderr=errfile,
+                                      close_fds=True)
+            except OSError as e:
+                LOG.error("Pipe2 failed - %s ", e)
+                raise
+
+            p1.stdout.close()
+            p2.communicate()
+            p1.wait()
+
+            errfile.seek(0)
+            px_stderr = errfile.read()
+
+        return p1.returncode or p2.returncode, px_stderr
 
     def _rbd_diff_transfer(self, src_name: str, src_pool: str,
                            dest_name: str, dest_pool: str,

cinder/backup/drivers/swift.py

@@ -192,7 +192,8 @@ def _headers(self, headers=None):
 
         sa_plugin = service_auth.get_service_auth_plugin()
         if sa_plugin is not None:
-            result['X-Service-Token'] = sa_plugin.get_token()
+            sa_session = service_auth.get_service_session()
+            result['X-Service-Token'] = sa_plugin.get_token(session=sa_session)
 
         return result
 

cinder/cmd/manage.py

@@ -645,6 +645,16 @@ def update_host(self, currenthost: str, newhost: str) -> None:
             db.volume_update(ctxt, v['id'],
                              {'host': newhost})
 
+    def update_service(self):
+        """Modify the service uuid associated with a volume.
+
+        In certain upgrade cases, we create new cinder services and delete the
+        records of old ones, however, the volumes created with old service
+        still contain the service uuid of the old services.
+        """
+        ctxt = context.get_admin_context()
+        db.volume_update_all_by_service(ctxt)
+
 
 class ConfigCommands(object):
     """Class for exposing the flags defined by flag_file(s)."""

cinder/db/api.py

@@ -443,6 +443,11 @@ def volume_get_all_by_host(context, host, filters=None):
     return IMPL.volume_get_all_by_host(context, host, filters=filters)
 
 
+def volume_update_all_by_service(context):
+    """Update all volumes associated with an old service."""
+    return IMPL.volume_update_all_by_service(context)
+
+
 def volume_get_all_by_group(context, group_id, filters=None):
     """Get all volumes belonging to a consistency group."""
     return IMPL.volume_get_all_by_group(context, group_id, filters=filters)

cinder/db/sqlalchemy/api.py

@@ -2862,6 +2862,32 @@ def volume_get_all_by_group(context, group_id, filters=None):
     return query.all()
 
 
+@require_admin_context
+@main_context_manager.writer
+def volume_update_all_by_service(context):
+    """Ensure volumes have the correct service_uuid value for their host.
+
+    In some deployment tools, when performing an upgrade, all service records
+    are recreated including c-vol service which gets a new record in the
+    services table, though its host name is constant. Later we then delete the
+    old service record.
+    As a consequence, the volumes have the right host name but the service
+    UUID needs to be updated to the ID of the new service record.
+
+    :param context: context to query under
+    """
+    # Get all cinder-volume services
+    services = service_get_all(context, binary='cinder-volume')
+    for service in services:
+        query = model_query(context, models.Volume)
+        query = query.filter(
+            _filter_host(
+                models.Volume.host, service.host),
+            models.Volume.service_uuid != service.uuid)
+        query.update(
+            {"service_uuid": service.uuid}, synchronize_session=False)
+
+
 @require_context
 @main_context_manager.reader
 def volume_get_all_by_generic_group(context, group_id, filters=None):
@@ -8678,9 +8704,8 @@ def use_quota_online_data_migration(
     calculate_use_quota,
 ):
     updated = 0
-    query = model_query(context, getattr(models, resource_name)).filter_by(
-        use_quota=None
-    )
+    query = model_query(context, getattr(models, resource_name),
+                        read_deleted='yes').filter_by(use_quota=None)
     if resource_name == 'Volume':
         query = query.options(joinedload(models.Volume.volume_admin_metadata))
     total = query.count()

cinder/image/image_utils.py

@@ -104,9 +104,9 @@
                      'an operator has configured glance property protections '
                      'to make some image properties read-only. Cinder will '
                      '*always* filter out image metadata in the namespaces '
-                     '`os_glance` and `img_signature`; this configuration '
-                     'option allows operators to specify *additional* '
-                     'namespaces to be excluded.',
+                     '`os_glance`, `img_signature` and `signature_verified`; '
+                     'this configuration option allows operators to specify '
+                     '*additional* namespaces to be excluded.',
                 default=[]),
 ]
 
@@ -130,7 +130,8 @@
 
 COMPRESSIBLE_IMAGE_FORMATS = ('qcow2',)
 
-GLANCE_RESERVED_NAMESPACES = ["os_glance", "img_signature"]
+GLANCE_RESERVED_NAMESPACES = ["os_glance", "img_signature",
+                              "signature_verified"]
 
 
 def validate_stores_id(context: context.RequestContext,

cinder/service.py

@@ -42,6 +42,7 @@
 from cinder.common import constants
 from cinder import context
 from cinder import coordination
+from cinder import db
 from cinder import exception
 from cinder.i18n import _
 from cinder import objects
@@ -366,6 +367,9 @@ def _create_service_ref(self,
         # If we have updated the service_ref with replication data from
         # the cluster it will be saved.
         service_ref.save()
+        # Update all volumes that are associated with an old service with
+        # the new service uuid
+        db.volume_update_all_by_service(context)
 
     def __getattr__(self, key: str):
         manager = self.__dict__.get('manager', None)

cinder/service_auth.py

@@ -19,6 +19,7 @@
 
 CONF = cfg.CONF
 _SERVICE_AUTH = None
+_SERVICE_SESSION = None
 
 SERVICE_USER_GROUP = 'service_user'
 
@@ -66,6 +67,16 @@ def get_service_auth_plugin():
     return None
 
 
+def get_service_session():
+    if CONF.service_user.send_service_user_token:
+        global _SERVICE_SESSION
+        if not _SERVICE_SESSION:
+            _SERVICE_SESSION = ks_loading.load_session_from_conf_options(
+                CONF, SERVICE_USER_GROUP, auth=get_service_auth_plugin())
+        return _SERVICE_SESSION
+    return None
+
+
 def get_auth_plugin(context, auth=None):
     if auth:
         user_auth = auth