diff --git a/gnocchi/indexer/alembic/versions/5c4f93e5bb4_mysql_float_to_timestamp.py b/gnocchi/indexer/alembic/versions/5c4f93e5bb4_mysql_float_to_timestamp.py index 74a790a61..46fe6548c 100644 --- a/gnocchi/indexer/alembic/versions/5c4f93e5bb4_mysql_float_to_timestamp.py +++ b/gnocchi/indexer/alembic/versions/5c4f93e5bb4_mysql_float_to_timestamp.py @@ -24,6 +24,7 @@ """ from alembic import op +from sqlalchemy.engine.reflection import Inspector import sqlalchemy as sa from sqlalchemy.sql import func @@ -38,8 +39,21 @@ def upgrade(): bind = op.get_bind() + inspector = Inspector.from_engine(bind) + if bind and bind.engine.name == "mysql": op.execute("SET time_zone = '+00:00'") + + previous_cks = {"resource": [], "resource_history": []} + for table in ("resource", "resource_history"): + existing_cks = [ + c['name'] for c in inspector.get_check_constraints(table) + ] + ck_name = "ck_{}_started_before_ended".format(table) + if ck_name in existing_cks: + op.drop_constraint(ck_name, table, type_="check") + previous_cks[table].append(ck_name) + # NOTE(jd) So that crappy engine that is MySQL does not have "ALTER # TABLE … USING …". We need to copy everything and convert… for table_name, column_name in (("resource", "started_at"), @@ -75,3 +89,7 @@ def upgrade(): existing_nullable=nullable, existing_type=existing_type, new_column_name=column_name) + + for table in ("resource", "resource_history"): + for ck_name in previous_cks[table]: + op.create_check_constraint(ck_name, table, "started_at <= ended_at") diff --git a/gnocchi/indexer/sqlalchemy.py b/gnocchi/indexer/sqlalchemy.py index 1265194ab..a842f32f4 100644 --- a/gnocchi/indexer/sqlalchemy.py +++ b/gnocchi/indexer/sqlalchemy.py @@ -260,27 +260,46 @@ def _safe_execute(self, connection, works): class SQLAlchemyIndexer(indexer.IndexerDriver): _RESOURCE_TYPE_MANAGER = ResourceClassMapper() + @staticmethod + def _set_url_database(url, database): + if hasattr(url, "set"): + return url.set(database=database) + else: + url.database = database + return url + + @staticmethod + def _set_url_drivername(url, drivername): + if hasattr(url, "set"): + return url.set(drivername=drivername) + else: + url.drivername = drivername + return url + @classmethod def _create_new_database(cls, url): """Used by testing to create a new database.""" purl = sqlalchemy_url.make_url( cls.dress_url( url)) - purl.database = purl.database + str(uuid.uuid4()).replace('-', '') + new_database = purl.database + str(uuid.uuid4()).replace('-', '') + purl = cls._set_url_database(purl, new_database) new_url = str(purl) sqlalchemy_utils.create_database(new_url) return new_url - @staticmethod - def dress_url(url): + @classmethod + def dress_url(cls, url): # If no explicit driver has been set, we default to pymysql if url.startswith("mysql://"): url = sqlalchemy_url.make_url(url) - url.drivername = "mysql+pymysql" + new_drivername = "mysql+pymysql" + url = cls._set_url_drivername(url, new_drivername) return str(url) if url.startswith("postgresql://"): url = sqlalchemy_url.make_url(url) - url.drivername = "postgresql+psycopg2" + new_drivername = "postgresql+psycopg2" + url = cls._set_url_drivername(url, new_drivername) return str(url) return url @@ -951,8 +970,6 @@ def delete_resources(self, resource_type='generic', target_cls = self._resource_type_to_mappers( session, resource_type)["resource"] - q = session.query(target_cls.id) - engine = session.connection() try: f = QueryTransformer.build_filter(engine.dialect.name, @@ -964,12 +981,16 @@ def delete_resources(self, resource_type='generic', raise indexer.ResourceAttributeError(resource_type, e.attribute) - q = q.filter(f) + q1 = session.query(target_cls.id) + q1 = q1.filter(f) session.query(Metric).filter( - Metric.resource_id.in_(q) + Metric.resource_id.in_(q1) ).update({"status": "delete"}, synchronize_session=False) + + q = session.query(target_cls) + q = q.filter(f) return q.delete(synchronize_session=False) @retry_on_deadlock diff --git a/gnocchi/indexer/sqlalchemy_base.py b/gnocchi/indexer/sqlalchemy_base.py index c9aae589e..331b51ef0 100644 --- a/gnocchi/indexer/sqlalchemy_base.py +++ b/gnocchi/indexer/sqlalchemy_base.py @@ -51,13 +51,17 @@ def process_bind_param(self, value, dialect): def process_result_value(self, value, dialect): values = super(ArchivePolicyDefinitionType, self).process_result_value(value, dialect) + if values is None: + return [] return [archive_policy.ArchivePolicyItem(**v) for v in values] class SetType(sqlalchemy_utils.JSONType): def process_result_value(self, value, dialect): - return set(super(SetType, - self).process_result_value(value, dialect)) + values = super(SetType, self).process_result_value(value, dialect) + if values is None: + return set() + return set(values) class ArchivePolicy(Base, GnocchiBase, archive_policy.ArchivePolicy): diff --git a/gnocchi/rest/auth_helper.py b/gnocchi/rest/auth_helper.py index 1362f83ae..d11fadf1b 100644 --- a/gnocchi/rest/auth_helper.py +++ b/gnocchi/rest/auth_helper.py @@ -118,8 +118,10 @@ def get_metric_policy_filter(request, rule): class BasicAuthHelper(object): @staticmethod def get_current_user(request): - auth = werkzeug.http.parse_authorization_header( - request.headers.get("Authorization")) + hdr = request.headers.get("Authorization") + auth_hdr = (hdr.decode('utf-8') if isinstance(hdr, bytes) + else hdr) + auth = werkzeug.http.parse_authorization_header(auth_hdr) if auth is None: api.abort(401) return auth.username diff --git a/setup.cfg b/setup.cfg index ef858a59b..0a1b0e2fb 100644 --- a/setup.cfg +++ b/setup.cfg @@ -58,13 +58,13 @@ keystone = mysql = pymysql oslo.db>=4.29.0 - sqlalchemy<1.4.0 + sqlalchemy sqlalchemy-utils alembic>=0.7.6,!=0.8.1,!=0.9.0 postgresql = psycopg2 oslo.db>=4.29.0 - sqlalchemy<1.4.0 + sqlalchemy sqlalchemy-utils alembic>=0.7.6,!=0.8.1,!=0.9.0 s3 = diff --git a/tox.ini b/tox.ini index 6930aab59..2a51a3dac 100644 --- a/tox.ini +++ b/tox.ini @@ -105,7 +105,6 @@ commands = {toxinidir}/run-upgrade-tests.sh mysql-ceph [testenv:pep8] basepython = python3 deps = hacking>=0.12 - commands = flake8 allowlist_externals = /usr/bin/flake8 @@ -117,7 +116,8 @@ commands = pifpaf -g GNOCCHI_INDEXER_URL run postgresql -- python setup.py testr exclude = .tox,.eggs,doc,gnocchi/rest/prometheus/remote_pb2.py,gnocchi/indexer/alembic/versions/ show-source = true enable-extensions = H904 -ignore = E501,E731,W503,W504 +# TODO(tobias-urdin): Remove H216 when we use unittest.mock +ignore = E501,E731,W503,W504,H216 [testenv:docs] basepython = python3