From c4351025320598bfc99c35b5514c6a42039e6d67 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Pawe=C5=82=20Szulc?= Date: Tue, 29 Oct 2024 15:13:32 +0100 Subject: [PATCH 1/6] Fix method to get limit choices --- src/ralph/lib/mixins/fields.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/ralph/lib/mixins/fields.py b/src/ralph/lib/mixins/fields.py index b78a207e6f..be3fbfeca3 100644 --- a/src/ralph/lib/mixins/fields.py +++ b/src/ralph/lib/mixins/fields.py @@ -232,6 +232,9 @@ def deconstruct(self): def limit_choices_to(self): return self.limit_choices() + def get_limit_choices_to(self): + return self.limit_choices() + def limit_choices(self): """ Add limit_choices_to search by content_type for models From 03da535881795dd111860169cbc81a19f94cbd12 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Pawe=C5=82=20Szulc?= Date: Thu, 31 Oct 2024 13:25:04 +0100 Subject: [PATCH 2/6] Fix data center asset export not getting filter from request --- src/ralph/admin/mixins.py | 3 --- src/ralph/data_center/admin.py | 7 ------- src/ralph/data_importer/resources.py | 8 ++++++++ 3 files changed, 8 insertions(+), 10 deletions(-) diff --git a/src/ralph/admin/mixins.py b/src/ralph/admin/mixins.py index c0496a115a..f359061786 100644 --- a/src/ralph/admin/mixins.py +++ b/src/ralph/admin/mixins.py @@ -421,9 +421,6 @@ def get_export_queryset(self, request): ) if resource_prefetch_related: queryset = queryset.prefetch_related(*resource_prefetch_related) - # cast to list to consider all prefetch_related (django-import-export - # use queryset.iterator() to "save memory", but then for every row - # sql queries are made to fetch all m2m relations) return list(queryset) def get_export_resource_class(self): diff --git a/src/ralph/data_center/admin.py b/src/ralph/data_center/admin.py index 32625631ae..f2d72b5e9c 100644 --- a/src/ralph/data_center/admin.py +++ b/src/ralph/data_center/admin.py @@ -490,13 +490,6 @@ class DataCenterAssetAdmin( }), ) - def get_export_queryset(self, request): - return DataCenterAsset.polymorphic_objects.select_related( - *self.list_select_related - ).polymorphic_prefetch_related( - DataCenterAsset=['tags', 'ethernet_set__ipaddress', 'parent__ethernet_set__ipaddress'], - ) - def get_multiadd_fields(self, obj=None): multiadd_fields = [ {'field': 'sn', 'allow_duplicates': False}, diff --git a/src/ralph/data_importer/resources.py b/src/ralph/data_importer/resources.py index 6f4faf5671..4e9634fab7 100644 --- a/src/ralph/data_importer/resources.py +++ b/src/ralph/data_importer/resources.py @@ -306,8 +306,16 @@ class DataCenterAssetResource(ResourceWithPrice, RalphModelResource): class Meta: model = physical.DataCenterAsset select_related = ( + 'model__manufacturer', 'model__category', 'service_env__service', 'service_env__environment', 'rack__server_room__data_center', + 'configuration_path', + 'property_of', + 'parent', + 'budget_info', + ) + prefetch_related = ( + 'tags', 'ethernet_set__ipaddress', 'parent__ethernet_set__ipaddress', ) exclude = ('content_type', 'asset_ptr', 'baseobject_ptr', 'connections') From d52c3aa35e40918e98ded4869a7e76e34cb40719 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Pawe=C5=82=20Szulc?= Date: Thu, 31 Oct 2024 13:29:58 +0100 Subject: [PATCH 3/6] Fix format to work with newer flake --- src/ralph/__init__.py | 2 +- src/ralph/accounts/helpers.py | 28 ++++++++++++++-------------- src/ralph/accounts/ldap.py | 1 + src/ralph/api/routers.py | 1 + src/ralph/assets/api/serializers.py | 1 + src/ralph/helpers.py | 1 + src/ralph/lib/metrics/collector.py | 1 + src/ralph/licences/models.py | 1 + src/ralph/networks/filters.py | 2 ++ src/ralph/settings/dev.py | 1 + 10 files changed, 24 insertions(+), 15 deletions(-) diff --git a/src/ralph/__init__.py b/src/ralph/__init__.py index df6f6264bf..5adeaeba19 100644 --- a/src/ralph/__init__.py +++ b/src/ralph/__init__.py @@ -7,7 +7,7 @@ def monkey_options_init(self, meta, app_label): self._old__init__(meta, app_label) self.default_permissions = ('add', 'change', 'delete', 'view') -# TODO: create PR to Django - default_permissions from settings + Options._old__init__ = Options.__init__ Options.__init__ = lambda self, meta, app_label=None: monkey_options_init( self, meta, app_label diff --git a/src/ralph/accounts/helpers.py b/src/ralph/accounts/helpers.py index a04b923983..7cbc9a9cb4 100644 --- a/src/ralph/accounts/helpers.py +++ b/src/ralph/accounts/helpers.py @@ -10,20 +10,20 @@ from ralph.lib.transitions.models import Transition from ralph.sim_cards.models import SIMCard -ACCEPTANCE_TRANSITION_ID = settings.ACCEPT_ASSETS_FOR_CURRENT_USER_CONFIG['TRANSITION_ID'] # noqa: E509 -ACCEPTANCE_SIM_TRANSITION_ID = settings.ACCEPT_ASSETS_FOR_CURRENT_USER_CONFIG['TRANSITION_SIM_ID'] # noqa: E509 -ACCEPTANCE_BACK_OFFICE_ACCEPT_STATUS = settings.ACCEPT_ASSETS_FOR_CURRENT_USER_CONFIG['BACK_OFFICE_ACCEPT_STATUS'] # noqa: E509 -ACCEPTANCE_SIMCARD_ACCEPT_STATUS = settings.ACCEPT_ASSETS_FOR_CURRENT_USER_CONFIG['SIMCARD_ACCEPT_STATUS'] # noqa: E509 -ACCEPTANCE_LOAN_TRANSITION_ID = settings.ACCEPT_ASSETS_FOR_CURRENT_USER_CONFIG['LOAN_TRANSITION_ID'] # noqa: E509 -ACCEPTANCE_BACK_OFFICE_ACCEPT_LOAN_STATUS = settings.ACCEPT_ASSETS_FOR_CURRENT_USER_CONFIG['BACK_OFFICE_ACCEPT_LOAN_STATUS'] # noqa: E509 -ACCEPTANCE_RETURN_TRANSITION_ID = settings.ACCEPT_ASSETS_FOR_CURRENT_USER_CONFIG['RETURN_TRANSITION_ID'] # noqa: E509 -ACCEPTANCE_BACK_OFFICE_RETURN_STATUS = settings.ACCEPT_ASSETS_FOR_CURRENT_USER_CONFIG['BACK_OFFICE_ACCEPT_RETURN_STATUS'] # noqa: E509 -ACCEPTANCE_ACCESS_CARD_TRANSITION_ID = settings.ACCEPT_ASSETS_FOR_CURRENT_USER_CONFIG['TRANSITION_ACCESS_CARD_ID'] # noqa: E509 -ACCEPTANCE_ACCESS_CARD_ACCEPT_STATUS = settings.ACCEPT_ASSETS_FOR_CURRENT_USER_CONFIG['ACCESS_CARD_ACCEPT_ACCEPT_STATUS'] # noqa: E509 -ACCEPTANCE_BACK_OFFICE_TEAM_ACCEPT_STATUS = settings.ACCEPT_ASSETS_FOR_CURRENT_USER_CONFIG['BACK_OFFICE_TEAM_ACCEPT_STATUS'] # noqa: E509 -ACCEPTANCE_TEAM_ACCEPT_TRANSITION_ID = settings.ACCEPT_ASSETS_FOR_CURRENT_USER_CONFIG['TRANSITION_TEAM_ACCEPT_ID'] # noqa: E509 -ACCEPTANCE_BACK_OFFICE_TEST_ACCEPT_STATUS = settings.ACCEPT_ASSETS_FOR_CURRENT_USER_CONFIG['BACK_OFFICE_TEST_ACCEPT_STATUS'] # noqa: E509 -ACCEPTANCE_TEST_ACCEPT_TRANSITION_ID = settings.ACCEPT_ASSETS_FOR_CURRENT_USER_CONFIG['TRANSITION_TEST_ACCEPT_ID'] # noqa: E509 +ACCEPTANCE_TRANSITION_ID = settings.ACCEPT_ASSETS_FOR_CURRENT_USER_CONFIG['TRANSITION_ID'] # noqa +ACCEPTANCE_SIM_TRANSITION_ID = settings.ACCEPT_ASSETS_FOR_CURRENT_USER_CONFIG['TRANSITION_SIM_ID'] # noqa +ACCEPTANCE_BACK_OFFICE_ACCEPT_STATUS = settings.ACCEPT_ASSETS_FOR_CURRENT_USER_CONFIG['BACK_OFFICE_ACCEPT_STATUS'] # noqa +ACCEPTANCE_SIMCARD_ACCEPT_STATUS = settings.ACCEPT_ASSETS_FOR_CURRENT_USER_CONFIG['SIMCARD_ACCEPT_STATUS'] # noqa +ACCEPTANCE_LOAN_TRANSITION_ID = settings.ACCEPT_ASSETS_FOR_CURRENT_USER_CONFIG['LOAN_TRANSITION_ID'] # noqa +ACCEPTANCE_BACK_OFFICE_ACCEPT_LOAN_STATUS = settings.ACCEPT_ASSETS_FOR_CURRENT_USER_CONFIG['BACK_OFFICE_ACCEPT_LOAN_STATUS'] # noqa +ACCEPTANCE_RETURN_TRANSITION_ID = settings.ACCEPT_ASSETS_FOR_CURRENT_USER_CONFIG['RETURN_TRANSITION_ID'] # noqa +ACCEPTANCE_BACK_OFFICE_RETURN_STATUS = settings.ACCEPT_ASSETS_FOR_CURRENT_USER_CONFIG['BACK_OFFICE_ACCEPT_RETURN_STATUS'] # noqa +ACCEPTANCE_ACCESS_CARD_TRANSITION_ID = settings.ACCEPT_ASSETS_FOR_CURRENT_USER_CONFIG['TRANSITION_ACCESS_CARD_ID'] # noqa +ACCEPTANCE_ACCESS_CARD_ACCEPT_STATUS = settings.ACCEPT_ASSETS_FOR_CURRENT_USER_CONFIG['ACCESS_CARD_ACCEPT_ACCEPT_STATUS'] # noqa +ACCEPTANCE_BACK_OFFICE_TEAM_ACCEPT_STATUS = settings.ACCEPT_ASSETS_FOR_CURRENT_USER_CONFIG['BACK_OFFICE_TEAM_ACCEPT_STATUS'] # noqa +ACCEPTANCE_TEAM_ACCEPT_TRANSITION_ID = settings.ACCEPT_ASSETS_FOR_CURRENT_USER_CONFIG['TRANSITION_TEAM_ACCEPT_ID'] # noqa +ACCEPTANCE_BACK_OFFICE_TEST_ACCEPT_STATUS = settings.ACCEPT_ASSETS_FOR_CURRENT_USER_CONFIG['BACK_OFFICE_TEST_ACCEPT_STATUS'] # noqa +ACCEPTANCE_TEST_ACCEPT_TRANSITION_ID = settings.ACCEPT_ASSETS_FOR_CURRENT_USER_CONFIG['TRANSITION_TEST_ACCEPT_ID'] # noqa def transition_exists(transition_id): diff --git a/src/ralph/accounts/ldap.py b/src/ralph/accounts/ldap.py index 5dfb36b3ab..caf3380247 100644 --- a/src/ralph/accounts/ldap.py +++ b/src/ralph/accounts/ldap.py @@ -68,6 +68,7 @@ def mirror_groups(self): self._user.groups.set(existing_groups + new_groups) + _LDAPUser._mirror_groups_original = _LDAPUser._mirror_groups _LDAPUser._mirror_groups = mirror_groups diff --git a/src/ralph/api/routers.py b/src/ralph/api/routers.py index e7448e1468..54684c7086 100644 --- a/src/ralph/api/routers.py +++ b/src/ralph/api/routers.py @@ -58,4 +58,5 @@ def get(self, request, *args, **kwargs): return APIRoot.as_view() + router = RalphRouter() diff --git a/src/ralph/assets/api/serializers.py b/src/ralph/assets/api/serializers.py index 9d026d3345..c5e5ba397a 100644 --- a/src/ralph/assets/api/serializers.py +++ b/src/ralph/assets/api/serializers.py @@ -317,6 +317,7 @@ class Meta: 'parent' ) + # TODO: Is there a better way to make it work since drf 3.5? del ConfigurationClassSimpleSerializer._declared_fields['tags'] diff --git a/src/ralph/helpers.py b/src/ralph/helpers.py index 6df2114820..f5572599fd 100644 --- a/src/ralph/helpers.py +++ b/src/ralph/helpers.py @@ -41,6 +41,7 @@ def generate_pdf_response(pdf_data, file_name): ) return response + CACHE_DEFAULT = object() diff --git a/src/ralph/lib/metrics/collector.py b/src/ralph/lib/metrics/collector.py index 3ed7fd766b..366c2cb308 100644 --- a/src/ralph/lib/metrics/collector.py +++ b/src/ralph/lib/metrics/collector.py @@ -25,6 +25,7 @@ def build_statsd_client( ipv6=ipv6 ) + if settings.COLLECT_METRICS and statsd is None: statsd = build_statsd_client() diff --git a/src/ralph/licences/models.py b/src/ralph/licences/models.py index 9193a4ed5f..c6af79886e 100644 --- a/src/ralph/licences/models.py +++ b/src/ralph/licences/models.py @@ -111,6 +111,7 @@ def get_queryset(self): } ) + LICENCES_RELATED_OBJECTS_PREFETCH_RELATED = [ 'users', # prefetch all baseobjects related with licence; this allows to call diff --git a/src/ralph/networks/filters.py b/src/ralph/networks/filters.py index 588ae74be2..f0ddb28eb1 100644 --- a/src/ralph/networks/filters.py +++ b/src/ralph/networks/filters.py @@ -24,6 +24,8 @@ def get_private_network_filter(): max_ip = int(network.broadcast_address) filter_ |= Q(min_ip__gte=min_ip, max_ip__lte=max_ip) return filter_ + + PRIVATE_NETWORK_FILTER = get_private_network_filter() diff --git a/src/ralph/settings/dev.py b/src/ralph/settings/dev.py index e819d2db2f..5a768d93cb 100644 --- a/src/ralph/settings/dev.py +++ b/src/ralph/settings/dev.py @@ -5,6 +5,7 @@ def only_true(request): '''For django debug toolbar.''' return True + DEBUG = True INSTALLED_APPS = INSTALLED_APPS + ( From b19881779248bd57ff79f92ace059627d008322e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Pawe=C5=82=20Szulc?= Date: Thu, 31 Oct 2024 14:06:30 +0100 Subject: [PATCH 4/6] Fix prefetches --- src/ralph/data_center/admin.py | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/src/ralph/data_center/admin.py b/src/ralph/data_center/admin.py index f2d72b5e9c..a1691b0979 100644 --- a/src/ralph/data_center/admin.py +++ b/src/ralph/data_center/admin.py @@ -28,6 +28,7 @@ from ralph.admin.mixins import ( BulkEditChangeListMixin, RalphAdmin, + RalphAdminImportExportMixin, RalphTabularInline ) from ralph.admin.views.extra import RalphDetailViewAdmin @@ -490,6 +491,15 @@ class DataCenterAssetAdmin( }), ) + def get_export_queryset(self, request): + # we skip method in RalphAdminImportExportMixin because it doesn't return a queryset + # sad :( + return super(RalphAdminImportExportMixin, self).get_export_queryset(request).select_related( + *self.list_select_related + ).polymorphic_prefetch_related( + DataCenterAsset=['tags', 'ethernet_set__ipaddress', 'parent__ethernet_set__ipaddress'], + ) + def get_multiadd_fields(self, obj=None): multiadd_fields = [ {'field': 'sn', 'allow_duplicates': False}, From 04002d96f0c3d72c9f0003d1bcaf2f156926d41d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Pawe=C5=82=20Szulc?= Date: Thu, 31 Oct 2024 14:53:41 +0100 Subject: [PATCH 5/6] Make prefetches work for filtered and not filtered export --- src/ralph/data_center/admin.py | 26 ++++++++++++++------ src/ralph/data_importer/tests/test_export.py | 10 ++++++++ 2 files changed, 29 insertions(+), 7 deletions(-) diff --git a/src/ralph/data_center/admin.py b/src/ralph/data_center/admin.py index a1691b0979..24af936b21 100644 --- a/src/ralph/data_center/admin.py +++ b/src/ralph/data_center/admin.py @@ -35,7 +35,7 @@ from ralph.admin.views.main import RalphChangeList from ralph.admin.views.multiadd import MulitiAddAdminMixin from ralph.assets.invoice_report import AssetInvoiceReportMixin -from ralph.assets.models.base import BaseObject +from ralph.assets.models.base import BaseObject, BaseObjectPolymorphicQuerySet from ralph.assets.models.components import Ethernet from ralph.assets.views import ComponentsAdminView from ralph.attachments.admin import AttachmentsMixin @@ -492,13 +492,25 @@ class DataCenterAssetAdmin( ) def get_export_queryset(self, request): - # we skip method in RalphAdminImportExportMixin because it doesn't return a queryset - # sad :( - return super(RalphAdminImportExportMixin, self).get_export_queryset(request).select_related( - *self.list_select_related - ).polymorphic_prefetch_related( - DataCenterAsset=['tags', 'ethernet_set__ipaddress', 'parent__ethernet_set__ipaddress'], + qs = ( + super(RalphAdminImportExportMixin, self) + .get_export_queryset(request) + .select_related( + *self.list_select_related + ) ) + if isinstance(qs, BaseObjectPolymorphicQuerySet): + return qs.polymorphic_prefetch_related( + DataCenterAsset=[ + 'tags', + 'ethernet_set__ipaddress', + 'parent__ethernet_set__ipaddress' + ] + ) + else: + return qs.prefetch_related( + 'tags', 'ethernet_set__ipaddress', 'parent__ethernet_set__ipaddress' + ) def get_multiadd_fields(self, obj=None): multiadd_fields = [ diff --git a/src/ralph/data_importer/tests/test_export.py b/src/ralph/data_importer/tests/test_export.py index 61b623bfb7..1a0b8cc4b6 100644 --- a/src/ralph/data_importer/tests/test_export.py +++ b/src/ralph/data_importer/tests/test_export.py @@ -114,6 +114,16 @@ def test_data_center_asset_export_queries_count(self): DataCenterAsset ), max_queries=12) + def test_data_center_asset_export_filtered(self): + self._init(10) + first_id = next(iter(self.data_center_assets_map.keys())) + with CaptureQueriesContext(connections['default']) as cqc: + export_data = self._export( + DataCenterAsset, filters={'id': first_id} + ) + queries = len(cqc) + self.assertEqual(len(export_data.dict), 1) + self.assertLessEqual(queries, 10) class DataCenterAssetExporterTestCaseWithParent(DataCenterAssetExporterTestCase): def _init(self, num=10): From 9d3e255e67beaefe9154e277cbe8d91a30bc4996 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Pawe=C5=82=20Szulc?= Date: Thu, 31 Oct 2024 15:26:23 +0100 Subject: [PATCH 6/6] loosen queries count constraint for export --- src/ralph/data_importer/tests/test_export.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/ralph/data_importer/tests/test_export.py b/src/ralph/data_importer/tests/test_export.py index 1a0b8cc4b6..814c903ace 100644 --- a/src/ralph/data_importer/tests/test_export.py +++ b/src/ralph/data_importer/tests/test_export.py @@ -123,7 +123,7 @@ def test_data_center_asset_export_filtered(self): ) queries = len(cqc) self.assertEqual(len(export_data.dict), 1) - self.assertLessEqual(queries, 10) + self.assertLessEqual(queries, 12) class DataCenterAssetExporterTestCaseWithParent(DataCenterAssetExporterTestCase): def _init(self, num=10):