diff --git a/debian/changelog b/debian/changelog index c21f3bf97c..03d60072cb 100644 --- a/debian/changelog +++ b/debian/changelog @@ -1,3 +1,50 @@ +ralph (20241104.2) bionic; urgency=medium + + [ awieckowski ] + * Increase max user department field length + * Add missing migration + + [ Paweł Szulc ] + * Fix ldap sync when bytes received + + -- Paweł Szulc Mon, 04 Nov 2024 13:06:26 +0000 + +ralph (20241104.1) bionic; urgency=medium + + * Fix method to get limit choices + * Fix data center asset export not getting filter from request + * Fix format to work with newer flake + * Fix prefetches + * Make prefetches work for filtered and not filtered export + * loosen queries count constraint for export + + -- Paweł Szulc Mon, 04 Nov 2024 10:17:53 +0000 + +ralph (20241029.1) bionic; urgency=medium + + [ awieckowski ] + * Fix LDAP Groups sync + + [ Paweł Szulc ] + * Fix bulk edit for some models + + -- Paweł Szulc Tue, 29 Oct 2024 13:00:03 +0000 + +ralph (20241028.1) bionic; urgency=medium + + * Fix virtual server - hypervisor + * Revert remove barcode from factory + * Fix virtual server - service env optional + + -- Paweł Szulc Mon, 28 Oct 2024 15:23:53 +0000 + +ralph (20241023.1) bionic; urgency=medium + + * Updated changelog for 20241011.1 version. + * Fix data not JSON-serializable + + -- Paweł Szulc Wed, 23 Oct 2024 10:29:45 +0000 + ralph (20241011.1) bionic; urgency=medium [ Paweł Szulc ] diff --git a/requirements/prod_ldap.txt b/requirements/prod_ldap.txt index 7aca55e51f..db5ba301a8 100644 --- a/requirements/prod_ldap.txt +++ b/requirements/prod_ldap.txt @@ -1,2 +1,2 @@ -r prod.txt -django-auth-ldap==1.2.7 +django-auth-ldap==1.6.1 diff --git a/src/ralph/__init__.py b/src/ralph/__init__.py index df6f6264bf..5adeaeba19 100644 --- a/src/ralph/__init__.py +++ b/src/ralph/__init__.py @@ -7,7 +7,7 @@ def monkey_options_init(self, meta, app_label): self._old__init__(meta, app_label) self.default_permissions = ('add', 'change', 'delete', 'view') -# TODO: create PR to Django - default_permissions from settings + Options._old__init__ = Options.__init__ Options.__init__ = lambda self, meta, app_label=None: monkey_options_init( self, meta, app_label diff --git a/src/ralph/accounts/helpers.py b/src/ralph/accounts/helpers.py index a04b923983..7cbc9a9cb4 100644 --- a/src/ralph/accounts/helpers.py +++ b/src/ralph/accounts/helpers.py @@ -10,20 +10,20 @@ from ralph.lib.transitions.models import Transition from ralph.sim_cards.models import SIMCard -ACCEPTANCE_TRANSITION_ID = settings.ACCEPT_ASSETS_FOR_CURRENT_USER_CONFIG['TRANSITION_ID'] # noqa: E509 -ACCEPTANCE_SIM_TRANSITION_ID = settings.ACCEPT_ASSETS_FOR_CURRENT_USER_CONFIG['TRANSITION_SIM_ID'] # noqa: E509 -ACCEPTANCE_BACK_OFFICE_ACCEPT_STATUS = settings.ACCEPT_ASSETS_FOR_CURRENT_USER_CONFIG['BACK_OFFICE_ACCEPT_STATUS'] # noqa: E509 -ACCEPTANCE_SIMCARD_ACCEPT_STATUS = settings.ACCEPT_ASSETS_FOR_CURRENT_USER_CONFIG['SIMCARD_ACCEPT_STATUS'] # noqa: E509 -ACCEPTANCE_LOAN_TRANSITION_ID = settings.ACCEPT_ASSETS_FOR_CURRENT_USER_CONFIG['LOAN_TRANSITION_ID'] # noqa: E509 -ACCEPTANCE_BACK_OFFICE_ACCEPT_LOAN_STATUS = settings.ACCEPT_ASSETS_FOR_CURRENT_USER_CONFIG['BACK_OFFICE_ACCEPT_LOAN_STATUS'] # noqa: E509 -ACCEPTANCE_RETURN_TRANSITION_ID = settings.ACCEPT_ASSETS_FOR_CURRENT_USER_CONFIG['RETURN_TRANSITION_ID'] # noqa: E509 -ACCEPTANCE_BACK_OFFICE_RETURN_STATUS = settings.ACCEPT_ASSETS_FOR_CURRENT_USER_CONFIG['BACK_OFFICE_ACCEPT_RETURN_STATUS'] # noqa: E509 -ACCEPTANCE_ACCESS_CARD_TRANSITION_ID = settings.ACCEPT_ASSETS_FOR_CURRENT_USER_CONFIG['TRANSITION_ACCESS_CARD_ID'] # noqa: E509 -ACCEPTANCE_ACCESS_CARD_ACCEPT_STATUS = settings.ACCEPT_ASSETS_FOR_CURRENT_USER_CONFIG['ACCESS_CARD_ACCEPT_ACCEPT_STATUS'] # noqa: E509 -ACCEPTANCE_BACK_OFFICE_TEAM_ACCEPT_STATUS = settings.ACCEPT_ASSETS_FOR_CURRENT_USER_CONFIG['BACK_OFFICE_TEAM_ACCEPT_STATUS'] # noqa: E509 -ACCEPTANCE_TEAM_ACCEPT_TRANSITION_ID = settings.ACCEPT_ASSETS_FOR_CURRENT_USER_CONFIG['TRANSITION_TEAM_ACCEPT_ID'] # noqa: E509 -ACCEPTANCE_BACK_OFFICE_TEST_ACCEPT_STATUS = settings.ACCEPT_ASSETS_FOR_CURRENT_USER_CONFIG['BACK_OFFICE_TEST_ACCEPT_STATUS'] # noqa: E509 -ACCEPTANCE_TEST_ACCEPT_TRANSITION_ID = settings.ACCEPT_ASSETS_FOR_CURRENT_USER_CONFIG['TRANSITION_TEST_ACCEPT_ID'] # noqa: E509 +ACCEPTANCE_TRANSITION_ID = settings.ACCEPT_ASSETS_FOR_CURRENT_USER_CONFIG['TRANSITION_ID'] # noqa +ACCEPTANCE_SIM_TRANSITION_ID = settings.ACCEPT_ASSETS_FOR_CURRENT_USER_CONFIG['TRANSITION_SIM_ID'] # noqa +ACCEPTANCE_BACK_OFFICE_ACCEPT_STATUS = settings.ACCEPT_ASSETS_FOR_CURRENT_USER_CONFIG['BACK_OFFICE_ACCEPT_STATUS'] # noqa +ACCEPTANCE_SIMCARD_ACCEPT_STATUS = settings.ACCEPT_ASSETS_FOR_CURRENT_USER_CONFIG['SIMCARD_ACCEPT_STATUS'] # noqa +ACCEPTANCE_LOAN_TRANSITION_ID = settings.ACCEPT_ASSETS_FOR_CURRENT_USER_CONFIG['LOAN_TRANSITION_ID'] # noqa +ACCEPTANCE_BACK_OFFICE_ACCEPT_LOAN_STATUS = settings.ACCEPT_ASSETS_FOR_CURRENT_USER_CONFIG['BACK_OFFICE_ACCEPT_LOAN_STATUS'] # noqa +ACCEPTANCE_RETURN_TRANSITION_ID = settings.ACCEPT_ASSETS_FOR_CURRENT_USER_CONFIG['RETURN_TRANSITION_ID'] # noqa +ACCEPTANCE_BACK_OFFICE_RETURN_STATUS = settings.ACCEPT_ASSETS_FOR_CURRENT_USER_CONFIG['BACK_OFFICE_ACCEPT_RETURN_STATUS'] # noqa +ACCEPTANCE_ACCESS_CARD_TRANSITION_ID = settings.ACCEPT_ASSETS_FOR_CURRENT_USER_CONFIG['TRANSITION_ACCESS_CARD_ID'] # noqa +ACCEPTANCE_ACCESS_CARD_ACCEPT_STATUS = settings.ACCEPT_ASSETS_FOR_CURRENT_USER_CONFIG['ACCESS_CARD_ACCEPT_ACCEPT_STATUS'] # noqa +ACCEPTANCE_BACK_OFFICE_TEAM_ACCEPT_STATUS = settings.ACCEPT_ASSETS_FOR_CURRENT_USER_CONFIG['BACK_OFFICE_TEAM_ACCEPT_STATUS'] # noqa +ACCEPTANCE_TEAM_ACCEPT_TRANSITION_ID = settings.ACCEPT_ASSETS_FOR_CURRENT_USER_CONFIG['TRANSITION_TEAM_ACCEPT_ID'] # noqa +ACCEPTANCE_BACK_OFFICE_TEST_ACCEPT_STATUS = settings.ACCEPT_ASSETS_FOR_CURRENT_USER_CONFIG['BACK_OFFICE_TEST_ACCEPT_STATUS'] # noqa +ACCEPTANCE_TEST_ACCEPT_TRANSITION_ID = settings.ACCEPT_ASSETS_FOR_CURRENT_USER_CONFIG['TRANSITION_TEST_ACCEPT_ID'] # noqa def transition_exists(transition_id): diff --git a/src/ralph/accounts/ldap.py b/src/ralph/accounts/ldap.py index d8cdabe5bd..caf3380247 100644 --- a/src/ralph/accounts/ldap.py +++ b/src/ralph/accounts/ldap.py @@ -66,7 +66,8 @@ def mirror_groups(self): new_groups = [Group.objects.get_or_create(name=name)[0] for name in target_group_names if name not in existing_group_names] - self._user.groups = existing_groups + new_groups + self._user.groups.set(existing_groups + new_groups) + _LDAPUser._mirror_groups_original = _LDAPUser._mirror_groups _LDAPUser._mirror_groups = mirror_groups diff --git a/src/ralph/accounts/management/commands/ldap_sync.py b/src/ralph/accounts/management/commands/ldap_sync.py index 1e619aac65..c592eeb207 100644 --- a/src/ralph/accounts/management/commands/ldap_sync.py +++ b/src/ralph/accounts/management/commands/ldap_sync.py @@ -24,6 +24,20 @@ ldap_module_exists = False +def decode_nested_dict(data): + if isinstance(data, dict): + return {key: decode_nested_dict(value) for key, value in data.items()} + elif isinstance(data, list): + return [decode_nested_dict(element) for element in data] + elif isinstance(data, bytes): + try: + return data.decode('utf-8') + except UnicodeDecodeError: + return data + else: + return data + + def _truncate(field_key, field_name, ldap_dict): """ Truncate user's field when it's longer then default django value, which is @@ -274,16 +288,8 @@ def populate_users(self): """Load users from ldap and populate them. Returns number of users.""" synced = 0 for user_dn, ldap_dict in self._get_users(): - if ldap_dict.get('c'): - try: - ldap_dict['c'] = [v.decode('utf-8') for v in ldap_dict['c']] - except UnicodeDecodeError: - logger.error( - "Can't decode country %s for user %s", - ldap_dict['c'], - user_dn - ) - continue + # decode bytes to str + ldap_dict = decode_nested_dict(ldap_dict) _truncate('sn', 'last_name', ldap_dict) user = self._create_or_update_user(user_dn, ldap_dict) self.nested_groups.handle(user) diff --git a/src/ralph/accounts/migrations/0010_auto_20241030_1235.py b/src/ralph/accounts/migrations/0010_auto_20241030_1235.py new file mode 100644 index 0000000000..41c9d3b7ca --- /dev/null +++ b/src/ralph/accounts/migrations/0010_auto_20241030_1235.py @@ -0,0 +1,18 @@ +# Generated by Django 2.0.13 on 2024-10-30 12:35 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('accounts', '0009_auto_20240621_1217'), + ] + + operations = [ + migrations.AlterField( + model_name='ralphuser', + name='department', + field=models.CharField(blank=True, max_length=128, verbose_name='department'), + ), + ] diff --git a/src/ralph/accounts/models.py b/src/ralph/accounts/models.py index d936cc4e35..bff526468b 100644 --- a/src/ralph/accounts/models.py +++ b/src/ralph/accounts/models.py @@ -102,7 +102,7 @@ class RalphUser( ) department = models.CharField( verbose_name=_('department'), - max_length=64, + max_length=128, blank=True, ) manager = models.CharField( diff --git a/src/ralph/admin/mixins.py b/src/ralph/admin/mixins.py index 3af0668dca..f359061786 100644 --- a/src/ralph/admin/mixins.py +++ b/src/ralph/admin/mixins.py @@ -421,9 +421,6 @@ def get_export_queryset(self, request): ) if resource_prefetch_related: queryset = queryset.prefetch_related(*resource_prefetch_related) - # cast to list to consider all prefetch_related (django-import-export - # use queryset.iterator() to "save memory", but then for every row - # sql queries are made to fetch all m2m relations) return list(queryset) def get_export_resource_class(self): @@ -541,8 +538,9 @@ def get_queryset(self, request): qs = super().get_queryset(request) id_list = request.GET.getlist(BULK_EDIT_VAR_IDS, []) if id_list: - qs = qs.filter(pk__in=id_list) - return qs + return self.model.objects.filter(id__in=id_list) + else: + return qs def get_list_display(self, request): """ diff --git a/src/ralph/api/routers.py b/src/ralph/api/routers.py index e7448e1468..54684c7086 100644 --- a/src/ralph/api/routers.py +++ b/src/ralph/api/routers.py @@ -58,4 +58,5 @@ def get(self, request, *args, **kwargs): return APIRoot.as_view() + router = RalphRouter() diff --git a/src/ralph/assets/api/serializers.py b/src/ralph/assets/api/serializers.py index 1abe37d43d..c5e5ba397a 100644 --- a/src/ralph/assets/api/serializers.py +++ b/src/ralph/assets/api/serializers.py @@ -53,9 +53,9 @@ def get_object_type(self, instance): class OwnersFromServiceEnvSerializerMixin(RalphAPISerializer): business_owners = SimpleRalphUserSerializer( - many=True, source='service_env.service.business_owners') + many=True, source='service_env.service.business_owners', required=False) technical_owners = SimpleRalphUserSerializer( - many=True, source='service_env.service.technical_owners') + many=True, source='service_env.service.technical_owners', required=False) class BusinessSegmentSerializer(RalphAPISerializer): @@ -317,6 +317,7 @@ class Meta: 'parent' ) + # TODO: Is there a better way to make it work since drf 3.5? del ConfigurationClassSimpleSerializer._declared_fields['tags'] diff --git a/src/ralph/assets/invoice_report.py b/src/ralph/assets/invoice_report.py index 9b44935b1f..ee828377d1 100644 --- a/src/ralph/assets/invoice_report.py +++ b/src/ralph/assets/invoice_report.py @@ -1,5 +1,6 @@ # -*- coding: utf-8 -*- import datetime +import json import logging from django.contrib import messages @@ -116,7 +117,7 @@ def _get_report_data(self, request, queryset): 'model': queryset.model._meta.model_name, 'base_info': { 'invoice_no': first_item.invoice_no, - 'invoice_date': first_item.invoice_date, + 'invoice_date': str(first_item.invoice_date), 'provider': first_item.provider, 'datetime': datetime.datetime.now().strftime( self._invoice_report_datetime_format @@ -148,6 +149,9 @@ def _get_pdf_content(self, data): template_content = f.read() service_pdf = ExternalService('PDF') + # Make sure data is JSON-serializable + # Will throw otherwise + data = json.loads(json.dumps(data)) result = service_pdf.run( template=template_content, data=data, diff --git a/src/ralph/data_center/admin.py b/src/ralph/data_center/admin.py index 32625631ae..24af936b21 100644 --- a/src/ralph/data_center/admin.py +++ b/src/ralph/data_center/admin.py @@ -28,13 +28,14 @@ from ralph.admin.mixins import ( BulkEditChangeListMixin, RalphAdmin, + RalphAdminImportExportMixin, RalphTabularInline ) from ralph.admin.views.extra import RalphDetailViewAdmin from ralph.admin.views.main import RalphChangeList from ralph.admin.views.multiadd import MulitiAddAdminMixin from ralph.assets.invoice_report import AssetInvoiceReportMixin -from ralph.assets.models.base import BaseObject +from ralph.assets.models.base import BaseObject, BaseObjectPolymorphicQuerySet from ralph.assets.models.components import Ethernet from ralph.assets.views import ComponentsAdminView from ralph.attachments.admin import AttachmentsMixin @@ -491,11 +492,25 @@ class DataCenterAssetAdmin( ) def get_export_queryset(self, request): - return DataCenterAsset.polymorphic_objects.select_related( - *self.list_select_related - ).polymorphic_prefetch_related( - DataCenterAsset=['tags', 'ethernet_set__ipaddress', 'parent__ethernet_set__ipaddress'], + qs = ( + super(RalphAdminImportExportMixin, self) + .get_export_queryset(request) + .select_related( + *self.list_select_related + ) ) + if isinstance(qs, BaseObjectPolymorphicQuerySet): + return qs.polymorphic_prefetch_related( + DataCenterAsset=[ + 'tags', + 'ethernet_set__ipaddress', + 'parent__ethernet_set__ipaddress' + ] + ) + else: + return qs.prefetch_related( + 'tags', 'ethernet_set__ipaddress', 'parent__ethernet_set__ipaddress' + ) def get_multiadd_fields(self, obj=None): multiadd_fields = [ diff --git a/src/ralph/data_importer/resources.py b/src/ralph/data_importer/resources.py index 6f4faf5671..4e9634fab7 100644 --- a/src/ralph/data_importer/resources.py +++ b/src/ralph/data_importer/resources.py @@ -306,8 +306,16 @@ class DataCenterAssetResource(ResourceWithPrice, RalphModelResource): class Meta: model = physical.DataCenterAsset select_related = ( + 'model__manufacturer', 'model__category', 'service_env__service', 'service_env__environment', 'rack__server_room__data_center', + 'configuration_path', + 'property_of', + 'parent', + 'budget_info', + ) + prefetch_related = ( + 'tags', 'ethernet_set__ipaddress', 'parent__ethernet_set__ipaddress', ) exclude = ('content_type', 'asset_ptr', 'baseobject_ptr', 'connections') diff --git a/src/ralph/data_importer/tests/test_export.py b/src/ralph/data_importer/tests/test_export.py index 61b623bfb7..814c903ace 100644 --- a/src/ralph/data_importer/tests/test_export.py +++ b/src/ralph/data_importer/tests/test_export.py @@ -114,6 +114,16 @@ def test_data_center_asset_export_queries_count(self): DataCenterAsset ), max_queries=12) + def test_data_center_asset_export_filtered(self): + self._init(10) + first_id = next(iter(self.data_center_assets_map.keys())) + with CaptureQueriesContext(connections['default']) as cqc: + export_data = self._export( + DataCenterAsset, filters={'id': first_id} + ) + queries = len(cqc) + self.assertEqual(len(export_data.dict), 1) + self.assertLessEqual(queries, 12) class DataCenterAssetExporterTestCaseWithParent(DataCenterAssetExporterTestCase): def _init(self, num=10): diff --git a/src/ralph/helpers.py b/src/ralph/helpers.py index 6df2114820..f5572599fd 100644 --- a/src/ralph/helpers.py +++ b/src/ralph/helpers.py @@ -41,6 +41,7 @@ def generate_pdf_response(pdf_data, file_name): ) return response + CACHE_DEFAULT = object() diff --git a/src/ralph/lib/metrics/collector.py b/src/ralph/lib/metrics/collector.py index 3ed7fd766b..366c2cb308 100644 --- a/src/ralph/lib/metrics/collector.py +++ b/src/ralph/lib/metrics/collector.py @@ -25,6 +25,7 @@ def build_statsd_client( ipv6=ipv6 ) + if settings.COLLECT_METRICS and statsd is None: statsd = build_statsd_client() diff --git a/src/ralph/lib/mixins/fields.py b/src/ralph/lib/mixins/fields.py index cf0a587631..642de66bda 100644 --- a/src/ralph/lib/mixins/fields.py +++ b/src/ralph/lib/mixins/fields.py @@ -232,6 +232,9 @@ def deconstruct(self): def limit_choices_to(self): return self.limit_choices() + def get_limit_choices_to(self): + return self.limit_choices() + def limit_choices(self): """ Add limit_choices_to search by content_type for models diff --git a/src/ralph/licences/models.py b/src/ralph/licences/models.py index 9193a4ed5f..c6af79886e 100644 --- a/src/ralph/licences/models.py +++ b/src/ralph/licences/models.py @@ -111,6 +111,7 @@ def get_queryset(self): } ) + LICENCES_RELATED_OBJECTS_PREFETCH_RELATED = [ 'users', # prefetch all baseobjects related with licence; this allows to call diff --git a/src/ralph/networks/filters.py b/src/ralph/networks/filters.py index 588ae74be2..f0ddb28eb1 100644 --- a/src/ralph/networks/filters.py +++ b/src/ralph/networks/filters.py @@ -24,6 +24,8 @@ def get_private_network_filter(): max_ip = int(network.broadcast_address) filter_ |= Q(min_ip__gte=min_ip, max_ip__lte=max_ip) return filter_ + + PRIVATE_NETWORK_FILTER = get_private_network_filter() diff --git a/src/ralph/reports/helpers.py b/src/ralph/reports/helpers.py index 798d7b043c..0c112ba350 100644 --- a/src/ralph/reports/helpers.py +++ b/src/ralph/reports/helpers.py @@ -1,4 +1,5 @@ import datetime +import json import os import tempfile @@ -25,17 +26,22 @@ def generate_report(name, requester, instances, language, context): service_pdf = ExternalService('PDF') for n in range(0, len(context), items_per_attachment): - result = service_pdf.run( - template=template_content, - data={ + # Make sure data is JSON-serializable + # Will throw otherwise + data = json.loads(json.dumps( + { 'id': ', '.join([str(obj.id) for obj in - instances[n:n+items_per_attachment]]), - 'now': datetime.datetime.now(), + instances[n:n + items_per_attachment]]), + 'now': str(datetime.datetime.now()), 'logged_user': obj_to_dict(requester), 'affected_user': obj_to_dict(instances[0].user), 'owner': obj_to_dict(instances[0].owner), - 'assets': context[n:n+items_per_attachment], + 'assets': context[n:n + items_per_attachment], } + )) + result = service_pdf.run( + template=template_content, + data=data ) filename = "_".join([ diff --git a/src/ralph/settings/dev.py b/src/ralph/settings/dev.py index e819d2db2f..5a768d93cb 100644 --- a/src/ralph/settings/dev.py +++ b/src/ralph/settings/dev.py @@ -5,6 +5,7 @@ def only_true(request): '''For django debug toolbar.''' return True + DEBUG = True INSTALLED_APPS = INSTALLED_APPS + ( diff --git a/src/ralph/virtual/admin.py b/src/ralph/virtual/admin.py index cea1675e49..4ae4f6aebc 100644 --- a/src/ralph/virtual/admin.py +++ b/src/ralph/virtual/admin.py @@ -130,7 +130,7 @@ class VirtualServerAdmin( ) list_display = [ 'hostname', 'type', 'sn', 'service_env', 'configuration_path', - 'get_parent', 'scan_status', 'scm_status_check' + 'parent_', 'scan_status', 'scm_status_check' ] raw_id_fields = ['parent', 'service_env', 'configuration_path'] fields = [ @@ -174,12 +174,14 @@ def get_queryset(self, request): ) @mark_safe - def get_parent(self, obj): - if not obj.parent_id: + def parent_(self, obj): + try: + parent = obj.polymorphic_parent + return '{}'.format( + parent.get_absolute_url(), parent.hostname + ) + except: # noqa # this happens when no parent or parent doesn't have a hostname return '-' - return '{}'.format( - obj.parent.get_absolute_url(), obj.parent.hostname - ) class CloudHostTabularInline(RalphTabularInline):