Merge branch 'ng' of github.com:allegro/ralph into django-2.1

Author: matyldv

debian/changelog

@@ -1,3 +1,50 @@
+ralph (20241104.2) bionic; urgency=medium
+
+  [ awieckowski ]
+  * Increase max user department field length
+  * Add missing migration
+
+  [ Paweł Szulc ]
+  * Fix ldap sync when bytes received
+
+ -- Paweł Szulc <[email protected]>  Mon, 04 Nov 2024 13:06:26 +0000
+
+ralph (20241104.1) bionic; urgency=medium
+
+  * Fix method to get limit choices
+  * Fix data center asset export not getting filter from request
+  * Fix format to work with newer flake
+  * Fix prefetches
+  * Make prefetches work for filtered and not filtered export
+  * loosen queries count constraint for export
+
+ -- Paweł Szulc <[email protected]>  Mon, 04 Nov 2024 10:17:53 +0000
+
+ralph (20241029.1) bionic; urgency=medium
+
+  [ awieckowski ]
+  * Fix LDAP Groups sync
+
+  [ Paweł Szulc ]
+  * Fix bulk edit for some models
+
+ -- Paweł Szulc <[email protected]>  Tue, 29 Oct 2024 13:00:03 +0000
+
+ralph (20241028.1) bionic; urgency=medium
+
+  * Fix virtual server - hypervisor
+  * Revert remove barcode from factory
+  * Fix virtual server - service env optional
+
+ -- Paweł Szulc <[email protected]>  Mon, 28 Oct 2024 15:23:53 +0000
+
+ralph (20241023.1) bionic; urgency=medium
+
+  * Updated changelog for 20241011.1 version.
+  * Fix data not JSON-serializable
+
+ -- Paweł Szulc <[email protected]>  Wed, 23 Oct 2024 10:29:45 +0000
+
 ralph (20241011.1) bionic; urgency=medium
 
   [ Paweł Szulc ]

requirements/prod_ldap.txt

@@ -1,2 +1,2 @@
 -r prod.txt
-django-auth-ldap==1.2.7
+django-auth-ldap==1.6.1

src/ralph/__init__.py

@@ -7,7 +7,7 @@ def monkey_options_init(self, meta, app_label):
     self._old__init__(meta, app_label)
     self.default_permissions = ('add', 'change', 'delete', 'view')
 
-# TODO: create PR to Django - default_permissions from settings
+
 Options._old__init__ = Options.__init__
 Options.__init__ = lambda self, meta, app_label=None: monkey_options_init(
     self, meta, app_label

src/ralph/accounts/helpers.py

@@ -10,20 +10,20 @@
 from ralph.lib.transitions.models import Transition
 from ralph.sim_cards.models import SIMCard
 
-ACCEPTANCE_TRANSITION_ID = settings.ACCEPT_ASSETS_FOR_CURRENT_USER_CONFIG['TRANSITION_ID']  # noqa: E509
-ACCEPTANCE_SIM_TRANSITION_ID = settings.ACCEPT_ASSETS_FOR_CURRENT_USER_CONFIG['TRANSITION_SIM_ID']  # noqa: E509
-ACCEPTANCE_BACK_OFFICE_ACCEPT_STATUS = settings.ACCEPT_ASSETS_FOR_CURRENT_USER_CONFIG['BACK_OFFICE_ACCEPT_STATUS']  # noqa: E509
-ACCEPTANCE_SIMCARD_ACCEPT_STATUS = settings.ACCEPT_ASSETS_FOR_CURRENT_USER_CONFIG['SIMCARD_ACCEPT_STATUS']  # noqa: E509
-ACCEPTANCE_LOAN_TRANSITION_ID = settings.ACCEPT_ASSETS_FOR_CURRENT_USER_CONFIG['LOAN_TRANSITION_ID']  # noqa: E509
-ACCEPTANCE_BACK_OFFICE_ACCEPT_LOAN_STATUS = settings.ACCEPT_ASSETS_FOR_CURRENT_USER_CONFIG['BACK_OFFICE_ACCEPT_LOAN_STATUS']  # noqa: E509
-ACCEPTANCE_RETURN_TRANSITION_ID = settings.ACCEPT_ASSETS_FOR_CURRENT_USER_CONFIG['RETURN_TRANSITION_ID']  # noqa: E509
-ACCEPTANCE_BACK_OFFICE_RETURN_STATUS = settings.ACCEPT_ASSETS_FOR_CURRENT_USER_CONFIG['BACK_OFFICE_ACCEPT_RETURN_STATUS']  # noqa: E509
-ACCEPTANCE_ACCESS_CARD_TRANSITION_ID = settings.ACCEPT_ASSETS_FOR_CURRENT_USER_CONFIG['TRANSITION_ACCESS_CARD_ID']  # noqa: E509
-ACCEPTANCE_ACCESS_CARD_ACCEPT_STATUS = settings.ACCEPT_ASSETS_FOR_CURRENT_USER_CONFIG['ACCESS_CARD_ACCEPT_ACCEPT_STATUS']  # noqa: E509
-ACCEPTANCE_BACK_OFFICE_TEAM_ACCEPT_STATUS = settings.ACCEPT_ASSETS_FOR_CURRENT_USER_CONFIG['BACK_OFFICE_TEAM_ACCEPT_STATUS']  # noqa: E509
-ACCEPTANCE_TEAM_ACCEPT_TRANSITION_ID = settings.ACCEPT_ASSETS_FOR_CURRENT_USER_CONFIG['TRANSITION_TEAM_ACCEPT_ID']  # noqa: E509
-ACCEPTANCE_BACK_OFFICE_TEST_ACCEPT_STATUS = settings.ACCEPT_ASSETS_FOR_CURRENT_USER_CONFIG['BACK_OFFICE_TEST_ACCEPT_STATUS']  # noqa: E509
-ACCEPTANCE_TEST_ACCEPT_TRANSITION_ID = settings.ACCEPT_ASSETS_FOR_CURRENT_USER_CONFIG['TRANSITION_TEST_ACCEPT_ID']  # noqa: E509
+ACCEPTANCE_TRANSITION_ID = settings.ACCEPT_ASSETS_FOR_CURRENT_USER_CONFIG['TRANSITION_ID']  # noqa
+ACCEPTANCE_SIM_TRANSITION_ID = settings.ACCEPT_ASSETS_FOR_CURRENT_USER_CONFIG['TRANSITION_SIM_ID']  # noqa
+ACCEPTANCE_BACK_OFFICE_ACCEPT_STATUS = settings.ACCEPT_ASSETS_FOR_CURRENT_USER_CONFIG['BACK_OFFICE_ACCEPT_STATUS']  # noqa
+ACCEPTANCE_SIMCARD_ACCEPT_STATUS = settings.ACCEPT_ASSETS_FOR_CURRENT_USER_CONFIG['SIMCARD_ACCEPT_STATUS']  # noqa
+ACCEPTANCE_LOAN_TRANSITION_ID = settings.ACCEPT_ASSETS_FOR_CURRENT_USER_CONFIG['LOAN_TRANSITION_ID']  # noqa
+ACCEPTANCE_BACK_OFFICE_ACCEPT_LOAN_STATUS = settings.ACCEPT_ASSETS_FOR_CURRENT_USER_CONFIG['BACK_OFFICE_ACCEPT_LOAN_STATUS']  # noqa
+ACCEPTANCE_RETURN_TRANSITION_ID = settings.ACCEPT_ASSETS_FOR_CURRENT_USER_CONFIG['RETURN_TRANSITION_ID']  # noqa
+ACCEPTANCE_BACK_OFFICE_RETURN_STATUS = settings.ACCEPT_ASSETS_FOR_CURRENT_USER_CONFIG['BACK_OFFICE_ACCEPT_RETURN_STATUS']  # noqa
+ACCEPTANCE_ACCESS_CARD_TRANSITION_ID = settings.ACCEPT_ASSETS_FOR_CURRENT_USER_CONFIG['TRANSITION_ACCESS_CARD_ID']  # noqa
+ACCEPTANCE_ACCESS_CARD_ACCEPT_STATUS = settings.ACCEPT_ASSETS_FOR_CURRENT_USER_CONFIG['ACCESS_CARD_ACCEPT_ACCEPT_STATUS']  # noqa
+ACCEPTANCE_BACK_OFFICE_TEAM_ACCEPT_STATUS = settings.ACCEPT_ASSETS_FOR_CURRENT_USER_CONFIG['BACK_OFFICE_TEAM_ACCEPT_STATUS']  # noqa
+ACCEPTANCE_TEAM_ACCEPT_TRANSITION_ID = settings.ACCEPT_ASSETS_FOR_CURRENT_USER_CONFIG['TRANSITION_TEAM_ACCEPT_ID']  # noqa
+ACCEPTANCE_BACK_OFFICE_TEST_ACCEPT_STATUS = settings.ACCEPT_ASSETS_FOR_CURRENT_USER_CONFIG['BACK_OFFICE_TEST_ACCEPT_STATUS']  # noqa
+ACCEPTANCE_TEST_ACCEPT_TRANSITION_ID = settings.ACCEPT_ASSETS_FOR_CURRENT_USER_CONFIG['TRANSITION_TEST_ACCEPT_ID']  # noqa
 
 
 def transition_exists(transition_id):

src/ralph/accounts/ldap.py

@@ -66,7 +66,8 @@ def mirror_groups(self):
         new_groups = [Group.objects.get_or_create(name=name)[0] for name
                       in target_group_names if name not in existing_group_names]
 
-        self._user.groups = existing_groups + new_groups
+        self._user.groups.set(existing_groups + new_groups)
+
 
 _LDAPUser._mirror_groups_original = _LDAPUser._mirror_groups
 _LDAPUser._mirror_groups = mirror_groups

src/ralph/accounts/management/commands/ldap_sync.py

@@ -24,6 +24,20 @@
     ldap_module_exists = False
 
 
+def decode_nested_dict(data):
+    if isinstance(data, dict):
+        return {key: decode_nested_dict(value) for key, value in data.items()}
+    elif isinstance(data, list):
+        return [decode_nested_dict(element) for element in data]
+    elif isinstance(data, bytes):
+        try:
+            return data.decode('utf-8')
+        except UnicodeDecodeError:
+            return data
+    else:
+        return data
+
+
 def _truncate(field_key, field_name, ldap_dict):
     """
     Truncate user's field when it's longer then default django value, which is
@@ -274,16 +288,8 @@ def populate_users(self):
         """Load users from ldap and populate them. Returns number of users."""
         synced = 0
         for user_dn, ldap_dict in self._get_users():
-            if ldap_dict.get('c'):
-                try:
-                    ldap_dict['c'] = [v.decode('utf-8') for v in ldap_dict['c']]
-                except UnicodeDecodeError:
-                    logger.error(
-                        "Can't decode country %s for user %s",
-                        ldap_dict['c'],
-                        user_dn
-                    )
-                    continue
+            # decode bytes to str
+            ldap_dict = decode_nested_dict(ldap_dict)
             _truncate('sn', 'last_name', ldap_dict)
             user = self._create_or_update_user(user_dn, ldap_dict)
             self.nested_groups.handle(user)

src/ralph/accounts/migrations/0010_auto_20241030_1235.py

@@ -0,0 +1,18 @@
+# Generated by Django 2.0.13 on 2024-10-30 12:35
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('accounts', '0009_auto_20240621_1217'),
+    ]
+
+    operations = [
+        migrations.AlterField(
+            model_name='ralphuser',
+            name='department',
+            field=models.CharField(blank=True, max_length=128, verbose_name='department'),
+        ),
+    ]

src/ralph/accounts/models.py

@@ -102,7 +102,7 @@ class RalphUser(
     )
     department = models.CharField(
         verbose_name=_('department'),
-        max_length=64,
+        max_length=128,
         blank=True,
     )
     manager = models.CharField(

src/ralph/admin/mixins.py

@@ -421,9 +421,6 @@ def get_export_queryset(self, request):
         )
         if resource_prefetch_related:
             queryset = queryset.prefetch_related(*resource_prefetch_related)
-        # cast to list to consider all prefetch_related (django-import-export
-        # use queryset.iterator() to "save memory", but then for every row
-        # sql queries are made to fetch all m2m relations)
         return list(queryset)
 
     def get_export_resource_class(self):
@@ -541,8 +538,9 @@ def get_queryset(self, request):
         qs = super().get_queryset(request)
         id_list = request.GET.getlist(BULK_EDIT_VAR_IDS, [])
         if id_list:
-            qs = qs.filter(pk__in=id_list)
-        return qs
+            return self.model.objects.filter(id__in=id_list)
+        else:
+            return qs
 
     def get_list_display(self, request):
         """

src/ralph/api/routers.py

@@ -58,4 +58,5 @@ def get(self, request, *args, **kwargs):
 
         return APIRoot.as_view()
 
+
 router = RalphRouter()

src/ralph/assets/api/serializers.py

@@ -53,9 +53,9 @@ def get_object_type(self, instance):
 
 class OwnersFromServiceEnvSerializerMixin(RalphAPISerializer):
     business_owners = SimpleRalphUserSerializer(
-        many=True, source='service_env.service.business_owners')
+        many=True, source='service_env.service.business_owners', required=False)
     technical_owners = SimpleRalphUserSerializer(
-        many=True, source='service_env.service.technical_owners')
+        many=True, source='service_env.service.technical_owners', required=False)
 
 
 class BusinessSegmentSerializer(RalphAPISerializer):
@@ -317,6 +317,7 @@ class Meta:
             'parent'
         )
 
+
 # TODO: Is there a better way to make it work since drf 3.5?
 del ConfigurationClassSimpleSerializer._declared_fields['tags']
 

src/ralph/assets/invoice_report.py

@@ -1,5 +1,6 @@
 # -*- coding: utf-8 -*-
 import datetime
+import json
 import logging
 
 from django.contrib import messages
@@ -116,7 +117,7 @@ def _get_report_data(self, request, queryset):
             'model': queryset.model._meta.model_name,
             'base_info': {
                 'invoice_no': first_item.invoice_no,
-                'invoice_date': first_item.invoice_date,
+                'invoice_date': str(first_item.invoice_date),
                 'provider': first_item.provider,
                 'datetime': datetime.datetime.now().strftime(
                     self._invoice_report_datetime_format
@@ -148,6 +149,9 @@ def _get_pdf_content(self, data):
             template_content = f.read()
 
         service_pdf = ExternalService('PDF')
+        # Make sure data is JSON-serializable
+        # Will throw otherwise
+        data = json.loads(json.dumps(data))
         result = service_pdf.run(
             template=template_content,
             data=data,

src/ralph/data_center/admin.py

@@ -28,13 +28,14 @@
 from ralph.admin.mixins import (
     BulkEditChangeListMixin,
     RalphAdmin,
+    RalphAdminImportExportMixin,
     RalphTabularInline
 )
 from ralph.admin.views.extra import RalphDetailViewAdmin
 from ralph.admin.views.main import RalphChangeList
 from ralph.admin.views.multiadd import MulitiAddAdminMixin
 from ralph.assets.invoice_report import AssetInvoiceReportMixin
-from ralph.assets.models.base import BaseObject
+from ralph.assets.models.base import BaseObject, BaseObjectPolymorphicQuerySet
 from ralph.assets.models.components import Ethernet
 from ralph.assets.views import ComponentsAdminView
 from ralph.attachments.admin import AttachmentsMixin
@@ -491,11 +492,25 @@ class DataCenterAssetAdmin(
     )
 
     def get_export_queryset(self, request):
-        return DataCenterAsset.polymorphic_objects.select_related(
-            *self.list_select_related
-        ).polymorphic_prefetch_related(
-            DataCenterAsset=['tags', 'ethernet_set__ipaddress', 'parent__ethernet_set__ipaddress'],
+        qs = (
+            super(RalphAdminImportExportMixin, self)
+            .get_export_queryset(request)
+            .select_related(
+                *self.list_select_related
+            )
         )
+        if isinstance(qs, BaseObjectPolymorphicQuerySet):
+            return qs.polymorphic_prefetch_related(
+                DataCenterAsset=[
+                    'tags',
+                    'ethernet_set__ipaddress',
+                    'parent__ethernet_set__ipaddress'
+                ]
+            )
+        else:
+            return qs.prefetch_related(
+                'tags', 'ethernet_set__ipaddress', 'parent__ethernet_set__ipaddress'
+            )
 
     def get_multiadd_fields(self, obj=None):
         multiadd_fields = [

src/ralph/data_importer/resources.py

@@ -306,8 +306,16 @@ class DataCenterAssetResource(ResourceWithPrice, RalphModelResource):
     class Meta:
         model = physical.DataCenterAsset
         select_related = (
+            'model__manufacturer', 'model__category',
             'service_env__service', 'service_env__environment',
             'rack__server_room__data_center',
+            'configuration_path',
+            'property_of',
+            'parent',
+            'budget_info',
+        )
+        prefetch_related = (
+            'tags', 'ethernet_set__ipaddress', 'parent__ethernet_set__ipaddress',
         )
         exclude = ('content_type', 'asset_ptr', 'baseobject_ptr', 'connections')
 

src/ralph/data_importer/tests/test_export.py

@@ -114,6 +114,16 @@ def test_data_center_asset_export_queries_count(self):
             DataCenterAsset
         ), max_queries=12)
 
+    def test_data_center_asset_export_filtered(self):
+        self._init(10)
+        first_id = next(iter(self.data_center_assets_map.keys()))
+        with CaptureQueriesContext(connections['default']) as cqc:
+            export_data = self._export(
+                DataCenterAsset, filters={'id': first_id}
+            )
+            queries = len(cqc)
+        self.assertEqual(len(export_data.dict), 1)
+        self.assertLessEqual(queries, 12)
 
 class DataCenterAssetExporterTestCaseWithParent(DataCenterAssetExporterTestCase):
     def _init(self, num=10):

src/ralph/helpers.py

@@ -41,6 +41,7 @@ def generate_pdf_response(pdf_data, file_name):
     )
     return response
 
+
 CACHE_DEFAULT = object()
 
 

src/ralph/lib/metrics/collector.py

@@ -25,6 +25,7 @@ def build_statsd_client(
         ipv6=ipv6
     )
 
+
 if settings.COLLECT_METRICS and statsd is None:
     statsd = build_statsd_client()
 

src/ralph/lib/mixins/fields.py

@@ -232,6 +232,9 @@ def deconstruct(self):
     def limit_choices_to(self):
         return self.limit_choices()
 
+    def get_limit_choices_to(self):
+        return self.limit_choices()
+
     def limit_choices(self):
         """
         Add limit_choices_to search by content_type for models

src/ralph/licences/models.py

@@ -111,6 +111,7 @@ def get_queryset(self):
             }
         )
 
+
 LICENCES_RELATED_OBJECTS_PREFETCH_RELATED = [
     'users',
     # prefetch all baseobjects related with licence; this allows to call

src/ralph/networks/filters.py

@@ -24,6 +24,8 @@ def get_private_network_filter():
         max_ip = int(network.broadcast_address)
         filter_ |= Q(min_ip__gte=min_ip, max_ip__lte=max_ip)
     return filter_
+
+
 PRIVATE_NETWORK_FILTER = get_private_network_filter()