Merge pull request #191 from openedx/iahmad/ENT-8117

feat: Optimized finalize_xblockskill_tags command for memory via chunking

Author: irfanuddinahmad

CHANGELOG.rst

@@ -13,6 +13,10 @@ Change Log
 
 Unreleased
 
+[1.46.2] - 2024-02-14
+---------------------
+* feat: Optimized finalize_xblockskill_tags command for memory via chunking
+
 [1.46.1] - 2024-01-05
 ---------------------
 * feat: Modify prefetch related to select related for whitelisted product skills.

taxonomy/__init__.py

@@ -15,6 +15,6 @@
 # 2. MINOR version when you add functionality in a backwards compatible manner, and
 # 3. PATCH version when you make backwards compatible bug fixes.
 # More details can be found at https://semver.org/
-__version__ = '1.46.1'
+__version__ = '1.46.2'
 
 default_app_config = 'taxonomy.apps.TaxonomyConfig'  # pylint: disable=invalid-name

taxonomy/management/commands/finalize_xblockskill_tags.py

@@ -7,7 +7,6 @@
 
 from django.conf import settings
 from django.core.management.base import BaseCommand
-from django.db import transaction
 from django.utils.translation import gettext as _
 
 from taxonomy.exceptions import InvalidCommandOptionsError
@@ -99,28 +98,27 @@ def handle(self, *args, **options):
             options,
         )
 
-        with transaction.atomic():
-            unverified_skills = XBlockSkillData.objects.filter(verified=False, is_blacklisted=False)
-            for xblock_skill in unverified_skills:
-                verified_count = xblock_skill.verified_count if xblock_skill.verified_count else 0
-                ignored_count = xblock_skill.ignored_count if xblock_skill.ignored_count else 0
-                total_count = int(verified_count + ignored_count)
-                if total_count <= 0:
-                    continue
-                if self._is_over_threshold(verified_count, total_count, min_verified_votes, ratio_verified_threshold):
-                    xblock_skill.verified = True
-                    xblock_skill.save()
-                    LOGGER.info(
-                        '[%s] skill tag for the xblock [%s] has been verified',
-                        xblock_skill.skill.name,
-                        xblock_skill.xblock.usage_key
-                    )
-                elif self._is_over_threshold(ignored_count, total_count, min_ignored_votes, ratio_ignored_threshold):
-                    xblock_skill.is_blacklisted = True
-                    xblock_skill.save()
-                    LOGGER.info(
-                        '[%s] skill tag for the xblock [%s] has been blacklisted',
-                        xblock_skill.skill.name,
-                        xblock_skill.xblock.usage_key
-                    )
+        for xblock_skill in XBlockSkillData.objects.filter(
+                verified=False, is_blacklisted=False).iterator(chunk_size=2000):
+            verified_count = xblock_skill.verified_count if xblock_skill.verified_count else 0
+            ignored_count = xblock_skill.ignored_count if xblock_skill.ignored_count else 0
+            total_count = int(verified_count + ignored_count)
+            if total_count <= 0:
+                continue
+            if self._is_over_threshold(verified_count, total_count, min_verified_votes, ratio_verified_threshold):
+                xblock_skill.verified = True
+                xblock_skill.save()
+                LOGGER.info(
+                    '[%s] skill tag for the xblock [%s] has been verified',
+                    xblock_skill.skill.name,
+                    xblock_skill.xblock.usage_key
+                )
+            elif self._is_over_threshold(ignored_count, total_count, min_ignored_votes, ratio_ignored_threshold):
+                xblock_skill.is_blacklisted = True
+                xblock_skill.save()
+                LOGGER.info(
+                    '[%s] skill tag for the xblock [%s] has been blacklisted',
+                    xblock_skill.skill.name,
+                    xblock_skill.xblock.usage_key
+                )
         LOGGER.info('Xblockskill tags verification task is completed')