✨ [FEAT] Add fetching related categories and parent sites in GeotrekSiteParser (refs #3569)

Author: Chatewgne

geotrek/outdoor/parsers.py

@@ -1,5 +1,6 @@
+from django.conf import settings
 from geotrek.common.parsers import (ApidaeBaseParser, AttachmentParserMixin, GeotrekParser, GlobalImportError, Parser)
-from geotrek.outdoor.models import Site
+from geotrek.outdoor.models import Practice, Rating, RatingScale, Sector, Site
 
 
 class GeotrekSiteParser(GeotrekParser):
@@ -9,11 +10,10 @@ class GeotrekSiteParser(GeotrekParser):
     model = Site
     replace_fields = {
         "eid": "uuid",
-        "geom": "geometry"
+        "geom": "geometry",
+        "scale": "ratingscale"
     }
     url_categories = {
-        "practice": "outdoor_practice",
-        "ratings": "outdoor_rating",
         "themes": "theme",
         "type": "outdoor_sitetype",
         'labels': 'label',
@@ -23,7 +23,10 @@ class GeotrekSiteParser(GeotrekParser):
     }
     categories_keys_api_v2 = {
         "practice": "name",
-        "ratings": "name",
+        "sector": "name",
+        "rating": "name",
+        "scale": "name",
+        "ratingscale": "name",
         "themes": "label",
         "type": "name",
         'labels': 'name',
@@ -33,21 +36,91 @@ class GeotrekSiteParser(GeotrekParser):
     }
     natural_keys = {
         "practice": "name",
-        "ratings": "name",
         "themes": "label",
         "type": "name",
         'labels': 'name',
         'source': 'name',
         'managers': 'organism',
         'structure': 'name',
     }
+    parents = {}
+
+    def get_id_from_mapping(self, mapping, value):
+        for dest_id, source_id in mapping.items():
+            if source_id == value:
+                return dest_id
+        return None
+
+    def init_outdoor_category(self, category, model, join_field=None, extra_fields={}):
+        response = self.request_or_retry(f"{self.url}/api/v2/outdoor_{category}")
+        results = response.json().get('results', [])
+        if category not in self.field_options.keys():
+            self.field_options[category] = {}
+        if "mapping" not in self.field_options[category].keys():
+            self.field_options[category]["mapping"] = {}
+        for result in results:
+            label = result["name"]
+            if isinstance(label, dict):
+                if label[settings.MODELTRANSLATION_DEFAULT_LANGUAGE]:
+                    replaced_label = self.replace_mapping(label[settings.MODELTRANSLATION_DEFAULT_LANGUAGE], f'outdoor_{category}')
+            else:
+                if label:
+                   replaced_label = self.replace_mapping(label, f'outdoor_{category}')
+            fields = {}
+            for field in extra_fields:
+                if isinstance(result[field], dict):
+                    if result[field][settings.MODELTRANSLATION_DEFAULT_LANGUAGE]:
+                        fields[field] = result[field][settings.MODELTRANSLATION_DEFAULT_LANGUAGE]
+                else:
+                    fields[field] = result[field]
+            if join_field:
+                mapping_key = self.replace_fields.get(join_field, join_field)
+                mapped_value = self.get_id_from_mapping(self.field_options[mapping_key]["mapping"], result[join_field])
+                if not mapped_value:
+                    continue # Ignore some results if related category was not retrieved
+                fields[f"{join_field}_id"] = mapped_value
+            category_obj, _ = model.objects.update_or_create(**{'name': replaced_label}, defaults=fields)
+            self.field_options[category]["mapping"][category_obj.pk] = result['id']
 
     def __init__(self, *args, **kwargs):
         super().__init__(*args, **kwargs)
+        self.init_outdoor_category('sector', Sector)
+        self.init_outdoor_category('practice', Practice, join_field='sector')
+        self.init_outdoor_category('ratingscale', RatingScale, join_field='practice')
+        self.init_outdoor_category('rating', Rating, join_field='scale', extra_fields=['description', 'order', 'color'])
         self.next_url = f"{self.url}/api/v2/outdoor_site"
-        print("AFTER INIT 9999999999999999999999999999999999999999999999999999999")
+
+    def filter_practice(self, src, val):
+        if val:
+            practice_id = self.get_id_from_mapping(self.field_options["practice"]["mapping"], val)
+            if practice_id:
+                return Practice.objects.get(pk=practice_id)
+        return None
+
+    def filter_ratings(self, src, val):
+        ratings = []
+        for subval in val:
+            rating_id = self.get_id_from_mapping(self.field_options["rating"]["mapping"], subval)
+            if rating_id:
+                ratings.append(Rating.objects.get(pk=rating_id).pk)
+        return ratings
+
+    def parse_row(self, row):
+        super().parse_row(row)
+        self.parents[row['uuid']] = row['parent_uuid']
 
     def end(self):
-        """Add children after all treks imported are created in database."""
-        #super().end()
-        print("MAKE LINK BETWEEN SITES")
+        """Add children after all Sites imported are created in database."""
+        for child, parent in self.parents.items():
+            try:
+                parent_site = Site.objects.get(eid=parent)
+            except Site.DoesNotExist:
+                self.add_warning(f"Trying to retrieve missing parent (UUID: {parent}) for child Site (UUID: {child})")
+                continue
+            try:
+                child_site = Site.objects.get(eid=child)
+            except Site.DoesNotExist:
+                self.add_warning(f"Trying to retrieve missing child (UUID: {child}) for parent Site (UUID: {parent})")
+                continue
+            child_site.parent = parent_site
+            child_site.save()

geotrek/outdoor/tests/data/geotrek_parser_v2/outdoor_rating.json

@@ -10,12 +10,12 @@
                 "fr": "3+"
             },
             "description": {
-                "en": null,
-                "fr": null
+                "en": "A description",
+                "fr": "Une description"
             },
             "scale": 6,
             "order": 302,
-            "color": "#D9D9D9"
+            "color": "#D9D9D8"
         },
         {
             "id": 36,

geotrek/outdoor/tests/test_parsers.py

@@ -19,7 +19,7 @@
 from geotrek.common.models import Organism, Theme, FileType, Attachment, Label
 from geotrek.common.tests.mixins import GeotrekParserTestMixin
 from geotrek.core.tests.factories import PathFactory
-from geotrek.outdoor.models import Site
+from geotrek.outdoor.models import Practice, Rating, RatingScale, Sector, Site
 from geotrek.outdoor.parsers import GeotrekSiteParser
 from geotrek.trekking.tests.factories import RouteFactory
 from geotrek.trekking.models import POI, POIType, Service, Trek, DifficultyLevel, Route
@@ -29,15 +29,12 @@
 )
 
 
-
 class TestGeotrekSiteParser(GeotrekSiteParser):
     url = "https://test.fr"
     provider = 'geotrek1'
     field_options = {
         'type': {'create': True, },
-        'ratings': {'create': True, },
         'themes': {'create': True},
-        'practice': {'create': True},
         'geom': {'required': True},
         'labels': {'create': True},
         'source': {'create': True},
@@ -72,7 +69,6 @@ class TestGeotrek2SiteParser(GeotrekSiteParser):
 #         'geom': {'required': True},
 #     }
 
-
 @override_settings(MODELTRANSLATION_DEFAULT_LANGUAGE="fr")
 @skipIf(settings.TREKKING_TOPOLOGY_ENABLED, 'Test without dynamic segmentation only')
 class SiteGeotrekParserTests(GeotrekParserTestMixin, TestCase):
@@ -85,14 +81,16 @@ def setUpTestData(cls):
     @mock.patch('requests.head')
     def test_create(self, mocked_head, mocked_get):
         self.mock_time = 0
-        self.mock_json_order = [('outdoor', 'outdoor_practice.json'),
-                                ('outdoor', 'outdoor_rating.json'),
-                                ('outdoor', 'theme.json'),
+        self.mock_json_order = [('outdoor', 'theme.json'),
                                 ('outdoor', 'outdoor_sitetype.json'),
                                 ('outdoor', 'label.json'),
                                 ('outdoor', 'source.json'),
                                 ('outdoor', 'organism.json'),
                                 ('outdoor', 'structure.json'),
+                                ('outdoor', 'outdoor_sector.json'),
+                                ('outdoor', 'outdoor_practice.json'),
+                                ('outdoor', 'outdoor_ratingscale.json'),
+                                ('outdoor', 'outdoor_rating.json'),
                                 ('outdoor', 'outdoor_site_ids.json'),
                                 ('outdoor', 'outdoor_site.json')]
 
@@ -104,17 +102,26 @@ def test_create(self, mocked_head, mocked_get):
 
         call_command('import', 'geotrek.outdoor.tests.test_parsers.TestGeotrekSiteParser', verbosity=0)
         self.assertEqual(Site.objects.count(), 6)
+        self.assertEqual(Sector.objects.count(), 2)
+        self.assertEqual(RatingScale.objects.count(), 1)
+        self.assertEqual(Rating.objects.count(), 3)
+        self.assertEqual(Practice.objects.count(), 1)
         site = Site.objects.get(name_fr="Racine", name_en="Root")
         # TODO : all the ones that are commented do not work
         self.assertEqual(site.published, True)
         self.assertEqual(site.published_fr, True)
         self.assertEqual(site.published_en, True)
         self.assertEqual(site.published_it, False)
         self.assertEqual(site.published_es, False)
+        self.assertEqual(str(site.practice.sector), 'Vertical')
         self.assertEqual(str(site.practice), 'Escalade')
         self.assertEqual(str(site.labels.first()), 'Label fr')
-        #self.assertEqual(str(site.ratings.all()), 'Très facile')
-        #self.assertEqual(str(site.practice.sector), 'Vertical')
+        self.assertEqual(site.ratings.count(), 3)
+        self.assertEqual(str(site.ratings.first()), 'Cotation : 3+')
+        self.assertEqual(site.ratings.first().description, 'Une description')
+        self.assertEqual(site.ratings.first().order, 302)
+        self.assertEqual(site.ratings.first().color, '#D9D9D8')
+        self.assertEqual(str(site.ratings.first().scale), 'Cotation (Escalade)')
         self.assertEqual(str(site.type), 'Ecole')
         self.assertAlmostEqual(site.geom[0][0][0][0], 970023.8976707931, places=5)
         self.assertAlmostEqual(site.geom[0][0][0][1], 6308806.903248067, places=5)
@@ -135,14 +142,16 @@ def test_create(self, mocked_head, mocked_get):
         self.assertEqual(site.orientation, ['NE', 'S'])
         self.assertEqual(site.ambiance, "Test ambiance fr")
         self.assertEqual(site.ambiance_en, "Test ambiance en")
-        #self.assertEqual(site.parent) # TODO use other to test this
         self.assertEqual(site.wind, ['N', 'E'])
+        self.assertEqual(str(site.structure), 'Test structure')
         # self.assertEqual(site.information_desks.count(), 1)
         # self.assertEqual(site.weblink.count(), 1)
         # self.assertEqual(site.excluded_pois.count(), 1)
         self.assertEqual(site.eid, "57a8fb52-213d-4dce-8224-bc997f892aae")
         # self.assertEqual(Attachment.objects.filter(object_id=site.pk).count(), 3)
         # self.assertEqual(Attachment.objects.get(object_id=site.pk, license__isnull=False).license.label, "License")
+        child_site = Site.objects.get(name_fr="Noeud 1", name_en="Node")
+        self.assertEqual(child_site.parent, site)
 
     @mock.patch('requests.get')
     @mock.patch('requests.head')