Import users through third party (Google, Slack, BambooHR etc)

back/admin/integrations/exceptions.py

@@ -0,0 +1,6 @@
+class GettingUsersError(Exception):
+    pass
+
+
+class KeyIsNotInDataError(Exception):
+    pass

back/admin/integrations/factories.py

@@ -13,6 +13,8 @@ class Meta:
 
 
 class CustomIntegrationFactory(IntegrationFactory):
+    integration = Integration.Type.CUSTOM
+    manifest_type = Integration.ManifestType.WEBHOOK
     manifest = {
         "form": [
             {
@@ -80,3 +82,46 @@ class CustomIntegrationFactory(IntegrationFactory):
             },
         ],
     }
+
+
+class CustomUserImportIntegrationFactory(IntegrationFactory):
+    integration = Integration.Type.CUSTOM
+    manifest_type = Integration.ManifestType.USER_IMPORT
+    manifest = {
+        "form": [],
+        "type": "import_users",
+        "execute": [
+            {
+                "url": "http://localhost/api/gateway.php/{{COMPANY_ID}}/v1/reports/{{REPORT_ID}}",
+                "method": "GET",
+            }
+        ],
+        "headers": {
+            "Accept": "application/json",
+            "Content-Type": "application/json",
+            "Authorization": "Basic {{KEY}}:x",
+        },
+        "data_from": "employees",
+        "data_structure": {
+            "email": "workEmail",
+            "last_name": "lastName",
+            "first_name": "firstName",
+        },
+        "initial_data_form": [
+            {
+                "id": "KEY",
+                "name": "The BambooHR api key",
+                "description": "Go to get one",
+            },
+            {
+                "id": "REPORT_ID",
+                "name": "The id of the report",
+                "description": "There is a number that will represent the ID.",
+            },
+            {
+                "id": "COMPANY_ID",
+                "name": "The id of the company",
+                "description": "The '<yourdomain>' is your domain name. ",
+            },
+        ],
+    }

back/admin/integrations/forms.py

@@ -5,21 +5,13 @@
 from django.contrib.auth import get_user_model
 from django.core.exceptions import ValidationError
 
+from admin.integrations.utils import get_value_from_notation
+
 from .models import Integration
 from .serializers import ManifestSerializer
 
 
 class IntegrationConfigForm(forms.ModelForm):
-    def _get_result(self, notation, value):
-        # if we don't need to go into props, then just return the value
-        if notation == "":
-            return value
-
-        notations = notation.split(".")
-        for notation in notations:
-            value = value[notation]
-        return value
-
     def _expected_example(self, form_item):
         def _add_items(form_item):
             items = []
@@ -92,10 +84,14 @@ def __init__(self, *args, **kwargs):
                         else forms.Select,
                         choices=[
                             (
-                                self._get_result(item.get("choice_value", "id"), x),
-                                self._get_result(item.get("choice_name", "name"), x),
+                                get_value_from_notation(
+                                    item.get("choice_value", "id"), x
+                                ),
+                                get_value_from_notation(
+                                    item.get("choice_name", "name"), x
+                                ),
                             )
-                            for x in self._get_result(
+                            for x in get_value_from_notation(
                                 item.get("data_from", ""), option_data
                             )
                         ],
@@ -130,7 +126,11 @@ class IntegrationForm(forms.ModelForm):
 
     class Meta:
         model = Integration
-        fields = ("name", "manifest")
+        fields = ("name", "manifest_type", "manifest")
+
+    def __init__(self, *args, **kwargs):
+        super().__init__(*args, **kwargs)
+        self.fields["manifest_type"].required = True
 
     def clean_manifest(self):
         manifest = self.cleaned_data["manifest"]

back/admin/integrations/import_users.py

@@ -0,0 +1,142 @@
+from admin.integrations.exceptions import KeyIsNotInDataError
+from django.contrib.auth import get_user_model
+from organization.models import Organization
+
+from admin.integrations.utils import get_value_from_notation
+from django.utils.translation import gettext_lazy as _
+from admin.integrations.exceptions import GettingUsersError
+
+
+class ImportUser:
+    """
+    Extension of the `Integration` model. This part is only used to get users
+    from a third party API endpoint and format them in a way that we can proccess
+    them.
+    """
+
+    def __init__(self, integration):
+        self.integration = integration
+
+    def extract_users_from_list_response(self, response):
+        # Building list of users from response. Dig into response to get to the users.
+        data_from = self.integration.manifest["data_from"]
+
+        try:
+            users = get_value_from_notation(data_from, response.json())
+        except KeyError:
+            # This is unlikely to go wrong - only when api changes or when
+            # configs are being setup
+            raise KeyIsNotInDataError(
+                _("Notation '%(notation)s' not in %(response)s")
+                % {
+                    "notation": data_from,
+                    "response": self.integration.clean_response(response.json()),
+                }
+            )
+        data_structure = self.integration.manifest["data_structure"]
+        user_details = []
+        for user_data in users:
+            user = {}
+            for prop, notation in data_structure.items():
+                try:
+                    user[prop] = get_value_from_notation(notation, user_data)
+                except KeyError:
+                    # This is unlikely to go wrong - only when api changes or when
+                    # configs are being setup
+                    raise KeyIsNotInDataError(
+                        _("Notation '%(notation)s' not in %(response)s")
+                        % {
+                            "notation": notation,
+                            "response": self.integration.clean_response(user_data),
+                        }
+                    )
+            user_details.append(user)
+        return user_details
+
+    def get_next_page(self, response):
+        # Some apis give us back a full URL, others just a token. If we get a full URL,
+        # follow that, if we get a token, then also specify the next_page. The token
+        # gets placed through the NEXT_PAGE_TOKEN variable.
+
+        # taken from response - full url including params for next page
+        next_page_from = self.integration.manifest.get("next_page_from")
+
+        # build up url ourself based on hardcoded url + token for next part
+        next_page_token_from = self.integration.manifest.get("next_page_token_from")
+        # hardcoded in manifest
+        next_page = self.integration.manifest.get("next_page")
+
+        # skip if none provided
+        if not next_page_from and not (next_page_token_from and next_page):
+            return
+
+        if next_page_from:
+            try:
+                return get_value_from_notation(next_page_from, response.json())
+            except KeyError:
+                # next page was not provided anymore, so we are done
+                return
+
+        # Build next url from next_page and next_page_token_from
+        try:
+            token = get_value_from_notation(next_page_token_from, response.json())
+        except KeyError:
+            # next page token was not provided anymore, so we are done
+            return
+
+        # Replace token variable with real token
+        self.integration.params["NEXT_PAGE_TOKEN"] = token
+        return self.integration._replace_vars(next_page)
+
+    def get_import_user_candidates(self, user):
+        success, response = self.integration.execute(user, {})
+        if not success:
+            raise GettingUsersError(self.integration.clean_response(response))
+
+        users = self.extract_users_from_list_response(response)
+
+        amount_pages_to_fetch = self.integration.manifest.get(
+            "amount_pages_to_fetch", 5
+        )
+        fetched_pages = 1
+        while amount_pages_to_fetch != fetched_pages:
+            # End everything if next page does not exist
+            next_page_url = self.get_next_page(response)
+            if next_page_url is None:
+                break
+
+            success, response = self.integration.run_request(
+                {"method": "GET", "url": next_page_url}
+            )
+            if not success:
+                raise GettingUsersError(
+                    _("Paginated URL fetch: %(response)s")
+                    % {"response": self.integration.clean_response(response)}
+                )
+
+            # Check if there are any new results. Google could send no users back
+            try:
+                data_from = self.integration.manifest["data_from"]
+                get_value_from_notation(data_from, response.json())
+            except KeyError:
+                break
+
+            users += self.extract_users_from_list_response(response)
+            fetched_pages += 1
+
+        # Remove users that are already in the system or have been ignored
+        existing_user_emails = list(
+            get_user_model().objects.all().values_list("email", flat=True)
+        )
+        ignored_user_emails = Organization.objects.get().ignored_user_emails
+        excluded_emails = (
+            existing_user_emails + ignored_user_emails + ["", None]
+        )  # also add blank emails to ignore
+
+        user_candidates = [
+            user_data
+            for user_data in users
+            if user_data.get("email", "") not in excluded_emails
+        ]
+
+        return user_candidates

back/admin/integrations/migrations/0020_integration_manifest_type.py

@@ -0,0 +1,31 @@
+# Generated by Django 4.2.5 on 2023-09-11 15:44
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+    def set_integration_manifest_type(apps, schema_editor):
+        Integration = apps.get_model("integrations", "integration")
+        Integration.objects.filter(integration=10).update(manifest_type=0)
+
+    dependencies = [
+        ("integrations", "0019_alter_integration_integration"),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name="integration",
+            name="manifest_type",
+            field=models.IntegerField(
+                blank=True,
+                choices=[
+                    (0, "Provision user accounts or trigger webhooks"),
+                    (1, "Import users"),
+                ],
+                null=True,
+            ),
+        ),
+        migrations.RunPython(
+            set_integration_manifest_type, reverse_code=migrations.RunPython.noop
+        ),
+    ]