diff --git a/back/admin/integrations/exceptions.py b/back/admin/integrations/exceptions.py new file mode 100644 index 000000000..1ce6a3f2a --- /dev/null +++ b/back/admin/integrations/exceptions.py @@ -0,0 +1,6 @@ +class GettingUsersError(Exception): + pass + + +class KeyIsNotInDataError(Exception): + pass diff --git a/back/admin/integrations/factories.py b/back/admin/integrations/factories.py index 9cbdacbd2..a1ca0bf45 100644 --- a/back/admin/integrations/factories.py +++ b/back/admin/integrations/factories.py @@ -13,6 +13,8 @@ class Meta: class CustomIntegrationFactory(IntegrationFactory): + integration = Integration.Type.CUSTOM + manifest_type = Integration.ManifestType.WEBHOOK manifest = { "form": [ { @@ -80,3 +82,46 @@ class CustomIntegrationFactory(IntegrationFactory): }, ], } + + +class CustomUserImportIntegrationFactory(IntegrationFactory): + integration = Integration.Type.CUSTOM + manifest_type = Integration.ManifestType.USER_IMPORT + manifest = { + "form": [], + "type": "import_users", + "execute": [ + { + "url": "http://localhost/api/gateway.php/{{COMPANY_ID}}/v1/reports/{{REPORT_ID}}", + "method": "GET", + } + ], + "headers": { + "Accept": "application/json", + "Content-Type": "application/json", + "Authorization": "Basic {{KEY}}:x", + }, + "data_from": "employees", + "data_structure": { + "email": "workEmail", + "last_name": "lastName", + "first_name": "firstName", + }, + "initial_data_form": [ + { + "id": "KEY", + "name": "The BambooHR api key", + "description": "Go to get one", + }, + { + "id": "REPORT_ID", + "name": "The id of the report", + "description": "There is a number that will represent the ID.", + }, + { + "id": "COMPANY_ID", + "name": "The id of the company", + "description": "The '' is your domain name. ", + }, + ], + } diff --git a/back/admin/integrations/forms.py b/back/admin/integrations/forms.py index ec3421c4e..5e7cfd2a9 100644 --- a/back/admin/integrations/forms.py +++ b/back/admin/integrations/forms.py @@ -5,21 +5,13 @@ from django.contrib.auth import get_user_model from django.core.exceptions import ValidationError +from admin.integrations.utils import get_value_from_notation + from .models import Integration from .serializers import ManifestSerializer class IntegrationConfigForm(forms.ModelForm): - def _get_result(self, notation, value): - # if we don't need to go into props, then just return the value - if notation == "": - return value - - notations = notation.split(".") - for notation in notations: - value = value[notation] - return value - def _expected_example(self, form_item): def _add_items(form_item): items = [] @@ -92,10 +84,14 @@ def __init__(self, *args, **kwargs): else forms.Select, choices=[ ( - self._get_result(item.get("choice_value", "id"), x), - self._get_result(item.get("choice_name", "name"), x), + get_value_from_notation( + item.get("choice_value", "id"), x + ), + get_value_from_notation( + item.get("choice_name", "name"), x + ), ) - for x in self._get_result( + for x in get_value_from_notation( item.get("data_from", ""), option_data ) ], @@ -130,7 +126,11 @@ class IntegrationForm(forms.ModelForm): class Meta: model = Integration - fields = ("name", "manifest") + fields = ("name", "manifest_type", "manifest") + + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) + self.fields["manifest_type"].required = True def clean_manifest(self): manifest = self.cleaned_data["manifest"] diff --git a/back/admin/integrations/import_users.py b/back/admin/integrations/import_users.py new file mode 100644 index 000000000..8c834baa2 --- /dev/null +++ b/back/admin/integrations/import_users.py @@ -0,0 +1,142 @@ +from admin.integrations.exceptions import KeyIsNotInDataError +from django.contrib.auth import get_user_model +from organization.models import Organization + +from admin.integrations.utils import get_value_from_notation +from django.utils.translation import gettext_lazy as _ +from admin.integrations.exceptions import GettingUsersError + + +class ImportUser: + """ + Extension of the `Integration` model. This part is only used to get users + from a third party API endpoint and format them in a way that we can proccess + them. + """ + + def __init__(self, integration): + self.integration = integration + + def extract_users_from_list_response(self, response): + # Building list of users from response. Dig into response to get to the users. + data_from = self.integration.manifest["data_from"] + + try: + users = get_value_from_notation(data_from, response.json()) + except KeyError: + # This is unlikely to go wrong - only when api changes or when + # configs are being setup + raise KeyIsNotInDataError( + _("Notation '%(notation)s' not in %(response)s") + % { + "notation": data_from, + "response": self.integration.clean_response(response.json()), + } + ) + data_structure = self.integration.manifest["data_structure"] + user_details = [] + for user_data in users: + user = {} + for prop, notation in data_structure.items(): + try: + user[prop] = get_value_from_notation(notation, user_data) + except KeyError: + # This is unlikely to go wrong - only when api changes or when + # configs are being setup + raise KeyIsNotInDataError( + _("Notation '%(notation)s' not in %(response)s") + % { + "notation": notation, + "response": self.integration.clean_response(user_data), + } + ) + user_details.append(user) + return user_details + + def get_next_page(self, response): + # Some apis give us back a full URL, others just a token. If we get a full URL, + # follow that, if we get a token, then also specify the next_page. The token + # gets placed through the NEXT_PAGE_TOKEN variable. + + # taken from response - full url including params for next page + next_page_from = self.integration.manifest.get("next_page_from") + + # build up url ourself based on hardcoded url + token for next part + next_page_token_from = self.integration.manifest.get("next_page_token_from") + # hardcoded in manifest + next_page = self.integration.manifest.get("next_page") + + # skip if none provided + if not next_page_from and not (next_page_token_from and next_page): + return + + if next_page_from: + try: + return get_value_from_notation(next_page_from, response.json()) + except KeyError: + # next page was not provided anymore, so we are done + return + + # Build next url from next_page and next_page_token_from + try: + token = get_value_from_notation(next_page_token_from, response.json()) + except KeyError: + # next page token was not provided anymore, so we are done + return + + # Replace token variable with real token + self.integration.params["NEXT_PAGE_TOKEN"] = token + return self.integration._replace_vars(next_page) + + def get_import_user_candidates(self, user): + success, response = self.integration.execute(user, {}) + if not success: + raise GettingUsersError(self.integration.clean_response(response)) + + users = self.extract_users_from_list_response(response) + + amount_pages_to_fetch = self.integration.manifest.get( + "amount_pages_to_fetch", 5 + ) + fetched_pages = 1 + while amount_pages_to_fetch != fetched_pages: + # End everything if next page does not exist + next_page_url = self.get_next_page(response) + if next_page_url is None: + break + + success, response = self.integration.run_request( + {"method": "GET", "url": next_page_url} + ) + if not success: + raise GettingUsersError( + _("Paginated URL fetch: %(response)s") + % {"response": self.integration.clean_response(response)} + ) + + # Check if there are any new results. Google could send no users back + try: + data_from = self.integration.manifest["data_from"] + get_value_from_notation(data_from, response.json()) + except KeyError: + break + + users += self.extract_users_from_list_response(response) + fetched_pages += 1 + + # Remove users that are already in the system or have been ignored + existing_user_emails = list( + get_user_model().objects.all().values_list("email", flat=True) + ) + ignored_user_emails = Organization.objects.get().ignored_user_emails + excluded_emails = ( + existing_user_emails + ignored_user_emails + ["", None] + ) # also add blank emails to ignore + + user_candidates = [ + user_data + for user_data in users + if user_data.get("email", "") not in excluded_emails + ] + + return user_candidates diff --git a/back/admin/integrations/migrations/0020_integration_manifest_type.py b/back/admin/integrations/migrations/0020_integration_manifest_type.py new file mode 100644 index 000000000..7a76543a2 --- /dev/null +++ b/back/admin/integrations/migrations/0020_integration_manifest_type.py @@ -0,0 +1,31 @@ +# Generated by Django 4.2.5 on 2023-09-11 15:44 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + def set_integration_manifest_type(apps, schema_editor): + Integration = apps.get_model("integrations", "integration") + Integration.objects.filter(integration=10).update(manifest_type=0) + + dependencies = [ + ("integrations", "0019_alter_integration_integration"), + ] + + operations = [ + migrations.AddField( + model_name="integration", + name="manifest_type", + field=models.IntegerField( + blank=True, + choices=[ + (0, "Provision user accounts or trigger webhooks"), + (1, "Import users"), + ], + null=True, + ), + ), + migrations.RunPython( + set_integration_manifest_type, reverse_code=migrations.RunPython.noop + ), + ] diff --git a/back/admin/integrations/models.py b/back/admin/integrations/models.py index 25533ad80..88f489bdb 100644 --- a/back/admin/integrations/models.py +++ b/back/admin/integrations/models.py @@ -39,11 +39,25 @@ def get_queryset(self): return super().get_queryset() def sequence_integration_options(self): - return self.get_queryset().filter(integration=Integration.Type.CUSTOM) + # any webhooks and account provisioning + return self.get_queryset().filter( + integration=Integration.Type.CUSTOM, + manifest_type=Integration.ManifestType.WEBHOOK, + ) def account_provision_options(self): + # only account provisioning (no general webhooks) + return self.get_queryset().filter( + integration=Integration.Type.CUSTOM, + manifest_type=Integration.ManifestType.WEBHOOK, + manifest__exists__isnull=False, + ) + + def import_users_options(self): + # only import user items return self.get_queryset().filter( - integration=Integration.Type.CUSTOM, manifest__exists__isnull=False + integration=Integration.Type.CUSTOM, + manifest_type=Integration.ManifestType.USER_IMPORT, ) @@ -56,8 +70,15 @@ class Type(models.IntegerChoices): ASANA = 4, _("Asana") # legacy CUSTOM = 10, _("Custom") + class ManifestType(models.IntegerChoices): + WEBHOOK = 0, _("Provision user accounts or trigger webhooks") + USER_IMPORT = 1, _("Import users") + name = models.CharField(max_length=300, default="", blank=True) integration = models.IntegerField(choices=Type.choices) + manifest_type = models.IntegerField( + choices=ManifestType.choices, null=True, blank=True + ) token = EncryptedTextField(max_length=10000, default="", blank=True) refresh_token = EncryptedTextField(max_length=10000, default="", blank=True) base_url = models.CharField(max_length=22300, default="", blank=True) @@ -83,6 +104,10 @@ class Type(models.IntegerChoices): bot_token = EncryptedTextField(max_length=10000, default="", blank=True) bot_id = models.CharField(max_length=100, default="") + @property + def has_user_context(self): + return self.manifest_type == Integration.ManifestType.WEBHOOK + def run_request(self, data): url = self._replace_vars(data["url"]) if "data" in data: @@ -151,7 +176,10 @@ def _replace_vars(self, text): def has_oauth(self): return "oauth" in self.manifest - def headers(self, headers={}): + def headers(self, headers=None): + if headers is None: + headers = {} + headers = ( self.manifest.get("headers", {}).items() if len(headers) == 0 @@ -209,20 +237,21 @@ def renew_key(self): self.expiring = timezone.now() + timedelta( seconds=response.json()["expires_in"] ) - self.save() + self.save(update_fields=["expiring", "extra_args"]) return success def execute(self, new_hire, params): self.params = params - self.params |= new_hire.extra_fields - self.new_hire = new_hire + if self.has_user_context: + self.params |= new_hire.extra_fields + self.new_hire = new_hire # Renew token if necessary if not self.renew_key(): - return False + return False, None # Add generated secrets - for item in self.manifest["initial_data_form"]: + for item in self.manifest.get("initial_data_form", []): if "name" in item and item["name"] == "generate": self.extra_args[item["id"]] = get_random_string(length=10) @@ -230,7 +259,8 @@ def execute(self, new_hire, params): for item in self.manifest["execute"]: success, response = self.run_request(item) - if not success: + # No need to retry or log when we are importing users + if not success and not self.has_user_context: response = self.clean_response(response=response) Notification.objects.create( notification_type=Notification.Type.FAILED_INTEGRATION, @@ -289,13 +319,15 @@ def execute(self, new_hire, params): ) return True, None - # Succesfully ran integration, add notification - Notification.objects.create( - notification_type=Notification.Type.RAN_INTEGRATION, - extra_text=self.name, - created_for=new_hire, - ) - return True, None + # Succesfully ran integration, add notification only when we are provisioning + # access + if self.has_user_context: + Notification.objects.create( + notification_type=Notification.Type.RAN_INTEGRATION, + extra_text=self.name, + created_for=new_hire, + ) + return True, response def config_form(self, data=None): from .forms import IntegrationConfigForm @@ -306,7 +338,9 @@ def clean_response(self, response): # if json, then convert to string to make it easier to replace values response = str(response) for name, value in self.extra_args.items(): - response = response.replace(str(value), f"***Secret value for {name}***") + response = response.replace( + str(value), _("***Secret value for %(name)s***") % {"name": name} + ) return response diff --git a/back/admin/integrations/serializers.py b/back/admin/integrations/serializers.py index fc3bf75c3..c8687e2ad 100644 --- a/back/admin/integrations/serializers.py +++ b/back/admin/integrations/serializers.py @@ -96,9 +96,21 @@ class ManifestOauthSerializer(ValidateMixin, serializers.Serializer): class ManifestSerializer(ValidateMixin, serializers.Serializer): + type = serializers.ChoiceField( + [ + ("import_users", "imports users from endpoint"), + ], + required=False, + ) form = ManifestFormSerializer(required=False, many=True) + data_from = serializers.CharField(required=False) + data_structure = serializers.JSONField(required=False) exists = ManifestExistSerializer(required=False) execute = ManifestExecuteSerializer(many=True) + next_page_token_from = serializers.CharField(required=False) + next_page = serializers.CharField(required=False) + next_page_from = serializers.CharField(required=False) + amount_pages_to_fetch = serializers.IntegerField(required=False) post_execute_notification = ManifestPostExecuteNotificationSerializer( many=True, required=False ) diff --git a/back/admin/integrations/tests.py b/back/admin/integrations/tests.py index 4555a65bc..0694b4e0e 100644 --- a/back/admin/integrations/tests.py +++ b/back/admin/integrations/tests.py @@ -28,7 +28,14 @@ def test_create_integration(client, django_user_model): assert "Enter a valid JSON." in response.content.decode() # Post with valid JSON - response = client.post(url, {"name": "test", "manifest": '{"execute": []}'}) + response = client.post( + url, + { + "name": "test", + "manifest": '{"execute": []}', + "manifest_type": Integration.ManifestType.WEBHOOK, + }, + ) assert "Enter a valid JSON." not in response.content.decode() assert Integration.objects.filter(integration=Integration.Type.CUSTOM).count() == 1 @@ -44,7 +51,7 @@ def test_update_integration(client, django_user_model, custom_integration_factor url = reverse("integrations:update", args=[integration.id]) response = client.get(url) - assert "Add new integration" in response.content.decode() + assert "Update existing integration" in response.content.decode() assert "TEAM_ID" in response.content.decode() assert integration.name in response.content.decode() diff --git a/back/admin/integrations/utils.py b/back/admin/integrations/utils.py new file mode 100644 index 000000000..5527e1f4e --- /dev/null +++ b/back/admin/integrations/utils.py @@ -0,0 +1,9 @@ +def get_value_from_notation(notation, value): + # if we don't need to go into props, then just return the value + if notation == "": + return value + + notations = notation.split(".") + for notation in notations: + value = value[notation] + return value diff --git a/back/admin/integrations/views.py b/back/admin/integrations/views.py index be84f6852..2bd08ae30 100644 --- a/back/admin/integrations/views.py +++ b/back/admin/integrations/views.py @@ -71,7 +71,7 @@ class IntegrationUpdateView( def get_context_data(self, **kwargs): context = super().get_context_data(**kwargs) - context["title"] = _("Add new integration") + context["title"] = _("Update existing integration") context["subtitle"] = _("settings") context["button_text"] = _("Update") return context @@ -158,10 +158,9 @@ def get_redirect_url(self, pk, *args, **kwargs): integration.expiring = timezone.now() + timedelta( seconds=response.json()["expires_in"] ) - integration.save() integration.enabled_oauth = True - integration.save() + integration.save(update_fields=["enabled_oauth", "extra_args", "expiring"]) return reverse_lazy("settings:integrations") diff --git a/back/admin/people/forms.py b/back/admin/people/forms.py index 8a0c7061a..5e1c764da 100644 --- a/back/admin/people/forms.py +++ b/back/admin/people/forms.py @@ -403,3 +403,23 @@ class Meta: labels = { "send_type": _("Send type"), } + + +class EmailIgnoreForm(forms.Form): + email = forms.EmailField() + + +class UserRoleForm(forms.ModelForm): + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) + self.fields["role"].label = "" + self.fields["role"].help_text = "" + # new hires are excluded as those should be created through a normal new hire + # create form or through the API to set all options (sequences, start day etc) + self.fields["role"].choices = tuple( + x for x in get_user_model().Role.choices if x[0] != 0 + ) + + class Meta: + model = get_user_model() + fields = ("role",) diff --git a/back/admin/people/serializers.py b/back/admin/people/serializers.py new file mode 100644 index 000000000..ee6af9f8b --- /dev/null +++ b/back/admin/people/serializers.py @@ -0,0 +1,8 @@ +from django.contrib.auth import get_user_model +from rest_framework import serializers + + +class UserImportSerializer(serializers.ModelSerializer): + class Meta: + model = get_user_model() + fields = ("first_name", "last_name", "email", "role") diff --git a/back/admin/people/templates/_import_user_table.html b/back/admin/people/templates/_import_user_table.html new file mode 100644 index 000000000..98bf47701 --- /dev/null +++ b/back/admin/people/templates/_import_user_table.html @@ -0,0 +1,57 @@ +{% load i18n %} +{% load general %} +{% load crispy_forms_tags %} + + + + + + + + + + + + + {% for user in users %} + + + + + + + + + {% empty %} + {% if not error %} + + + + {% else %} + + + + {% endif %} + {% endfor %} + +
{% translate "First name" %}{% translate "Last name" %}{% translate "Email" %}Role
+ + + {{ user.first_name }} + + + {{ user.last_name }} + + + {{ user.email }} + + + {{ role_form|crispy }} + + + {% translate "Don't show this user again" %} + +
{% translate "No users found" %}
{{ error }}
+ +
+

{% translate "Managers and administrators will receive an email with login credentials" %}

diff --git a/back/admin/people/templates/colleague_import.html b/back/admin/people/templates/colleague_import.html new file mode 100644 index 000000000..34e615093 --- /dev/null +++ b/back/admin/people/templates/colleague_import.html @@ -0,0 +1,90 @@ +{% extends 'admin_base.html' %} +{% load i18n %} +{% load crispy_forms_tags %} + +{% block content %} +
+
+
+
+
+ + {% translate "Getting users..." %} +
+
+
+
+{% endblock %} + +{% block extra_js %} + +{% endblock %} diff --git a/back/admin/people/templates/colleagues.html b/back/admin/people/templates/colleagues.html index d3459e9e5..c17e693d7 100644 --- a/back/admin/people/templates/colleagues.html +++ b/back/admin/people/templates/colleagues.html @@ -15,6 +15,13 @@ {% endif %} +{% if request.user.is_admin %} + {% for import_user_option in import_users_options %} + + {% blocktranslate with service=import_user_option.name %}Import users with {{ service }}{% endblocktranslate %} + + {% endfor %} +{% endif %} {% endblock %} {% block content %} diff --git a/back/admin/people/tests.py b/back/admin/people/tests.py index fa08e10b9..4ad9f317d 100644 --- a/back/admin/people/tests.py +++ b/back/admin/people/tests.py @@ -8,6 +8,7 @@ from django.urls import reverse from django.utils import timezone from freezegun import freeze_time +from rest_framework.test import APIClient from admin.appointments.factories import AppointmentFactory from admin.integrations.models import Integration @@ -1523,11 +1524,11 @@ def test_new_hire_access_list( integration1 = integration_factory(integration=Integration.Type.SLACK_BOT) # Should show up integration2 = custom_integration_factory( - name="Asana", integration=Integration.Type.CUSTOM + name="Asana", ) integration3 = custom_integration_factory( - name="Google", integration=Integration.Type.CUSTOM + name="Google", ) # Remove exists, so should not show up integration3.manifest = {} @@ -1553,9 +1554,7 @@ def test_new_hire_access_per_integration( new_hire1 = new_hire_factory(email="stan@example.com") new_hire2 = new_hire_factory() - integration1 = custom_integration_factory( - name="Asana", integration=Integration.Type.CUSTOM - ) + integration1 = custom_integration_factory(name="Asana") with patch( "admin.integrations.models.Integration.user_exists", Mock(return_value=True) @@ -1618,9 +1617,7 @@ def test_new_hire_access_per_integration_config_form( ) new_hire1 = new_hire_factory(email="stan@example.com") - integration1 = custom_integration_factory( - name="Asana", integration=Integration.Type.CUSTOM - ) + integration1 = custom_integration_factory(name="Asana") integration1.manifest["extra_user_info"] = [ { "id": "PERSONAL_EMAIL", @@ -2420,3 +2417,341 @@ def test_employee_toggle_resources( assert "Add" in response.content.decode() assert not employee1.resources.filter(id=resource1.id).exists() + + +@pytest.mark.django_db +def test_visibility_import_employees_button( + client, + django_user_model, + custom_user_import_integration_factory, + custom_integration_factory, +): + client.force_login(django_user_model.objects.create(role=1)) + + custom_integration_factory(name="Asana") + custom_user_import_integration_factory(name="Google import") + + url = reverse("people:colleagues") + response = client.get(url, follow=True) + + assert "Import users with Google import" in response.content.decode() + assert "Asana" not in response.content.decode() + + +@pytest.mark.django_db +def test_importing_employees( + client, django_user_model, custom_user_import_integration_factory +): + client.force_login(django_user_model.objects.create(role=1)) + + integration = custom_user_import_integration_factory(name="Google import") + + url = reverse("people:import", args=[integration.id]) + response = client.get(url, follow=True) + + assert "Import people" in response.content.decode() + # shows it's loading items + assert "Getting users" in response.content.decode() + + +@pytest.mark.django_db +def test_ignore_user_from_importing_employees( + client, django_user_model, custom_user_import_integration_factory +): + client.force_login(django_user_model.objects.create(role=1)) + org = Organization.objects.get() + assert org.ignored_user_emails == [] + + custom_user_import_integration_factory(name="Google import") + + url = reverse("people:import-ignore-hx") + client.post(url, data={"email": "stan@chiefonboarding.com"}, follow=True) + + org.refresh_from_db() + assert org.ignored_user_emails == ["stan@chiefonboarding.com"] + + +@pytest.mark.django_db +# first two will be ignored. Last two will show +@patch( + "admin.integrations.models.Integration.run_request", + Mock( + return_value=( + True, + Mock( + json=lambda: { + "directory": { + "employees": [ + { + "detail": { + "workEmail": "stan@chiefonboarding.com", + "firstName": "stan", + "lastName": "Do", + } + }, + { + "detail": { + "workEmail": "test@chiefonboarding.com", + "firstName": "stan", + "lastName": "Do", + } + }, + { + "detail": { + "workEmail": "jake@chiefonboarding.com", + "firstName": "Jake", + "lastName": "Weller", + } + }, + { + "detail": { + "workEmail": "brian@chiefonboarding.com", + "firstName": "Brian", + "lastName": "Boss", + } + }, + ] + } + } + ), + ) + ), +) +def test_fetching_employees( + client, django_user_model, custom_user_import_integration_factory, employee_factory +): + # create two users who are already in the system (should not show up) + employee_factory(email="stan@chiefonboarding.com") + employee_factory(email="john@chiefonboarding.com") + org = Organization.objects.get() + + # two emails who have been ignored by the user + org.ignored_user_emails = ["test@chiefonboarding.com", "bla@chiefonboarding.com"] + org.save() + + client.force_login(django_user_model.objects.create(role=1)) + + integration = custom_user_import_integration_factory( + manifest={ + "type": "import_users", + "execute": [ + {"url": "http://localhost:8000/test_api/users", "method": "GET"} + ], + "data_from": "directory.employees", + "data_structure": { + "email": "detail.workEmail", + "last_name": "detail.lastName", + "first_name": "detail.firstName", + }, + "initial_data_form": [], + }, + name="Google import", + ) + + url = reverse("people:import-users-hx", args=[integration.id]) + response = client.get(url, follow=True) + + assert "brian@chiefonboarding.com" in response.content.decode() + assert "jake@chiefonboarding.com" in response.content.decode() + # ignored due to already exist or on ignore list + assert "stan@chiefonboarding.com" not in response.content.decode() + assert "bla@chiefonboarding" not in response.content.decode() + + +@pytest.mark.django_db +@patch( + "admin.integrations.models.Integration.run_request", + Mock( + side_effect=( + [ + True, + Mock( + json=lambda: { + "directory": { + "employees": [ + { + "detail": { + "workEmail": "stan@chiefonboarding.com", + "firstName": "stan", + "lastName": "Do", + } + }, + { + "detail": { + "workEmail": "test@chiefonboarding.com", + "firstName": "stan", + "lastName": "Do", + } + }, + { + "detail": { + "workEmail": "jake@chiefonboarding.com", + "firstName": "Jake", + "lastName": "Weller", + } + }, + { + "detail": { + "workEmail": "brian@chiefonboarding.com", + "firstName": "Brian", + "lastName": "Boss", + } + }, + ] + }, + "nextPageToken": "244", + } + ), + ], + # second call + [ + True, + Mock( + json=lambda: { + "directory": { + "employees": [ + { + "detail": { + "workEmail": "chris@chiefonboarding.com", + "firstName": "chris", + "lastName": "Do", + } + }, + { + "detail": { + "workEmail": "emma@chiefonboarding.com", + "firstName": "emma", + "lastName": "Do", + } + }, + ] + } + } + ), + ], + ) + ), +) +def test_fetching_employees_paginated_response( + client, django_user_model, custom_user_import_integration_factory +): + client.force_login(django_user_model.objects.create(role=1)) + + integration = custom_user_import_integration_factory( + manifest={ + "type": "import_users", + "execute": [{"url": "http://localhost/test_api/users", "method": "GET"}], + "data_from": "directory.employees", + "data_structure": { + "email": "detail.workEmail", + "last_name": "detail.lastName", + "first_name": "detail.firstName", + }, + "initial_data_form": [], + "next_page_token_from": "nextPageToken", + "next_page": "https://localhost/test_api/users?pt={{ NEXT_PAGE_TOKEN }}", + }, + name="Google import", + ) + + url = reverse("people:import-users-hx", args=[integration.id]) + response = client.get(url, follow=True) + + assert "brian@chiefonboarding.com" in response.content.decode() + assert "jake@chiefonboarding.com" in response.content.decode() + assert "stan@chiefonboarding.com" in response.content.decode() + assert "test@chiefonboarding" in response.content.decode() + assert "emma@chiefonboarding" in response.content.decode() + assert "chris@chiefonboarding" in response.content.decode() + + +@pytest.mark.django_db +@patch( + "admin.integrations.models.Integration.run_request", + Mock(return_value=(True, Mock(json=lambda: {"directory": {"users": []}}))), +) +def test_fetching_employees_incorrect_notation( + client, django_user_model, custom_user_import_integration_factory +): + # create two users who are already in the system (should not show up) + client.force_login(django_user_model.objects.create(role=1)) + + integration = custom_user_import_integration_factory( + manifest={ + "type": "import_users", + "execute": [ + {"url": "http://localhost:8000/test_api/users", "method": "GET"} + ], + "data_from": "directory.employees", + "data_structure": { + "email": "detail.workEmail", + "last_name": "detail.lastName", + "first_name": "detail.firstName", + }, + "initial_data_form": [], + }, + name="Google import", + ) + + # directory.employees should have been directory.users based on the mock data + + url = reverse("people:import-users-hx", args=[integration.id]) + response = client.get(url, follow=True) + assert ( + "Notation 'directory.employees' not in" in response.content.decode() + ) + + +@pytest.mark.django_db +def test_import_users_create_users( + django_user_model, custom_user_import_integration_factory, mailoutbox +): + client = APIClient() + client.force_login(django_user_model.objects.create(role=1)) + + assert django_user_model.objects.all().count() == 1 + + custom_user_import_integration_factory(name="Google import") + + url = reverse("people:import-create") + client.post( + url, + data=[ + { + "first_name": "stan", + "last_name": "Do", + "email": "stan@chiefonboarding.com", + "role": 1, + }, + { + "first_name": "Peter", + "last_name": "Bla", + "email": "bla@chiefonboarding.com", + "role": 1, + }, + { + "first_name": "Jane", + "last_name": "Do", + "email": "jane@chiefonboarding.com", + "role": 3, + }, + ], + format="json", + follow=True, + ) + + # 4 users: 3 imported users + 1 admin user who created the users + assert django_user_model.objects.all().count() == 4 + + # 2 emails are send out because only two are role 1 (or 2) + assert len(mailoutbox) == 2 + + assert len(mailoutbox[0].to) == 1 + assert "stan@chiefonboarding.com" in mailoutbox[0].to[0] + assert len(mailoutbox[1].to) == 1 + assert "bla@chiefonboarding.com" in mailoutbox[1].to[0] + + # has a unique link - make sure we are not calling bulk_create as that would ignore + # the `save` method + user = django_user_model.objects.get(email="stan@chiefonboarding.com") + assert len(user.unique_url) == 8 diff --git a/back/admin/people/urls.py b/back/admin/people/urls.py index e9e068d6e..4946dec41 100644 --- a/back/admin/people/urls.py +++ b/back/admin/people/urls.py @@ -163,4 +163,24 @@ views.ColleagueToggleResourceView.as_view(), name="toggle_resource", ), + path( + "colleagues//import/", + views.ColleagueImportView.as_view(), + name="import", + ), + path( + "colleagues//import/users/", + views.ColleagueImportFetchUsersHXView.as_view(), + name="import-users-hx", + ), + path( + "colleagues/import/ignore/", + views.ColleagueImportIgnoreUserHXView.as_view(), + name="import-ignore-hx", + ), + path( + "colleagues/import/create/", + views.ColleagueImportAddUsersView.as_view(), + name="import-create", + ), ] diff --git a/back/admin/people/views.py b/back/admin/people/views.py index bd5b258ea..e336d9a3a 100644 --- a/back/admin/people/views.py +++ b/back/admin/people/views.py @@ -9,19 +9,32 @@ from django.views.generic.detail import DetailView from django.views.generic.edit import CreateView, DeleteView, UpdateView from django.views.generic.list import ListView +from django_q.tasks import async_task +from rest_framework import generics +from rest_framework.authentication import SessionAuthentication +from admin.integrations.exceptions import GettingUsersError, KeyIsNotInDataError from admin.integrations.models import Integration +from admin.integrations.import_users import ImportUser +from admin.people.serializers import UserImportSerializer from admin.resources.models import Resource -from organization.models import WelcomeMessage +from api.permissions import AdminPermission +from organization.models import Organization, WelcomeMessage from slack_bot.utils import Slack, actions, button, paragraph from users.emails import email_new_admin_cred from users.mixins import ( + AdminPermMixin, IsAdminOrNewHireManagerMixin, LoginRequiredMixin, ManagerPermMixin, ) -from .forms import ColleagueCreateForm, ColleagueUpdateForm +from .forms import ( + ColleagueCreateForm, + ColleagueUpdateForm, + EmailIgnoreForm, + UserRoleForm, +) # See new_hire_views.py for new hire functions! @@ -39,6 +52,7 @@ def get_context_data(self, **kwargs): context["slack_active"] = Integration.objects.filter( integration=Integration.Type.SLACK_BOT ).exists() + context["import_users_options"] = Integration.objects.import_users_options() context["add_action"] = reverse_lazy("people:colleague_create") return context @@ -259,3 +273,85 @@ def post(self, request, pk, *args, **kwargs): context["button_name"] = button_name context["exists"] = user.is_active return render(request, self.template_name, context) + + +class ColleagueImportView(LoginRequiredMixin, AdminPermMixin, DetailView): + """Generic view to start showing the options based on what it fetched from the + server + """ + + template_name = "colleague_import.html" + context_object_name = "integration" + + def get_queryset(self): + return Integration.objects.import_users_options() + + def get_context_data(self, **kwargs): + context = super().get_context_data(**kwargs) + context["subtitle"] = _("Import new users from a third party") + context["title"] = _("Import people") + return context + + +class ColleagueImportFetchUsersHXView(LoginRequiredMixin, AdminPermMixin, View): + """HTMLX view to get all users and return a table""" + + def get(self, request, pk, *args, **kwargs): + integration = get_object_or_404( + Integration.objects.import_users_options(), id=pk + ) + try: + # we are passing in the user who is requesting it, but we likely don't need + # them. + users = ImportUser(integration).get_import_user_candidates( + self.request.user + ) + except (KeyIsNotInDataError, GettingUsersError) as e: + return render(request, "_import_user_table.html", {"error": e}) + + return render( + request, + "_import_user_table.html", + {"users": users, "role_form": UserRoleForm}, + ) + + +class ColleagueImportIgnoreUserHXView(LoginRequiredMixin, AdminPermMixin, View): + """HTMLX view to put people on the ignore list""" + + def post(self, request, *args, **kwargs): + form = EmailIgnoreForm(request.POST) + # We always expect an email here, if it's not, then all data is likely incorrect + # as we specifically call "email" from the API + if form.is_valid(): + org = Organization.objects.get() + org.ignored_user_emails += [form.cleaned_data["email"]] + org.save() + + return HttpResponse() + + +class ColleagueImportAddUsersView(LoginRequiredMixin, generics.CreateAPIView): + permission_classes = (AdminPermission,) + authentication_classes = [ + SessionAuthentication, + ] + serializer_class = UserImportSerializer + + def create(self, request, *args, **kwargs): + serializer = self.get_serializer(data=request.data, many=True) + serializer.is_valid(raise_exception=True) + users = serializer.save() + + # users is a list, so manually checking instead of filter queryset + for user in users: + if user.is_admin_or_manager: + async_task(email_new_admin_cred, user) + + success_message = _( + "Users got imported succesfully. " + "Admins and managers will receive an email shortly." + ) + return HttpResponse( + f"

{success_message}

" + ) diff --git a/back/admin/sequences/tests.py b/back/admin/sequences/tests.py index bb60e49cc..c9405fbdc 100644 --- a/back/admin/sequences/tests.py +++ b/back/admin/sequences/tests.py @@ -856,13 +856,13 @@ def test_sequence_item_test_message(client, admin_factory, mailoutbox): @pytest.mark.django_db def test_sequence_default_templates_integrations( - client, admin_factory, integration_factory + client, admin_factory, integration_factory, custom_integration_factory ): admin = admin_factory() client.force_login(admin) url = reverse("sequences:template_list") - integration_factory(integration=Integration.Type.CUSTOM) - integration_factory(integration=Integration.Type.CUSTOM) + custom_integration_factory() + custom_integration_factory() integration_factory(integration=Integration.Type.SLACK_ACCOUNT_CREATION) integration_factory(integration=Integration.Type.GOOGLE_LOGIN) diff --git a/back/admin/settings/templates/token_create.html b/back/admin/settings/templates/token_create.html index 398fdae28..5801dcd50 100644 --- a/back/admin/settings/templates/token_create.html +++ b/back/admin/settings/templates/token_create.html @@ -8,7 +8,7 @@
{% csrf_token %} {{ form|crispy }} - +
{% if object.has_oauth and not object.enabled_oauth %} {% translate "Redirect URL:" %} {{ request.scheme }}://{{ request.get_host }}{% url 'integrations:oauth-callback' object.id %} diff --git a/back/back/templatetags/general.py b/back/back/templatetags/general.py index 738f6f3e1..76436d5b5 100644 --- a/back/back/templatetags/general.py +++ b/back/back/templatetags/general.py @@ -1,3 +1,4 @@ +import hashlib import json from datetime import timedelta @@ -89,3 +90,12 @@ def show_start_card(conditions, idx, new_hire): return True return False + + +@register.filter(name="hash") +def hash(text): + """ + Hashes the value. Could be used to get uuids (for data that is unique). + """ + text = text.encode() + return hashlib.sha256(text).hexdigest() diff --git a/back/conftest.py b/back/conftest.py index d045c1f0b..c302ae221 100644 --- a/back/conftest.py +++ b/back/conftest.py @@ -6,7 +6,11 @@ from admin.admin_tasks.factories import AdminTaskFactory from admin.appointments.factories import AppointmentFactory from admin.badges.factories import BadgeFactory -from admin.integrations.factories import CustomIntegrationFactory, IntegrationFactory +from admin.integrations.factories import ( + CustomIntegrationFactory, + CustomUserImportIntegrationFactory, + IntegrationFactory, +) from admin.introductions.factories import IntroductionFactory from admin.notes.factories import NoteFactory from admin.preboarding.factories import PreboardingFactory @@ -95,8 +99,9 @@ def run_around_tests(request, settings): register(PendingSlackMessageFactory) register(PendingTextMessageFactory) register(BadgeFactory) -register(CustomIntegrationFactory) register(IntegrationFactory) +register(CustomIntegrationFactory) +register(CustomUserImportIntegrationFactory) register(IntegrationConfigFactory) register(ConditionWithItemsFactory) register(FileFactory) diff --git a/back/organization/migrations/0036_organization_ignored_user_emails.py b/back/organization/migrations/0036_organization_ignored_user_emails.py new file mode 100644 index 000000000..458369eff --- /dev/null +++ b/back/organization/migrations/0036_organization_ignored_user_emails.py @@ -0,0 +1,23 @@ +# Generated by Django 4.2.5 on 2023-09-11 16:00 + +import django.contrib.postgres.fields +from django.db import migrations, models + + +class Migration(migrations.Migration): + dependencies = [ + ("organization", "0035_merge_20230613_1354"), + ] + + operations = [ + migrations.AddField( + model_name="organization", + name="ignored_user_emails", + field=django.contrib.postgres.fields.ArrayField( + base_field=models.EmailField(max_length=254), + default=list, + help_text="Emails which get ignored by the importer", + size=None, + ), + ), + ] diff --git a/back/organization/models.py b/back/organization/models.py index 91918efe0..6d95b144d 100644 --- a/back/organization/models.py +++ b/back/organization/models.py @@ -152,6 +152,11 @@ class Organization(models.Model): "See documentation if you want to use your own." ), ) + ignored_user_emails = ArrayField( + models.EmailField(), + default=list, + help_text="Emails which get ignored by the importer", + ) object = ObjectManager() objects = models.Manager() diff --git a/docs/Integrations.md b/docs/Integrations.md index 6ae4c8b9b..0723366af 100644 --- a/docs/Integrations.md +++ b/docs/Integrations.md @@ -3,15 +3,20 @@ order: 65 --- # Integrations / Webhooks (Beta) -ChiefOnboarding allows you to create integrations or webhooks that can be triggered either manually or through a sequence. You can view some examples at https://integrations.chiefonboarding.com. You are free to copy them. If you created a cool new integrations, then please post it there. That way we can help eachother out by not re-inventing the wheel! Thanks a lot! +ChiefOnboarding allows you to create integrations or trigger webhooks that can be triggered either manually or through a sequence. You can view some examples at https://integrations.chiefonboarding.com. You are free to copy them. If you created a cool new integrations, then please post it there. That way we can help eachother out by not re-inventing the wheel! Thanks a lot! +There are two types of integrations: +1. An integration to trigger a URL on an third party service +2. An "import users" integration, which allows you to import users from a third party to ChiefOnboarding (manually) + +## Integration to trigger a third party url Here is an example of a manifest of an integration (in this case to add a user to an Asana team): :::code source="static/manifest.json" ::: Let's go over the items: -## Form +### Form This is the form that is shown to you when you add this to a sequence. The form should consist of items that you would like to have different for different type of people. In the example above, the form allows you to pick a specific team to add the user to. For example: you probably don't want to add a developer to the HR team in Asana. So, the team is what you can customize every time you add the integration to a user or in a sequence. You can customize where we get the info from and what an admin should fill in. The `form` should always be an array and allows the following properties: @@ -39,7 +44,7 @@ For `choice`: `items`: (if you are not fetching items from a url) You can add an array here with objects in them with the props: `id` and `name`. For example: `[{"id": "233", "name": "option 1"}, {"id": "234", "name": "option 2"}]`. -## Exists +### Exists Exists is an option to check if a user is already part of a team. If you add this property to your manifest then it will show up under new hire -> access. From there, you will be able to manaully enable/disable an account for them. Generally, you should skip this option if you are making any calls not related to account provisioning. `url`: The url to check if the user exists. Everything that comes back is parsed to a string and then checked against. @@ -52,7 +57,7 @@ Exists is an option to check if a user is already part of a team. If you add thi `fail_when_4xx_response_code`: Default: True. If the server response with a 4xx status code, then that's considered a failing request. In some cases, apis will return a 404 if the user does not exist. In that case, set this to `False`, so it can check for the `expected` value. -## Execute +### Execute These requests will be ran when this integration gets triggered. `url`: The url where the request will be made to. @@ -63,10 +68,10 @@ These requests will be ran when this integration gets triggered. `headers`: (optionally) This will overwrite the default headers. -## Headers +### Headers These headers will be send with every request. These could include some sort of token variable for authentication. -## Oauth +### Oauth If you need to use OAuth2 to get a token, then you will need to use this. Just create a prop called `oauth` and then in that use these properties: `authenticate_url`: This is the url that is used to send the user to the login/authorize the connection (this should be a url to the third party). This is always a `GET` request. It expects an url here. @@ -78,7 +83,7 @@ If you need to use OAuth2 to get a token, then you will need to use this. Just c `without_code`: Default: `False`. Enable this is a valid callback won't return a `code` query in the url. In some cases, we don't get it and also not need it. -## Initial data form +### Initial data form This is a form that you can create to fill in when you add this integration to your instance. Any sensitive info should be filled in here, instead of in the manifest itself. Data that gets filled in here will be saved encrypted in the database. The manifest itself does not get encrypted. So, again, any tokens, authentication, sensitive info should be filled in through this form and not hardcoded! @@ -91,7 +96,7 @@ You can obviously add as many as you want. You can use these variables by using `description`: Any other info you want to leave to make it clear where to find this value. Mainly used for documentation and/or sharing. -## Post execute notification +### Post execute notification Defined as `post_execute_notification`. Gives you the ability to send a text message or email to someone after this integration has been completed. `type`: Either `email`, or `text`. Depends if you want to send an email or text message. @@ -103,7 +108,7 @@ Defined as `post_execute_notification`. Gives you the ability to send a text mes `message`: The message that should be send (plain text). -## Variables +### Variables Throughout the manifest you can use the variables that you have defined in the `initial_data_form` or the `form` wrapped around in double curly brackets. On top of that, you can also use new hire values. You can use: `email`: New hire's email address @@ -133,5 +138,74 @@ Please do not overwrite these with your own ids ## Notes * If triggering an integration fails, then it will retry the entire integration again one hour after failing. If it fails again, it will not retry. -* Integrations/Webhooks are currently in beta. Features will be added to it (OAuth support soon!). * If you are using any of the integrations from the repo at: https://integrations.chiefonboarding.com then you have to validate them yourself. This is a user repository and we do not actively moderate the submissions there. Please always validate the urls where requests are going to make sure it's legit. + + +## Import user integration +You can create custom import integrations to pull users from a third party and put them in ChiefOnboarding. This is fairly universal and will work with most APIs. A sample integration config will look like this: + +:::code source="static/import_user_manifest.json" ::: + +The setup is very similar to what we have for other integrations to trigger a webhook. The most notable differences are: + +`"type": "import_users"`: this is to indicate that this integration is only used to import users into ChiefOnboarding. + +`data_from`: this is to indicate where the users are located in the response. You can use a dot notation if need to go deep into the json to get the data. + +We then also need to define where the data is stored in the users array (for each object). We can do that with `data_structure`. In there, you can define the values you want to copy. +Please note that only `email`, `first_name` and `last_name` can be used for now. + +``` +"data_structure": { + "email": "workEmail", + "last_name": "lastName", + "first_name": "firstName" +}, +``` + +So for the current config, we expect a JSON response from the third party like this: +``` +{ + employees: [ + { + "firstName: "John", + "lastName: "Do", + "workEmail": "john@do.com" + }, + { + "firstName: "Jane", + "lastName: "Do", + "workEmail": "jane@do.com" + } + ] +} +``` + +Other values in the JSON will be ignored. + +### Paginated response +Sometimes, we might not get all items at once. We have something to cover that too. + +`amount_pages_to_fetch`: Default: 5. Maximum amount of page to fetch. It will stop earlier if there are no users found anymore. There is a limit to this number. Please see the note below. + +There are two ways of fetching a new page: +1. Sometimes an API will only return a token and we will have to build the url ourselves. (Google does this for example) +2. In most cases, you will get a full url that you can use to fetch the new page of users. + +For case 1, you will need to provide two items (the url and where the token is): + +`next_page_token_from`: the place to look for the next page token. You can use the dot notation to do go deeper into the JSON. If it's not found, it will stop. +`next_page`: This should be a fixed url that you should put here. You can use `{{ NEXT_PAGE_TOKEN }}` as the variable to include the previously found `next_page_token_from` value. + +For example, this is necessary for Google: + +``` +"next_page": "https://admin.googleapis.com/admin/directory/v1/users?customer=my_customer&maxResults=500&pageToken={{ NEXT_PAGE_TOKEN }}", +"next_page_token_from": "nextPageToken" +``` + +For case 2, you will only need to provide the place where we need to look for the URL with this item: + +`next_page_from`: the place to look for the next page url. You can use the dot notation to do go deeper into the JSON. If it's not found, it will stop. + +Note: fetching users is being done live when you visit the page. If you set a high amount of pages to be fetched, then this might cause a timeout on the server. It makes rendering all users on the client also very sluggish. We recommend to not load more than 5000 users or not more than 10 pages (if a timeout of 30 seconds is set on your server (like Heroku for example)), whichever comes first. If you do need more, then we recommend going for an alternative method of importing users. diff --git a/docs/static/import_user_manifest.json b/docs/static/import_user_manifest.json new file mode 100644 index 000000000..4a0fe14fd --- /dev/null +++ b/docs/static/import_user_manifest.json @@ -0,0 +1,38 @@ +{ + "form": [], + "type": "import_users", + "execute": [ + { + "url": "https://api.bamboohr.com/api/gateway.php/{{COMPANY_ID}}/v1/reports/{{REPORT_ID}}", + "method": "GET" + } + ], + "headers": { + "Accept": "application/json", + "Content-Type": "application/json", + "Authorization": "Basic {{KEY}}:x" + }, + "data_from": "employees", + "data_structure": { + "email": "workEmail", + "last_name": "lastName", + "first_name": "firstName" + }, + "initial_data_form": [ + { + "id": "KEY", + "name": "The BambooHR api key", + "description": "Go to: https://.bamboohr.com/settings/permissions/api.php to get one" + }, + { + "id": "REPORT_ID", + "name": "The id of the report", + "description": "Go to: https://.bamboohr.com/app/reports/ to find the id of the report. click on the report and then look at the url. There is a number that will represent the ID of the report." + }, + { + "id": "COMPANY_ID", + "name": "The id of the company", + "description": "When you login you get a domain like this: https://.bamboohr.com/. The '' is your domain name. " + } + ] +}