Feat/update dependencies (#204)

jacquesfize · web-flow · commit 2d52602a2b3c · 2023-12-07T13:23:33.000+01:00
Update to SQLAlchemy 1.4. Part of PnX-SI/GeoNature#2751
diff --git a/.github/workflows/pytest.yml b/.github/workflows/pytest.yml
@@ -16,23 +16,23 @@ on:
 
 jobs:
   build:
-
     runs-on: ubuntu-latest
 
     strategy:
       fail-fast: false
       matrix:
+        debian-version: ["11", "12"]
         include:
-          - name: "Debian 10"
-            python-version: "3.7"
-            postgres-version: 11
-            postgis-version: 2.5
-          - name: "Debian 11"
+          - debian-version: "11"
             python-version: "3.9"
             postgres-version: 13
             postgis-version: 3.2
+          - debian-version: "12"
+            python-version: "3.9"
+            postgres-version: 15
+            postgis-version: 3.3
 
-    name: ${{ matrix.name }}
+    name: Debian ${{ matrix.debian-version }}
 
     services:
       postgres:
@@ -59,68 +59,66 @@ jobs:
           --health-retries 5
 
     steps:
-    - name: Add postgis_raster database extension
-      if: ${{ matrix.postgis-version >= 3 }}
-      run: |
-        psql -h localhost -U geonatadmin -d geonature2db -tc 'CREATE EXTENSION "postgis_raster";'
-      env:
-        PGPASSWORD: geonatpasswd
-    - name: Add database extensions
-      run: |
-        psql -h localhost -U geonatadmin -d geonature2db -tc 'CREATE EXTENSION "hstore";'
-        psql -h localhost -U geonatadmin -d geonature2db -tc 'CREATE EXTENSION "uuid-ossp";'
-        psql -h localhost -U geonatadmin -d geonature2db -tc 'CREATE EXTENSION "pg_trgm";'
-        psql -h localhost -U geonatadmin -d geonature2db -tc 'CREATE EXTENSION "unaccent";'
-      env:
-        PGPASSWORD: geonatpasswd
-    - uses: actions/checkout@v2
-      with:
-        submodules: recursive
-    - name: Set up Python ${{ matrix.python-version }}
-      uses: actions/setup-python@v2
-      with:
-        python-version: ${{ matrix.python-version }}
-    - name: Install GDAL
-      run: |
-        sudo apt update
-        sudo apt install -y libgdal-dev
-    - name: Install dependencies
-      run: |
-        python -m pip install --upgrade pip
-        python -m pip install \
-          -e ..[tests] \
-          -r requirements-dev.txt
-      working-directory: ./dependencies/GeoNature/backend
-    - name: Install database
-      run: |
-        geonature db upgrade geonature@head -x local-srid=2154
-        geonature db autoupgrade -x local-srid=2154
-        geonature taxref import-v15 --skip-bdc-statuts
-        geonature db upgrade geonature-samples@head
-        geonature db upgrade nomenclatures_taxonomie_data@head
-        geonature db upgrade ref_geo_fr_departments@head
-        geonature db upgrade ref_geo_fr_municipalities@head
-        geonature db upgrade ref_geo_inpn_grids_10@head
-      env:
-        GEONATURE_CONFIG_FILE: dependencies/GeoNature/config/test_config.toml
-    - name: Install export module backend
-      run: |
-        pip install \
-          --editable . \
-          --editable file:dependencies/Utils-Flask-SQLAlchemy#egg=utils-flask-sqlalchemy \
-          --editable file:dependencies/Utils-Flask-SQLAlchemy-Geo#egg=utils-flask-sqlalchemy-geo
-    - name: Install import module database
-      run: |
-        geonature upgrade-modules-db EXPORTS
-      env:
-        GEONATURE_CONFIG_FILE: dependencies/GeoNature/config/test_config.toml
-    - name: Test with pytest
-      run: |
-        pytest -v --cov --cov-report xml
-      env:
-        GEONATURE_CONFIG_FILE: dependencies/GeoNature/config/test_config.toml
-    - name: Upload coverage to Codecov
-      if: ${{ matrix.name == 'Debian 11' }}
-      uses: codecov/codecov-action@v2
-      with:
-        flags: pytest
+      - name: Add postgis_raster database extension
+        if: ${{ matrix.postgis-version >= 3 }}
+        run: |
+          psql -h localhost -U geonatadmin -d geonature2db -tc 'CREATE EXTENSION "postgis_raster";'
+        env:
+          PGPASSWORD: geonatpasswd
+      - name: Add database extensions
+        run: |
+          psql -h localhost -U geonatadmin -d geonature2db -tc 'CREATE EXTENSION "hstore";'
+          psql -h localhost -U geonatadmin -d geonature2db -tc 'CREATE EXTENSION "uuid-ossp";'
+          psql -h localhost -U geonatadmin -d geonature2db -tc 'CREATE EXTENSION "pg_trgm";'
+          psql -h localhost -U geonatadmin -d geonature2db -tc 'CREATE EXTENSION "unaccent";'
+        env:
+          PGPASSWORD: geonatpasswd
+      - uses: actions/checkout@v2
+        with:
+          submodules: recursive
+      - name: Set up Python ${{ matrix.python-version }}
+        uses: actions/setup-python@v2
+        with:
+          python-version: ${{ matrix.python-version }}
+      - name: Install GDAL
+        run: |
+          sudo apt update
+          sudo apt install -y libgdal-dev
+      - name: Install dependencies
+        run: |
+          python -m pip install --upgrade pip
+          python -m pip install \
+            -e ..[tests] \
+            -r requirements-dev.txt
+        working-directory: ./dependencies/GeoNature/backend
+      - name: Install database
+        run: |
+          geonature db upgrade geonature@head -x local-srid=2154
+          geonature db autoupgrade -x local-srid=2154
+          geonature taxref import-v15 --skip-bdc-statuts
+          geonature db upgrade geonature-samples@head
+          geonature db upgrade nomenclatures_taxonomie_data@head
+          geonature db upgrade ref_geo_fr_departments@head
+          geonature db upgrade ref_geo_fr_municipalities@head
+          geonature db upgrade ref_geo_inpn_grids_10@head
+        env:
+          GEONATURE_CONFIG_FILE: dependencies/GeoNature/config/test_config.toml
+      - name: Install export module backend
+        run: |
+          pip install -r requirements-dev.in
+          pip install .
+      - name: Install import module database
+        run: |
+          geonature upgrade-modules-db EXPORTS
+        env:
+          GEONATURE_CONFIG_FILE: dependencies/GeoNature/config/test_config.toml
+      - name: Test with pytest
+        run: |
+          pytest -v --cov --cov-report xml
+        env:
+          GEONATURE_CONFIG_FILE: dependencies/GeoNature/config/test_config.toml
+      - name: Upload coverage to Codecov
+        if: ${{ matrix.name == 'Debian 11' }}
+        uses: codecov/codecov-action@v2
+        with:
+          flags: pytest
diff --git a/backend/gn_module_export/admin.py b/backend/gn_module_export/admin.py
@@ -1,4 +1,5 @@
-from flask import Markup, current_app, flash
+from flask import current_app, flash
+from markupsafe import Markup
 from flask_admin.babel import gettext
 from flask_admin.contrib.sqla import ModelView
 from flask_admin.helpers import is_form_submitted
diff --git a/backend/gn_module_export/blueprint.py b/backend/gn_module_export/blueprint.py
@@ -217,7 +217,7 @@ def get_one_export_api(id_export):
     offset : numéro de page
     token : clé de l'API permettant de se passer de la connexion via un
         cookie. Peut aussi être transmis dans un entête HTTP
-        `Authorization: Bearer <token>`.
+        `AuthorizationExport: Bearer <token>`.
 
     FILTRES :
         nom_col=val: Si nom_col fait partie des colonnes
@@ -265,15 +265,16 @@ def get_one_export_api(id_export):
     token = request.args.get("token", default=None, type=str)
 
     # Try to extract token from Bearer Authorization HTTP Header
-    bearer = request.headers.get("Authorization", default=None, type=str)
-    if token is None and bearer is not None:
-        token = bearer.lstrip("Bearer ")
+    headerToken = request.headers.get("api-key", default=None, type=str)
+    if not token and headerToken:
+        token = headerToken
 
     user = g.current_user
 
-    export = Export.query.get_or_404(id_export)
+    export = DB.get_or_404(Export, id_export)
+
     scope = None
-    if user:
+    if user.is_authenticated:
         scope = get_scopes_by_action(user.id_role, "EXPORTS")["R"]
     if not export.has_instance_permission(user=user, token=token, scope=scope):
         raise Forbidden
diff --git a/backend/gn_module_export/commands.py b/backend/gn_module_export/commands.py
@@ -16,6 +16,7 @@
     ExportGenerationNotNeeded,
     ExportRequest,
 )
+from geonature.utils.env import db
 
 
 @click.command()
@@ -38,30 +39,33 @@ def generate(export_id, export_format, user_id, skip_newer_than):
     """
     scheduled_export = None
     user = None
-
     if user_id:
-        user = User.query.get(user_id)
+        user = db.session.get(User, user_id)
         if not user:
             raise ClickException(f"User {user_id} not found.")
     else:
         # If not user_id => scheduled
+        print(1)
         scheduled_export = (
             ExportSchedules.query.filter(ExportSchedules.id_export == export_id)
             .filter(ExportSchedules.format == export_format)
             .first()
         )
+        print(2)
         if not scheduled_export:
             raise ClickException(f"Schedule export {export_id} format {export_format} not found.")
         # Parameter skip_newer_than overide scheduled_export.skip_newer_than property
         if not skip_newer_than:
             skip_newer_than = scheduled_export.skip_newer_than
     try:
+        print(3)
         export_request = ExportRequest(
             id_export=export_id,
             user=user,
             format=export_format,
             skip_newer_than=skip_newer_than,
         )
+        print(4)
     except NotFound:
         raise ClickException(f"Export {export_id} not found.")
     except Forbidden:
diff --git a/backend/gn_module_export/models.py b/backend/gn_module_export/models.py
@@ -125,13 +125,11 @@ def has_instance_permission(
             return True
         if token:
             return token in map(lambda cor: cor.token, self.cor_roles_exports)
-        if user is None:
-            user = g.current_user
-        if user is None:  # no user provided and no user connected
+        if not user.is_authenticated:  # no user provided and no user connected
             return False
         if scope == 3:
             return True
-        if scope in (1, 2):
+        if 0 < scope < 3:
             allowed_id_roles = list(map(lambda user: user.id_role, self.allowed_roles))
             ids_group_of_user = list(map(lambda group: group.id_role, user.groups))
             return user.id_role in allowed_id_roles or set(ids_group_of_user) & set(
diff --git a/backend/gn_module_export/tasks.py b/backend/gn_module_export/tasks.py
@@ -10,6 +10,8 @@
 from flask import current_app
 from geonature.utils.celery import celery_app
 
+from geonature.utils.env import db
+
 from .models import Export, ExportSchedules
 from .utils_export import (
     export_data_file,
@@ -56,7 +58,7 @@ def generate_scheduled_exports(self):
 @celery_app.task(bind=True, throws=ExportGenerationNotNeeded)
 def generate_export(self, export_id, file_name, export_url, format, id_role, filters):
     logger.info(f"Generate export {export_id}...")
-    export = Export.query.get(export_id)
+    export = db.session.get(Export, export_id)
     if export is None:
         logger.warning("Export {export_id} does not exist")
     export_data_file(export_id, file_name, export_url, format, id_role, filters)
diff --git a/backend/gn_module_export/tests/conftest.py b/backend/gn_module_export/tests/conftest.py
@@ -1,2 +1,3 @@
 from geonature.tests.fixtures import *
 from geonature.tests.fixtures import _session, app, users
+from geonature.tests.test_permissions import g_permissions
diff --git a/backend/gn_module_export/tests/test_blueprint.py b/backend/gn_module_export/tests/test_blueprint.py
@@ -6,7 +6,6 @@
 from werkzeug.datastructures import Headers
 
 from geonature.tests.fixtures import *
-from geonature.tests.utils import set_logged_user_cookie
 from pypnusershub.tests.utils import set_logged_user_cookie
 
 from .fixtures import *
@@ -56,7 +55,7 @@ def test_private_export_with_token_in_header_Authorization_Bearer(self, users, e
         headers = Headers()
         headers.extend(
             {
-                "Authorization": f"Bearer {token}",
+                "api-key": token,
             }
         )
 
@@ -67,7 +66,7 @@ def test_private_export_with_token_in_header_Authorization_Bearer(self, users, e
             ),
             headers=headers,
         )
-        assert response.request.headers["Authorization"] == f"Bearer {token}"
+        assert response.request.headers["api-key"] == token
         assert response.status_code == 200
 
     def test_private_export_with_token_in_header_Authorization(self, users, exports):
@@ -76,7 +75,7 @@ def test_private_export_with_token_in_header_Authorization(self, users, exports)
         headers = Headers()
         headers.extend(
             {
-                "Authorization": token,
+                "api-key": token,
             }
         )
 
@@ -87,7 +86,7 @@ def test_private_export_with_token_in_header_Authorization(self, users, exports)
             ),
             headers=headers,
         )
-        assert response.request.headers["Authorization"] == token
+        assert response.request.headers["api-key"] == token
         assert response.status_code == 200
 
     def test_private_export_with_allowed_id_role(self, users, exports):
diff --git a/backend/gn_module_export/tests/test_commands.py b/backend/gn_module_export/tests/test_commands.py
@@ -8,7 +8,7 @@
 from .fixtures import *
 
 
-@pytest.mark.usefixtures("client_class", "temporary_transaction")
+@pytest.mark.usefixtures("client_class", "temporary_transaction", "g_permissions")
 class TestExportsCommands:
     # deletes the files hand created files
     def test_generate_user(self, export_directories, exports, users):
@@ -23,6 +23,7 @@ def test_generate_user(self, export_directories, exports, users):
                 users["admin_user"].id_role,
             ],
         )
+        # print(result.stdout)
         assert result.exit_code == 0
 
         # Test skip-newer-than
@@ -67,4 +68,5 @@ def test_schedule_user(self, export_directories, exports_schedule):
             generate,
             [str(exports_schedule.export.id), "--format", "csv"],
         )
+        print("-----", result.stdout)
         assert result.exit_code == 0
diff --git a/backend/gn_module_export/tests/test_tasks.py b/backend/gn_module_export/tests/test_tasks.py
@@ -7,7 +7,8 @@
 import os
 
 from geonature.tests.fixtures import *
-from geonature.tests.utils import set_logged_user_cookie
+
+# from geonature.tests.utils import set_logged_user_cookie
 from pypnusershub.tests.utils import set_logged_user_cookie
 
 from geonature.core.notifications.models import Notification
@@ -22,7 +23,7 @@
 from .fixtures import *
 
 
-@pytest.mark.usefixtures("client_class", "temporary_transaction")
+@pytest.mark.usefixtures("client_class", "temporary_transaction", "g_permissions")
 class TestExportsTasks:
     # deletes the files hand created files
     def test_export_clean(self, export_directories):
diff --git a/backend/gn_module_export/utils_export.py b/backend/gn_module_export/utils_export.py
diff --git a/dependencies/GeoNature b/dependencies/GeoNature
diff --git a/requirements-dev.in b/requirements-dev.in

Original file line number	Diff line number	Diff line change
`@@ -1,2 +1,3 @@`
`1`	`1`	`from geonature.tests.fixtures import *`
`2`	`2`	`from geonature.tests.fixtures import _session, app, users`
	`3`	`+from geonature.tests.test_permissions import g_permissions`
Original file line number	Diff line number	Diff line change
`@@ -6,7 +6,6 @@`
`6`	`6`	`from werkzeug.datastructures import Headers`
`7`	`7`
`8`	`8`	`from geonature.tests.fixtures import *`
`9`		`-from geonature.tests.utils import set_logged_user_cookie`
`10`	`9`	`from pypnusershub.tests.utils import set_logged_user_cookie`
`11`	`10`
`12`	`11`	`from .fixtures import *`
`@@ -56,7 +55,7 @@ def test_private_export_with_token_in_header_Authorization_Bearer(self, users, e`
`56`	`55`	`headers = Headers()`
`57`	`56`	`headers.extend(`
`58`	`57`	`{`
`59`		`- "Authorization": f"Bearer {token}",`
	`58`	`+ "api-key": token,`
`60`	`59`	`}`
`61`	`60`	`)`
`62`	`61`
`@@ -67,7 +66,7 @@ def test_private_export_with_token_in_header_Authorization_Bearer(self, users, e`
`67`	`66`	`),`
`68`	`67`	`headers=headers,`
`69`	`68`	`)`
`70`		`- assert response.request.headers["Authorization"] == f"Bearer {token}"`
	`69`	`+ assert response.request.headers["api-key"] == token`
`71`	`70`	`assert response.status_code == 200`
`72`	`71`
`73`	`72`	`def test_private_export_with_token_in_header_Authorization(self, users, exports):`
`@@ -76,7 +75,7 @@ def test_private_export_with_token_in_header_Authorization(self, users, exports)`
`76`	`75`	`headers = Headers()`
`77`	`76`	`headers.extend(`
`78`	`77`	`{`
`79`		`- "Authorization": token,`
	`78`	`+ "api-key": token,`
`80`	`79`	`}`
`81`	`80`	`)`
`82`	`81`
`@@ -87,7 +86,7 @@ def test_private_export_with_token_in_header_Authorization(self, users, exports)`
`87`	`86`	`),`
`88`	`87`	`headers=headers,`
`89`	`88`	`)`
`90`		`- assert response.request.headers["Authorization"] == token`
	`89`	`+ assert response.request.headers["api-key"] == token`
`91`	`90`	`assert response.status_code == 200`
`92`	`91`
`93`	`92`	`def test_private_export_with_allowed_id_role(self, users, exports):`