Corrections + Ajout de tests

Remove export as csv test

Fix/migrations (#285)

* Move create cor_type_site bib_type_site cor_module_type to geonature

* Remove id_nomenclature_type_site and migrate model to GeoNature

* Move t_observations to GeoNature

* Rename cor_type_site to cor_site_type

* Bump GeoNature

db downgrade does not work with data (#280)

Fix test export route (#293)

fix: organism_actors attributes

fix: clean python files imports + BLACK 24 (#290)

* fix: remove relative imports and unused libs

* fix: remove unused modules

* Add import fixture monitoring_users

* Reorder imports

* Black

---------

Co-authored-by: Mathieu ROUDAUT <[email protected]>
Co-authored-by: amandine-sahl <[email protected]>

fix: use table models migrated in geonature core BibTypesSites (#292)

* fix: use table models migrated in geonature core

* fix: imports in tests files

* Import TNomenclature from pypnnomenclature

---------

Co-authored-by: Mathieu ROUDAUT <[email protected]>
Co-authored-by: amandine-sahl <[email protected]>

Fix/clean transfert permissions ALL TO MONTORING_XXX (#291)

* Script sql de migration des permissions

* Changelog

Fix : sqlalchemy query

Fix/detail site url - cf #298 (#299)

* Correction url lien vers détail site

* Harmonisation popup carto

Fix/synchro on synthese (#288)

* remove silented errors when insert in synthese

* add migration for trigger delete in synthese
---------

Co-authored-by: amandine-sahl <[email protected]>

Feat/multiple geom type (#296)

* allow multiple geom type

* Sites avec plusieurs type de géométrie par défaut

---------

Co-authored-by: amandine-sahl <[email protected]>

Correction affichage propriétés spécifiques - cf #303 (#304)

Fix : migration modification for permissions (#305)

* Fix : migration modification for permissions

Ajout interaction carte liste (#300)

Fix downgrade declare available types sites

feat: add unit tests for generic monitoring routes and utils (#295)

* feat: add test for config route

* feat: add data_utils tests

* fix: response code for data utils route

* fix: rename test to match source file

* fix: remove useless file

* fix: remove legacy code

* feat: add tests for utils and error

Harmonisation des processus d'exécution de sql (#308)

* Harmonisation des processus d'exécution de sql

* Add test forbidden sql instructions

fix: filter ,sort and page on datatable (#310)

* fix: filter ,sort and page on datatable

Missing changes to check according to event on table(sort, filter, page)
Refact code for ngOnChanges

Reviewed-by: andriac

* style(front): apply prettier

Reviewed-by: andriacap

backend/gn_module_monitoring/blueprint.py

@@ -4,11 +4,12 @@
 """
 
 from flask import Blueprint, current_app
-from geonature.core.admin.admin import admin as flask_admin
+
 from geonature.utils.env import DB
+from geonature.core.admin.admin import admin as flask_admin
 
 from gn_module_monitoring.monitoring.admin import BibTypeSiteView
-from .command.cmd import commands
+from gn_module_monitoring.command.cmd import commands
 
 blueprint = Blueprint(
     "monitorings", __name__, template_folder=current_app.config["MEDIA_FOLDER"] + "/monitorings"

backend/gn_module_monitoring/command/cmd.py

@@ -1,29 +1,26 @@
-import os
-from pydoc import cli
 import click
 
 from pathlib import Path
 from flask.cli import with_appcontext
 from sqlalchemy.sql import text, select
 
-from geonature.utils.env import DB, BACKEND_DIR
+from geonature.utils.env import DB
 from geonature.core.gn_synthese.models import TSources
 from geonature.core.gn_synthese.utils.process import import_from_table
 from geonature.core.gn_commons.models import TModules
 
+from gn_module_monitoring.config.repositories import get_config
+from gn_module_monitoring.config.utils import monitoring_module_config_path
+from gn_module_monitoring.monitoring.models import TMonitoringModules
+from gn_module_monitoring.modules.repositories import get_simple_module
 
-from ..monitoring.models import TMonitoringModules
-from ..config.repositories import get_config
-from ..config.utils import json_from_file, monitoring_module_config_path
-from ..modules.repositories import get_simple_module
-
-from .utils import (
-    process_export_csv,
+from gn_module_monitoring.command.utils import (
     process_available_permissions,
     remove_monitoring_module,
     add_nomenclature,
     available_modules,
     installed_modules,
+    process_sql_files,
 )
 
 
@@ -33,9 +30,18 @@
 def cmd_process_all(module_code):
     """
     Met à jour les paramètres de configuration pour un module
+        Fichiers sql synthese et export
     """
-    # process export csv
-    process_export_csv(module_code)
+    if module_code:
+        modules = [module_code]
+    else:
+        modules = [module["module_code"] for module in installed_modules()]
+
+    for module in modules:
+        # process Synthese
+        process_sql_files(dir=None, module_code=module, depth=1)
+        # process Exports
+        process_sql_files(dir="exports/csv", module_code=module, depth=None, allowed_files=None)
 
 
 @click.command("process_export_csv")
@@ -45,7 +51,13 @@ def cmd_process_export_csv(module_code):
     """
     Met à jour les fichiers pour les exports pdf
     """
-    process_export_csv(module_code)
+    if module_code:
+        modules = [module_code]
+    else:
+        modules = [module["module_code"] for module in installed_modules()]
+
+    for module in modules:
+        process_sql_files(dir="exports/csv", module_code=module, depth=None, allowed_files=None)
 
 
 @click.command("install")
@@ -98,8 +110,10 @@ def cmd_install_monitoring_module(module_code):
     except Exception:
         pass
 
-    # process export csv
-    process_export_csv(module_code)
+    # process Synthese
+    process_sql_files(dir=None, module_code=module_code, depth=1)
+    # process Exports
+    process_sql_files(dir=None, module_code=module_code, depth=None, allowed_files=None)
 
     config = get_config(module_code, force=True)
 
@@ -141,23 +155,6 @@ def cmd_install_monitoring_module(module_code):
     process_available_permissions(module_code, session=DB.session)
     DB.session.commit()
 
-    #  run specific sql
-    if (module_config_dir_path / "synthese.sql").exists:
-        click.secho("Execution du script synthese.sql")
-        sql_script = module_config_dir_path / "synthese.sql"
-        txt = (
-            Path(sql_script)
-            .read_text()
-            .replace(":'module_code'", "'{}'".format(module_code))
-            .replace(":module_code", "{}".format(module_code))
-        )
-        try:
-            DB.session.execute(text(txt))
-            DB.session.commit()
-        except Exception as e:
-            print(e)
-            click.secho("Erreur dans le script synthese.sql", fg="red")
-
     # insert nomenclature
     add_nomenclature(module_code)
 
@@ -195,14 +192,13 @@ def cmd_process_available_permission_module(module_code):
         module_code ([string]): code du sous module
 
     """
-
     if module_code:
-        process_available_permissions(module_code, session=DB.session)
-        DB.session.commit()
-        return
+        modules = [module_code]
+    else:
+        modules = [module["module_code"] for module in installed_modules()]
 
-    for module in installed_modules():
-        process_available_permissions(module["module_code"], session=DB.session)
+    for module in modules:
+        process_available_permissions(module, session=DB.session)
     DB.session.commit()
 
 

backend/gn_module_monitoring/command/utils.py

@@ -1,14 +1,14 @@
 import os
+
 from pathlib import Path
 
 from flask import current_app
 from sqlalchemy import and_, text, delete, select
 from sqlalchemy.exc import IntegrityError
 from sqlalchemy.orm.exc import NoResultFound
-
 from sqlalchemy.dialects.postgresql import insert as pg_insert
 
-from geonature.utils.env import DB, BACKEND_DIR
+from geonature.utils.env import DB
 from geonature.core.gn_permissions.models import (
     PermObject,
     PermissionAvailable,
@@ -19,11 +19,15 @@
 
 from pypnnomenclature.models import TNomenclatures, BibNomenclaturesTypes
 
-from ..config.utils import json_from_file, monitoring_module_config_path, SUB_MODULE_CONFIG_DIR
+from gn_module_monitoring.config.utils import (
+    json_from_file,
+    monitoring_module_config_path,
+    SUB_MODULE_CONFIG_DIR,
+)
 
-from ..config.repositories import get_config
+from gn_module_monitoring.config.repositories import get_config
 
-from ..modules.repositories import get_module, get_source_by_code, get_modules
+from gn_module_monitoring.modules.repositories import get_module, get_source_by_code, get_modules
 
 
 """
@@ -33,6 +37,19 @@
 """
 
 
+FORBIDDEN_SQL_INSTRUCTION = [
+    "INSERT ",
+    "DROP ",
+    "DELETE ",
+    "UPDATE ",
+    "EXECUTE ",
+    "TRUNCATE ",
+    "ALTER ",
+    "GRANT ",
+    "COPY ",
+    "PERFORM ",
+]
+
 PERMISSION_LABEL = {
     "MONITORINGS_MODULES": {"label": "modules", "actions": ["R", "U", "E"]},
     "MONITORINGS_GRP_SITES": {"label": "groupes de sites", "actions": ["C", "R", "U", "D"]},
@@ -49,45 +66,63 @@
 }
 
 
-def process_for_all_module(process_func):
-    """
-    boucle sur les répertoire des module
-        et exécute la fonction <process_func> pour chacun
-        (sauf generic)
-    """
-    for module in get_modules():
-        process_func(module.module_code)
-    return
-
-
-def process_export_csv(module_code=None):
-    """
-    fonction qui va chercher les fichier sql de exports/csv et qui les joue
-    """
-
-    if not module_code:
-        """
-        pour tous les modules
-        """
-        return process_for_all_module(process_export_csv)
-
-    export_csv_dir = Path(monitoring_module_config_path(module_code)) / "exports/csv"
-
-    if not export_csv_dir.is_dir():
+def process_sql_files(
+    dir=None, module_code=None, depth=1, allowed_files=["export.sql", "synthese.sql"]
+):
+    sql_dir = Path(monitoring_module_config_path(module_code))
+    if dir:
+        sql_dir = sql_dir / "exports/csv"
+    if not sql_dir.is_dir():
         return
 
-    for root, dirs, files in os.walk(export_csv_dir, followlinks=True):
+    if not allowed_files:
+        allowed_files = []
+    count_depth = 0
+    for root, dirs, files in os.walk(sql_dir, followlinks=True):
+        count_depth = count_depth + 1
         for f in files:
             if not f.endswith(".sql"):
                 continue
-            txt = Path(Path(root) / Path(f)).read_text()
+            if not f in allowed_files and allowed_files:
+                continue
+            # Vérification commandes non autorisée
             try:
-                DB.session.execute(text(txt).bindparams(module_code=module_code))
-                DB.session.commit()
-                print("{} - export csv file : {}".format(module_code, f))
-
+                execute_sql_file(root, f, module_code, FORBIDDEN_SQL_INSTRUCTION)
+                print("{} - exécution du fichier : {}".format(module_code, f))
             except Exception as e:
-                print("{} - export csv erreur dans le script {} : {}".format(module_code, f, e))
+                print(e)
+
+        # Limite profondeur de la recherche dans les répertoires
+        if depth:
+            if count_depth >= depth:
+                break
+
+
+def execute_sql_file(dir, file, module_code, forbidden_instruction=[]):
+    """
+    Execution d'un fichier sql dans la base de donnée
+    dir : nom du répertoire
+    file : nom du fichier à éxécuter
+    module_code : code du module
+    forbidden_instruction : liste d'instructions sql qui sont proscrites du fichier.
+
+    """
+    sql_content = Path(Path(dir) / file).read_text()
+    for sql_cmd in forbidden_instruction:
+        if sql_cmd.lower() in sql_content.lower():
+            raise Exception(
+                "erreur dans le script {} instruction sql non autorisée {}".format(
+                    module_code, file, sql_cmd
+                )
+            )
+
+    try:
+        DB.engine.execute(
+            text(sql_content),
+            module_code=module_code,
+        )
+    except Exception as e:
+        raise Exception("{} - erreur dans le script {} : {}".format(module_code, file, e))
 
 
 def process_available_permissions(module_code, session):

backend/gn_module_monitoring/config/generic/site.json

@@ -6,7 +6,7 @@
   "genre": "M",
   "geom_field_name": "geom",
   "uuid_field_name": "uuid_base_site",
-  "geometry_type": "Point",
+  "geometry_type": ["Point", "LineString", "Polygon"],
   "display_properties": [
     "base_site_name",
     "base_site_code",

backend/gn_module_monitoring/config/repositories.py

@@ -2,14 +2,13 @@
     module de gestion de la configuarion des protocoles de suivi
 """
 
-import json
 import os
+
 from flask import current_app
-from .utils import (
+
+from gn_module_monitoring.config.utils import (
     customize_config,
     config_from_files,
-    get_directory_last_modif,
-    get_base_last_modif,
     json_config_from_file,
     get_id_table_location,
     process_config_display,
@@ -178,7 +177,6 @@ def get_config(module_code=None, force=False, customSpecConfig=None):
 
     # customize config
     if module:
-        custom = {}
         config["custom"] = {}
         for field_name in [
             "module_code",

backend/gn_module_monitoring/config/utils.py

@@ -1,16 +1,16 @@
 import os, datetime, time
 import importlib
-from pathlib import Path
 import json
+from pathlib import Path
 
 from sqlalchemy import and_, select
-from sqlalchemy.exc import NoResultFound
 
-from geonature.core.gn_commons.models import BibTablesLocation, TModules
-from geonature.utils.errors import GeoNatureError
 from geonature.utils.env import DB
+from geonature.utils.errors import GeoNatureError
 from geonature.utils.config import config as gn_config
-from ..monitoring.models import TMonitoringModules
+from geonature.core.gn_commons.models import BibTablesLocation, TModules
+
+from gn_module_monitoring.monitoring.models import TMonitoringModules
 
 SUB_MODULE_CONFIG_DIR = Path(gn_config["MEDIA_FOLDER"]) / "monitorings/"
 

backend/gn_module_monitoring/migrations/2003e18f248a_add_type_to_gn_modules.py

@@ -5,6 +5,7 @@
 Create Date: 2022-12-19 14:01:42.559701
 
 """
+
 from alembic import op
 import sqlalchemy as sa
 
@@ -29,7 +30,7 @@ def upgrade():
 def downgrade():
     op.execute(
         """
-        UPDATE gn_commons.t_modules AS tm SET type=NULL
+        UPDATE gn_commons.t_modules AS tm SET type=''
         FROM gn_monitoring.t_module_complements AS tmc
         WHERE tm.id_module = tmc.id_module;
     """