feat: Implement smt plugin and operations

Author: GermanMT

.prospector.yaml

@@ -6,7 +6,7 @@ pylint:
   disable:
     - super-init-not-called
     - unsubscriptable-object
-    - dangerous-default-value
+    - too-many-arguments
 
 mccabe:
   run: true

app/apis/pypi_service.py

@@ -45,21 +45,17 @@ async def requires_packages(pkg_name: str, version_dist: str) -> dict[str, dict[
     ).json()['info']['requires_dist']
 
     if response:
-        require_packages = {}
+        require_packages: dict[str, Any] = {}
 
         for dist in response:
             data = dist.split(';')
 
-            '''
-            TODO: En el futuro sería interesante construir el grado teniendo en cuenta los extras
-            '''
+            # TODO: En el futuro sería interesante construir el grado teniendo en cuenta los extras
             if len(data) > 1:
                 if 'extra' in data[1]:
                     continue
 
-            '''
-            TODO: Eliminamos que se puedan requerir extras
-            '''
+            # TODO: Eliminamos que se puedan requerir extras
             if '[' in data[0]:
                 pos_1 = await get_first_position(data[0], ['['])
                 pos_2 = await get_first_position(data[0], [']']) + 1
@@ -73,9 +69,13 @@ async def requires_packages(pkg_name: str, version_dist: str) -> dict[str, dict[
             raw_ctcs = data[pos:]
 
             if dist in require_packages:
-                if not isinstance(require_packages[dist], str):
-                    require_packages[dist].update(await parse_constraints(raw_ctcs))
+                if isinstance(require_packages[dist], dict):
+                    ctcs = await parse_constraints(raw_ctcs)
+                    if isinstance(ctcs, dict):
+                        require_packages[dist].update(ctcs)
             else:
+                if '.' not in raw_ctcs:
+                    continue
                 require_packages[dist] = await parse_constraints(raw_ctcs)
 
         return require_packages

app/controllers/generate_controller.py

@@ -107,7 +107,8 @@ async def generate_packages(package_name: str, version: dict[str, Any]) -> None:
 
                 print(package_name)
                 await no_exist_package(
-                    package_name, constraints,
+                    package_name,
+                    constraints,
                     'pypi',
                     version['_id'],
                     'version'

app/controllers/operation_controller.py

@@ -1,13 +1,15 @@
-from typing import Any
-
 from fastapi import APIRouter, status
 from fastapi.responses import JSONResponse
 
-from app.controllers.network_info import NetworkInfo
-from app.controllers.network_to_smt import NetworkToSMT
-from app.controllers.operations import (FilterConfigs, MaximizeImpact,
-                                        MinimizeImpact, NumberOfProducts,
-                                        ValidModel)
+from flamapy.metamodels.dn_metamodel.operations import NetworkInfo
+from flamapy.metamodels.smt_metamodel.transformations import NetworkToSMT
+from flamapy.metamodels.smt_metamodel.operations import (
+    FilterConfigs,
+    MaximizeImpact,
+    MinimizeImpact,
+    NumberOfProducts,
+    ValidModel
+)
 from app.controllers.serialize_controller import serialize_network
 from app.services.version_service import get_release_by_values
 from app.utils.json_encoder import json_encoder
@@ -32,25 +34,25 @@ async def valid_file(network_id: str, file_name: str, agregator: str) -> JSONRes
     smt_transform = NetworkToSMT(dependency_network, agregator)
     smt_transform.transform()
     smt_model = smt_transform.destination_model
-    operation = ValidModel()
-    operation.execute(smt_model, file_name)
+    operation = ValidModel(file_name)
+    operation.execute(smt_model)
     result = {'is_valid': operation.get_result()}
     return JSONResponse(status_code=status.HTTP_200_OK, content=json_encoder(result))
 
 
-@router.post(
-    '/operation/number_of_products/{network_id}',
-    response_description='Number of products operation'
-)
-async def number_of_products(network_id: str, file_name: str, agregator: str) -> JSONResponse:
-    dependency_network = await serialize_network(network_id)
-    smt_transform = NetworkToSMT(dependency_network, agregator)
-    smt_transform.transform()
-    smt_model = smt_transform.destination_model
-    operation = NumberOfProducts()
-    operation.execute(smt_model, file_name)
-    result = {'number_of_products': operation.get_result()}
-    return JSONResponse(status_code=status.HTTP_200_OK, content=json_encoder(result))
+# @router.post(
+#     '/operation/number_of_products/{network_id}',
+#     response_description='Number of products operation'
+# )
+# async def number_of_products(network_id: str, file_name: str, agregator: str) -> JSONResponse:
+#     dependency_network = await serialize_network(network_id)
+#     smt_transform = NetworkToSMT(dependency_network, agregator)
+#     smt_transform.transform()
+#     smt_model = smt_transform.destination_model
+#     operation = NumberOfProducts(file_name)
+#     operation.execute(smt_model)
+#     result = {'number_of_products': operation.get_result()}
+#     return JSONResponse(status_code=status.HTTP_200_OK, content=json_encoder(result))
 
 
 @router.post(
@@ -59,16 +61,16 @@ async def number_of_products(network_id: str, file_name: str, agregator: str) ->
 )
 async def minimize_impact(
     network_id: str,
-    file_name: str,
     agregator: str,
-    op_configs: dict[str, int] = {'limit': 10}
+    file_name: str,
+    limit: int
 ) -> JSONResponse:
     dependency_network = await serialize_network(network_id)
     smt_transform = NetworkToSMT(dependency_network, agregator)
     smt_transform.transform()
     smt_model = smt_transform.destination_model
-    operation = MinimizeImpact(**op_configs)
-    operation.execute(smt_model, file_name)
+    operation = MinimizeImpact(file_name, limit)
+    operation.execute(smt_model)
     result = await get_release_by_values(operation.get_result())
     return JSONResponse(status_code=status.HTTP_200_OK, content=json_encoder({'result': result}))
 
@@ -79,16 +81,16 @@ async def minimize_impact(
 )
 async def maximize_impact(
     network_id: str,
-    file_name: str,
     agregator: str,
-    op_configs: dict[str, int] = {'limit': 10}
+    file_name: str,
+    limit: int
 ) -> JSONResponse:
     dependency_network = await serialize_network(network_id)
     smt_transform = NetworkToSMT(dependency_network, agregator)
     smt_transform.transform()
     smt_model = smt_transform.destination_model
-    operation = MaximizeImpact(**op_configs)
-    operation.execute(smt_model, file_name)
+    operation = MaximizeImpact(file_name, limit)
+    operation.execute(smt_model)
     result = await get_release_by_values(operation.get_result())
     return JSONResponse(status_code=status.HTTP_200_OK, content=json_encoder({'result': result}))
 
@@ -99,19 +101,17 @@ async def maximize_impact(
 )
 async def filter_configs(
     network_id: str,
-    file_name: str,
     agregator: str,
-    op_configs: dict[str, Any] = {
-        'max_threshold': 10.,
-        'min_threshold': 0.,
-        'limit': 10
-    }
+    file_name: str,
+    max_threshold: float,
+    min_threshold: float,
+    limit: int
 ) -> JSONResponse:
     dependency_network = await serialize_network(network_id)
     smt_transform = NetworkToSMT(dependency_network, agregator)
     smt_transform.transform()
     smt_model = smt_transform.destination_model
-    operation = FilterConfigs(**op_configs)
-    operation.execute(smt_model, file_name)
+    operation = FilterConfigs(file_name, max_threshold, min_threshold, limit)
+    operation.execute(smt_model)
     result = await get_release_by_values(operation.get_result())
     return JSONResponse(status_code=status.HTTP_200_OK, content=json_encoder({'result': result}))

app/controllers/serialize_controller.py

@@ -1,21 +1,26 @@
 from typing import Any
 
 from bson import ObjectId
-from flamapy.metamodels.dn_metamodel.models import (DependencyNetwork, Package,
-                                                    RequirementFile, Version)
+from flamapy.metamodels.dn_metamodel.models import (
+    DependencyNetwork,
+    RequirementFile,
+    Package,
+    Version
+)
 from flamapy.metamodels.dn_metamodel.transformations import SerializeNetwork
 
 from app.services.package_edge_service import read_package_edge_by_id
 from app.services.serialize_service import aggregate_network_by_id
 
-all_package_edges: list[dict[str, Any]] = []
+all_package_edges: dict[str, dict[str, Any]] = {}
 all_package_edges_ids: list[ObjectId] = []
+all_versions: dict[str, Version] = {}
+all_packages: dict[str, Package] = {}
 
 
 async def serialize_network(network_id: str) -> DependencyNetwork | None:
     network = await aggregate_network_by_id(network_id)
     requirement_files = network['requirement_files']
-    del network['_id']
     network['requirement_files'] = []
     serializer = SerializeNetwork(source_model=network)
     serializer.transform()
@@ -31,50 +36,57 @@ async def read_requirement_files(
         if not requirement_file['package_edges']:
             continue
         package_edges = requirement_file['package_edges']
-        del requirement_file['package_edges']
-        del requirement_file['_id']
         requirement_file['packages'] = []
         requirement_file = RequirementFile(**requirement_file)
         dn_model.requirement_files.append(requirement_file)
         await read_packages(package_edges, requirement_file)
         all_package_edges.clear()
         all_package_edges_ids.clear()
+        all_versions.clear()
+        all_packages.clear()
 
 
 async def read_packages(
     package_egdes: list[dict[str, Any]],
     parent: RequirementFile | Version
 ) -> None:
     for package_edge in package_egdes:
-        package = Package(**{'name': package_edge['package_name'], 'versions': []})
-        parent.packages.append(package)
-        await read_versions(package_edge['versions'], package)
+        key = package_edge['package_name'] + str(package_edge['constraints'])
+        if key in all_packages:
+            parent.packages.append(all_packages[key])
+        else:
+            package = Package(**{
+                'name': package_edge['package_name'],
+                'constraints': package_edge['constraints'],
+                'versions': []
+            })
+            all_packages[key] = package
+            parent.packages.append(package)
+            await read_versions(package_edge['versions'], package)
 
 
 async def read_versions(versions: list[dict[str, Any]], package: Package) -> None:
     for version in versions:
-        if '_id' not in version:
-            continue
-        package_edges = await search_package_edge(version['package_edges'])
-        del version['package_edges']
-        del version['_id']
+        package_edges = []
+        if 'package_edges' in version:
+            package_edges = await search_package_edge(version['package_edges'])
         version['packages'] = []
         version = Version(**version)
         package.versions.append(version)
-        await read_packages(package_edges, version)
+        if package_edges:
+            await read_packages(package_edges, version)
 
 
 async def search_package_edge(package_edge_ids: list[ObjectId]) -> list[dict[str, Any]]:
     package_edges = []
     for package_edge_id in package_edge_ids:
+        key = str(package_edge_id)
         if package_edge_id not in all_package_edges_ids:
             package_edge = await read_package_edge_by_id(package_edge_id, 'pypi')
-            all_package_edges.append(package_edge)
+            all_package_edges[key] = package_edge
             all_package_edges_ids.append(package_edge_id)
             package_edges.append(package_edge)
             continue
-        for package_edge in all_package_edges:
-            if package_edge['_id'] == package_edge_id:
-                package_edges.append(package_edge)
-                break
+        package_edge = all_package_edges[key]
+        package_edges.append(package_edge)
     return package_edges

app/utils/get_session.py

@@ -7,6 +7,5 @@ async def get_session() -> Session:
     session = Session()
     retry = Retry(connect=4, backoff_factor=0.5)
     adapter = HTTPAdapter(max_retries=retry)
-    session.mount('http://', adapter)
-    session.mount('https://', adapter)
+    session.mount('http*://', adapter)
     return session

mongo_seed/seeds/cves.bson.gz

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:efcab8082b6b1efb19fe6c796683ae2e2a310f94a0e0bd32cd9164910b249994
-size 134086565
+oid sha256:3278c194abf20f166b8eb929d769ad550ec11bb78d9e79889ae56246a302c908
+size 151949839

mongo_seed/seeds/env_variables.bson.gz

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:a2117dfff06661366070174e0b87493eae8a8668ab5d17687183d3228f0544e3
+oid sha256:0ad6777181562035fb433e7d0835930fb491cbe67d7ab68bb9c6368568e6438c
 size 120

mongo_seed/seeds/package_edges.bson.gz

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:4ad7d17ded9348ba908cb0020e736226b486b7554fd24e55e1d2e7ee8904e6a7
-size 1455980
+oid sha256:47d9bdaca10021bea64eb5f933dd62fcafa4267f50f2a2fdb158dbaf0ce30acd
+size 1457336

mongo_seed/seeds/packages.bson.gz

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:01884f838c4b62e1b21e0128a20722d9fdaca1a7309dcb252897311e55ddd41c
-size 158079
+oid sha256:83c94f5bb26eb712f836d9c6e30991095510031ed1d7f0de4c947e1eb0947134
+size 156132