feat: Divide database

Author: GermanMT

app/apis/git_service.py

@@ -1,20 +1,11 @@
 from requests import post
 
-from functools import lru_cache
-
-from app.config import Settings
+from app.config import settings
 
 from app.utils.ctc_parser import parse_constraints
 from app.utils.managers import managers
 
 
-@lru_cache()
-def get_settings():
-    return Settings()
-
-
-settings: Settings = get_settings()
-
 headers = {
     'Accept': 'application/vnd.github.hawkgirl-preview+json',
     'Authorization': f'Bearer {settings.GIT_GRAPHQL_API_KEY}',

app/config.py

@@ -1,10 +1,19 @@
 from pydantic import BaseSettings
 
+from functools import lru_cache
+
 
 class Settings(BaseSettings):
 
     DATABASE_URL: str | None = None
     GIT_GRAPHQL_API_KEY: str | None = None
 
     class Config:
-        env_file = '.env'
+        env_file = '.env'
+
+@lru_cache()
+def get_settings():
+    return Settings()
+
+
+settings: Settings = get_settings()

app/controllers/generate_controller.py

@@ -25,35 +25,35 @@ async def generate_packages(package_name: str, release: dict) -> list[ObjectId]:
 
             if package is not None:
 
-                package_edge_id = await exist_package(package, package_edge)
+                package_edge_id = await exist_package(package, package_edge, 'pypi')
 
             else:
 
-                package_edge_id = await no_exist_package(package_name, package_edge)
+                package_edge_id = await no_exist_package(package_name, package_edge, 'pypi')
 
             package_edges.append(package_edge_id)
 
     return package_edges
 
-async def exist_package(package: dict, package_edge: dict) -> ObjectId:
+async def exist_package(package: dict, package_edge: dict, db: str) -> ObjectId:
     package_edge['package'] = package['_id']
 
     package_edge['versions'] = await filter_versions_db(package_edge['constraints'], package['versions'])
 
-    new_package_edge = await add_package_edge(package_edge)
+    new_package_edge = await add_package_edge(package_edge, db)
 
     return new_package_edge['_id']
 
-async def no_exist_package(package_name: str, package_edge: dict) -> ObjectId:
+async def no_exist_package(package_name: str, package_edge: dict, db: str) -> ObjectId:
     package = {'name': package_name}
 
     new_package = await add_package(package)
 
     package_edge['package'] = new_package['_id']
 
-    new_package_edge = await add_package_edge(package_edge)
+    new_package_edge = await add_package_edge(package_edge, db)
 
-    no_existing_versions = await generate_versions(new_package, new_package_edge)
+    no_existing_versions = await generate_versions(new_package, new_package_edge, db)
 
     await relate_versions(no_existing_versions, new_package['name'])
 
@@ -66,7 +66,7 @@ async def relate_versions(no_existing_versions: list, package_name: str) -> None
 
         await update_version_package_edges(version[0], version_package_edges)
 
-async def generate_versions(package: dict, package_edge: dict) -> list:
+async def generate_versions(package: dict, package_edge: dict, db: str) -> list:
     no_existing_versions: list = []
     package_versions: list = []
 
@@ -94,6 +94,6 @@ async def generate_versions(package: dict, package_edge: dict) -> list:
 
     await update_package_versions(package['_id'], package_versions)
 
-    await update_package_edge_versions(package_edge['_id'], filtered_versions)
+    await update_package_edge_versions(package_edge['_id'], filtered_versions, db)
 
     return no_existing_versions

app/controllers/graph_controller.py

@@ -8,7 +8,7 @@
 
 from app.models.graph_model import GraphModel
 
-from app.services.graph_service import add_graph, update_graph_requirement_files
+from app.services.graph_service import read_graph, add_graph, update_graph_requirement_files, update_graph_is_completed
 from app.services.package_service import get_package_by_name
 from app.services.requirement_file_service import add_requirement_file, update_requirement_file
 
@@ -17,6 +17,12 @@
 
 router = APIRouter()
 
+@router.get('/graph/{graph_id}', response_description='Get graph', response_model = GraphModel)
+async def get_graph(graph_id: str):
+    new_graph = await read_graph(graph_id)
+
+    return JSONResponse(status_code = status.HTTP_200_OK, content = JSONEncoder().encode(new_graph))
+
 @router.post('/graph', response_description='Init graph', response_model = GraphModel)
 async def init_graph(background_tasks: BackgroundTasks, graph: GraphModel = Body(...)):
     graph_json = jsonable_encoder(graph)
@@ -52,14 +58,16 @@ async def generate_graph(graph: dict) -> None:
 
             if package is not None:
 
-                package_edge_id = await exist_package(package, package_edge)
+                package_edge_id = await exist_package(package, package_edge, 'depex')
 
             else:
 
-                package_edge_id = await no_exist_package(package_name, package_edge)
+                package_edge_id = await no_exist_package(package_name, package_edge, 'depex')
 
             package_edges.append(package_edge_id)
 
         await update_requirement_file(new_req_file['_id'], package_edges)
 
-    await update_graph_requirement_files(graph['_id'], requirement_files)
+    await update_graph_requirement_files(graph['_id'], requirement_files)
+
+    await update_graph_is_completed(graph['_id'])

app/services/db/database.py

@@ -0,0 +1,16 @@
+import motor.motor_asyncio
+
+from app.config import settings
+
+
+client = motor.motor_asyncio.AsyncIOMotorClient(settings.DATABASE_URL)
+
+depex_db = client.depex
+graph_collection = depex_db.get_collection('graphs')
+depex_package_edge_collection = depex_db.get_collection('package_edges')
+requirement_file_collection = depex_db.get_collection('requirement_files')
+
+pypi_db = client.pypi
+pypi_package_edge_collection = pypi_db.get_collection('package_edges')
+package_collection = pypi_db.get_collection('packages')
+version_collection = pypi_db.get_collection('versions')

app/services/db/__init__.py renamed to app/services/dbs/__init__.py

@@ -1,13 +1,21 @@
 from bson import ObjectId
 
-from app.services.db.database import graph_collection
+from app.services.dbs.databases import graph_collection
 
 
+async def read_graph(graph_id: str) -> dict:
+    new_graph = await graph_collection.find_one({'_id': ObjectId(graph_id)})
+    return new_graph
+
 async def add_graph(graph_data: dict) -> dict:
     graph = await graph_collection.insert_one(graph_data)
     new_graph = await graph_collection.find_one({'_id': graph.inserted_id})
     return new_graph
 
 async def update_graph_requirement_files(graph_id: ObjectId, requirement_files: list) -> dict:
     updated_graph = await graph_collection.find_one_and_update({'_id': graph_id}, {'$set': {'requirement_files': requirement_files}})
+    return updated_graph
+
+async def update_graph_is_completed(graph_id: ObjectId) -> dict:
+    updated_graph = await graph_collection.find_one_and_update({'_id': graph_id}, {'$set': {'is_complete': True}})
     return updated_graph

app/services/dbs/databases.py

@@ -1,13 +1,25 @@
 from bson import ObjectId
 
-from app.services.db.database import package_edge_collection
+from app.services.dbs.databases import depex_package_edge_collection, pypi_package_edge_collection
 
 
-async def add_package_edge(package_edge_data: dict) -> dict:
-    package_edge = await package_edge_collection.insert_one(package_edge_data)
-    new_package_edge = await package_edge_collection.find_one({'_id': package_edge.inserted_id})
+async def add_package_edge(package_edge_data: dict, db: str) -> dict:
+    match db:
+        case 'depex':
+            collection = depex_package_edge_collection
+        case 'pypi':
+            collection = pypi_package_edge_collection
+
+    package_edge = await collection.insert_one(package_edge_data)
+    new_package_edge = await collection.find_one({'_id': package_edge.inserted_id})
     return new_package_edge
 
-async def update_package_edge_versions(package_edge_id: ObjectId, versions: list) -> dict:
-    updated_package_edge = await package_edge_collection.find_one_and_update({'_id': package_edge_id}, {'$set': {'versions': versions}})
+async def update_package_edge_versions(package_edge_id: ObjectId, versions: list, db: str) -> dict:
+    match db:
+        case 'depex':
+            collection = depex_package_edge_collection
+        case 'pypi':
+            collection = pypi_package_edge_collection
+
+    updated_package_edge = await collection.find_one_and_update({'_id': package_edge_id}, {'$set': {'versions': versions}})
     return updated_package_edge

app/services/graph_service.py

@@ -1,6 +1,6 @@
 from bson import ObjectId
 
-from app.services.db.database import package_collection
+from app.services.dbs.databases import package_collection
 
 
 async def add_package(package_data: dict) -> dict: