Merge pull request #1416 from rommapp/feature/screenscraper-integration

feat: Screenscraper integration

.vscode/tasks.json

@@ -35,7 +35,7 @@
     {
       "label": "Setup testing environment",
       "type": "shell",
-      "command": "export $(cat .env | grep DB_ROOT_PASSWD | xargs) && docker exec -i mariadb mariadb -u root -p$DB_ROOT_PASSWD < backend/romm_test/setup.sql",
+      "command": "export $(cat .env | grep DB_ROOT_PASSWD | xargs) && docker exec -i romm-db-dev mariadb -u root -p$DB_ROOT_PASSWD < backend/romm_test/setup.sql",
       "problemMatcher": []
     },
     {

backend/alembic/versions/0030_user_email_null.py

@@ -1,6 +1,6 @@
 """Change empty string in users.email to NULL.
 
-Revision ID: 951473b0c581
+Revision ID: 0030_user_email_null
 Revises: 0029_platforms_custom_name
 Create Date: 2025-01-14 01:30:39.696257
 

backend/alembic/versions/0035_screenscraper.py

@@ -0,0 +1,46 @@
+"""empty message
+
+Revision ID: 0035_screenscraper
+Revises: 0034_virtual_collections_db_view
+Create Date: 2025-01-02 18:58:55.557123
+
+"""
+
+import sqlalchemy as sa
+from alembic import op
+from utils.database import CustomJSON
+
+# revision identifiers, used by Alembic.
+revision = "0035_screenscraper"
+down_revision = "0034_virtual_collections_db_view"
+branch_labels = None
+depends_on = None
+
+
+def upgrade() -> None:
+    # ### commands auto generated by Alembic - please adjust! ###
+    with op.batch_alter_table("platforms", schema=None) as batch_op:
+        batch_op.add_column(sa.Column("ss_id", sa.Integer(), nullable=True))
+
+    with op.batch_alter_table("roms", schema=None) as batch_op:
+        batch_op.add_column(sa.Column("ss_id", sa.Integer(), nullable=True))
+        batch_op.add_column(sa.Column("ss_metadata", CustomJSON(), nullable=True))
+        batch_op.add_column(sa.Column("url_manual", sa.Text(), nullable=True)),
+        batch_op.add_column(sa.Column("path_manual", sa.Text(), nullable=True)),
+        batch_op.create_index("idx_roms_ss_id", ["ss_id"], unique=False)
+    # ### end Alembic commands ###
+
+
+def downgrade() -> None:
+    # ### commands auto generated by Alembic - please adjust! ###
+    with op.batch_alter_table("roms", schema=None) as batch_op:
+        batch_op.drop_index("idx_roms_ss_id")
+        batch_op.drop_column("ss_id")
+        batch_op.drop_column("ss_metadata")
+        batch_op.drop_column("url_manual")
+        batch_op.drop_column("path_manual")
+        batch_op.drop_index("idx_roms_ss_id")
+
+    with op.batch_alter_table("platforms", schema=None) as batch_op:
+        batch_op.drop_column("ss_id")
+    # ### end Alembic commands ###

backend/alembic/versions/0036_screenscraper_platforms_id.py

@@ -0,0 +1,34 @@
+"""empty message
+
+Revision ID: 0036_screenscraper_platforms_id
+Revises: 0035_screenscraper
+Create Date: 2025-01-02 18:58:55.557123
+
+"""
+
+import sqlalchemy as sa
+from alembic import op
+from handler.metadata.ss_handler import SLUG_TO_SS_ID
+
+# revision identifiers, used by Alembic.
+revision = "0036_screenscraper_platforms_id"
+down_revision = "0035_screenscraper"
+branch_labels = None
+depends_on = None
+
+
+def upgrade() -> None:
+    # ### commands auto generated by Alembic - please adjust! ###
+    connection = op.get_bind()
+    for slug, ss_id in SLUG_TO_SS_ID.items():
+        connection.execute(
+            sa.text("UPDATE platforms SET ss_id = :ss_id WHERE slug = :slug"),
+            {"ss_id": ss_id["id"], "slug": slug},
+        )
+    # ### end Alembic commands ###
+
+
+def downgrade() -> None:
+    # ### commands auto generated by Alembic - please adjust! ###
+    pass
+    # ### end Alembic commands ###

backend/config/__init__.py

@@ -57,6 +57,10 @@ def str_to_bool(value: str) -> bool:
     "IGDB_CLIENT_SECRET", os.environ.get("CLIENT_SECRET", "")
 )
 
+# SCREENSCRAPER
+SCREENSCRAPER_USER: Final = os.environ.get("SCREENSCRAPER_USER", "")
+SCREENSCRAPER_PASSWORD: Final = os.environ.get("SCREENSCRAPER_PASSWORD", "")
+
 # STEAMGRIDDB
 STEAMGRIDDB_API_KEY: Final = os.environ.get("STEAMGRIDDB_API_KEY", "")
 

backend/endpoints/collections.py

@@ -78,8 +78,8 @@ async def add_collection(
             )
     else:
         path_cover_s, path_cover_l = await fs_resource_handler.get_cover(
-            overwrite=True,
             entity=_added_collection,
+            overwrite=True,
             url_cover=_added_collection.url_cover,
         )
 
@@ -247,8 +247,8 @@ async def update_collection(
                     {"url_cover": data.get("url_cover", collection.url_cover)}
                 )
                 path_cover_s, path_cover_l = await fs_resource_handler.get_cover(
-                    overwrite=True,
                     entity=collection,
+                    overwrite=True,
                     url_cover=data.get("url_cover", ""),  # type: ignore
                 )
                 cleaned_data.update(

backend/endpoints/heartbeat.py

@@ -18,6 +18,7 @@
 from handler.metadata.igdb_handler import IGDB_API_ENABLED
 from handler.metadata.moby_handler import MOBY_API_ENABLED
 from handler.metadata.sgdb_handler import STEAMGRIDDB_API_ENABLED
+from handler.metadata.ss_handler import SS_API_ENABLED
 from utils import get_version
 from utils.router import APIRouter
 
@@ -40,9 +41,12 @@ def heartbeat() -> HeartbeatResponse:
             "SHOW_SETUP_WIZARD": len(db_user_handler.get_admin_users()) == 0,
         },
         "METADATA_SOURCES": {
-            "ANY_SOURCE_ENABLED": IGDB_API_ENABLED or MOBY_API_ENABLED,
+            "ANY_SOURCE_ENABLED": IGDB_API_ENABLED
+            or MOBY_API_ENABLED
+            or SS_API_ENABLED,
             "IGDB_API_ENABLED": IGDB_API_ENABLED,
             "MOBY_API_ENABLED": MOBY_API_ENABLED,
+            "SS_API_ENABLED": SS_API_ENABLED,
             "STEAMGRIDDB_ENABLED": STEAMGRIDDB_API_ENABLED,
         },
         "FILESYSTEM": {

backend/endpoints/responses/heartbeat.py

@@ -25,6 +25,7 @@ class MetadataSourcesDict(TypedDict):
     ANY_SOURCE_ENABLED: bool
     IGDB_API_ENABLED: bool
     MOBY_API_ENABLED: bool
+    SS_API_ENABLED: bool
     STEAMGRIDDB_ENABLED: bool
 
 

backend/endpoints/responses/platform.py

@@ -17,6 +17,7 @@ class PlatformSchema(BaseModel):
     igdb_id: int | None = None
     sgdb_id: int | None = None
     moby_id: int | None = None
+    ss_id: int | None = None
     category: str | None = None
     generation: int | None = None
     family_name: str | None = None

backend/endpoints/responses/rom.py

@@ -9,6 +9,7 @@
 from fastapi import Request
 from handler.metadata.igdb_handler import IGDBMetadata
 from handler.metadata.moby_handler import MobyMetadata
+from handler.metadata.ss_handler import SSMetadata
 from models.rom import Rom, RomFileCategory, RomUserStatus
 from pydantic import computed_field
 
@@ -26,6 +27,11 @@
     dict((k, NotRequired[v]) for k, v in get_type_hints(MobyMetadata).items()),
     total=False,
 )
+RomSSMetadata = TypedDict(  # type: ignore[misc]
+    "RomSSMetadata",
+    dict((k, NotRequired[v]) for k, v in get_type_hints(SSMetadata).items()),
+    total=False,
+)
 
 
 def rom_user_schema_factory() -> RomUserSchema:
@@ -120,6 +126,7 @@ class RomSchema(BaseModel):
     igdb_id: int | None
     sgdb_id: int | None
     moby_id: int | None
+    ss_id: int | None
 
     platform_id: int
     platform_slug: str
@@ -152,10 +159,16 @@ class RomSchema(BaseModel):
     age_ratings: list[str]
     igdb_metadata: RomIGDBMetadata | None
     moby_metadata: RomMobyMetadata | None
+    ss_metadata: RomSSMetadata | None
 
     path_cover_small: str | None
     path_cover_large: str | None
     url_cover: str | None
+
+    has_manual: bool
+    path_manual: str | None
+    url_manual: str | None
+
     is_unidentified: bool
 
     revision: str | None

backend/endpoints/responses/search.py

@@ -4,11 +4,13 @@
 class SearchRomSchema(BaseModel):
     igdb_id: int | None = None
     moby_id: int | None = None
+    ss_id: int | None = None
     slug: str
     name: str
     summary: str
     igdb_url_cover: str = ""
     moby_url_cover: str = ""
+    ss_url_cover: str = ""
     platform_id: int
 
 

backend/endpoints/rom.py

@@ -1,4 +1,5 @@
 import binascii
+import os
 from base64 import b64encode
 from datetime import datetime, timezone
 from io import BytesIO
@@ -31,7 +32,8 @@
 from handler.database import db_platform_handler, db_rom_handler
 from handler.filesystem import fs_resource_handler, fs_rom_handler
 from handler.filesystem.base_handler import CoverSize
-from handler.metadata import meta_igdb_handler, meta_moby_handler
+from handler.metadata import meta_igdb_handler, meta_moby_handler, meta_ss_handler
+from logger.formatter import highlight as hl
 from logger.logger import log
 from models.rom import Rom, RomFile, RomUser
 from PIL import Image
@@ -492,16 +494,19 @@ async def update_rom(
                 "igdb_id": None,
                 "sgdb_id": None,
                 "moby_id": None,
+                "ss_id": None,
                 "name": rom.fs_name,
                 "summary": "",
                 "url_screenshots": [],
                 "path_screenshots": [],
                 "path_cover_s": "",
                 "path_cover_l": "",
                 "url_cover": "",
+                "url_manual": "",
                 "slug": "",
                 "igdb_metadata": {},
                 "moby_metadata": {},
+                "ss_metadata": {},
                 "revision": "",
             },
         )
@@ -515,6 +520,7 @@ async def update_rom(
     cleaned_data: dict[str, Any] = {
         "igdb_id": data.get("igdb_id", rom.igdb_id),
         "moby_id": data.get("moby_id", rom.moby_id),
+        "ss_id": data.get("ss_id", rom.ss_id),
     }
 
     moby_id = cleaned_data["moby_id"]
@@ -527,9 +533,23 @@ async def update_rom(
         )
         cleaned_data.update({"path_screenshots": path_screenshots})
 
-    igdb_id = cleaned_data["igdb_id"]
-    if igdb_id and int(igdb_id) != rom.igdb_id:
-        igdb_rom = await meta_igdb_handler.get_rom_by_id(int(igdb_id))
+    if (
+        cleaned_data.get("ss_id", "")
+        and int(cleaned_data.get("ss_id", "")) != rom.ss_id
+    ):
+        ss_rom = await meta_ss_handler.get_rom_by_id(cleaned_data["ss_id"])
+        cleaned_data.update(ss_rom)
+        path_screenshots = await fs_resource_handler.get_rom_screenshots(
+            rom=rom,
+            url_screenshots=cleaned_data.get("url_screenshots", []),
+        )
+        cleaned_data.update({"path_screenshots": path_screenshots})
+
+    if (
+        cleaned_data.get("igdb_id", "")
+        and int(cleaned_data.get("igdb_id", "")) != rom.igdb_id
+    ):
+        igdb_rom = await meta_igdb_handler.get_rom_by_id(cleaned_data["igdb_id"])
         cleaned_data.update(igdb_rom)
         path_screenshots = await fs_resource_handler.get_rom_screenshots(
             rom=rom,
@@ -613,14 +633,29 @@ async def update_rom(
             ):
                 cleaned_data.update({"url_cover": data.get("url_cover", rom.url_cover)})
                 path_cover_s, path_cover_l = await fs_resource_handler.get_cover(
-                    overwrite=True,
                     entity=rom,
+                    overwrite=True,
                     url_cover=str(data.get("url_cover") or ""),
                 )
                 cleaned_data.update(
                     {"path_cover_s": path_cover_s, "path_cover_l": path_cover_l}
                 )
 
+    if data.get("url_manual", "") != rom.url_manual or not (
+        await fs_resource_handler.manual_exists(rom)
+    ):
+        cleaned_data.update({"url_manual": data.get("url_manual", rom.url_manual)})
+        url_manual = await fs_resource_handler.get_manual(
+            rom=rom,
+            overwrite=True,
+            url_manual=str(data.get("url_manual") or ""),
+        )
+        cleaned_data.update({"url_manual": url_manual})
+
+    log.debug(
+        f"Updating {hl(cleaned_data.get('name', ''))} [{id}] with data {cleaned_data}"
+    )
+
     db_rom_handler.update_rom(id, cleaned_data)
     rom = db_rom_handler.get_rom(id)
     if not rom:
@@ -629,6 +664,60 @@ async def update_rom(
     return DetailedRomSchema.from_orm_with_request(rom, request)
 
 
+@protected_route(router.post, "/{id}/manuals", [Scope.ROMS_WRITE])
+async def add_rom_manuals(request: Request, id: int):
+    """Upload manuals for a rom
+
+    Args:
+        request (Request): Fastapi Request object
+
+    Raises:
+        HTTPException: No files were uploaded
+    """
+    rom = db_rom_handler.get_rom(id)
+    if not rom:
+        raise RomNotFoundInDatabaseException(id)
+
+    filename = request.headers.get("x-upload-filename")
+
+    manuals_path = f"{RESOURCES_BASE_PATH}/{rom.fs_resources_path}/manual"
+    file_location = Path(f"{manuals_path}/{rom.id}.pdf")
+    log.info(f"Uploading {file_location}")
+
+    if not os.path.exists(manuals_path):
+        await Path(manuals_path).mkdir(parents=True, exist_ok=True)
+
+    parser = StreamingFormDataParser(headers=request.headers)
+    parser.register("x-upload-platform", NullTarget())
+    parser.register(filename, FileTarget(str(file_location)))
+
+    async def cleanup_partial_file():
+        if await file_location.exists():
+            await file_location.unlink()
+
+    try:
+        async for chunk in request.stream():
+            parser.data_received(chunk)
+    except ClientDisconnect:
+        log.error("Client disconnected during upload")
+        await cleanup_partial_file()
+    except Exception as exc:
+        log.error("Error uploading files", exc_info=exc)
+        await cleanup_partial_file()
+        raise HTTPException(
+            status_code=status.HTTP_500_INTERNAL_SERVER_ERROR,
+            detail="There was an error uploading the file(s)",
+        ) from exc
+
+    path_manual = await fs_resource_handler.get_manual(
+        rom=rom, overwrite=False, url_manual=None
+    )
+
+    db_rom_handler.update_rom(id, {"path_manual": path_manual})
+
+    return Response(status_code=status.HTTP_201_CREATED)
+
+
 @protected_route(router.post, "/delete", [Scope.ROMS_WRITE])
 async def delete_roms(
     request: Request,