chore: refactor

alembic/env.py

@@ -13,8 +13,6 @@
 
 from alembic import context
 
-from db import db, init_schemas
-
 # this is the Alembic Config object, which provides
 # access to the values within the .ini file in use.
 config = context.config
@@ -26,7 +24,7 @@
 
 
 ## to init db.metadata
-init_schemas()
+from db import db
 
 # add your model's MetaData object here
 # for 'autogenerate' support

llmpa/backends/base.py

@@ -15,7 +15,7 @@ def insert_path():
 insert_path()
 del insert_path
 
-from schemas.file import FileInfo
+from schemas import FileInfo
 
 
 JsonType = Dict[str, Any]

llmpa/backends/local/__main__.py

@@ -0,0 +1,17 @@
+from . import LocalBackend
+
+
+def main():
+    import os
+    import sys
+
+    ## if len(sys.argv) < 2:
+    ##     print(f"Usage: {os.path.basename(__file__)} <filepath>")
+    ##     sys.exit(1)
+
+    backend = LocalBackend()
+    print(backend.embedding_text("gpt2", "Hello, world!"))
+
+
+if __name__ == "__main__":
+    main()

llmpa/backends/local/models/image.py

@@ -6,11 +6,6 @@
 
 from transformers import AutoModel, AutoFeatureExtractor
 
-# from tensorflow.keras.applications import (
-#     EfficientNetV2B0,
-#     ResNet50,
-# )
-
 
 # EmbeddingExtractor Class with model name as a parameter
 class EmbeddingExtractor:

llmpa/db/__init__.py

@@ -1 +1,2 @@
-from .postgres import db, async_engine, async_session, init_db, init_schemas
+from .postgres import db, async_engine, async_session, init_db
+from . import schemas

llmpa/db/postgres.py

@@ -13,12 +13,6 @@
 async_session = sessionmaker(async_engine, class_=AsyncSession, expire_on_commit=False)
 
 
-def init_schemas():
-    from schemas import File, Task, User
-
-    __models = [File, Task, User]
-
-
 def init_db(app: Flask):
     global async_engine, async_session
 

llmpa/db/schemas/__init__.py

@@ -0,0 +1,3 @@
+from .file import File
+from .task import Task
+from .user import User

llmpa/schemas/base.py → llmpa/db/schemas/base.py

@@ -1,7 +1,7 @@
 from datetime import datetime
 from sqlalchemy.ext.declarative import declared_attr
 
-from db import db
+from ..postgres import db
 
 
 ## for backup

llmpa/schemas/file.py → llmpa/db/schemas/file.py

@@ -1,7 +1,7 @@
 import uuid
 from sqlalchemy.dialects.postgresql import UUID
 
-from db import db
+from ..postgres import db
 from .base import timestamped, soft_deletable
 
 
@@ -27,23 +27,3 @@ def __repr__(self):
         return f"<File {self.name}>"
 
 
-class FileInfo:
-    def __init__(
-        self,
-        fileId: str,
-        name: str,
-        path: str,
-        originName: str,
-        size: int,
-        type: str,
-        mimetype: str,
-    ):
-        self.name = name
-        self.path = path
-        self.originName = originName
-        self.size = size
-        self.type = type
-        self.mimetype = mimetype
-
-    def __repr__(self):
-        return f"<FileInfo {self.name} ({self.fileType})>"

llmpa/schemas/task.py → llmpa/db/schemas/task.py

@@ -2,7 +2,7 @@
 from sqlalchemy.dialects.postgresql import JSON, UUID
 
 from agents.task import TaskStatus
-from db import db
+from ..postgres import db
 from .base import timestamped, soft_deletable
 
 

llmpa/schemas/user.py → llmpa/db/schemas/user.py

@@ -3,7 +3,7 @@
 from sqlalchemy import event
 from sqlalchemy.dialects.postgresql import UUID
 
-from db import db
+from ..postgres import db
 from .base import timestamped, soft_deletable
 
 

llmpa/schemas/__init__.py

@@ -1,5 +1,2 @@
-from .file import File
-from .task import Task
-from .user import User
+from .fileinfo import FileInfo
 
-__all__ = ["File", "Task", "User"]

llmpa/schemas/fileinfo.py

@@ -0,0 +1,22 @@
+class FileInfo:
+    def __init__(
+        self,
+        fileId: str,
+        name: str,
+        path: str,
+        originName: str,
+        size: int,
+        type: str,
+        mimetype: str,
+    ):
+        self.name = name
+        self.path = path
+        self.originName = originName
+        self.size = size
+        self.type = type
+        self.mimetype = mimetype
+
+    def __repr__(self):
+        return f"<FileInfo {self.name} ({self.type})>"
+
+

llmpa/utils.py

@@ -0,0 +1,42 @@
+import os
+
+from .fileparser.mimetype import detect
+from .schemas.fileinfo import FileInfo
+
+def load_file_info(filepath: str):
+    if not os.path.exists(filepath):
+        raise FileNotFoundError(f"File '{filepath}' does not exist")
+
+    # Get file information
+    name = os.path.basename(filepath)
+    path = os.path.abspath(filepath)
+    originName = name
+    size = os.path.getsize(filepath)
+    file_type = os.path.splitext(filepath)[1][1:]  # File extension as type
+    mimetype = detect(filepath)
+
+    return FileInfo(
+        fileId=str(os.path.getctime(filepath)),  # Using creation time as fileId (or customize)
+        name=name,
+        path=path,
+        originName=originName,
+        size=size,
+        type=file_type,
+        mimetype=mimetype or "unknown"
+    )
+
+def main():
+    import os
+    import sys
+
+    if len(sys.argv) < 2:
+        print(f"Usage: {os.path.basename(__file__)} <filepath>")
+        sys.exit(1)
+
+    filepath = sys.argv[1]
+    file_info = load_file_info(filepath)
+    print(file_info)
+
+
+if __name__ == "__main__":
+    main()