merge

Author: GermanMT

.gitattributes

@@ -1 +1 @@
-*.gz filter=lfs diff=lfs merge=lfs -text
+*.gz filter=lfs diff=lfs merge=lfs -text

.github/workflows/analisys.yml

@@ -3,19 +3,19 @@ name: Python analisys
 on: [push, pull_request]
 
 jobs:
-  prospector:
+  ruff:
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v2
-      - name: Set up Python 3.11
+      - name: Set up Python 3.12
         uses: actions/setup-python@v2
         with:
-          python-version: '3.11'
+          python-version: '3.12'
       - name: Install dependencies
         run: |
           python -m pip install --upgrade pip
           pip install -r requirements.txt
-          pip install prospector[with_everything]==1.10.2
-      - name: Analysing the code with prospector
+          pip install ruff==0.1.5
+      - name: Analysing the code with ruff
         run: |
-          prospector
+          ruff check .

.github/workflows/github-release.yml

@@ -35,4 +35,4 @@ jobs:
           token: ${{secrets.GIT_TOKEN}}
           prerelease: ${{steps.get_variables.outputs.IS_PRERELEASE}}
           tag_name: ${{steps.get_variables.outputs.VERSION}}
-          body: ${{steps.changelog.outputs.changes}}
+          body: ${{steps.changelog.outputs.changes}}

.gitignore

@@ -130,4 +130,4 @@ venv.bak/
 dmypy.json
 
 # Pyre type checker
-.pyre/
+.pyre/

.prospector.yaml

@@ -12,4 +12,4 @@ RUN pip install -r requirements.txt
 
 EXPOSE 8000
 
-CMD uvicorn app.main:app --host 0.0.0.0 --port 8000
+CMD uvicorn app.main:app --host 0.0.0.0 --port 8000

Dockerfile

@@ -1,9 +1,11 @@
+from .github_service import get_last_commit_date_github, get_repo_data
+from .gitlab_service import get_last_commit_date_gitlab
 from .manager_service import get_all_versions, requires_packages
-from .git_service import get_repo_data, get_last_commit_date
 
 __all__ = [
-    'get_all_versions',
-    'requires_packages',
-    'get_repo_data',
-    'get_last_commit_date'
-]
+    "get_all_versions",
+    "requires_packages",
+    "get_repo_data",
+    "get_last_commit_date_github",
+    "get_last_commit_date_gitlab",
+]

app/apis/__init__.py

@@ -0,0 +1,160 @@
+from datetime import datetime
+from time import sleep
+from typing import Any
+
+from dateutil.parser import parse
+from requests import ConnectionError, ConnectTimeout, post
+
+from app.config import settings
+from app.utils import get_manager, parse_pip_constraints
+
+headers_github = {
+    "Accept": "application/vnd.github.hawkgirl-preview+json",
+    "Authorization": f"Bearer {settings.GITHUB_GRAPHQL_API_KEY}",
+}
+
+
+async def get_repo_data(
+    owner: str,
+    name: str,
+    all_packages: dict[str, Any] | None = None,
+    end_cursor: str | None = None,
+) -> dict[str, Any]:
+    if not all_packages:
+        all_packages = {}
+    if not end_cursor:
+        query = f"""
+        {{
+            repository(owner: "{owner}", name: "{name}") {{
+                dependencyGraphManifests {{
+                    nodes {{
+                        filename
+                        dependencies {{
+                            pageInfo {{
+                                hasNextPage
+                                endCursor
+                            }}
+                            nodes {{
+                                packageName
+                                requirements
+                            }}
+                        }}
+                    }}
+                }}
+            }}
+        }}
+        """
+    else:
+        query = f"""
+        {{
+            repository(owner: "{owner}", name: "{name}") {{
+                dependencyGraphManifests {{
+                    nodes {{
+                        filename
+                        dependencies (after: "{end_cursor}") {{
+                            pageInfo {{
+                                hasNextPage
+                                endCursor
+                            }}
+                            nodes {{
+                                packageName
+                                requirements
+                            }}
+                        }}
+                    }}
+                }}
+            }}
+        }}
+        """
+    while True:
+        try:
+            response = post(
+                "https://api.github.com/graphql",
+                json={"query": query},
+                headers=headers_github,
+            ).json()
+            break
+        except (ConnectTimeout, ConnectionError):
+            sleep(5)
+    page_info, all_packages = await json_reader(response, all_packages)
+    if not page_info["hasNextPage"]:
+        return all_packages
+    return await get_repo_data(owner, name, all_packages, page_info["endCursor"])
+
+
+async def json_reader(
+    response: Any, all_packages: dict[str, Any]
+) -> tuple[dict[str, Any], dict[str, Any]]:
+    page_info = {"hasNextPage": None}
+    for file_node in response["data"]["repository"]["dependencyGraphManifests"][
+        "nodes"
+    ]:
+        requirement_file_name = file_node["filename"]
+        if requirement_file_name == "package-lock.json":
+            continue
+        page_info = file_node["dependencies"]["pageInfo"]
+        requirement_file_manager = await get_manager(requirement_file_name)
+        if not requirement_file_manager:
+            continue
+        if requirement_file_name not in all_packages:
+            all_packages[requirement_file_name] = {
+                "package_manager": requirement_file_manager,
+                "dependencies": {},
+            }
+        for depependency_node in file_node["dependencies"]["nodes"]:
+            if requirement_file_manager == "MVN":
+                if "=" in depependency_node["requirements"]:
+                    depependency_node["requirements"] = (
+                        "["
+                        + depependency_node["requirements"]
+                        .replace("=", "")
+                        .replace(" ", "")
+                        + "]"
+                    )
+            if requirement_file_manager == "PIP":
+                depependency_node["requirements"] = await parse_pip_constraints(
+                    depependency_node["requirements"]
+                )
+            all_packages[requirement_file_name]["dependencies"].update(
+                {depependency_node["packageName"]: depependency_node["requirements"]}
+            )
+    return (page_info, all_packages)
+
+
+async def get_last_commit_date_github(owner: str, name: str) -> datetime:
+    query = f"""
+    {{
+        repository(owner: "{owner}", name: "{name}") {{
+            defaultBranchRef {{
+                target {{
+                    ... on Commit {{
+                        history(first: 1) {{
+                            edges {{
+                                node {{
+                                    author {{
+                                    date
+                                    }}
+                                }}
+                            }}
+                        }}
+                    }}
+                }}
+            }}
+        }}
+    }}
+    """
+    while True:
+        try:
+            response = post(
+                "https://api.github.com/graphql",
+                json={"query": query},
+                headers=headers_github,
+            ).json()
+            break
+        except (ConnectTimeout, ConnectionError):
+            sleep(5)
+    return parse(
+        response["data"]["repository"]["defaultBranchRef"]["target"]["history"][
+            "edges"
+        ][0]["node"]["author"]["date"]
+    )

app/apis/git_service.py

@@ -0,0 +1,33 @@
+from datetime import datetime
+from time import sleep
+
+from dateutil.parser import parse
+from requests import ConnectionError, ConnectTimeout, post
+
+from app.config import settings
+
+headers_gitlab = {
+    "Content-Type": "application/json",
+    "Authorization": f"Bearer {settings.GITLAB_GRAPHQL_API_KEY}",
+}
+
+
+async def get_last_commit_date_gitlab(owner: str, name: str) -> datetime:
+    query = f"""
+    {{
+        project(fullPath: "{owner}/{name}") {{
+            lastActivityAt
+        }}
+    }}
+    """
+    while True:
+        try:
+            response = post(
+                "https://gitlab.com/api/graphql",
+                json={"query": query},
+                headers=headers_gitlab,
+            ).json()
+            break
+        except (ConnectTimeout, ConnectionError):
+            sleep(5)
+    return parse(response["data"]["project"]["lastActivityAt"])