complete migration

Author: Fogapod

example.settings.toml

@@ -2,13 +2,6 @@
 token = "NTU2NTI4MDcwNjMxMjI3NDAy.XI1xJF.xl6iW44dagmEjnZVjzBHTL_w-pM"
 prefix = "!"
 
-[database]
-host = "localhost"
-port = 10700
-user = "pink"
-password = "pass"
-database = "pink"
-
 [redis]
 host = "127.0.0.1"
 port = 6379
@@ -23,6 +16,9 @@ ids = [439205512425504771, 80528701850124288]
 # either "combine"  or "overwrite"
 mode = "combine"
 
+[db]
+path = "/data/pink.db"
+
 [cog.servermyserver]
 server = 391987311468085248
 user_log_channel = 399649344443383810

requirements/base.in

@@ -3,9 +3,6 @@
 # discord framework/extensions
 discord.py[speed]>=2.0
 
-# databases
-asyncpg
-
 # api libs for commands
 googletrans-py==4.0.0
 
@@ -18,7 +15,7 @@ Pillow
 # error reporting
 sentry-sdk
 
-# perfomance
+# performance
 orjson
 uvloop
 

requirements/base.txt

@@ -9,7 +9,6 @@ aiosignal==1.3.1
     # via aiohttp
 anyio==4.3.0
     # via httpx
-asyncpg==0.29.0
 attrs==23.2.0
     # via aiohttp
 brotli==1.1.0

requirements/dev.in

@@ -5,6 +5,4 @@ mypy
 ruff
 
 # typing
-# https://github.com/MagicStack/asyncpg/issues/569
-asyncpg-stubs
 types-redis

requirements/dev.txt

@@ -8,9 +8,6 @@ aiosignal==1.3.1
     # via aiohttp
 anyio==4.3.0
     # via httpx
-asyncpg==0.29.0
-    # via asyncpg-stubs
-asyncpg-stubs==0.29.1
 attrs==23.2.0
     # via aiohttp
 brotli==1.1.0
@@ -80,9 +77,7 @@ types-redis==4.6.0.20240425
 types-setuptools==69.5.0.20240423
     # via types-cffi
 typing-extensions==4.11.0
-    # via
-    #   asyncpg-stubs
-    #   mypy
+    # via mypy
 urllib3==2.2.1
     # via sentry-sdk
 uvloop==0.19.0

scripts/migrate_postgres_to_sqlite.py

@@ -22,11 +22,11 @@ async def main() -> None:
     with Path("schema.sql").open() as f:
         db.executescript(f.read())
 
-    prefixes = [r async for r in pg.fetch("SELECT * FROM prefixes")]
+    prefixes = await pg.fetch("SELECT * FROM prefixes")
     print("inserting", len(prefixes), "prefixes")
     db.executemany("INSERT INTO prefixes (guild_id, prefix) VALUES (?, ?)", prefixes)
 
-    accents = [r async for r in pg.fetch("SELECT * FROM accents")]
+    accents = await pg.fetch("SELECT * FROM accents")
     print("inserting", len(accents), "accents")
     db.executemany("INSERT INTO accents (guild_id, user_id, name, severity) VALUES (?, ?, ?, ?)", accents)
 

src/__main__.py

@@ -3,7 +3,6 @@
 import platform
 
 import aiohttp
-import asyncpg
 import discord
 import redis.asyncio as redis
 
@@ -39,13 +38,6 @@ async def main() -> None:
         log.warning("skipped sentry initialization")
 
     session = aiohttp.ClientSession(headers={"user-agent": "PINK bot"})
-    pg = asyncpg.create_pool(
-        host=settings.database.host,
-        port=settings.database.port,
-        user=settings.database.user,
-        password=settings.database.password,
-        database=settings.database.database,
-    )
     rd: redis.Redis[bytes] = redis.Redis(
         host=settings.redis.host,
         port=settings.redis.port,
@@ -54,7 +46,6 @@ async def main() -> None:
 
     pink = PINK(
         session=session,
-        pg=pg,
         redis=rd,
         version=version,
         command_prefix=settings.bot.prefix,
@@ -70,7 +61,7 @@ async def main() -> None:
         ),
     )
 
-    async with pink, session, pg, rd:
+    async with pink, session, rd:
         await pink.start(settings.bot.token)
 
 

src/bot.py

@@ -3,14 +3,14 @@
 import asyncio
 import logging
 import re
+import sqlite3
 import time
 
 from collections.abc import Iterator
 from pathlib import Path
 from typing import Any, Optional
 
 import aiohttp
-import asyncpg
 import discord
 import sentry_sdk
 
@@ -45,21 +45,26 @@ def __init__(self, bot: PINK, *, prefix: str):
         self.prefix_re = mention_or_prefix_regex(bot.user.id, self.prefix)  # type: ignore
 
     @classmethod
-    def from_pg(cls, bot: PINK, data: asyncpg.Record) -> Prefix:
+    def from_db(cls, bot: PINK, data: sqlite3.Row) -> Prefix:
         return cls(bot, prefix=data["prefix"])
 
-    async def write(self, ctx: Context) -> None:
-        await ctx.pg.fetchrow(
-            "INSERT INTO prefixes (guild_id, prefix) VALUES ($1, $2) "
+    def write(self, ctx: Context) -> None:
+        ctx.db.execute(
+            "INSERT INTO prefixes (guild_id, prefix) VALUES (?, ?) "
             "ON CONFLICT (guild_id) DO UPDATE "
             "SET prefix = EXCLUDED.prefix",
-            ctx.guild.id,  # type: ignore
-            self.prefix,
+            (
+                ctx.guild.id,  # type: ignore
+                self.prefix,
+            ),
         )
 
     @staticmethod
-    async def delete(ctx: Context) -> None:
-        await ctx.pg.fetchrow("DELETE FROM prefixes WHERE guild_id = $1", ctx.guild.id)  # type: ignore
+    def delete(ctx: Context) -> None:
+        ctx.db.execute(
+            "DELETE FROM prefixes WHERE guild_id = ?",
+            (ctx.guild.id,),  # type: ignore
+        )
 
     def __repr__(self) -> str:
         return f"<{type(self).__name__} prefix={self.prefix}>"
@@ -70,38 +75,56 @@ def __init__(
         self,
         *,
         session: aiohttp.ClientSession,
-        pg: asyncpg.Pool[asyncpg.Record],
         redis: Redis[bytes],
         version: Version,
         **kwargs: Any,
     ) -> None:
         super().__init__(**kwargs)
 
         self.session = session
-        self.pg = pg
         self.redis = redis
         self.version = version
 
         self.prefixes: dict[int, Prefix] = {}
         self.owner_ids: set[int] = set()
 
+    def init_db(self) -> None:
+        with Path("schema.sql").open() as f:
+            db = self.db_cursor()
+            db.executescript(f.read())
+
+    def db_cursor(self) -> sqlite3.Cursor:
+        conn = sqlite3.connect(settings.db.path)
+        conn.isolation_level = None
+        conn.execute("PRAGMA journal_mode=wal")
+        conn.execute("PRAGMA foreign_keys=ON")
+        conn.row_factory = sqlite3.Row
+
+        return conn.cursor()
+
     # --- overloads ---
     async def setup_hook(self) -> None:
         self.launched_at = time.monotonic()
 
-        await self._load_prefixes()
+        self.init_db()
 
-        await self._load_cogs()
-
-        asyncio.gather(
+        await asyncio.gather(
+            self._load_prefixes(),
+            self._load_cogs(),
             self._fetch_owners(),
         )
 
     async def _load_prefixes(self) -> None:
-        self._default_prefix_re = mention_or_prefix_regex(self.user.id, settings.bot.prefix)  # type: ignore
+        self._default_prefix_re = mention_or_prefix_regex(
+            self.user.id,  # type: ignore
+            settings.bot.prefix,
+        )
+
+        db = self.db_cursor()
+        db.execute("SELECT guild_id, prefix FROM prefixes")
 
-        for guild in await self.pg.fetch("SELECT guild_id, prefix FROM prefixes"):
-            self.prefixes[guild["guild_id"]] = Prefix.from_pg(self, guild)
+        for guild in db.fetchall():
+            self.prefixes[guild["guild_id"]] = Prefix.from_db(self, guild)
 
     async def _fetch_owners(self) -> None:
         owners = settings.owners.ids.copy()

src/cogs/accents/cog.py

@@ -60,7 +60,9 @@ async def cog_load(self) -> None:
         # current name: PINK
         self.accent_wh_name = f"{self.bot.user.name} bot accent webhook"  # type: ignore
 
-        for settings in await self.bot.pg.fetch("SELECT guild_id, user_id, name, severity FROM accents"):
+        db = self.bot.db_cursor()
+        db.execute("SELECT guild_id, user_id, name, severity FROM accents")
+        for settings in db.fetchall():
             if (accent_cls := ALL_ACCENTS.get(settings["name"].lower())) is None:
                 log.error(
                     "unknown accent: guild=%s user=%s %s", settings["guild_id"], settings["user_id"], settings["name"]
@@ -247,24 +249,32 @@ async def _add_accents(self, ctx: Context, member: discord.Member, accents: _Use
 
         self.set_user_accents(member, all_accents)
 
-        for accent in all_accents:
-            await self.bot.pg.fetchrow(
-                "INSERT INTO accents (guild_id, user_id, name, severity) VALUES ($1, $2, $3, $4) "
-                "ON CONFLICT (guild_id, user_id, name) DO UPDATE "
-                "SET name = EXCLUDED.name",
+        rows = (
+            (
                 ctx.guild.id,  # type: ignore
                 member.id,
                 accent.name(),
                 accent.severity,
             )
+            for accent in all_accents
+        )
+
+        ctx.db.executemany(
+            "INSERT INTO accents (guild_id, user_id, name, severity) VALUES (?, ?, ?, ?) "
+            "ON CONFLICT (guild_id, user_id, name) DO UPDATE "
+            "SET name = EXCLUDED.name",
+            rows,
+        )
 
     async def _remove_accents(self, ctx: Context, member: discord.Member, accents: _UserAccentsType) -> None:
         # a special case. empty iterable means remove everything
         if not accents:
-            await self.bot.pg.fetchrow(
-                "DELETE FROM accents WHERE guild_id = $1 AND user_id = $2",
-                ctx.guild.id,  # type: ignore
-                member.id,
+            await ctx.db.execute(
+                "DELETE FROM accents WHERE guild_id = ? AND user_id = ?",
+                (
+                    ctx.guild.id,  # type: ignore
+                    member.id,
+                ),
             )
 
             self.set_user_accents(member, [])
@@ -286,13 +296,15 @@ async def _remove_accents(self, ctx: Context, member: discord.Member, accents: _
 
         self.set_user_accents(member, name_to_accent.values())
 
-        for accent in to_remove:
-            await self.bot.pg.fetchrow(
-                "DELETE FROM accents WHERE guild_id = $1 AND user_id = $2 AND name = $3",
+        rows = (
+            (
                 ctx.guild.id,  # type: ignore
                 member.id,
                 accent.name(),
             )
+            for accent in to_remove
+        )
+        ctx.db.executemany("DELETE FROM accents WHERE guild_id = ? AND user_id = ? AND name = ?", rows)
 
     async def _update_nick(self, ctx: Context) -> None:
         new_nick = ctx.me.name

src/cogs/meta.py

@@ -351,7 +351,7 @@ async def set(self, ctx: Context, *, prefix: str) -> None:
             return
 
         settings = Prefix(ctx.bot, prefix=prefix.lower())
-        await settings.write(ctx)
+        settings.write(ctx)
 
         ctx.bot.prefixes[ctx.guild.id] = settings
 
@@ -367,8 +367,7 @@ async def unset(self, ctx: Context) -> None:
         assert ctx.guild is not None
 
         if ctx.guild.id in ctx.bot.prefixes:
-            await Prefix.delete(ctx)
-
+            Prefix.delete(ctx)
             del ctx.bot.prefixes[ctx.guild.id]
 
         await ctx.ok()

src/cogs/techadmin.py

@@ -6,14 +6,14 @@
 import io
 import random
 import re
+import sqlite3
 import textwrap
 import traceback
 
 from contextlib import redirect_stdout
 from pprint import pformat
 from typing import Any, Optional, Union
 
-import asyncpg
 import discord
 
 from discord.ext import commands
@@ -167,8 +167,9 @@ async def sql(self, ctx: Context, *, code: Code) -> None:
 
         async with ctx.typing():
             try:
-                data = await ctx.pg.fetch(query)
-            except asyncpg.PostgresError as e:
+                db = ctx.db.execute(query)
+                data = db.fetchall()
+            except sqlite3.Error as e:
                 return await ctx.send(f"Error: **{type(e).__name__}**: `{e}`")
 
             if not data:
@@ -266,15 +267,15 @@ async def _exec(self, _: Context, arguments: str) -> str:
 
         return result
 
-    async def _sql_table(self, result: list[asyncpg.Record]) -> str:
+    async def _sql_table(self, result: list[sqlite3.Row]) -> str:
         # convert to list because otherwise iterator is exhausted
         columns = list(result[0].keys())
 
         no_named_columns = not list(filter(lambda c: c != PG_UNNAMED_COLUMN, columns))
         col_widths = [0 if no_named_columns and c == PG_UNNAMED_COLUMN else len(c) for c in columns]
 
         for row in result:
-            for i, value in enumerate(row.values()):
+            for i, value in enumerate(row):
                 col_widths[i] = max((col_widths[i], len(str(value))))
 
         if no_named_columns:
@@ -289,8 +290,7 @@ def sanitize_value(value: Any) -> str:
             return str(value).replace("\n", "\\n")
 
         body = "\n".join(
-            " | ".join(f"{sanitize_value(value):<{col_widths[i]}}" for i, value in enumerate(row.values()))
-            for row in result
+            " | ".join(f"{sanitize_value(value):<{col_widths[i]}}" for i, value in enumerate(row)) for row in result
         )
 
         return f"```prolog\n{header}{body}```"