From 2b9dd5857c0207cb082b45887237e645a9d53f2d Mon Sep 17 00:00:00 2001 From: DanielFGray Date: Sun, 18 Aug 2024 20:28:04 -0500 Subject: [PATCH 1/8] initial sh tracking attempt --- app/db.d.ts | 14 ++++ app/discord/activityTracker.ts | 91 ++++++++++++++++++++++ app/discord/gateway.ts | 2 + migrations/20240808232207_message_stats.ts | 21 +++++ 4 files changed, 128 insertions(+) create mode 100644 app/discord/activityTracker.ts create mode 100644 migrations/20240808232207_message_stats.ts diff --git a/app/db.d.ts b/app/db.d.ts index 3673806..77b4e07 100644 --- a/app/db.d.ts +++ b/app/db.d.ts @@ -9,6 +9,19 @@ export interface Guilds { settings: string | null; } +export interface MessageStats { + author_id: string; + channel_category: string | null; + channel_id: string; + char_count: number; + guild_id: string; + message_id: string | null; + react_count: Generated; + recipient_id: string | null; + sent_at: string; + word_count: number; +} + export interface Sessions { data: string | null; expires: string | null; @@ -24,6 +37,7 @@ export interface Users { export interface DB { guilds: Guilds; + message_stats: MessageStats; sessions: Sessions; users: Users; } diff --git a/app/discord/activityTracker.ts b/app/discord/activityTracker.ts new file mode 100644 index 0000000..3263ce2 --- /dev/null +++ b/app/discord/activityTracker.ts @@ -0,0 +1,91 @@ +import { Events, ChannelType } from "discord.js"; +import type { Client, Message, PartialMessage } from "discord.js"; +import db from "~/db.server"; + +export async function startActivityTracking(client: Client) { + client.on(Events.MessageCreate, async (msg) => { + const info = await getMessageStats(msg); + if (!info) return; + await db + .insertInto("message_stats") + .values({ + ...info, + message_id: msg.id, + author_id: msg.author!.id, + guild_id: msg.guildId, + channel_id: msg.channelId, + recipient_id: msg.mentions?.repliedUser?.id ?? null, + // TODO: cache this? + channel_category: (await msg.channel.fetch()).parentId, + }) + .execute(); + reportByGuild(msg.guildId!); + }); + + client.on(Events.MessageUpdate, async (msg) => { + const info = await getMessageStats(msg); + console.log(msg, info); + if (!info) return; + await updateStatsById(msg.id).set(info).execute(); + reportByGuild(msg.guildId!); + }); + + client.on(Events.MessageDelete, async (msg) => { + const info = await getMessageStats(msg); + if (!info) return; + await db + .deleteFrom("message_stats") + .where("message_id", "=", msg.id) + .execute(); + reportByGuild(msg.guildId!); + }); + + client.on(Events.MessageReactionAdd, async (msg) => { + await updateStatsById(msg.message.id) + .set({ react_count: (eb) => eb(eb.ref("react_count"), "+", 1) }) + .execute(); + reportByGuild(msg.message.guildId!); + }); + + client.on(Events.MessageReactionRemove, async (msg) => { + await updateStatsById(msg.message.id) + .set({ react_count: (eb) => eb(eb.ref("react_count"), "-", 1) }) + .execute(); + reportByGuild(msg.message.guildId!); + }); +} + +function updateStatsById(id: string) { + return db.updateTable("message_stats").where("message_id", "=", id); +} + +async function getMessageStats(msg: Message | PartialMessage) { + // TODO: more filters + if (msg.channel.type !== ChannelType.GuildText || msg.author?.bot) { + return; + } + return { + char_count: msg.content?.length ?? 0, + word_count: msg.content?.split(/\s+/).length ?? 0, + react_count: msg.reactions.cache.size, + sent_at: String(msg.createdTimestamp), + }; +} + +async function reportByGuild(guildId: string) { + const result = await db + .selectFrom("message_stats") + .select((eb) => [ + eb.fn.countAll().as("message_count"), + eb.fn.sum("char_count").as("char_total"), + eb.fn.sum("word_count").as("word_total"), + eb.fn.sum("react_count").as("react_total"), + eb.fn.avg("char_count").as("avg_chars"), + eb.fn.avg("word_count").as("avg_words"), + eb.fn.avg("react_count").as("avg_reacts"), + ]) + .where("guild_id", "=", guildId) + .groupBy("author_id") + .execute(); + console.log(result); +} diff --git a/app/discord/gateway.ts b/app/discord/gateway.ts index d15438d..193fce8 100644 --- a/app/discord/gateway.ts +++ b/app/discord/gateway.ts @@ -8,6 +8,7 @@ import { import automod from "~/discord/automod"; import onboardGuild from "~/discord/onboardGuild"; +import { startActivityTracking } from "~/discord/activityTracker"; import * as convene from "~/commands/convene"; import * as setup from "~/commands/setup"; @@ -27,6 +28,7 @@ export default function init() { onboardGuild(client), automod(client), deployCommands(client), + startActivityTracking(client), ]); }); diff --git a/migrations/20240808232207_message_stats.ts b/migrations/20240808232207_message_stats.ts new file mode 100644 index 0000000..2dc0c97 --- /dev/null +++ b/migrations/20240808232207_message_stats.ts @@ -0,0 +1,21 @@ +import type { Kysely } from "kysely"; + +export async function up(db: Kysely) { + return db.schema + .createTable("message_stats") + .addColumn("message_id", "text", (c) => c.primaryKey()) + .addColumn("author_id", "text", (c) => c.notNull()) + .addColumn("guild_id", "text", (c) => c.notNull()) + .addColumn("channel_id", "text", (c) => c.notNull()) + .addColumn("channel_category", "text") + .addColumn("recipient_id", "text") + .addColumn("char_count", "integer", (c) => c.notNull()) + .addColumn("word_count", "integer", (c) => c.notNull()) + .addColumn("react_count", "integer", (c) => c.notNull().defaultTo(0)) + .addColumn("sent_at", "text", (c) => c.notNull()) + .execute(); +} + +export async function down(db: Kysely) { + return db.schema.dropTable("message_stats").execute(); +} From 0a864296b8a9bb69a01eb3bc946211398e77f3aa Mon Sep 17 00:00:00 2001 From: DanielFGray Date: Fri, 6 Sep 2024 13:07:39 -0500 Subject: [PATCH 2/8] add cache for channel info requests --- app/discord/activityTracker.ts | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/app/discord/activityTracker.ts b/app/discord/activityTracker.ts index 3263ce2..94e5e5c 100644 --- a/app/discord/activityTracker.ts +++ b/app/discord/activityTracker.ts @@ -1,8 +1,18 @@ import { Events, ChannelType } from "discord.js"; -import type { Client, Message, PartialMessage } from "discord.js"; +import type { Client, Message, PartialMessage, TextChannel } from "discord.js"; import db from "~/db.server"; export async function startActivityTracking(client: Client) { + const channelCache = new Map(); + + async function getOrFetchChannel(msg: Message) { + return channelCache.has(msg.channelId) + ? channelCache.get(msg.channelId) + : channelCache + .set(msg.channelId, await msg.channel.fetch()) + .get(msg.channelId)!; + } + client.on(Events.MessageCreate, async (msg) => { const info = await getMessageStats(msg); if (!info) return; @@ -16,7 +26,7 @@ export async function startActivityTracking(client: Client) { channel_id: msg.channelId, recipient_id: msg.mentions?.repliedUser?.id ?? null, // TODO: cache this? - channel_category: (await msg.channel.fetch()).parentId, + channel_category: (await getOrFetchChannel(msg))!.parentId, }) .execute(); reportByGuild(msg.guildId!); From f0171a211080a5aa5ce24c28b397584d71752607 Mon Sep 17 00:00:00 2001 From: DanielFGray Date: Fri, 6 Sep 2024 16:00:06 -0500 Subject: [PATCH 3/8] fix out-of-order migration --- ...08232207_message_stats.ts => 20240906155529-message_stats.ts} | 1 + 1 file changed, 1 insertion(+) rename migrations/{20240808232207_message_stats.ts => 20240906155529-message_stats.ts} (99%) diff --git a/migrations/20240808232207_message_stats.ts b/migrations/20240906155529-message_stats.ts similarity index 99% rename from migrations/20240808232207_message_stats.ts rename to migrations/20240906155529-message_stats.ts index 2dc0c97..7048173 100644 --- a/migrations/20240808232207_message_stats.ts +++ b/migrations/20240906155529-message_stats.ts @@ -19,3 +19,4 @@ export async function up(db: Kysely) { export async function down(db: Kysely) { return db.schema.dropTable("message_stats").execute(); } + From 12c045d324799548c4cedd5d8cc2983a09ef7fd9 Mon Sep 17 00:00:00 2001 From: DanielFGray Date: Fri, 6 Sep 2024 19:32:57 -0500 Subject: [PATCH 4/8] update types and comments --- app/discord/activityTracker.ts | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/app/discord/activityTracker.ts b/app/discord/activityTracker.ts index 94e5e5c..ba7c6ba 100644 --- a/app/discord/activityTracker.ts +++ b/app/discord/activityTracker.ts @@ -5,12 +5,13 @@ import db from "~/db.server"; export async function startActivityTracking(client: Client) { const channelCache = new Map(); - async function getOrFetchChannel(msg: Message) { + async function getOrFetchChannel(msg: Message) { + // TODO: cache eviction? return channelCache.has(msg.channelId) ? channelCache.get(msg.channelId) : channelCache - .set(msg.channelId, await msg.channel.fetch()) - .get(msg.channelId)!; + .set(msg.channelId, (await msg.channel.fetch()) as TextChannel) + .get(msg.channelId); } client.on(Events.MessageCreate, async (msg) => { @@ -22,11 +23,10 @@ export async function startActivityTracking(client: Client) { ...info, message_id: msg.id, author_id: msg.author!.id, - guild_id: msg.guildId, + guild_id: msg.guildId!, channel_id: msg.channelId, recipient_id: msg.mentions?.repliedUser?.id ?? null, - // TODO: cache this? - channel_category: (await getOrFetchChannel(msg))!.parentId, + channel_category: (await getOrFetchChannel(msg))?.parent?.name, }) .execute(); reportByGuild(msg.guildId!); From 9687d8406f44906d7f7b1c4d634ff53f56ab097e Mon Sep 17 00:00:00 2001 From: DanielFGray Date: Fri, 6 Sep 2024 19:38:22 -0500 Subject: [PATCH 5/8] tidy up --- app/discord/activityTracker.ts | 10 ++-------- migrations/20240906155529-message_stats.ts | 1 - 2 files changed, 2 insertions(+), 9 deletions(-) diff --git a/app/discord/activityTracker.ts b/app/discord/activityTracker.ts index ba7c6ba..d16fe37 100644 --- a/app/discord/activityTracker.ts +++ b/app/discord/activityTracker.ts @@ -29,15 +29,12 @@ export async function startActivityTracking(client: Client) { channel_category: (await getOrFetchChannel(msg))?.parent?.name, }) .execute(); - reportByGuild(msg.guildId!); }); client.on(Events.MessageUpdate, async (msg) => { const info = await getMessageStats(msg); - console.log(msg, info); if (!info) return; await updateStatsById(msg.id).set(info).execute(); - reportByGuild(msg.guildId!); }); client.on(Events.MessageDelete, async (msg) => { @@ -47,21 +44,18 @@ export async function startActivityTracking(client: Client) { .deleteFrom("message_stats") .where("message_id", "=", msg.id) .execute(); - reportByGuild(msg.guildId!); }); client.on(Events.MessageReactionAdd, async (msg) => { await updateStatsById(msg.message.id) .set({ react_count: (eb) => eb(eb.ref("react_count"), "+", 1) }) .execute(); - reportByGuild(msg.message.guildId!); }); client.on(Events.MessageReactionRemove, async (msg) => { await updateStatsById(msg.message.id) .set({ react_count: (eb) => eb(eb.ref("react_count"), "-", 1) }) .execute(); - reportByGuild(msg.message.guildId!); }); } @@ -82,7 +76,7 @@ async function getMessageStats(msg: Message | PartialMessage) { }; } -async function reportByGuild(guildId: string) { +export async function reportByGuild(guildId: string) { const result = await db .selectFrom("message_stats") .select((eb) => [ @@ -97,5 +91,5 @@ async function reportByGuild(guildId: string) { .where("guild_id", "=", guildId) .groupBy("author_id") .execute(); - console.log(result); + return result } diff --git a/migrations/20240906155529-message_stats.ts b/migrations/20240906155529-message_stats.ts index 7048173..2dc0c97 100644 --- a/migrations/20240906155529-message_stats.ts +++ b/migrations/20240906155529-message_stats.ts @@ -19,4 +19,3 @@ export async function up(db: Kysely) { export async function down(db: Kysely) { return db.schema.dropTable("message_stats").execute(); } - From 1031367546ff8e4c178a336fcad9bddefcb41e17 Mon Sep 17 00:00:00 2001 From: DanielFGray Date: Fri, 4 Oct 2024 16:48:46 -0500 Subject: [PATCH 6/8] fix: fetch message content --- app/discord/activityTracker.ts | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/app/discord/activityTracker.ts b/app/discord/activityTracker.ts index d16fe37..e001501 100644 --- a/app/discord/activityTracker.ts +++ b/app/discord/activityTracker.ts @@ -68,9 +68,10 @@ async function getMessageStats(msg: Message | PartialMessage) { if (msg.channel.type !== ChannelType.GuildText || msg.author?.bot) { return; } + const { content } = await msg.fetch(); return { - char_count: msg.content?.length ?? 0, - word_count: msg.content?.split(/\s+/).length ?? 0, + char_count: content?.length ?? 0, + word_count: content?.split(/\s+/).length ?? 0, react_count: msg.reactions.cache.size, sent_at: String(msg.createdTimestamp), }; From 5e778c07c92cdf42170438a4096ec20ec491dba9 Mon Sep 17 00:00:00 2001 From: DanielFGray Date: Fri, 4 Oct 2024 16:55:33 -0500 Subject: [PATCH 7/8] fix: properly store dates as ints --- app/discord/activityTracker.ts | 2 +- migrations/20240906155529-message_stats.ts | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/app/discord/activityTracker.ts b/app/discord/activityTracker.ts index e001501..02a49fb 100644 --- a/app/discord/activityTracker.ts +++ b/app/discord/activityTracker.ts @@ -73,7 +73,7 @@ async function getMessageStats(msg: Message | PartialMessage) { char_count: content?.length ?? 0, word_count: content?.split(/\s+/).length ?? 0, react_count: msg.reactions.cache.size, - sent_at: String(msg.createdTimestamp), + sent_at: msg.createdTimestamp, }; } diff --git a/migrations/20240906155529-message_stats.ts b/migrations/20240906155529-message_stats.ts index 2dc0c97..4ef7a09 100644 --- a/migrations/20240906155529-message_stats.ts +++ b/migrations/20240906155529-message_stats.ts @@ -12,7 +12,7 @@ export async function up(db: Kysely) { .addColumn("char_count", "integer", (c) => c.notNull()) .addColumn("word_count", "integer", (c) => c.notNull()) .addColumn("react_count", "integer", (c) => c.notNull().defaultTo(0)) - .addColumn("sent_at", "text", (c) => c.notNull()) + .addColumn("sent_at", "integer", (c) => c.notNull()) .execute(); } From 0c44dbf69d2cd22b46afb1d0164b2e9d7ba84b40 Mon Sep 17 00:00:00 2001 From: DanielFGray Date: Fri, 4 Oct 2024 19:54:36 -0500 Subject: [PATCH 8/8] fix: types --- app/discord/activityTracker.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/discord/activityTracker.ts b/app/discord/activityTracker.ts index 02a49fb..e001501 100644 --- a/app/discord/activityTracker.ts +++ b/app/discord/activityTracker.ts @@ -73,7 +73,7 @@ async function getMessageStats(msg: Message | PartialMessage) { char_count: content?.length ?? 0, word_count: content?.split(/\s+/).length ?? 0, react_count: msg.reactions.cache.size, - sent_at: msg.createdTimestamp, + sent_at: String(msg.createdTimestamp), }; }