diff --git a/database/000_recreate_playerdata_dev.sql b/database/000_recreate_playerdata_dev.sql new file mode 100644 index 0000000..51e9bdc --- /dev/null +++ b/database/000_recreate_playerdata_dev.sql @@ -0,0 +1,3 @@ +DROP DATABASE IF EXISTS playerdata_dev; +CREATE DATABASE playerdata_dev; +GRANT SELECT, INSERT, UPDATE, DELETE, EVENT, TRIGGER ON `playerdata\_dev`.* TO 'event_admin'@'localhost'; ALTER USER 'event_admin'@'localhost' ; diff --git a/database/001_insert_apiUser.sql b/database/001_insert_apiUser.sql new file mode 100644 index 0000000..98e46a5 --- /dev/null +++ b/database/001_insert_apiUser.sql @@ -0,0 +1,19 @@ +insert ignore into apiPermissions (id, permission) values +(1, 'verify_ban'), +(2, 'verify_players'), +(3, 'request_highscores') +; + +insert ignore into apiUser (id, username, token, ratelimit, is_active) values +(1, 'verify_ban', 'verify_ban', -1, 1), +(2, 'verify_players', 'verify_players', -1, 1), +(3, 'request_highscores', 'request_highscores', -1, 1) +; + +insert ignore into apiUserPerms (user_id, permission_id) VALUES +(1,1), +(1,2), +(1,3), +(2,2), +(3,3) +; \ No newline at end of file diff --git a/database/002_insert_players.sql b/database/002_insert_players.sql new file mode 100644 index 0000000..66b50fc --- /dev/null +++ b/database/002_insert_players.sql @@ -0,0 +1,50 @@ +/* + Insert players +*/ +insert ignore into playerdata_dev.Labels +select * from playerdata.Labels order by id desc; +update playerdata_dev.Labels set id=0 where label='Unknown'; + +insert ignore into playerdata_dev.LabelJagex +select * from playerdata.LabelJagex; + +insert ignore into playerdata_dev.Players +select p.* from playerdata.Labels lb +join lateral ( + select + pl.* + from playerdata.Players pl + where lb.id = pl.label_id + limit 100 +) p on (1=1) +; + +/* +insert ignore into playerdata_dev.Players +select * from playerdata.Players pl +where 1=1 + and pl.label_id = 0 -- unkown player + and pl.id not in (select id from playerdata_dev.Players) +limit 100 +; + +insert ignore into playerdata_dev.Players +select * from playerdata.Players pl +where 1=1 + and pl.label_id = 1 -- real player + and pl.id not in (select id from playerdata_dev.Players) +limit 15000 +; +*/ + +/* + Encrypt player names +*/ +update playerdata_dev.Players +set + name=HEX(AES_ENCRYPT(name, "Victor_careers_onto_THE9_free0_endorser.")), + normalized_name=HEX(AES_ENCRYPT(normalized_name, "Victor_careers_onto_THE9_free0_endorser.")) +; +/* + To decrypt the encryption: ES_DECRYPT(BINARY(UNHEX(setting_value)), 'key') +*/ \ No newline at end of file diff --git a/database/003_insert_playerdata_dev.sql b/database/003_insert_playerdata_dev.sql new file mode 100644 index 0000000..d950aab --- /dev/null +++ b/database/003_insert_playerdata_dev.sql @@ -0,0 +1,17 @@ +/* + Generic data +*/ +insert ignore into playerdata_dev.PredictionsFeedback +select pr.* from playerdata.PredictionsFeedback pr +join playerdata_dev.Players pl on pr.voter_id = pl.id or pr.subject_id=pl.id; + +insert ignore into playerdata_dev.reportLatest +select rp.* from playerdata.reportLatest rp +join playerdata_dev.Players pl on rp.reported_id = pl.id; + +insert ignore into playerdata_dev.Predictions +select pr.* from playerdata.Predictions pr +join playerdata_dev.Players pl on pr.id = pl.id; + +ALTER TABLE playerdata_dev.Predictions CHANGE `name` `name` VARCHAR(64) CHARACTER SET utf8mb4 COLLATE utf8mb4_0900_ai_ci NULL DEFAULT NULL; +update playerdata_dev.Predictions set name=HEX(AES_ENCRYPT(name, "Victor_careers_onto_THE9_free0_endorser.")); \ No newline at end of file diff --git a/database/004_insert_playerHiscoreData.sql b/database/004_insert_playerHiscoreData.sql new file mode 100644 index 0000000..65558ae --- /dev/null +++ b/database/004_insert_playerHiscoreData.sql @@ -0,0 +1,12 @@ +insert ignore into playerdata_dev.playerHiscoreData +select phd.* from playerdata.playerHiscoreData phd +join playerdata_dev.Players pl on phd.Player_id = pl.id; + +insert ignore into playerdata_dev.playerHiscoreDataLatest +select phd.* from playerdata.playerHiscoreDataLatest phd +join playerdata_dev.Players pl on phd.Player_id = pl.id; + +insert ignore into playerdata_dev.playerHiscoreDataXPChange +select phd.* from playerdata.playerHiscoreDataXPChange phd +join playerdata_dev.Players pl on phd.Player_id = pl.id; + diff --git a/database/005_insert_reports.sql b/database/005_insert_reports.sql new file mode 100644 index 0000000..373cbf6 --- /dev/null +++ b/database/005_insert_reports.sql @@ -0,0 +1,21 @@ +insert ignore into playerdata_dev.Reports +select a.* from playerdata_dev.Players pl +join lateral ( + select + plr.* + from playerdata.Reports plr + where plr.reportedid = pl.id + limit 100 +) a on (1=1) +; + +insert ignore into playerdata_dev.Reports +select a.* from playerdata_dev.Players pl +join lateral ( + select + plr.* + from playerdata.Reports plr + where plr.reportingid = pl.id + limit 100 +) a on (1=1) +; \ No newline at end of file diff --git a/database/create.sh b/database/create.sh new file mode 100644 index 0000000..6f48e2e --- /dev/null +++ b/database/create.sh @@ -0,0 +1,14 @@ +#!/bin/sh + +sudo mysql -u root < /home/ubuntu/000_recreate_playerdata_dev.sql +sudo mysqldump -u root --no-data --routines --triggers --events --databases playerdata| sudo mysql -u root playerdata_dev +sudo mysql -u root playerdata_dev < /home/ubuntu/001_insert_apiUser.sql +sudo mysql -u root playerdata_dev < /home/ubuntu/002_insert_players.sql +sudo mysql -u root playerdata_dev < /home/ubuntu/003_insert_playerdata_dev.sql +sudo mysql -u root playerdata_dev < /home/ubuntu/004_insert_playerHiscoreData.sql +sudo mysql -u root playerdata_dev < /home/ubuntu/005_insert_reports.sql +sudo mysqldump -u root --routines --events --databases playerdata_dev | sed 's/playerdata_dev/playerdata/g'| gzip > 100_playerdata_dev.sql.gz + +# sudo mysql -u root playerdata_dev < /home/ubuntu/insert_playerReports.sql +# sudo mysql -u root playerdata_dev < /home/ubuntu/insert_playerReportsManual.sql +# sudo mysql -u root playerdata_dev < /home/ubuntu/insert_playerLocationsDetail.sql diff --git a/database/xx_insert_playerLocationsDetail.sql b/database/xx_insert_playerLocationsDetail.sql new file mode 100644 index 0000000..0ddbd9d --- /dev/null +++ b/database/xx_insert_playerLocationsDetail.sql @@ -0,0 +1,21 @@ +insert ignore into playerdata_dev.playerLocations +select a.* from playerdata_dev.Players pl +join lateral ( + select + plo.* + from playerdata.playerLocations plo + where plo.reported_id = pl.id + limit 100 +) a on (1=1) +; + +insert ignore into playerdata_dev.playerLocationsDetail +select a.* from playerdata_dev.Players pl +join lateral ( + select + pld.* + from playerdata.playerLocationsDetail pld + where pld.reported_id = pl.id + limit 100 +) a on (1=1) +; \ No newline at end of file diff --git a/database/xx_insert_playerReports.sql b/database/xx_insert_playerReports.sql new file mode 100644 index 0000000..021fbfe --- /dev/null +++ b/database/xx_insert_playerReports.sql @@ -0,0 +1,21 @@ +insert ignore into playerdata_dev.playerReports +select a.* from playerdata_dev.Players pl +join lateral ( + select + plr.* + from playerdata.playerReports plr + where plr.reported_id = pl.id + limit 100 +) a on (1=1) +; + +insert ignore into playerdata_dev.playerReports +select a.* from playerdata_dev.Players pl +join lateral ( + select + plr.* + from playerdata.playerReports plr + where plr.reporting_id = pl.id + limit 100 +) a on (1=1) +; \ No newline at end of file diff --git a/database/xx_insert_playerReportsManual.sql b/database/xx_insert_playerReportsManual.sql new file mode 100644 index 0000000..49cfc19 --- /dev/null +++ b/database/xx_insert_playerReportsManual.sql @@ -0,0 +1,21 @@ +insert ignore into playerdata_dev.playerReportsManual +select a.* from playerdata_dev.Players pl +join lateral ( + select + prm.* + from playerdata.playerReportsManual prm + where prm.reporting_id = pl.id + limit 100 +) a on (1=1) +; + +insert ignore into playerdata_dev.playerReportsManual +select a.* from playerdata_dev.Players pl +join lateral ( + select + prm.* + from playerdata.playerReportsManual prm + where prm.reported_id = pl.id + limit 100 +) a on (1=1) +; \ No newline at end of file diff --git a/notes.md b/notes.md new file mode 100644 index 0000000..eacd2f1 --- /dev/null +++ b/notes.md @@ -0,0 +1,18 @@ +## setup +``` +python -m venv .venv +.venv\Scripts\activate +python -m pip install --upgrade pip +pip install -r requirements.txt +``` +# for admin purposes saving & upgrading +run this in cmd + +``` +venv\Scripts\activate +call pip freeze > requirements.txt +powershell "(Get-Content requirements.txt) | ForEach-Object { $_ -replace '==', '>=' } | Set-Content requirements.txt" +call pip install -r requirements.txt --upgrade +call pip freeze > requirements.txt +powershell "(Get-Content requirements.txt) | ForEach-Object { $_ -replace '>=', '==' } | Set-Content requirements.txt" +``` \ No newline at end of file diff --git a/report_migration/migrate_report_data.py b/report_migration/migrate_report_data.py index b781bc8..e4a4fbd 100644 --- a/report_migration/migrate_report_data.py +++ b/report_migration/migrate_report_data.py @@ -20,14 +20,45 @@ autoflush=False, ) +with open('report_data.sql','r') as x: + migration_query = x.read() + + +with open('report_data_where_not_exists.sql','r') as x: + migration_query_where_not_exists = x.read() + + +with open('reset_tables.sql','r') as x: + reset_tables_query = x.read() + + + async def main(): + await migrate_selection(1, 100_000) + +async def reset_tables(): # get a session async with Session() as session: session: AsyncSession - # get a transaction async with session.begin(): - # do something - pass + await session.execute(sqla.text(reset_tables_query+" COMMIT;")) +async def migrate_selection(startId, endId): + # get a session + async with Session() as session: + session: AsyncSession + params = {"startId": startId, "endId": endId} + async with session.begin(): + await session.execute(sqla.text(migration_query+" COMMIT;"), params=params) + +async def migrate_selection_where_not_exists(startId, endId): + # get a session + async with Session() as session: + session: AsyncSession + params = {"startId": startId, "endId": endId} + async with session.begin(): + await session.execute(sqla.text(migration_query_where_not_exists+" COMMIT;"), params=params) + + if __name__ == "__main__": asyncio.run(main()) \ No newline at end of file diff --git a/report_migration/migrate_test.py b/report_migration/migrate_test.py new file mode 100644 index 0000000..49a36d7 --- /dev/null +++ b/report_migration/migrate_test.py @@ -0,0 +1,54 @@ +import asyncio +import time +from migrate_report_data import reset_tables, migrate_selection, migrate_selection_where_not_exists + + +async def reset_and_time(func, *args, **kwargs): + """ + Helper function to reset tables and measure execution time of a function. + """ + await reset_tables() # Reset tables before timing + start_time = time.time() + await func(*args, **kwargs) + end_time = time.time() + return end_time - start_time + + +async def test_migrate_selection(): + """ + Test migrate_selection and measure average time for 10 calls. + """ + print("Testing migrate_selection...") + total_time = 0 + for i in range(10): + elapsed = await reset_and_time(migrate_selection, 1, 100_000) + print(f"Run {i + 1}: {elapsed:.4f} seconds") + total_time += elapsed + avg_time = total_time / 10 + print(f"Average time for migrate_selection: {avg_time:.4f} seconds") + + +async def test_migrate_selection_where_not_exists(): + """ + Test migrate_selection_where_not_exists and measure average time for 10 calls. + """ + print("\nTesting migrate_selection_where_not_exists...") + total_time = 0 + for i in range(10): + elapsed = await reset_and_time(migrate_selection_where_not_exists, 1, 100_000) + print(f"Run {i + 1}: {elapsed:.4f} seconds") + total_time += elapsed + avg_time = total_time / 10 + print(f"Average time for migrate_selection_where_not_exists: {avg_time:.4f} seconds") + + +async def main(): + """ + Main function to run all tests sequentially. + """ + await test_migrate_selection() + await test_migrate_selection_where_not_exists() + + +if __name__ == "__main__": + asyncio.run(main()) diff --git a/report_migration/report_data.sql b/report_migration/report_data.sql new file mode 100644 index 0000000..2d9849c --- /dev/null +++ b/report_migration/report_data.sql @@ -0,0 +1,154 @@ +-- Step 1: Create temp_batch table +DROP TEMPORARY TABLE IF EXISTS temp_batch; + +CREATE TEMPORARY TABLE IF NOT EXISTS temp_batch ( + ID BIGINT NOT NULL, + created_at TIMESTAMP, + reportedID INT, + reportingID INT, + region_id INT, + x_coord INT, + y_coord INT, + z_coord INT, + timestamp TIMESTAMP, + manual_detect TINYINT, + on_members_world INT, + on_pvp_world TINYINT, + world_number INT, + equip_head_id INT, + equip_amulet_id INT, + equip_torso_id INT, + equip_legs_id INT, + equip_boots_id INT, + equip_cape_id INT, + equip_hands_id INT, + equip_weapon_id INT, + equip_shield_id INT, + equip_ge_value BIGINT, + report_sighting_id INT DEFAULT NULL, + report_location_id INT DEFAULT NULL, + report_gear_id INT DEFAULT NULL +); + +-- Step 2: Insert data into temp_batch (manually adjust startId, endId, and batch_size as needed) + +INSERT INTO temp_batch (ID, created_at, reportedID, reportingID, region_id, x_coord, y_coord, z_coord, + timestamp, manual_detect, on_members_world, on_pvp_world, world_number, + equip_head_id, equip_amulet_id, equip_torso_id, equip_legs_id, equip_boots_id, + equip_cape_id, equip_hands_id, equip_weapon_id, equip_shield_id, equip_ge_value) +SELECT ID, created_at, reportedID, reportingID, region_id, x_coord, y_coord, z_coord, + timestamp, manual_detect, on_members_world, on_pvp_world, world_number, + equip_head_id, equip_amulet_id, equip_torso_id, equip_legs_id, equip_boots_id, + equip_cape_id, equip_hands_id, equip_weapon_id, equip_shield_id, equip_ge_value +FROM Reports +WHERE ID > :startId AND ID <= :endId -- Python Variables +ORDER BY ID ASC; + +-- Step 3: Update temp_batch with existing foreign keys from report_sighting +UPDATE temp_batch AS tb +LEFT JOIN report_sighting AS rs +ON tb.reportingID = rs.reporting_id AND tb.reportedID = rs.reported_id AND tb.manual_detect = rs.manual_detect +SET tb.report_sighting_id = rs.report_sighting_id; + +UPDATE temp_batch AS tb +LEFT JOIN report_gear AS rg +ON + tb.equip_head_id = rg.equip_head_id AND + tb.equip_amulet_id = rg.equip_amulet_id AND + tb.equip_torso_id = rg.equip_torso_id AND + tb.equip_legs_id = rg.equip_legs_id AND + tb.equip_boots_id = rg.equip_boots_id AND + tb.equip_cape_id = rg.equip_cape_id AND + tb.equip_hands_id = rg.equip_hands_id AND + tb.equip_weapon_id = rg.equip_weapon_id AND + tb.equip_shield_id = rg.equip_shield_id + +SET tb.report_gear_id = rg.report_gear_id; + +UPDATE temp_batch AS tb +LEFT JOIN report_location AS rl +ON tb.region_id = rl.region_id +AND tb.x_coord = rl.x_coord +AND tb.y_coord = rl.y_coord +AND tb.z_coord = rl.z_coord +SET tb.report_location_id = rl.report_location_id; + + + +-- Debug: Check temp_batch after update +-- SELECT * FROM temp_batch WHERE report_sighting_id IS NULL; + +-- Step 4: Insert missing rows into report_sighting +INSERT INTO report_sighting (reporting_id, reported_id, manual_detect) +SELECT DISTINCT reportingID, reportedID, manual_detect +FROM temp_batch +WHERE report_sighting_id IS NULL +ORDER BY reportingID, reportedID, manual_detect; + +-- Debug: Check rows inserted into report_sighting +-- SELECT * FROM report_sighting ORDER BY report_sighting_id DESC LIMIT 10; + +-- Step 5: Update temp_batch with newly inserted report_sighting IDs +UPDATE temp_batch AS tb +JOIN report_sighting AS rs +ON tb.reportingID = rs.reporting_id AND tb.reportedID = rs.reported_id AND tb.manual_detect = rs.manual_detect +SET tb.report_sighting_id = rs.report_sighting_id +WHERE tb.report_sighting_id IS NULL; + +-- Debug: Verify report_sighting_id is updated +-- SELECT * FROM temp_batch; + +-- Step 6: Insert missing rows into report_gear +INSERT INTO report_gear (equip_head_id, equip_amulet_id, equip_torso_id, equip_legs_id, + equip_boots_id, equip_cape_id, equip_hands_id, equip_weapon_id, equip_shield_id) +SELECT DISTINCT equip_head_id, equip_amulet_id, equip_torso_id, equip_legs_id, + equip_boots_id, equip_cape_id, equip_hands_id, equip_weapon_id, equip_shield_id +FROM temp_batch +WHERE report_gear_id IS NULL +ORDER BY equip_head_id, equip_amulet_id, equip_torso_id, equip_legs_id, + equip_boots_id, equip_cape_id, equip_hands_id, equip_weapon_id, equip_shield_id; + +-- Debug: Check rows inserted into report_gear +-- SELECT * FROM report_gear ORDER BY report_gear_id DESC LIMIT 10; + +-- Step 7: Update temp_batch with newly inserted report_gear IDs +UPDATE temp_batch AS tb +JOIN report_gear AS rg +ON tb.equip_head_id = rg.equip_head_id AND tb.equip_amulet_id = rg.equip_amulet_id + AND tb.equip_torso_id = rg.equip_torso_id AND tb.equip_legs_id = rg.equip_legs_id + AND tb.equip_boots_id = rg.equip_boots_id AND tb.equip_cape_id = rg.equip_cape_id + AND tb.equip_hands_id = rg.equip_hands_id AND tb.equip_weapon_id = rg.equip_weapon_id + AND tb.equip_shield_id = rg.equip_shield_id +SET tb.report_gear_id = rg.report_gear_id +WHERE tb.report_gear_id IS NULL; + +-- Debug: Verify report_gear_id is updated +-- SELECT * FROM temp_batch; + +-- Step 8: Insert missing rows into report_location +INSERT INTO report_location (region_id, x_coord, y_coord, z_coord) +SELECT DISTINCT region_id, x_coord, y_coord, z_coord +FROM temp_batch +WHERE report_location_id IS NULL +ORDER BY region_id, x_coord, y_coord, z_coord; + +-- Debug: Check rows inserted into report_location +-- SELECT * FROM report_location ORDER BY report_location_id DESC LIMIT 10; + +-- Step 9: Update temp_batch with newly inserted report_location IDs +UPDATE temp_batch AS tb +JOIN report_location AS rl +ON tb.region_id = rl.region_id AND tb.x_coord = rl.x_coord + AND tb.y_coord = rl.y_coord AND tb.z_coord = rl.z_coord +SET tb.report_location_id = rl.report_location_id +WHERE tb.report_location_id IS NULL; + +-- Debug: Verify report_location_id is updated +-- SELECT * FROM temp_batch; + +-- Step 10: Final insert into the report table +INSERT IGNORE INTO report (report_sighting_id, report_location_id, report_gear_id, created_at, reported_at, + on_members_world, on_pvp_world, world_number, region_id) +SELECT DISTINCT report_sighting_id, report_location_id, report_gear_id, created_at, timestamp, + on_members_world, on_pvp_world, world_number, region_id +FROM temp_batch; diff --git a/report_migration/report_data_where_not_exists.sql b/report_migration/report_data_where_not_exists.sql new file mode 100644 index 0000000..fc0b9f4 --- /dev/null +++ b/report_migration/report_data_where_not_exists.sql @@ -0,0 +1,201 @@ +-- Step 1: Create temp_batch table +DROP TEMPORARY TABLE IF EXISTS temp_batch; + +CREATE TEMPORARY TABLE IF NOT EXISTS temp_batch +( + ID BIGINT NOT NULL, + created_at TIMESTAMP, + reportedID INT, + reportingID INT, + region_id INT, + x_coord INT, + y_coord INT, + z_coord INT, + timestamp TIMESTAMP, + manual_detect TINYINT, + on_members_world INT, + on_pvp_world TINYINT, + world_number INT, + equip_head_id INT, + equip_amulet_id INT, + equip_torso_id INT, + equip_legs_id INT, + equip_boots_id INT, + equip_cape_id INT, + equip_hands_id INT, + equip_weapon_id INT, + equip_shield_id INT, + equip_ge_value BIGINT, + report_sighting_id INT DEFAULT NULL, + report_location_id INT DEFAULT NULL, + report_gear_id INT DEFAULT NULL +); + +-- Step 2: Insert data into temp_batch (manually adjust startId, endId, and batch_size as needed) +SET @startId = 1; +SET @endId = 10000; +SET @batch_size = 1000; + +INSERT INTO temp_batch (ID, created_at, reportedID, reportingID, region_id, x_coord, y_coord, z_coord, + timestamp, manual_detect, on_members_world, on_pvp_world, world_number, + equip_head_id, equip_amulet_id, equip_torso_id, equip_legs_id, equip_boots_id, + equip_cape_id, equip_hands_id, equip_weapon_id, equip_shield_id, equip_ge_value) +SELECT ID, + created_at, + reportedID, + reportingID, + region_id, + x_coord, + y_coord, + z_coord, + timestamp, + manual_detect, + on_members_world, + on_pvp_world, + world_number, + equip_head_id, + equip_amulet_id, + equip_torso_id, + equip_legs_id, + equip_boots_id, + equip_cape_id, + equip_hands_id, + equip_weapon_id, + equip_shield_id, + equip_ge_value +FROM Reports +WHERE ID > 1 + AND ID <= 100000 +ORDER BY ID ASC; + +-- Step 3: Update temp_batch with existing foreign keys from (report_sighting, report_gear, report_location) + + +-- Step 4: Insert missing rows into report_sighting +INSERT INTO report_sighting (reporting_id, reported_id, manual_detect) +SELECT DISTINCT reportingID, reportedID, manual_detect +FROM temp_batch tb +WHERE NOT EXISTS (SELECT 1 + FROM report_sighting rs + WHERE 1 + AND tb.reportingID = rs.reporting_id + AND tb.reportedID = rs.reported_id + AND IFNULL(tb.manual_detect, 0) = rs.manual_detect) +ORDER BY reportingID, reportedID, manual_detect; + +-- Debug: Check rows inserted into report_sighting +SELECT * +FROM report_sighting +ORDER BY report_sighting_id DESC +LIMIT 10; + +-- Step 5: Update temp_batch with newly inserted report_sighting IDs +UPDATE temp_batch AS tb + JOIN report_sighting AS rs + ON tb.reportingID = rs.reporting_id AND tb.reportedID = rs.reported_id AND tb.manual_detect = rs.manual_detect +SET tb.report_sighting_id = rs.report_sighting_id +WHERE tb.report_sighting_id IS NULL; + +-- Debug: Verify report_sighting_id is updated +SELECT * +FROM temp_batch; + +-- Step 6: Insert missing rows into report_gear +INSERT INTO report_gear (equip_head_id, equip_amulet_id, equip_torso_id, equip_legs_id, + equip_boots_id, equip_cape_id, equip_hands_id, equip_weapon_id, equip_shield_id) +SELECT DISTINCT equip_head_id, + equip_amulet_id, + equip_torso_id, + equip_legs_id, + equip_boots_id, + equip_cape_id, + equip_hands_id, + equip_weapon_id, + equip_shield_id +FROM temp_batch tb +WHERE NOT EXISTS (SELECT 1 + FROM report_gear rg + WHERE 1 + AND tb.equip_head_id = rg.equip_head_id + AND tb.equip_amulet_id = rg.equip_amulet_id + AND tb.equip_torso_id = rg.equip_torso_id + AND tb.equip_legs_id = rg.equip_legs_id + AND tb.equip_boots_id = rg.equip_boots_id + AND tb.equip_cape_id = rg.equip_cape_id + AND tb.equip_hands_id = rg.equip_hands_id + AND tb.equip_weapon_id = rg.equip_weapon_id + AND tb.equip_shield_id = rg.equip_shield_id) +ORDER BY equip_head_id, equip_amulet_id, equip_torso_id, equip_legs_id, + equip_boots_id, equip_cape_id, equip_hands_id, equip_weapon_id, equip_shield_id; + +-- Debug: Check rows inserted into report_gear +SELECT * +FROM report_gear +ORDER BY report_gear_id DESC +LIMIT 10; + +-- Step 7: Update temp_batch with newly inserted report_gear IDs +UPDATE temp_batch AS tb + JOIN report_gear AS rg + ON 1 + AND tb.equip_head_id = rg.equip_head_id + AND tb.equip_amulet_id = rg.equip_amulet_id + AND tb.equip_torso_id = rg.equip_torso_id + AND tb.equip_legs_id = rg.equip_legs_id + AND tb.equip_boots_id = rg.equip_boots_id + AND tb.equip_cape_id = rg.equip_cape_id + AND tb.equip_hands_id = rg.equip_hands_id + AND tb.equip_weapon_id = rg.equip_weapon_id + AND tb.equip_shield_id = rg.equip_shield_id +SET tb.report_gear_id = rg.report_gear_id +WHERE tb.report_gear_id IS NULL; + +-- Debug: Verify report_gear_id is updated +SELECT * +FROM temp_batch; + +-- Step 8: Insert missing rows into report_location +INSERT INTO report_location (region_id, x_coord, y_coord, z_coord) +SELECT DISTINCT region_id, x_coord, y_coord, z_coord +FROM temp_batch tb +WHERE NOT EXISTS (SELECT 1 + FROM report_location rl + WHERE 1 + AND tb.region_id = rl.region_id + AND tb.x_coord = rl.x_coord + AND tb.y_coord = rl.y_coord + AND tb.z_coord = rl.z_coord) +ORDER BY region_id, x_coord, y_coord, z_coord; + +-- Debug: Check rows inserted into report_location +SELECT * +FROM report_location +ORDER BY report_location_id DESC +LIMIT 10; + +-- Step 9: Update temp_batch with newly inserted report_location IDs +UPDATE temp_batch AS tb + JOIN report_location AS rl + ON tb.region_id = rl.region_id AND tb.x_coord = rl.x_coord + AND tb.y_coord = rl.y_coord AND tb.z_coord = rl.z_coord +SET tb.report_location_id = rl.report_location_id +WHERE tb.report_location_id IS NULL; + +-- Debug: Verify report_location_id is updated +SELECT * +FROM temp_batch; + +-- Step 10: Final insert into the report table +INSERT IGNORE INTO report (report_sighting_id, report_location_id, report_gear_id, created_at, reported_at, + on_members_world, on_pvp_world, world_number, region_id) +SELECT DISTINCT report_sighting_id, + report_location_id, + report_gear_id, + created_at, + timestamp, + on_members_world, + on_pvp_world, + world_number, + region_id +FROM temp_batch; + diff --git a/report_migration/reset_tables.sql b/report_migration/reset_tables.sql new file mode 100644 index 0000000..8fad1fb --- /dev/null +++ b/report_migration/reset_tables.sql @@ -0,0 +1,54 @@ +drop table if exists report; + +drop table if exists report_gear; + +drop table if exists report_location; + +drop table if exists report_sighting; + + +CREATE TABLE `report_sighting` ( + `report_sighting_id` INT UNSIGNED NOT NULL AUTO_INCREMENT, + `reporting_id` INT UNSIGNED NOT NULL, + `reported_id` INT UNSIGNED NOT NULL, + `manual_detect` TINYINT(1) DEFAULT 0, + PRIMARY key (`report_sighting_id`), + UNIQUE KEY unique_sighting (`reporting_id`, `reported_id`, `manual_detect`), + KEY idx_reported_id (`reported_id`) +); + +CREATE TABLE `report_gear` ( + `report_gear_id` INT UNSIGNED NOT NULL AUTO_INCREMENT, + `equip_head_id` SMALLINT UNSIGNED DEFAULT NULL, + `equip_amulet_id` SMALLINT UNSIGNED DEFAULT NULL, + `equip_torso_id` SMALLINT UNSIGNED DEFAULT NULL, + `equip_legs_id` SMALLINT UNSIGNED DEFAULT NULL, + `equip_boots_id` SMALLINT UNSIGNED DEFAULT NULL, + `equip_cape_id` SMALLINT UNSIGNED DEFAULT NULL, + `equip_hands_id` SMALLINT UNSIGNED DEFAULT NULL, + `equip_weapon_id` SMALLINT UNSIGNED DEFAULT NULL, + `equip_shield_id` SMALLINT UNSIGNED DEFAULT NULL, + PRIMARY key (`report_gear_id`), + UNIQUE KEY unique_gear (`equip_head_id`,`equip_amulet_id`,`equip_torso_id`,`equip_legs_id`,`equip_boots_id`,`equip_cape_id`,`equip_hands_id`,`equip_weapon_id`,`equip_shield_id`) +); +CREATE TABLE `report_location` ( + `report_location_id` INT UNSIGNED NOT NULL AUTO_INCREMENT, + `region_id` MEDIUMINT UNSIGNED NOT NULL, + `x_coord` MEDIUMINT UNSIGNED NOT NULL, + `y_coord` MEDIUMINT UNSIGNED NOT NULL, + `z_coord` MEDIUMINT UNSIGNED NOT NULL, + PRIMARY key (`report_location_id`), + UNIQUE KEY unique_location (`region_id`, `x_coord`, `y_coord`, `z_coord`) +); +CREATE TABLE `report` ( + `report_sighting_id` INT UNSIGNED NOT NULL, + `report_location_id` INT UNSIGNED NOT NULL, + `report_gear_id` INT UNSIGNED NOT NULL, + `created_at` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP, + `reported_at` timestamp NOT NULL, + `on_members_world` TINYINT(1) DEFAULT NULL, + `on_pvp_world` TINYINT(1) DEFAULT NULL, + `world_number` SMALLINT UNSIGNED DEFAULT NULL, + `region_id` MEDIUMINT UNSIGNED NOT NULL, + PRIMARY key (`report_sighting_id`, `report_location_id`, `region_id`) +); diff --git a/test.sql b/test.sql new file mode 100644 index 0000000..e69de29