Index annotations

build.gradle

@@ -27,6 +27,8 @@ application {
 }
 
 dependencies {
+    implementation 'org.slf4j:slf4j-simple:2.0.16'
+
     implementation 'org.ow2.asm:asm:9.7.1'
     implementation 'com.electronwill.night-config:toml:3.6.6'
     implementation 'com.google.code.gson:gson:2.11.0'

src/main/java/net/neoforged/waifu/MainDatabase.java

@@ -2,6 +2,7 @@
 
 import org.flywaydb.core.Flyway;
 import org.jdbi.v3.core.Jdbi;
+import org.jdbi.v3.sqlobject.SqlObjectPlugin;
 import org.jdbi.v3.sqlobject.statement.SqlQuery;
 import org.jdbi.v3.sqlobject.statement.SqlUpdate;
 import org.jdbi.v3.sqlobject.transaction.Transactional;
@@ -16,7 +17,7 @@ public class MainDatabase {
     private final DBTrans transactional;
 
     public MainDatabase(Path path) {
-        if (Files.exists(path)) {
+        if (!Files.exists(path)) {
             try {
                 var parent = path.getParent();
                 if (parent != null) Files.createDirectories(parent);
@@ -33,7 +34,11 @@ public MainDatabase(Path path) {
         dataSource.setDatabaseName("WAIFU main");
         dataSource.setEnforceForeignKeys(true);
 
-        this.transactional = Jdbi.create(dataSource).onDemand(DBTrans.class);
+        var jdbi = Jdbi.create(dataSource);
+
+        jdbi.installPlugin(new SqlObjectPlugin());
+
+        this.transactional = jdbi.onDemand(DBTrans.class);
     }
 
     public void runFlyway() {

src/main/java/net/neoforged/waifu/ModIndexer.java

@@ -26,7 +26,7 @@
 import java.util.concurrent.ExecutorService;
 
 public class ModIndexer<T extends IndexDatabase.DatabaseMod> {
-    private static final boolean KEEP_CACHES = Boolean.parseBoolean(System.getenv().getOrDefault("KEEP_PLATFORM_CACHES", "true")));
+    private static final boolean KEEP_CACHES = Boolean.parseBoolean(System.getenv().getOrDefault("KEEP_PLATFORM_CACHES", "true"));
     private final Path baseCacheFolder;
     private final IndexDatabase<T> db;
 
@@ -149,7 +149,7 @@ private void runCurrent(ProgressMonitor<IndexCandidate> monitor, List<Completabl
     }
 
     private Runnable indexAndPrepareUpload(@Nullable PlatformModFile platform, ModFileInfo file, T mod, boolean refs, DataSanitizer sanitizer) throws IOException {
-        List<ClassData> classes = IndexingClassVisitor.collect(file.getRootDirectory(), refs);
+        List<ClassData> classes = IndexingClassVisitor.collect(file.getRootDirectory(), refs, refs); // TODO - do we want a separate parameter?
 
         var tags = TagCollector.collect(file.getPath("data"));
 

src/main/java/net/neoforged/waifu/db/ClassData.java

@@ -3,13 +3,13 @@
 import org.jetbrains.annotations.Nullable;
 import org.objectweb.asm.Type;
 
-import java.util.ArrayList;
 import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
 
 public record ClassData(
         String name, @Nullable String superClass, String[] interfaces,
+        List<AnnotationInfo> annotations,
 
         Map<String, FieldInfo> fields,
         Map<String, MethodInfo> methods,
@@ -18,19 +18,35 @@ public record ClassData(
         Map<Reference, Integer> fieldRefs
 ) {
 
-    public record FieldInfo(String name, Type desc, int accessLevel) {}
+    public record FieldInfo(String name, Type desc, int accessLevel, List<AnnotationInfo> annotations) {}
 
     public record MethodInfo(
-            String name, String desc, int accessLevel
+            String name, String desc, int accessLevel, List<AnnotationInfo> annotations
     ) {}
 
     public record Reference(String owner, String name, String desc) {}
 
     public ClassData copy() {
+        // TODO - we don't copy annotations - fix?
         return new ClassData(
-                name, superClass, interfaces,
+                name, superClass, interfaces, annotations,
                 new HashMap<>(fields), new HashMap<>(methods),
                 new HashMap<>(methodRefs), new HashMap<>(fieldRefs)
         );
     }
+
+    public record AnnotationInfo(
+            Type type,
+            Map<String, Object> members
+    ) {}
+
+    public record EnumValue(
+            Type enumType,
+            String value
+    ) {
+        @Override
+        public String toString() {
+            return enumType.getClassName() + ":" + value;
+        }
+    }
 }

src/main/java/net/neoforged/waifu/db/SQLDatabase.java

@@ -13,6 +13,7 @@
 import org.jdbi.v3.postgres.PostgresPlugin;
 import org.jdbi.v3.sqlobject.SqlObjectPlugin;
 import org.jetbrains.annotations.Nullable;
+import org.objectweb.asm.Type;
 
 import java.nio.file.Files;
 import java.sql.DriverManager;
@@ -143,15 +144,16 @@ public void insertClasses(List<ClassData> classes) {
                     if (classes.isEmpty()) return;
 
                     try {
-                        var stmt = con.prepareStatement("select * from insert_class(?, ?, ?, ?, ?, ?, ?)");
+                        var stmt = con.prepareStatement("select * from insert_class(?, ?, ?, ?, ?, ?, ?, ?)");
                         for (var aClass : classes) {
                             stmt.setInt(1, modId);
                             stmt.setString(2, aClass.name());
                             stmt.setString(3, aClass.superClass());
                             stmt.setArray(4, con.createArrayOf("text", aClass.interfaces()));
-                            stmt.setString(5, fields(aClass));
-                            stmt.setString(6, methods(aClass));
-                            stmt.setString(7, refs(aClass));
+                            stmt.setString(5, Utils.GSON.toJson(formatAnnotations(aClass.annotations())));
+                            stmt.setString(6, fields(aClass));
+                            stmt.setString(7, methods(aClass));
+                            stmt.setString(8, refs(aClass));
                             stmt.addBatch();
                         }
 
@@ -240,6 +242,9 @@ private static String methods(ClassData cd) {
             var sub = new JsonArray();
             sub.add(method.name());
             sub.add(method.desc());
+            if (!method.annotations().isEmpty()) {
+                sub.add(formatAnnotations(method.annotations()));
+            }
             json.add(sub);
         }
 
@@ -285,12 +290,70 @@ private static String fields(ClassData cd) {
             var sub = new JsonArray();
             sub.add(fields.name());
             sub.add(fields.desc().getInternalName());
+            if (!fields.annotations().isEmpty()) {
+                sub.add(formatAnnotations(fields.annotations()));
+            }
             json.add(sub);
         }
 
         return Utils.GSON.toJson(json);
     }
 
+    @SuppressWarnings("DuplicatedCode")
+    private static JsonArray formatAnnotations(List<ClassData.AnnotationInfo> anns) {
+        var json = new JsonArray();
+        if (anns.isEmpty()) {
+            return json;
+        }
+
+        for (ClassData.AnnotationInfo ann : anns) {
+            var js = new JsonArray();
+            js.add(ann.type().getInternalName());
+            var builder = new StringBuilder();
+            var itr = ann.members().entrySet().iterator();
+            while (itr.hasNext()) {
+                var next = itr.next();
+                builder.append(next.getKey()).append("=");
+                appendMember(builder, next.getValue());
+                if (itr.hasNext()) builder.append(',');
+            }
+            js.add(builder.toString());
+
+            json.add(js);
+        }
+        return json;
+    }
+
+    @SuppressWarnings("DuplicatedCode")
+    private static void appendMember(StringBuilder builder, Object member) {
+        switch (member) {
+            case ClassData.AnnotationInfo ai -> {
+                builder.append("@").append(ai.type().getInternalName())
+                        .append("(");
+                var itr = ai.members().entrySet().iterator();
+                while (itr.hasNext()) {
+                    var next = itr.next();
+                    builder.append(next.getKey()).append("=");
+                    appendMember(builder, next.getValue());
+                    if (itr.hasNext()) builder.append(',');
+                }
+                builder.append(")");
+            }
+            case List<?> list -> {
+                builder.append("[");
+                var itr = list.iterator();
+                while (itr.hasNext()) {
+                    appendMember(builder, itr.next());
+                    if (itr.hasNext()) builder.append(',');
+                }
+                builder.append(']');
+            }
+            case String str -> builder.append('"').append(str).append("'");
+            case Type tp -> builder.append(tp.getInternalName()).append(".class");
+            default -> builder.append(member);
+        }
+    }
+
     public class SqlMod implements DatabaseMod {
         private final int id;
         private final String mavenCoordinates;

src/main/java/net/neoforged/waifu/discord/DiscordBot.java

@@ -106,11 +106,11 @@ public void run() {
                     } else if (startedIndex) {
                         embed.addField("Step", "Indexing mods", false);
                         embed.addField("Found mods", currentCounter.getAmount() + " mods found", false);
-                        embed.appendDescription("Indexed: %s/%s".formatted(indexed.get(), expected.get()));
+                        embed.appendDescription("Indexed: %s/%s\n".formatted(indexed.get(), expected.get()));
                         embed.appendDescription("Stored: %s/%s".formatted(stored.get(), expected.get()));
 
                         if (failed.get() != 0) {
-                            embed.appendDescription("Failed: %s".formatted(failed.get()));
+                            embed.appendDescription("\n\nFailed: %s".formatted(failed.get()));
                         }
                     } else if (currentCounter != null) {
                         embed.addField("Step", "Searching mods", false);

src/main/java/net/neoforged/waifu/index/IndexingClassVisitor.java

@@ -1,6 +1,7 @@
 package net.neoforged.waifu.index;
 
 import net.neoforged.waifu.db.ClassData;
+import org.objectweb.asm.AnnotationVisitor;
 import org.objectweb.asm.ClassReader;
 import org.objectweb.asm.ClassVisitor;
 import org.objectweb.asm.FieldVisitor;
@@ -21,26 +22,28 @@
 public class IndexingClassVisitor extends ClassVisitor {
     private final List<ClassData> classList;
     private final boolean includeReferences;
+    private final boolean includeAnnotations;
 
     private ClassData current;
 
-    public IndexingClassVisitor(List<ClassData> classList, boolean includeReferences) {
+    public IndexingClassVisitor(List<ClassData> classList, boolean includeReferences, boolean includeAnnotations) {
         super(Opcodes.ASM9);
         this.classList = classList;
         this.includeReferences = includeReferences;
+        this.includeAnnotations = includeAnnotations;
     }
 
-    public static List<ClassData> collect(Path directory, boolean includeReferences) throws IOException {
+    public static List<ClassData> collect(Path directory, boolean includeReferences, boolean includeAnnotations) throws IOException {
         List<ClassData> classes = new ArrayList<>();
-        var indexer = new IndexingClassVisitor(classes, includeReferences);
+        var indexer = new IndexingClassVisitor(classes, includeReferences, includeAnnotations);
 
         Files.walkFileTree(directory, new SimpleFileVisitor<>() {
             @Override
             public FileVisitResult visitFile(Path file, BasicFileAttributes attrs) throws IOException {
                 var fileName = file.getFileName().toString();
                 if (fileName.endsWith(".class")) {
                     try (var is = Files.newInputStream(file)) {
-                        new ClassReader(is).accept(indexer, ClassReader.SKIP_DEBUG);
+                        new ClassReader(is).accept(indexer, includeReferences ? ClassReader.SKIP_DEBUG : (ClassReader.SKIP_CODE | ClassReader.SKIP_FRAMES));
                     }
                 }
 
@@ -54,33 +57,101 @@ public FileVisitResult visitFile(Path file, BasicFileAttributes attrs) throws IO
     public void visit(int version, int access, String name, String signature, String superName, String[] interfaces) {
         if (!name.endsWith("package-info") && !name.endsWith("module-info")) {
             current = new ClassData(
-                    name, superName, interfaces, new HashMap<>(), new HashMap<>(), new HashMap<>(), new HashMap<>(1 << 3)
+                    name, superName, interfaces, new ArrayList<>(0), new HashMap<>(), new HashMap<>(), new HashMap<>(), new HashMap<>(1 << 3)
             );
             classList.add(current);
         }
     }
 
     @Override
     public MethodVisitor visitMethod(int access, String name, String descriptor, String signature, String[] exceptions) {
-        var method = new ClassData.MethodInfo(name, descriptor, access);
+        var method = new ClassData.MethodInfo(name, descriptor, access, new ArrayList<>(0));
         current.methods().put(name + descriptor, method);
-        return includeReferences ? new MethodVisitor(Opcodes.ASM9) {
+        return includeReferences || includeAnnotations ? new MethodVisitor(Opcodes.ASM9) {
 
             @Override
             public void visitMethodInsn(int opcode, String owner, String name, String descriptor, boolean isInterface) {
-                current.methodRefs().merge(new ClassData.Reference(owner, name, descriptor), 1, Integer::sum);
+                if (includeReferences) {
+                    current.methodRefs().merge(new ClassData.Reference(owner, name, descriptor), 1, Integer::sum);
+                }
             }
 
             @Override
             public void visitFieldInsn(int opcode, String owner, String name, String descriptor) {
-                current.fieldRefs().merge(new ClassData.Reference(owner, name, Type.getType(descriptor).getInternalName()), 1, Integer::sum);
+                if (includeReferences) {
+                    current.fieldRefs().merge(new ClassData.Reference(owner, name, Type.getType(descriptor).getInternalName()), 1, Integer::sum);
+                }
+            }
+
+            @Override
+            public AnnotationVisitor visitAnnotation(String descriptor, boolean visible) {
+                if (includeAnnotations && visible) {
+                    var info = new ClassData.AnnotationInfo(Type.getType(descriptor), new HashMap<>(2));
+                    method.annotations().add(info);
+                    return visitor(info);
+                }
+                return null;
             }
         } : null;
     }
 
+    @Override
+    public AnnotationVisitor visitAnnotation(String descriptor, boolean visible) {
+        if (current != null && includeAnnotations && visible) {
+            var info = new ClassData.AnnotationInfo(Type.getType(descriptor), new HashMap<>(2));
+            current.annotations().add(info);
+            return visitor(info);
+        }
+        return null;
+    }
+
     @Override
     public FieldVisitor visitField(int access, String name, String descriptor, String signature, Object value) {
-        current.fields().put(name, new ClassData.FieldInfo(name, Type.getType(descriptor), access));
-        return super.visitField(access, name, descriptor, signature, value);
+        var field = new ClassData.FieldInfo(name, Type.getType(descriptor), access, new ArrayList<>(0));
+        current.fields().put(name, field);
+        return includeAnnotations ? new FieldVisitor(Opcodes.ASM9) {
+            @Override
+            public AnnotationVisitor visitAnnotation(String descriptor, boolean visible) {
+                if (visible) {
+                    var info = new ClassData.AnnotationInfo(Type.getType(descriptor), new HashMap<>(2));
+                    field.annotations().add(info);
+                    return visitor(info);
+                }
+                return null;
+            }
+        } : null;
+    }
+
+    private AnnotationVisitor visitor(ClassData.AnnotationInfo annotationInfo) {
+        return new AnnotationVisitor(Opcodes.ASM9) {
+            @Override
+            public void visit(String name, Object value) {
+                annotationInfo.members().put(name, value);
+            }
+
+            @Override
+            public AnnotationVisitor visitAnnotation(String name, String descriptor) {
+                var newAn = new ClassData.AnnotationInfo(Type.getType(descriptor), new HashMap<>(2));
+                annotationInfo.members().put(name, newAn);
+                return visitor(newAn);
+            }
+
+            @Override
+            public void visitEnum(String name, String descriptor, String value) {
+                annotationInfo.members().put(name, new ClassData.EnumValue(Type.getType(descriptor), value));
+            }
+
+            @Override
+            public AnnotationVisitor visitArray(String name) {
+                var lst = new ArrayList<>(2);
+                annotationInfo.members().put(name, lst);
+                return new AnnotationVisitor(Opcodes.ASM9) {
+                    @Override
+                    public void visit(String name, Object value) {
+                        lst.add(value);
+                    }
+                };
+            }
+        };
     }
 }

src/main/java/net/neoforged/waifu/platform/impl/mr/ModrinthPlatform.java

@@ -110,7 +110,7 @@ public String getSlug() {
 
             @Override
             public PlatformModFile getLatestFile(String gameVersion) {
-                var file = sendRequest("/project/" + id + "/version?loaders=neoforge&game_versions=" + gameVersion, new TypeToken<List<Version>>() {})
+                var file = sendRequest("/project/" + id + "/version?loaders=" + URLEncoder.encode("[\"neoforge\"]", StandardCharsets.UTF_8) + "&game_versions=" + URLEncoder.encode("[\"" + gameVersion + "\"]", StandardCharsets.UTF_8), new TypeToken<List<Version>>() {})
                         .get(0);
                 return createModFile(this, file);
             }