diff --git a/src/main/java/net/fabricmc/loom/configuration/providers/minecraft/AnnotationsApplyVisitor.java b/src/main/java/net/fabricmc/loom/configuration/providers/minecraft/AnnotationsApplyVisitor.java
index b22f28702..96a377484 100644
--- a/src/main/java/net/fabricmc/loom/configuration/providers/minecraft/AnnotationsApplyVisitor.java
+++ b/src/main/java/net/fabricmc/loom/configuration/providers/minecraft/AnnotationsApplyVisitor.java
@@ -46,31 +46,34 @@
 public record AnnotationsApplyVisitor(AnnotationsData annotationsData) implements TinyRemapper.ApplyVisitorProvider {
 	@Override
 	public ClassVisitor insertApplyVisitor(TrClass cls, ClassVisitor next) {
-		return new AnnotationsApplyClassVisitor(next, cls.getName(), annotationsData);
+		ClassAnnotationData classData = annotationsData.classes().get(cls.getName());
+
+		if (classData == null) {
+			return next;
+		}
+
+		return new AnnotationsApplyClassVisitor(next, classData);
 	}
 
 	public static class AnnotationsApplyClassVisitor extends ClassVisitor {
 		private final ClassAnnotationData classData;
 		private boolean hasAddedAnnotations;
 
-		public AnnotationsApplyClassVisitor(ClassVisitor cv, String className, AnnotationsData annotationsData) {
+		public AnnotationsApplyClassVisitor(ClassVisitor cv, ClassAnnotationData classData) {
 			super(Constants.ASM_VERSION, cv);
-			this.classData = annotationsData.classes().get(className);
+			this.classData = classData;
 			hasAddedAnnotations = false;
 		}
 
 		@Override
 		public void visit(int version, int access, String name, String signature, String superName, String[] interfaces) {
-			if (classData != null) {
-				access = classData.modifyAccessFlags(access);
-			}
-
+			access = classData.modifyAccessFlags(access);
 			super.visit(version, access, name, signature, superName, interfaces);
 		}
 
 		@Override
 		public AnnotationVisitor visitTypeAnnotation(int typeRef, TypePath typePath, String descriptor, boolean visible) {
-			if (classData != null && classData.typeAnnotationsToRemove().contains(new TypeAnnotationKey(typeRef, typePath.toString(), Type.getType(descriptor).getInternalName()))) {
+			if (classData.typeAnnotationsToRemove().contains(new TypeAnnotationKey(typeRef, typePath.toString(), Type.getType(descriptor).getInternalName()))) {
 				return null;
 			}
 
@@ -79,7 +82,7 @@ public AnnotationVisitor visitTypeAnnotation(int typeRef, TypePath typePath, Str
 
 		@Override
 		public AnnotationVisitor visitAnnotation(String descriptor, boolean visible) {
-			if (classData != null && classData.annotationsToRemove().contains(Type.getType(descriptor).getInternalName())) {
+			if (classData.annotationsToRemove().contains(Type.getType(descriptor).getInternalName())) {
 				return null;
 			}
 
diff --git a/src/test/groovy/net/fabricmc/loom/test/unit/processor/AnnotationsApplyTest.groovy b/src/test/groovy/net/fabricmc/loom/test/unit/processor/AnnotationsApplyTest.groovy
index 9e212001f..c540c1879 100644
--- a/src/test/groovy/net/fabricmc/loom/test/unit/processor/AnnotationsApplyTest.groovy
+++ b/src/test/groovy/net/fabricmc/loom/test/unit/processor/AnnotationsApplyTest.groovy
@@ -46,9 +46,9 @@ class AnnotationsApplyTest extends Specification {
 		def annotationData = AnnotationsData.read(new StringReader(ANNOTATIONS_DATA))
 
 		def annotatedNode1 = new ClassNode()
-		def classVisitor1 = new AnnotationsApplyVisitor.AnnotationsApplyClassVisitor(annotatedNode1, 'net/fabricmc/loom/test/unit/processor/AnnotationsApplyTest$ExampleClass1', annotationData)
+		def classVisitor1 = new AnnotationsApplyVisitor.AnnotationsApplyClassVisitor(annotatedNode1, annotationData.classes().get('net/fabricmc/loom/test/unit/processor/AnnotationsApplyTest$ExampleClass1'))
 		def annotatedNode2 = new ClassNode()
-		def classVisitor2 = new AnnotationsApplyVisitor.AnnotationsApplyClassVisitor(annotatedNode2, 'net/fabricmc/loom/test/unit/processor/AnnotationsApplyTest$ExampleClass2', annotationData)
+		def classVisitor2 = new AnnotationsApplyVisitor.AnnotationsApplyClassVisitor(annotatedNode2, annotationData.classes().get('net/fabricmc/loom/test/unit/processor/AnnotationsApplyTest$ExampleClass2'))
 
 		when:
 		def classReader1 = new ClassReader(getClassBytes(ExampleClass1))