c18n: Introduce fast path in trampolines

These fast paths check if the compartment transition is a
self-transition and if so, jump directly to the target without modifying
the trusted stack.

Calls through the PLT continue to use the old slow path as
self-transitions can already be ruled out when relocating jump slots.

Introduce the LD_COMPARTMENT_NO_FAST_PATH environment variable which
forces all trampolines to use the slow path.

Author: dpgao

libexec/rtld-elf/aarch64/rtld_c18n_asm.S

@@ -104,7 +104,7 @@ ENTRY(create_untrusted_stk)
 	/*
 	 * NON-STANDARD CALLING CONVENTION
 	 *
-	 * w19: Callee's compartment ID
+	 * w13: Callee's compartment ID
 	 * c26: Callee to be tail-called
 	 *
 	 * The function resolves the callee's stack, installs it, and tail-calls
@@ -123,7 +123,7 @@ ENTRY(create_untrusted_stk)
 
 	save_arguments
 
-	mov	w0, w19
+	mov	w0, w13
 	bl	resolve_untrusted_stk_impl
 	mov	c10, c0
 
@@ -166,16 +166,27 @@ ENTRY(tramp_hook)
 	 *
 	 * c10-c11 hold the first two arguments of tramp_hook.
 	 *
-	 * All argument registers and callee-saved registers must be preserved.
+	 * All argument registers, callee-saved registers, and arguments used
+	 * by the stack resolver must be preserved.
 	 */
 
 	save_arguments
 
+	/*
+	 * Save arguments used by the stack resolver.
+	 */
+	mov	w19, w13
+
 	mov	c0, c10
 	mov	c1, c11
 	mrs	c2, TRUSTED_STACK
 	bl	tramp_hook_impl
 
+	/*
+	 * Restore arguments used by the stack resolver.
+	 */
+	mov	w13, w19
+
 	restore_arguments
 
 #ifdef __ARM_MORELLO_PURECAP_BENCHMARK_ABI
@@ -217,13 +228,54 @@ TRAMP(tramp_push_frame)
 	 */
 	ldp	x10, x11, [TRUSTED_STACK_C, #TRUSTED_FRAME_CALLER]
 	/*
-	 * Get the stack lookup table.
+	 * Extract the caller's compartment ID.
 	 */
-	mrs	STACK_TABLE_C, STACK_TABLE
+	ubfx	x12, x10, #32, #16
+	/*
+	 * Get the callee's compartment ID.
+	 */
+1:	movz	w13, #0			/* To be patched at runtime */
+
+	/*
+	 * Tail-call the target if the caller's and callee's compartment IDs
+	 * match. Otherwise, go to the middle of the slow path.
+	 */
+	cmp	w12, w13
+	b.ne	5f
+	/*
+	 * Load the target capability and overwrite c18.
+	 */
+2:	ldr	c18, #0			/* To be patched at runtime */
+#ifdef __ARM_MORELLO_PURECAP_BENCHMARK_ABI
+	br	x18
+#else
+	brr	c18
+#endif
+
+	/*
+	 * Entry point of the slow path.
+	 */
+3:	mrs	TRUSTED_STACK_C, TRUSTED_STACK
+
+	/*
+	 * Load the caller's compartment ID and the landing address from the
+	 * previous trusted frame.
+	 */
+	ldp	x10, x11, [TRUSTED_STACK_C, #TRUSTED_FRAME_CALLER]
 	/*
 	 * Extract the caller's compartment ID.
 	 */
 	ubfx	x12, x10, #32, #16
+	/*
+	 * Get the callee's compartment ID.
+	 */
+4:	movz	w13, #0			/* To be patched at runtime */
+
+5:
+	/*
+	 * Get the stack lookup table.
+	 */
+	mrs	STACK_TABLE_C, STACK_TABLE
 	/*
 	 * Load the caller's old stack top from the stack lookup table.
 	 */
@@ -239,21 +291,17 @@ TRAMP(tramp_push_frame)
 	/*
 	 * Get the length of the stack lookup table.
 	 */
-	gclen	x13, STACK_TABLE_C
+	gclen	x14, STACK_TABLE_C
 
 	stp	c19, c20, [TRUSTED_STACK_C, #(-CAP_WIDTH * TRUSTED_FRAME_SIZE + CAP_WIDTH * 2)]
-	/*
-	 * Get the callee's compartment ID.
-	 */
-1:	movz	w19, #0			/* To be patched at runtime */
 	/*
 	 * Use subs instead of cmp to clear a capability tag.
 	 */
-	subs	x14, x13, x19
+	subs	x19, x14, x13
 	/*
 	 * If the stack lookup table index is out-of-bounds, set it to zero.
 	 */
-	csel	w20, w19, wzr, hi
+	csel	w20, w13, wzr, hi
 	/*
 	 * Load the callee's stack if the stack lookup table index is within
 	 * bounds. Otherwise the resolver will be loaded.
@@ -288,18 +336,18 @@ TRAMP(tramp_push_frame)
 	/*
 	 * Get the landing address.
 	 */
-2:	adr	c24, #0			/* To be patched at runtime */
+6:	adr	c24, #0			/* To be patched at runtime */
 
 	stp	c25, c26, [TRUSTED_STACK_C, #(-CAP_WIDTH * TRUSTED_FRAME_SIZE + CAP_WIDTH * 8)]
 	/*
 	 * Compute the number of return value registers. If the call is a tail-
 	 * call, it is the minimum of that of the caller and the callee.
 	 */
-3:	ubfm	x25, x23, #48, #0	/* To be patched at runtime */
+7:	ubfm	x25, x23, #48, #0	/* To be patched at runtime */
 	/*
 	 * Load the target capability.
 	 */
-4:	ldr	c26, #0			/* To be patched at runtime */
+8:	ldr	c26, #0			/* To be patched at runtime */
 
 	/*
 	 * Save the caller's current stack top and old stack top.
@@ -347,10 +395,10 @@ TRAMP(tramp_push_frame)
 	 */
 	str	x24, [TRUSTED_STACK_C, #TRUSTED_FRAME_LANDING]
 	/*
-	 * Combine the caller's compartment ID and the number of return value
+	 * Combine the callee's compartment ID and the number of return value
 	 * registers.
 	 */
-	orr	w24, w19, w25, lsl #16
+	orr	w24, w13, w25, lsl #16
 	/*
 	 * Save the callee's compartment ID and the number of return value
 	 * registers.
@@ -369,10 +417,20 @@ TRAMP(tramp_push_frame)
 	set_untrusted_stk	c15
 TRAMPEND(tramp_push_frame)
 
-PATCH_POINT(tramp_push_frame, cid, 1b)
-PATCH_POINT(tramp_push_frame, landing, 2b)
-PATCH_POINT(tramp_push_frame, n_rets, 3b)
-PATCH_POINT(tramp_push_frame, target, 4b)
+PATCH_POINT(tramp_push_frame, cid_fast, 1b)
+PATCH_POINT(tramp_push_frame, target_fast, 2b)
+PATCH_POINT(tramp_push_frame, cid, 4b)
+PATCH_POINT(tramp_push_frame, landing, 6b)
+PATCH_POINT(tramp_push_frame, n_rets, 7b)
+PATCH_POINT(tramp_push_frame, target, 8b)
+
+.section	.rodata
+.globl	c18n_tramp_entry_slow_offset
+.balign	8
+.type	c18n_tramp_entry_slow_offset,%object
+c18n_tramp_entry_slow_offset:
+	.quad	3b - tramp_push_frame
+.size	c18n_tramp_entry_slow_offset, . - c18n_tramp_entry_slow_offset
 
 /*
  * Save the address of the current frame to c29 so that unwinders can locate it.

libexec/rtld-elf/aarch64/rtld_c18n_machdep.c

@@ -147,6 +147,8 @@ tramp_compile(char **entry, const struct tramp_data *data)
 	callee = compart_id_for_address(data->defobj, (ptraddr_t)data->target);
 
 	COPY(push_frame);
+	PATCH_MOV(push_frame, cid_fast, cid_to_index(callee).val);
+	PATCH_LDR_IMM(push_frame, target_fast, target_off);
 	PATCH_MOV(push_frame, cid, cid_to_index(callee).val);
 	landing_off = PATCH_OFF(push_frame, landing);
 	/*

libexec/rtld-elf/aarch64/rtld_c18n_machdep.h

@@ -39,6 +39,8 @@
 
 #define	SIG_FRAME_SIZE			1360
 
+#define	INST_ALIGN		4
+
 #ifdef __ARM_MORELLO_PURECAP_BENCHMARK_ABI
 #define	TRUSTED_STACK		rddc_el0
 #define	UNTRUSTED_STACK		csp

libexec/rtld-elf/rtld.c

@@ -441,6 +441,7 @@ static struct ld_env_var_desc ld_env_vars[] = {
 	LD_ENV_DESC(COMPARTMENT_UNWIND, false),
 	LD_ENV_DESC(COMPARTMENT_STATS, false),
 	LD_ENV_DESC(COMPARTMENT_SWITCH_COUNT, false),
+	LD_ENV_DESC(COMPARTMENT_NO_FAST_PATH, false),
 #endif
 };
 
@@ -852,6 +853,7 @@ _rtld(Elf_Addr *sp, func_ptr_type *exit_proc, Obj_Entry **objp)
     ld_compartment_unwind = ld_get_env_var(LD_COMPARTMENT_UNWIND);
     ld_compartment_stats = ld_get_env_var(LD_COMPARTMENT_STATS);
     ld_compartment_switch_count = ld_get_env_var(LD_COMPARTMENT_SWITCH_COUNT);
+    ld_compartment_no_fast_path = ld_get_env_var(LD_COMPARTMENT_NO_FAST_PATH);
     /*
      * DISABLE takes precedence over ENABLE.
      */

libexec/rtld-elf/rtld.h

@@ -510,6 +510,7 @@ enum {
 	LD_COMPARTMENT_UNWIND,
 	LD_COMPARTMENT_STATS,
 	LD_COMPARTMENT_SWITCH_COUNT,
+	LD_COMPARTMENT_NO_FAST_PATH,
 #endif
 };
 

libexec/rtld-elf/rtld_c18n.c

@@ -145,10 +145,16 @@ const char *ld_compartment_stats;
 /* Export count of compartment switches to statistics */
 const char *ld_compartment_switch_count;
 
+/* Do not use the fast paths in trampolines for self-transitions */
+const char *ld_compartment_no_fast_path;
+
 /* Compartmentalisation information exported to the kernel */
 static struct cheri_c18n_info *c18n_info;
 struct rtld_c18n_stats *c18n_stats;
 
+/* Offset of a trampoline's slow path entry point relative to the fast path */
+extern const size_t c18n_tramp_entry_slow_offset;
+
 #define	INC_NUM_COMPART		(c18n_stats->rcs_compart++, comparts.size++)
 #define	INC_NUM_BYTES(n)						\
 	atomic_fetch_add_explicit(&c18n_stats->rcs_bytes_total, (n),	\
@@ -1438,10 +1444,12 @@ tramp_create(const struct tramp_data *data)
 }
 
 static void *
-tramp_make_entry(struct tramp_header *header)
+tramp_make_entry(struct tramp_header *header, bool slow)
 {
-	void *entry = header->entry;
+	uint8_t *entry = header->entry;
 
+	if (ld_compartment_no_fast_path != NULL || slow)
+		entry += c18n_tramp_entry_slow_offset;
 	entry = cheri_clearperm(entry, FUNC_PTR_REMOVE_PERMS);
 #ifndef __ARM_MORELLO_PURECAP_BENCHMARK_ABI
 	entry = cheri_capmode(entry);
@@ -1597,7 +1605,7 @@ tramp_intern(const Plt_Entry *plt, compart_id_t caller,
 	 * Defensive programming: if the requester supplies an untagged target
 	 * capability, return an untagged trampoline.
 	 */
-	tramp_entry = tramp_make_entry(header);
+	tramp_entry = tramp_make_entry(header, plt != NULL);
 	if (!cheri_gettag(data->target))
 		tramp_entry = cheri_cleartag(tramp_entry);
 
@@ -1636,18 +1644,36 @@ tramp_reflect(const void *data)
 #ifndef __ARM_MORELLO_PURECAP_BENCHMARK_ABI
 	data = (const char *)data - 1;
 #endif
+	/*
+	 * INVARIANT: The pointer being reflected never points to before the
+	 * function pointer entry point of the trampoline.
+	 *
+	 * When the fast path is enabled, the function pointer entry point is
+	 * the first instruction of the trampoline. Otherwise, the function
+	 * pointer entry point is `c18n_tramp_entry_slow_offset` bytes after
+	 * the first instruction of the trampoline, and the fast path is never
+	 * exposed. The return entry point is after either function pointer
+	 * entry point.
+	 */
+	if (ld_compartment_no_fast_path != NULL)
+		data = (const char *)data - c18n_tramp_entry_slow_offset;
 	data = __containerof(data, struct tramp_header, entry);
 
 	for (page = atomic_load_explicit(&tramp_pgs.head, memory_order_acquire);
-	    page != NULL; page = SLIST_NEXT(page, link)) {
+	     page != NULL; page = SLIST_NEXT(page, link)) {
 		ret = cheri_buildcap(page, (uintptr_t)data);
 		if (!cheri_gettag(ret))
 			continue;
-		if (cheri_gettag(ret->defobj))
+		/*
+		 * INVARIANT: The rederived pointer never points to before the
+		 * actual trampoline header.
+		 */
+		if (__builtin_is_aligned(ret, _Alignof(typeof(*ret))) &&
+		    cheri_gettag(ret->defobj))
 			/*
-			 * At this point, the provided data must have been (a)
-			 * tagged and (b) pointing to the entry point of a
-			 * trampoline.
+			 * If the rederived pointer is correctly aligned and
+			 * the `defobj` field is tagged, then it must point to
+			 * the actual trampoline header.
 			 */
 			return (ret);
 		else {

libexec/rtld-elf/rtld_c18n.h

@@ -43,6 +43,7 @@ extern const char *ld_compartment_sig;
 extern const char *ld_compartment_unwind;
 extern const char *ld_compartment_stats;
 extern const char *ld_compartment_switch_count;
+extern const char *ld_compartment_no_fast_path;
 extern struct rtld_c18n_stats *c18n_stats;
 
 /*
@@ -207,10 +208,15 @@ struct tramp_header {
 	 * that the tagged value is visible to the trampoline when it is run.
 	 */
 	_Atomic(void *) target;
+	/*
+	 * INVARIANT: This field must be the last tagged member of the
+	 * trampoline. Trampoline reflection relies on this to locate the
+	 * header.
+	 */
 	const Obj_Entry *defobj;
 	size_t symnum;
 	struct func_sig sig;
-	uint32_t entry[];
+	_Alignas(INST_ALIGN) uint8_t entry[];
 };
 
 /*