diff --git a/pal/src/host/linux-sgx/host_exception.c b/pal/src/host/linux-sgx/host_exception.c index dca0451c75..c551866ae2 100644 --- a/pal/src/host/linux-sgx/host_exception.c +++ b/pal/src/host/linux-sgx/host_exception.c @@ -93,25 +93,76 @@ static bool interrupted_in_aex(void) { } static void handle_sync_signal(int signum, siginfo_t* info, struct ucontext* uc) { - enum pal_event event = signal_to_pal_event(signum); - __UNUSED(info); + enum pal_event event = signal_to_pal_event(signum); + uint64_t rip = ucontext_get_ip(uc); + /* send dummy signal to RPC threads so they interrupt blocked syscalls */ if (g_rpc_queue) for (size_t i = 0; i < g_rpc_queue->rpc_threads_cnt; i++) DO_SYSCALL(tkill, g_rpc_queue->rpc_threads[i], SIGUSR2); + if (event == PAL_EVENT_MEMFAULT && interrupted_in_aex() && rip == (uint64_t)&eenter_pointer) { + /* + * This is a #GP on EENTER instruction inside sgx_raise(), called during AEX handling by + * maybe_raise_pending_signal(). This implies that some async signal arrived and was + * injected by AEX logic while the enclave thread is being executed in CSSA=1 (stage-1 + * exception handler). + * + * We ignore this #GP fault by skipping EENTER. This newly arrived async signal will be + * delivered at some later AEX event, when the enclave thread starts executing in CSSA=0. + * + * Since last_async_event was reset to NO_EVENT before sgx_raise(), we must restore it to + * this failed-to-deliver async signal. We extract async signal number from RDI register. + * See also maybe_raise_pending_signal(). + */ + enum pal_event faulted_event = uc->uc_mcontext.rdi; /* convention, see .Lcssa1_exception */ + if (faulted_event != PAL_EVENT_INTERRUPTED && faulted_event != PAL_EVENT_QUIT) { + log_error("#GP on EENTER instruction not because of async signal, impossible!"); + BUG(); + } + if (pal_get_host_tcb()->last_async_event != PAL_EVENT_QUIT) { + /* Do not overwrite `PAL_EVENT_QUIT`. For explanation, see handle_async_signal(). */ + pal_get_host_tcb()->last_async_event = faulted_event; + } + + ucontext_set_ip(uc, rip + /*sizeof(ENCLU)=*/3); /* skip EENTER */ + return; + } + if (interrupted_in_enclave(uc)) { - /* exception happened in app/LibOS/trusted PAL code, handle signal inside enclave */ + /* + * Exception happened in app/LibOS/trusted PAL code, mark this sync signal as pending. This + * singal will be delivered right after this untrusted-runtime signal handler returns + * control to the AEX logic, which will call maybe_raise_pending_signal(). + * + * We do not deliver the signal immediately to the enclave (but instead mark it as pending) + * because we want to support AEX-Notify hardware feature in SGX. In particular, AEX-Notify + * must execute in-enclave flows in regular context of the untrusted runtime, because + * AEX-Notify uses EDECCSSA instruction to go from CSSA=1 context to CSSA=0 context (i.e., + * AEX-Notify does not exit the SGX enclave and thus does not give an opportunity to the + * untrusted runtime to switch from signal-handling context to regular context). + * + * Therefore, we must execute the in-enclave stage-1 signal handler in regular context of + * the untrusted runtime. This is achieved by interposing on the AEX flow (which executes + * right after the host kernel handles control from this signal handler back to regular + * context). + * + * We don't need to use atomics when accessing last_sync_event since we are in the + * signal-handling context, and thus no other signal can arrive while we're here. + */ + if (pal_get_host_tcb()->last_sync_event != PAL_EVENT_NO_EVENT) { + log_error("Nested sync signal, impossible!"); + BUG(); + } + pal_get_host_tcb()->last_sync_event = event; + pal_get_host_tcb()->sync_signal_cnt++; - sgx_raise(event); return; } /* exception happened in untrusted PAL code (during syscall handling): fatal in Gramine */ - - unsigned long rip = ucontext_get_ip(uc); char buf[LOCATION_BUF_SIZE]; pal_describe_location(rip, buf, sizeof(buf)); @@ -153,13 +204,11 @@ static void handle_async_signal(int signum, siginfo_t* info, struct ucontext* uc for (size_t i = 0; i < g_rpc_queue->rpc_threads_cnt; i++) DO_SYSCALL(tkill, g_rpc_queue->rpc_threads[i], SIGUSR2); - if (interrupted_in_enclave(uc) || interrupted_in_aex()) { - /* signal arrived while in app/LibOS/trusted PAL code or when handling another AEX, handle - * signal inside enclave */ + if (interrupted_in_enclave(uc)) pal_get_host_tcb()->async_signal_cnt++; - sgx_raise(event); - return; - } + + /* see comments in handle_sync_signal() on why we do not deliver the signal immediately to the + * enclave (but instead mark it as pending) */ assert(event == PAL_EVENT_INTERRUPTED || event == PAL_EVENT_QUIT); if (pal_get_host_tcb()->last_async_event != PAL_EVENT_QUIT) { @@ -276,7 +325,7 @@ void pal_describe_location(uintptr_t addr, char* buf, size_t buf_size) { } #ifdef DEBUG -/* called on each AEX and OCALL (in normal context), see host_entry.S */ +/* called on each AEX and OCALL (in regular context), see host_entry.S */ void maybe_dump_and_reset_stats(void) { if (!g_sgx_enable_stats) return; @@ -288,6 +337,57 @@ void maybe_dump_and_reset_stats(void) { } #endif +/* + * The handle_sync_signal() and handle_async_signal() functions, executed in signal-handling + * context, added pending sync/async signal to the thread -- now the AEX flow, executed in regular + * context, must inform the enclave about these signals. + * + * This function is executed as part of the AEX flow, and may result in EENTER -> in-enclave stage-1 + * signal handler -> EEXIT (if there is any pending signal, and enclave is not in the middle of + * another stage-1 signal handler). When the function returns, the AEX flow continues and ends up in + * ERESUME, that resumes "regular context" inside the enclave (which may be stage-2 signal handler). + * + * Only one of potentially two signals (one sync and one async) will be injected into the enclave at + * a time by this function. The hope is that the second (async) signal will be added at some later + * AEX event. + * + * Note that async signals are special in Gramine, there are only two of them: SIGCONT (aka + * PAL_EVENT_INTERRUPTED) which is dummy (can be ignored) and SIGTERM (aka PAL_EVENT_QUIT) which is + * injected only once anyway. Thus we don't need a queue of pending async signals, and a single slot + * for a pending async signal is sufficient (which is the `pal_get_host_tcb()->last_async_event` + * variable). + * + * Also note that new sync signals cannot occur while in this function, but new async signals can + * occur (since we are in regular context and cannot block async signals), thus handling async + * signals must be aware of concurrent signal handling code, i.e., last_async_event must be accessed + * atomically. We also access last_sync_event atomically, just for uniformity (though it is not + * strictly required). + */ void maybe_raise_pending_signal(void) { - /* TODO: check if there is any sync or async pending signal and raise it */ + enum pal_event event; + + event = __atomic_exchange_n(&pal_get_host_tcb()->last_sync_event, PAL_EVENT_NO_EVENT, + __ATOMIC_RELAXED); + if (event != PAL_EVENT_NO_EVENT) { + /* + * Sync event must always be consumed by the enclave. There is no scenario where the + * in-enclave stage-1 handling of another sync/async event would generate a sync event. + */ + sgx_raise(event); + return; + } + + event = __atomic_exchange_n(&pal_get_host_tcb()->last_async_event, PAL_EVENT_NO_EVENT, + __ATOMIC_RELAXED); + if (event != PAL_EVENT_NO_EVENT) { + /* + * Async event may be *not* consumed by the enclave. This can happen if the enclave was + * already in the middle of stage-1 handler and thus EENTER generates #GP (because this + * EENTER would imply CSSA=2 which Gramine always programmes as prohibited in Intel SGX). + * In such case, this async event is ignored and will be delivered on some later AEX. + * See also handle_sync_signal(). + */ + sgx_raise(event); + return; + } } diff --git a/pal/src/host/linux-sgx/pal_tcb.h b/pal/src/host/linux-sgx/pal_tcb.h index 95728910bf..d6c724d0c9 100644 --- a/pal/src/host/linux-sgx/pal_tcb.h +++ b/pal/src/host/linux-sgx/pal_tcb.h @@ -99,7 +99,8 @@ typedef struct pal_host_tcb { atomic_ulong sync_signal_cnt; /* # of sync signals, corresponds to # of SIGSEGV/SIGILL/.. */ atomic_ulong async_signal_cnt; /* # of async signals, corresponds to # of SIGINT/SIGCONT/.. */ uint64_t profile_sample_time; /* last time sgx_profile_sample() recorded a sample */ - int32_t last_async_event; /* last async signal, reported to the enclave on ocall return */ + int32_t last_async_event; /* last async signal, reported to enclave on ocall return/AEX */ + int32_t last_sync_event; /* last sync signal, reported to enclave on ocall return/AEX */ int* start_status_ptr; /* pointer to return value of clone_thread */ bool reset_stats; /* if true, dump SGX stats and reset them on next AEX/OCALL */ } PAL_HOST_TCB;