-Zindirect-branch-cs-prefix on top of -Zretpoline*

compiler/rustc_codegen_gcc/src/gcc_util.rs

@@ -94,7 +94,9 @@ pub(crate) fn global_gcc_features(sess: &Session, diagnostics: bool) -> Vec<Stri
                     sess.dcx().emit_warn(unknown_feature);
                 }
                 Some(&(_, stability, _)) => {
-                    if let Err(reason) = stability.toggle_allowed() {
+                    if let Err(reason) =
+                        stability.toggle_allowed(|flag| sess.opts.target_feature_flag_enabled(flag))
+                    {
                         sess.dcx().emit_warn(ForbiddenCTargetFeature {
                             feature,
                             enabled: if enable { "enabled" } else { "disabled" },

compiler/rustc_codegen_llvm/src/context.rs

@@ -443,6 +443,15 @@ pub(crate) unsafe fn create_module<'ll>(
         }
     }
 
+    if sess.opts.unstable_opts.indirect_branch_cs_prefix {
+        llvm::add_module_flag_u32(
+            llmod,
+            llvm::ModuleFlagMergeBehavior::Override,
+            "indirect_branch_cs_prefix",
+            1,
+        );
+    }
+
     match (sess.opts.unstable_opts.small_data_threshold, sess.target.small_data_threshold_support())
     {
         // Set up the small-data optimization limit for architectures that use

compiler/rustc_codegen_llvm/src/llvm_util.rs

@@ -832,7 +832,9 @@ pub(crate) fn global_llvm_features(
                         sess.dcx().emit_warn(unknown_feature);
                     }
                     Some((_, stability, _)) => {
-                        if let Err(reason) = stability.toggle_allowed() {
+                        if let Err(reason) = stability
+                            .toggle_allowed(|flag| sess.opts.target_feature_flag_enabled(flag))
+                        {
                             sess.dcx().emit_warn(ForbiddenCTargetFeature {
                                 feature,
                                 enabled: if enable { "enabled" } else { "disabled" },

compiler/rustc_codegen_ssa/src/target_features.rs

@@ -64,7 +64,9 @@ pub(crate) fn from_target_feature_attr(
 
             // Only allow target features whose feature gates have been enabled
             // and which are permitted to be toggled.
-            if let Err(reason) = stability.toggle_allowed() {
+            if let Err(reason) =
+                stability.toggle_allowed(|flag| tcx.sess.opts.target_feature_flag_enabled(flag))
+            {
                 tcx.dcx().emit_err(errors::ForbiddenTargetFeatureAttr {
                     span: item.span(),
                     feature,

compiler/rustc_interface/src/tests.rs

@@ -802,6 +802,7 @@ fn test_unstable_options_tracking_hash() {
     tracked!(function_sections, Some(false));
     tracked!(human_readable_cgu_names, true);
     tracked!(incremental_ignore_spans, true);
+    tracked!(indirect_branch_cs_prefix, true);
     tracked!(inline_mir, Some(true));
     tracked!(inline_mir_hint_threshold, Some(123));
     tracked!(inline_mir_threshold, Some(123));

compiler/rustc_session/messages.ftl

@@ -49,6 +49,8 @@ session_hexadecimal_float_literal_not_supported = hexadecimal float literal is n
 session_incompatible_linker_flavor = linker flavor `{$flavor}` is incompatible with the current target
     .note = compatible flavors are: {$compatible_list}
 
+session_indirect_branch_cs_prefix_requires_x86_or_x86_64 = `-Zindirect-branch-cs-prefix` is only supported on x86 and x86_64
+
 session_instrumentation_not_supported = {$us} instrumentation is not supported for this target
 
 session_int_literal_too_large = integer literal is too large

compiler/rustc_session/src/config.rs

@@ -2649,6 +2649,16 @@ pub fn build_session_options(early_dcx: &mut EarlyDiagCtxt, matches: &getopts::M
 
     let prints = collect_print_requests(early_dcx, &mut cg, &unstable_opts, matches);
 
+    // -Zretpoline-external-thunk also requires -Zretpoline
+    if unstable_opts.retpoline_external_thunk {
+        unstable_opts.retpoline = true;
+        target_modifiers.insert(
+            OptionsTargetModifiers::UnstableOptions(UnstableOptionsTargetModifiers::retpoline),
+            "true".to_string(),
+        );
+    }
+    Options::fill_target_features_by_flags(&unstable_opts, &mut cg);
+
     let cg = cg;
 
     let sysroot_opt = matches.opt_str("sysroot").map(|m| PathBuf::from(&m));

compiler/rustc_session/src/errors.rs

@@ -471,6 +471,10 @@ pub(crate) struct FunctionReturnRequiresX86OrX8664;
 #[diag(session_function_return_thunk_extern_requires_non_large_code_model)]
 pub(crate) struct FunctionReturnThunkExternRequiresNonLargeCodeModel;
 
+#[derive(Diagnostic)]
+#[diag(session_indirect_branch_cs_prefix_requires_x86_or_x86_64)]
+pub(crate) struct IndirectBranchCsPrefixRequiresX86OrX8664;
+
 #[derive(Diagnostic)]
 #[diag(session_unsupported_regparm)]
 pub(crate) struct UnsupportedRegparm {

compiler/rustc_session/src/options.rs

@@ -290,6 +290,43 @@ macro_rules! top_level_options {
                 mods.sort_by(|a, b| a.opt.cmp(&b.opt));
                 mods
             }
+
+            pub fn target_feature_flag_enabled(&self, flag: &str) -> bool {
+                match flag {
+                    "retpoline" => self.unstable_opts.retpoline,
+                    "retpoline-external-thunk" => self.unstable_opts.retpoline_external_thunk,
+                    _ => false,
+                }
+            }
+
+            pub fn fill_target_features_by_flags(
+                unstable_opts: &UnstableOptions, cg: &mut CodegenOptions
+            ) {
+                // -Zretpoline without -Zretpoline-external-thunk enables
+                // retpoline-indirect-branches and retpoline-indirect-calls target features
+                if unstable_opts.retpoline && !unstable_opts.retpoline_external_thunk {
+                    if !cg.target_feature.is_empty() {
+                        cg.target_feature.push(',');
+                    }
+                    cg.target_feature.push_str(
+                        "+retpoline-indirect-branches,\
+                        +retpoline-indirect-calls"
+                    );
+                }
+                // -Zretpoline-external-thunk (maybe, with -Zretpoline too) enables
+                // retpoline-external-thunk, retpoline-indirect-branches and
+                // retpoline-indirect-calls target features
+                if unstable_opts.retpoline_external_thunk {
+                    if !cg.target_feature.is_empty() {
+                        cg.target_feature.push(',');
+                    }
+                    cg.target_feature.push_str(
+                        "+retpoline-external-thunk,\
+                        +retpoline-indirect-branches,\
+                        +retpoline-indirect-calls"
+                    );
+                }
+            }
         }
     );
 }
@@ -2249,6 +2286,8 @@ options! {
         - hashes of green query instances
         - hash collisions of query keys
         - hash collisions when creating dep-nodes"),
+    indirect_branch_cs_prefix: bool = (false, parse_bool, [TRACKED TARGET_MODIFIER],
+        "add `cs` prefix to `call` and `jmp` to indirect thunks (default: no)"),
     inline_llvm: bool = (true, parse_bool, [TRACKED],
         "enable LLVM inlining (default: yes)"),
     inline_mir: Option<bool> = (None, parse_opt_bool, [TRACKED],
@@ -2446,6 +2485,11 @@ options! {
     remark_dir: Option<PathBuf> = (None, parse_opt_pathbuf, [UNTRACKED],
         "directory into which to write optimization remarks (if not specified, they will be \
 written to standard error output)"),
+    retpoline: bool = (false, parse_bool, [TRACKED TARGET_MODIFIER],
+        "enables retpoline-indirect-branches and retpoline-indirect-calls target features (default: no)"),
+    retpoline_external_thunk: bool = (false, parse_bool, [TRACKED TARGET_MODIFIER],
+        "enables retpoline-external-thunk, retpoline-indirect-branches and retpoline-indirect-calls \
+        target features (default: no)"),
     sanitizer: SanitizerSet = (SanitizerSet::empty(), parse_sanitizers, [TRACKED],
         "use a sanitizer"),
     sanitizer_cfi_canonical_jump_tables: Option<bool> = (Some(true), parse_opt_bool, [TRACKED],

compiler/rustc_session/src/session.rs

@@ -1376,6 +1376,12 @@ fn validate_commandline_args_with_session_available(sess: &Session) {
         }
     }
 
+    if sess.opts.unstable_opts.indirect_branch_cs_prefix {
+        if sess.target.arch != "x86" && sess.target.arch != "x86_64" {
+            sess.dcx().emit_err(errors::IndirectBranchCsPrefixRequiresX86OrX8664);
+        }
+    }
+
     if let Some(regparm) = sess.opts.unstable_opts.regparm {
         if regparm > 3 {
             sess.dcx().emit_err(errors::UnsupportedRegparm { regparm });

compiler/rustc_target/src/target_features.rs

@@ -34,6 +34,9 @@ pub enum Stability {
     /// particular for features are actually ABI configuration flags (not all targets are as nice as
     /// RISC-V and have an explicit way to set the ABI separate from target features).
     Forbidden { reason: &'static str },
+    /// This feature can not be set via `-Ctarget-feature` or `#[target_feature]`, it can only be set
+    /// by target modifier flag. Target modifier flags are tracked to be consistent in linked modules.
+    EnabledByTargetModifierFlag { reason: &'static str, flag: &'static str },
 }
 use Stability::*;
 
@@ -49,6 +52,7 @@ impl<CTX> HashStable<CTX> for Stability {
             Stability::Forbidden { reason } => {
                 reason.hash_stable(hcx, hasher);
             }
+            Stability::EnabledByTargetModifierFlag { .. } => {}
         }
     }
 }
@@ -74,15 +78,23 @@ impl Stability {
             Stability::Unstable(nightly_feature) => Some(nightly_feature),
             Stability::Stable { .. } => None,
             Stability::Forbidden { .. } => panic!("forbidden features should not reach this far"),
+            Stability::EnabledByTargetModifierFlag { .. } => None,
         }
     }
 
     /// Returns whether the feature may be toggled via `#[target_feature]` or `-Ctarget-feature`.
     /// (It might still be nightly-only even if this returns `true`, so make sure to also check
     /// `requires_nightly`.)
-    pub fn toggle_allowed(&self) -> Result<(), &'static str> {
+    pub fn toggle_allowed(&self, flag_enabled: impl Fn(&str) -> bool) -> Result<(), &'static str> {
         match self {
             Stability::Forbidden { reason } => Err(reason),
+            Stability::EnabledByTargetModifierFlag { reason, flag } => {
+                if !flag_enabled(*flag) {
+                    Err(reason)
+                } else {
+                    Ok(())
+                }
+            }
             _ => Ok(()),
         }
     }
@@ -453,6 +465,30 @@ static X86_FEATURES: &[(&str, Stability, ImpliedFeatures)] = &[
     ("prfchw", Unstable(sym::prfchw_target_feature), &[]),
     ("rdrand", Stable, &[]),
     ("rdseed", Stable, &[]),
+    (
+        "retpoline-external-thunk",
+        Stability::EnabledByTargetModifierFlag {
+            reason: "use `retpoline-external-thunk` target modifier flag instead",
+            flag: "retpoline-external-thunk",
+        },
+        &[],
+    ),
+    (
+        "retpoline-indirect-branches",
+        Stability::EnabledByTargetModifierFlag {
+            reason: "use `retpoline` target modifier flag instead",
+            flag: "retpoline",
+        },
+        &[],
+    ),
+    (
+        "retpoline-indirect-calls",
+        Stability::EnabledByTargetModifierFlag {
+            reason: "use `retpoline` target modifier flag instead",
+            flag: "retpoline",
+        },
+        &[],
+    ),
     ("rtm", Unstable(sym::rtm_target_feature), &[]),
     ("sha", Stable, &["sse2"]),
     ("sha512", Unstable(sym::sha512_sm_x86), &["avx2"]),

src/doc/unstable-book/src/compiler-flags/indirect-branch-cs-prefix.md

@@ -0,0 +1,19 @@
+# `indirect-branch-cs-prefix`
+
+The tracking issue for this feature is: https://github.com/rust-lang/rust/issues/116852.
+
+------------------------
+
+Option `-Zindirect-branch-cs-prefix` controls whether a `cs` prefix is added to
+`call` and `jmp` to indirect thunks.
+
+It is equivalent to [Clang]'s and [GCC]'s `-mindirect-branch-cs-prefix`. The
+Linux kernel uses it for RETPOLINE builds. For details, see
+[LLVM commit 6f867f910283] ("[X86] Support ``-mindirect-branch-cs-prefix`` for
+call and jmp to indirect thunk") which introduces the feature.
+
+Only x86 is supported.
+
+[Clang]: https://clang.llvm.org/docs/ClangCommandLineReference.html#cmdoption-clang-mindirect-branch-cs-prefix
+[GCC]: https://gcc.gnu.org/onlinedocs/gcc/x86-Options.html#index-mindirect-branch-cs-prefix
+[LLVM commit 6f867f910283]: https://github.com/llvm/llvm-project/commit/6f867f9102838ebe314c1f3661fdf95700386e5a

src/librustdoc/json/mod.rs

@@ -148,7 +148,7 @@ fn target(sess: &rustc_session::Session) -> types::Target {
             .copied()
             .filter(|(_, stability, _)| {
                 // Describe only target features which the user can toggle
-                stability.toggle_allowed().is_ok()
+                stability.toggle_allowed(|flag| sess.opts.target_feature_flag_enabled(flag)).is_ok()
             })
             .map(|(name, stability, implied_features)| {
                 types::TargetFeature {
@@ -164,7 +164,13 @@ fn target(sess: &rustc_session::Session) -> types::Target {
                             // Imply only target features which the user can toggle
                             feature_stability
                                 .get(name)
-                                .map(|stability| stability.toggle_allowed().is_ok())
+                                .map(|stability| {
+                                    stability
+                                        .toggle_allowed(|flag| {
+                                            sess.opts.target_feature_flag_enabled(flag)
+                                        })
+                                        .is_ok()
+                                })
                                 .unwrap_or(false)
                         })
                         .map(String::from)

tests/assembly/x86_64-indirect-branch-cs-prefix.rs

@@ -0,0 +1,29 @@
+// Test that the `cs` prefix is (not) added into a `call` and a `jmp` to the
+// indirect thunk when the `-Zindirect-branch-cs-prefix` flag is (not) set.
+
+//@ revisions: unset set
+//@ assembly-output: emit-asm
+//@ compile-flags: -Copt-level=3 -Cunsafe-allow-abi-mismatch=retpoline,retpoline-external-thunk,indirect-branch-cs-prefix -Zretpoline-external-thunk
+//@ [set] compile-flags: -Zindirect-branch-cs-prefix
+//@ only-x86_64
+// FIXME: Are these required?
+//@ ignore-apple Symbol is called `___x86_indirect_thunk` (Darwin's extra underscore)
+//@ ignore-sgx Tests incompatible with LVI mitigations
+
+#![crate_type = "lib"]
+
+// CHECK-LABEL: foo:
+#[no_mangle]
+pub fn foo(g: fn()) {
+    // unset-NOT: cs
+    // unset: callq {{__x86_indirect_thunk.*}}
+    // set: cs
+    // set-NEXT: callq {{__x86_indirect_thunk.*}}
+    g();
+
+    // unset-NOT: cs
+    // unset: jmp {{__x86_indirect_thunk.*}}
+    // set: cs
+    // set-NEXT: jmp {{__x86_indirect_thunk.*}}
+    g();
+}

tests/codegen/indirect-branch-cs-prefix.rs

@@ -0,0 +1,18 @@
+// Test that the `indirect_branch_cs_prefix` module attribute is (not)
+// emitted when the `-Zindirect-branch-cs-prefix` flag is (not) set.
+
+//@ add-core-stubs
+//@ revisions: unset set
+//@ needs-llvm-components: x86
+//@ compile-flags: --target x86_64-unknown-linux-gnu
+//@ [set] compile-flags: -Zindirect-branch-cs-prefix
+
+#![crate_type = "lib"]
+#![feature(no_core, lang_items)]
+#![no_core]
+
+extern crate minicore;
+use minicore::*;
+
+// unset-NOT: !{{[0-9]+}} = !{i32 4, !"indirect_branch_cs_prefix", i32 1}
+// set: !{{[0-9]+}} = !{i32 4, !"indirect_branch_cs_prefix", i32 1}

tests/codegen/retpoline.rs

@@ -0,0 +1,29 @@
+// ignore-tidy-linelength
+// Test that the
+// `retpoline-external-thunk`, `retpoline-indirect-branches`, `retpoline-indirect-calls`
+// target features are (not) emitted when the `retpoline/retpoline-external-thunk` flag is (not) set.
+
+//@ revisions: disabled enabled_retpoline enabled_retpoline_external_thunk
+//@ needs-llvm-components: x86
+//@ compile-flags: --target x86_64-unknown-linux-gnu
+//@ [enabled_retpoline] compile-flags: -Zretpoline
+//@ [enabled_retpoline_external_thunk] compile-flags: -Zretpoline-external-thunk
+
+#![crate_type = "lib"]
+#![feature(no_core, lang_items)]
+#![no_core]
+
+#[lang = "sized"]
+trait Sized {}
+
+#[no_mangle]
+pub fn foo() {
+    // CHECK: @foo() unnamed_addr #0
+
+    // disabled-NOT: attributes #0 = { {{.*}}"target-features"="{{[^"]*}}+retpoline-external-thunk{{.*}} }
+    // disabled-NOT: attributes #0 = { {{.*}}"target-features"="{{[^"]*}}+retpoline-indirect-branches{{.*}} }
+    // disabled-NOT: attributes #0 = { {{.*}}"target-features"="{{[^"]*}}+retpoline-indirect-calls{{.*}} }
+
+    // enabled_retpoline: attributes #0 = { {{.*}}"target-features"="{{[^"]*}}+retpoline-indirect-branches,+retpoline-indirect-calls{{.*}} }
+    // enabled_retpoline_external_thunk: attributes #0 = { {{.*}}"target-features"="{{[^"]*}}+retpoline-external-thunk,+retpoline-indirect-branches,+retpoline-indirect-calls{{.*}} }
+}

tests/ui/check-cfg/target_feature.stderr

@@ -202,6 +202,9 @@ LL |     cfg!(target_feature = "_UNEXPECTED_VALUE");
 `relax`
 `relaxed-simd`
 `reserve-x18`
+`retpoline-external-thunk`
+`retpoline-indirect-branches`
+`retpoline-indirect-calls`
 `rtm`
 `sb`
 `scq`

tests/ui/invalid-compile-flags/indirect-branch-cs-prefix/requires-x86-or-x86_64.aarch64.stderr

@@ -0,0 +1,4 @@
+error: `-Zindirect-branch-cs-prefix` is only supported on x86 and x86_64
+
+error: aborting due to 1 previous error
+