Implement projection and shim for AFIDT

compiler/rustc_middle/src/ty/instance.rs

@@ -678,23 +678,26 @@ impl<'tcx> Instance<'tcx> {
                 //
                 // 1) The underlying method expects a caller location parameter
                 // in the ABI
-                if resolved.def.requires_caller_location(tcx)
-                        // 2) The caller location parameter comes from having `#[track_caller]`
-                        // on the implementation, and *not* on the trait method.
-                        && !tcx.should_inherit_track_caller(def)
-                        // If the method implementation comes from the trait definition itself
-                        // (e.g. `trait Foo { #[track_caller] my_fn() { /* impl */ } }`),
-                        // then we don't need to generate a shim. This check is needed because
-                        // `should_inherit_track_caller` returns `false` if our method
-                        // implementation comes from the trait block, and not an impl block
-                        && !matches!(
-                            tcx.opt_associated_item(def),
-                            Some(ty::AssocItem {
-                                container: ty::AssocItemContainer::Trait,
-                                ..
-                            })
-                        )
-                {
+                let needs_track_caller_shim = resolved.def.requires_caller_location(tcx)
+                    // 2) The caller location parameter comes from having `#[track_caller]`
+                    // on the implementation, and *not* on the trait method.
+                    && !tcx.should_inherit_track_caller(def)
+                    // If the method implementation comes from the trait definition itself
+                    // (e.g. `trait Foo { #[track_caller] my_fn() { /* impl */ } }`),
+                    // then we don't need to generate a shim. This check is needed because
+                    // `should_inherit_track_caller` returns `false` if our method
+                    // implementation comes from the trait block, and not an impl block
+                    && !matches!(
+                        tcx.opt_associated_item(def),
+                        Some(ty::AssocItem {
+                            container: ty::AssocItemContainer::Trait,
+                            ..
+                        })
+                    );
+                // We also need to generate a shim if this is an AFIT.
+                let needs_rpitit_shim =
+                    tcx.return_position_impl_trait_in_trait_shim_data(def).is_some();
+                if needs_track_caller_shim || needs_rpitit_shim {
                     if tcx.is_closure_like(def) {
                         debug!(
                             " => vtable fn pointer created for closure with #[track_caller]: {:?} for method {:?} {:?}",

compiler/rustc_middle/src/ty/mod.rs

@@ -148,6 +148,7 @@ mod opaque_types;
 mod parameterized;
 mod predicate;
 mod region;
+mod return_position_impl_trait_in_trait;
 mod rvalue_scopes;
 mod structural_impls;
 #[allow(hidden_glob_reexports)]

compiler/rustc_middle/src/ty/return_position_impl_trait_in_trait.rs

@@ -0,0 +1,88 @@
+use rustc_hir::def_id::DefId;
+
+use crate::ty::{self, ExistentialPredicateStableCmpExt, TyCtxt};
+
+impl<'tcx> TyCtxt<'tcx> {
+    /// Given a `def_id` of a trait or impl method, compute whether that method needs to
+    /// have an RPITIT shim applied to it for it to be object safe. If so, return the
+    /// `def_id` of the RPITIT, and also the args of trait method that returns the RPITIT.
+    ///
+    /// NOTE that these args are not, in general, the same as than the RPITIT's args. They
+    /// are a subset of those args,  since they do not include the late-bound lifetimes of
+    /// the RPITIT. Depending on the context, these will need to be dealt with in different
+    /// ways -- in codegen, it's okay to fill them with ReErased.
+    pub fn return_position_impl_trait_in_trait_shim_data(
+        self,
+        def_id: DefId,
+    ) -> Option<(DefId, ty::EarlyBinder<'tcx, ty::GenericArgsRef<'tcx>>)> {
+        let assoc_item = self.opt_associated_item(def_id)?;
+
+        let (trait_item_def_id, opt_impl_def_id) = match assoc_item.container {
+            ty::AssocItemContainer::Impl => {
+                (assoc_item.trait_item_def_id?, Some(self.parent(def_id)))
+            }
+            ty::AssocItemContainer::Trait => (def_id, None),
+        };
+
+        let sig = self.fn_sig(trait_item_def_id);
+
+        let ty::Alias(ty::Projection, alias_ty) = *sig.skip_binder().skip_binder().output().kind()
+        else {
+            return None;
+        };
+
+        if !self.is_impl_trait_in_trait(alias_ty.def_id) {
+            return None;
+        }
+
+        let args = if let Some(impl_def_id) = opt_impl_def_id {
+            // Rebase the args from the RPITIT onto the impl trait ref, so we can later
+            // substitute them with the method args of the *impl* method, since that's
+            // the instance we're building a vtable shim for.
+            ty::GenericArgs::identity_for_item(self, trait_item_def_id).rebase_onto(
+                self,
+                self.parent(trait_item_def_id),
+                self.impl_trait_ref(impl_def_id)
+                    .expect("expected impl trait ref from parent of impl item")
+                    .instantiate_identity()
+                    .args,
+            )
+        } else {
+            // This is when we have a default trait implementation.
+            ty::GenericArgs::identity_for_item(self, trait_item_def_id)
+        };
+
+        Some((alias_ty.def_id, ty::EarlyBinder::bind(args)))
+    }
+
+    /// Given a `DefId` of an RPITIT and its args, return the existential predicates
+    /// that corresponds to the RPITIT's bounds with the self type erased.
+    pub fn item_bounds_to_existential_predicates(
+        self,
+        def_id: DefId,
+        args: ty::GenericArgsRef<'tcx>,
+    ) -> &'tcx ty::List<ty::PolyExistentialPredicate<'tcx>> {
+        let mut bounds: Vec<_> = self
+            .item_super_predicates(def_id)
+            .iter_instantiated(self, args)
+            .filter_map(|clause| {
+                clause
+                    .kind()
+                    .map_bound(|clause| match clause {
+                        ty::ClauseKind::Trait(trait_pred) => Some(ty::ExistentialPredicate::Trait(
+                            ty::ExistentialTraitRef::erase_self_ty(self, trait_pred.trait_ref),
+                        )),
+                        ty::ClauseKind::Projection(projection_pred) => {
+                            Some(ty::ExistentialPredicate::Projection(
+                                ty::ExistentialProjection::erase_self_ty(self, projection_pred),
+                            ))
+                        }
+                        _ => None,
+                    })
+                    .transpose()
+            })
+            .collect();
+        bounds.sort_by(|a, b| a.skip_binder().stable_cmp(self, &b.skip_binder()));
+        self.mk_poly_existential_predicates(&bounds)
+    }
+}

compiler/rustc_mir_transform/src/shim.rs

@@ -9,6 +9,7 @@ use rustc_index::{Idx, IndexVec};
 use rustc_middle::mir::patch::MirPatch;
 use rustc_middle::mir::*;
 use rustc_middle::query::Providers;
+use rustc_middle::ty::adjustment::PointerCoercion;
 use rustc_middle::ty::{
     self, CoroutineArgs, CoroutineArgsExt, EarlyBinder, GenericArgs, Ty, TyCtxt,
 };
@@ -710,6 +711,13 @@ fn build_call_shim<'tcx>(
     };
 
     let def_id = instance.def_id();
+
+    let rpitit_shim = if let ty::InstanceKind::ReifyShim(..) = instance {
+        tcx.return_position_impl_trait_in_trait_shim_data(def_id)
+    } else {
+        None
+    };
+
     let sig = tcx.fn_sig(def_id);
     let sig = sig.map_bound(|sig| tcx.instantiate_bound_regions_with_erased(sig));
 
@@ -765,9 +773,34 @@ fn build_call_shim<'tcx>(
     let mut local_decls = local_decls_for_sig(&sig, span);
     let source_info = SourceInfo::outermost(span);
 
+    let mut destination = Place::return_place();
+    if let Some((rpitit_def_id, fn_args)) = rpitit_shim {
+        let rpitit_args =
+            fn_args.instantiate_identity().extend_to(tcx, rpitit_def_id, |param, _| {
+                match param.kind {
+                    ty::GenericParamDefKind::Lifetime => tcx.lifetimes.re_erased.into(),
+                    ty::GenericParamDefKind::Type { .. }
+                    | ty::GenericParamDefKind::Const { .. } => {
+                        unreachable!("rpitit should have no addition ty/ct")
+                    }
+                }
+            });
+        let dyn_star_ty = Ty::new_dynamic(
+            tcx,
+            tcx.item_bounds_to_existential_predicates(rpitit_def_id, rpitit_args),
+            tcx.lifetimes.re_erased,
+            ty::DynStar,
+        );
+        destination = local_decls.push(local_decls[RETURN_PLACE].clone()).into();
+        local_decls[RETURN_PLACE].ty = dyn_star_ty;
+        let mut inputs_and_output = sig.inputs_and_output.to_vec();
+        *inputs_and_output.last_mut().unwrap() = dyn_star_ty;
+        sig.inputs_and_output = tcx.mk_type_list(&inputs_and_output);
+    }
+
     let rcvr_place = || {
         assert!(rcvr_adjustment.is_some());
-        Place::from(Local::new(1 + 0))
+        Place::from(Local::new(1))
     };
     let mut statements = vec![];
 
@@ -854,7 +887,7 @@ fn build_call_shim<'tcx>(
         TerminatorKind::Call {
             func: callee,
             args,
-            destination: Place::return_place(),
+            destination,
             target: Some(BasicBlock::new(1)),
             unwind: if let Some(Adjustment::RefMut) = rcvr_adjustment {
                 UnwindAction::Cleanup(BasicBlock::new(3))
@@ -882,7 +915,24 @@ fn build_call_shim<'tcx>(
         );
     }
     // BB #1/#2 - return
-    block(&mut blocks, vec![], TerminatorKind::Return, false);
+    // NOTE: If this is an RPITIT in dyn, we also want to coerce
+    // the return type of the function into a `dyn*`.
+    let stmts = if rpitit_shim.is_some() {
+        vec![Statement {
+            source_info,
+            kind: StatementKind::Assign(Box::new((
+                Place::return_place(),
+                Rvalue::Cast(
+                    CastKind::PointerCoercion(PointerCoercion::DynStar, CoercionSource::Implicit),
+                    Operand::Move(destination),
+                    sig.output(),
+                ),
+            ))),
+        }]
+    } else {
+        vec![]
+    };
+    block(&mut blocks, stmts, TerminatorKind::Return, false);
     if let Some(Adjustment::RefMut) = rcvr_adjustment {
         // BB #3 - drop if closure panics
         block(

compiler/rustc_monomorphize/src/lib.rs

@@ -40,7 +40,10 @@ fn custom_coerce_unsize_info<'tcx>(
             ..
         })) => Ok(tcx.coerce_unsized_info(impl_def_id)?.custom_kind.unwrap()),
         impl_source => {
-            bug!("invalid `CoerceUnsized` impl_source: {:?}", impl_source);
+            bug!(
+                "invalid `CoerceUnsized` from {source_ty} to {target_ty}: impl_source: {:?}",
+                impl_source
+            );
         }
     }
 }

compiler/rustc_trait_selection/src/traits/project.rs

@@ -7,8 +7,8 @@ use rustc_data_structures::stack::ensure_sufficient_stack;
 use rustc_errors::ErrorGuaranteed;
 use rustc_hir::def::DefKind;
 use rustc_hir::lang_items::LangItem;
-use rustc_infer::infer::DefineOpaqueTypes;
 use rustc_infer::infer::resolve::OpportunisticRegionResolver;
+use rustc_infer::infer::{DefineOpaqueTypes, RegionVariableOrigin};
 use rustc_infer::traits::{ObligationCauseCode, PredicateObligations};
 pub use rustc_middle::traits::Reveal;
 use rustc_middle::traits::select::OverflowError;
@@ -19,6 +19,7 @@ use rustc_middle::ty::visit::{MaxUniverse, TypeVisitable, TypeVisitableExt};
 use rustc_middle::ty::{self, Term, Ty, TyCtxt, TypingMode, Upcast};
 use rustc_middle::{bug, span_bug};
 use rustc_span::symbol::sym;
+use thin_vec::thin_vec;
 use tracing::{debug, instrument};
 
 use super::{
@@ -62,6 +63,9 @@ enum ProjectionCandidate<'tcx> {
     /// Bounds specified on an object type
     Object(ty::PolyProjectionPredicate<'tcx>),
 
+    /// Built-in bound for a dyn async fn in trait
+    ObjectRpitit,
+
     /// From an "impl" (or a "pseudo-impl" returned by select)
     Select(Selection<'tcx>),
 }
@@ -852,6 +856,17 @@ fn assemble_candidates_from_object_ty<'cx, 'tcx>(
         env_predicates,
         false,
     );
+
+    // `dyn Trait` automagically project their AFITs to `dyn* Future`.
+    if tcx.is_impl_trait_in_trait(obligation.predicate.def_id)
+        && let Some(out_trait_def_id) = data.principal_def_id()
+        && let rpitit_trait_def_id = tcx.parent(obligation.predicate.def_id)
+        && tcx
+            .supertrait_def_ids(out_trait_def_id)
+            .any(|trait_def_id| trait_def_id == rpitit_trait_def_id)
+    {
+        candidate_set.push_candidate(ProjectionCandidate::ObjectRpitit);
+    }
 }
 
 #[instrument(
@@ -1270,6 +1285,8 @@ fn confirm_candidate<'cx, 'tcx>(
         ProjectionCandidate::Select(impl_source) => {
             confirm_select_candidate(selcx, obligation, impl_source)
         }
+
+        ProjectionCandidate::ObjectRpitit => confirm_object_rpitit_candidate(selcx, obligation),
     };
 
     // When checking for cycle during evaluation, we compare predicates with
@@ -2057,6 +2074,45 @@ fn confirm_impl_candidate<'cx, 'tcx>(
     }
 }
 
+fn confirm_object_rpitit_candidate<'cx, 'tcx>(
+    selcx: &mut SelectionContext<'cx, 'tcx>,
+    obligation: &ProjectionTermObligation<'tcx>,
+) -> Progress<'tcx> {
+    let tcx = selcx.tcx();
+    let mut obligations = thin_vec![];
+
+    // Compute an intersection lifetime for all the input components of this GAT.
+    let intersection =
+        selcx.infcx.next_region_var(RegionVariableOrigin::MiscVariable(obligation.cause.span));
+    for component in obligation.predicate.args {
+        match component.unpack() {
+            ty::GenericArgKind::Lifetime(lt) => {
+                obligations.push(obligation.with(tcx, ty::OutlivesPredicate(lt, intersection)));
+            }
+            ty::GenericArgKind::Type(ty) => {
+                obligations.push(obligation.with(tcx, ty::OutlivesPredicate(ty, intersection)));
+            }
+            ty::GenericArgKind::Const(_ct) => {
+                // Consts have no outlives...
+            }
+        }
+    }
+
+    Progress {
+        term: Ty::new_dynamic(
+            tcx,
+            tcx.item_bounds_to_existential_predicates(
+                obligation.predicate.def_id,
+                obligation.predicate.args,
+            ),
+            intersection,
+            ty::DynStar,
+        )
+        .into(),
+        obligations,
+    }
+}
+
 // Get obligations corresponding to the predicates from the where-clause of the
 // associated type itself.
 fn assoc_ty_own_obligations<'cx, 'tcx>(